Hey, great to get such a fast response!
- Your setup and configuration in Kubernetes
weaviate.yaml
apiVersion: v1
kind: Service
metadata:
name: weaviate-headless
namespace: weaviate
labels:
app: weaviate
spec:
clusterIP: None
selector:
app: weaviate
ports:
- name: http
port: 8080
- name: grpc
port: 50051
- name: metrics
port: 2112
# ADD these so pods can talk raft/gossip on 7100 and 7101
- name: gossip
port: 7100
- name: raft
port: 7101
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: weaviate
namespace: weaviate
labels:
app: weaviate
spec:
serviceName: weaviate-headless
replicas: 3
selector:
matchLabels:
app: weaviate
template:
metadata:
labels:
app: weaviate
spec:
containers:
- name: weaviate
image: semitechnologies/weaviate:1.30.2
args:
- --host
- 0.0.0.0
- --port
- "8080"
- --scheme
- http
- --read-timeout=60s
- --write-timeout=120s
ports:
- containerPort: 8080
name: http
- containerPort: 50051
name: grpc
- containerPort: 2112
name: metrics
# ADD these so the pod actually opens 7100 and 7101
- containerPort: 7100
name: gossip
- containerPort: 7101
name: raft
env:
# Async indexing (optional but helpful for bulk imports)
- name: ASYNC_INDEXING
value: "true"
# Auth settings
- name: AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED
value: "false"
- name: AUTHENTICATION_APIKEY_ENABLED
value: "true"
- name: AUTHENTICATION_APIKEY_USERS
value: ""
- name: AUTHENTICATION_APIKEY_ALLOWED_KEYS
value: ""
- name: PERSISTENCE_LSM_ACCESS_STRATEGY
value: "mmap"
# Enabled modules
- name: ENABLE_MODULES
value: "text2vec-cohere,text2vec-huggingface,text2vec-palm,text2vec-openai,generative-openai,generative-cohere,generative-palm,ref2vec-centroid,reranker-cohere,qna-openai"
- name: CLUSTER_HOSTNAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: CLUSTER_GOSSIP_BIND_PORT
value: "7100"
- name: CLUSTER_DATA_BIND_PORT
value: "7101"
- name: CLUSTER_JOIN
value: "weaviate-headless.weaviate.svc.cluster.local."
- name: PERSISTENCE_DATA_PATH
value: "/var/lib/weaviate"
resources:
requests:
cpu: "2"
memory: "16Gi"
limits:
cpu: "3"
memory: "24Gi"
volumeMounts:
- name: weaviate-data
mountPath: /var/lib/weaviate
volumeClaimTemplates:
- metadata:
name: weaviate-data
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: "premium-rwo"
resources:
requests:
storage: 500Gi
weaviate-loadbalancer.yaml
apiVersion: v1
kind: Service
metadata:
name: weaviate-loadbalancer
namespace: weaviate
labels:
app: weaviate
spec:
type: LoadBalancer
selector:
app: weaviate
ports:
- name: http
port: 80
targetPort: 8080
- name: grpc
port: 50051
targetPort: 50051
sysctl-patch.yaml
# sysctl-patch.yaml (updated)
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: sysctl-patch
namespace: weaviate
spec:
selector:
matchLabels:
app: sysctl-patch
template:
metadata:
labels:
app: sysctl-patch
spec:
hostPID: true
containers:
- name: sysctl
image: busybox:1.36
securityContext:
privileged: true
command:
- sh
- -c
- |
sysctl -w vm.max_map_count=262144
# (optional) bump file-max too:
# sysctl -w fs.file-max=8388608
# keep this container alive so the DaemonSet Pod isn't restarted
tail -f /dev/null
nodeSelector:
kubernetes.io/os: linux
tolerations:
- operator: Exists
{"classes":[{"class":"Memory","invertedIndexConfig":{"bm25":{"b":0.75,"k1":1.2},"cleanupIntervalSeconds":60,"stopwords":{"additions":null,"preset":"en","removals":null}},"moduleConfig":{"text2vec-cohere":{"baseUrl":"https://api.cohere.ai","model":"embed-multilingual-v3.0","truncate":"END","vectorizeClassName":true}},"multiTenancyConfig":{"autoTenantActivation":true,"autoTenantCreation":true,"enabled":true},"properties":[{"dataType":["geoCoordinates"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"moduleConfig":{"text2vec-cohere":{"skip":false,"vectorizePropertyName":true}},"name":"coordinates"},{"dataType":["date"],"indexFilterable":true,"indexRangeFilters":true,"indexSearchable":false,"moduleConfig":{"text2vec-cohere":{"skip":true,"vectorizePropertyName":true}},"name":"endTime"},{"dataType":["MemoryChunk"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"moduleConfig":{"text2vec-cohere":{"skip":false,"vectorizePropertyName":false}},"name":"hasChunks"},{"dataType":["text[]"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"moduleConfig":{"text2vec-cohere":{"skip":true,"vectorizePropertyName":true}},"name":"keywords","tokenization":"word"},{"dataType":["object"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"moduleConfig":{"text2vec-cohere":{"skip":true,"vectorizePropertyName":true}},"name":"location","nestedProperties":[{"dataType":["text"],"description":"Formatted address","indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"name":"address","tokenization":"word"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"name":"placeId","tokenization":"word"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"name":"country","tokenization":"word"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"name":"zip","tokenization":"word"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"name":"street","tokenization":"word"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"name":"streetNumber","tokenization":"word"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"name":"city","tokenization":"word"},{"dataType":["text"],"description":"This nested property was generated by Weaviate's auto-schema feature on Wed Apr 2 15:15:20 2025","indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"name":"street_number","tokenization":"word"},{"dataType":["text"],"description":"This nested property was generated by Weaviate's auto-schema feature on Wed Apr 2 15:15:20 2025","indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"name":"place_id","tokenization":"word"}]},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"moduleConfig":{"text2vec-cohere":{"skip":false,"vectorizePropertyName":true}},"name":"longSummary","tokenization":"lowercase"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"moduleConfig":{"text2vec-cohere":{"skip":true,"vectorizePropertyName":true}},"name":"memoryId","tokenization":"field"},{"dataType":["object[]"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"moduleConfig":{"text2vec-cohere":{"skip":true,"vectorizePropertyName":true}},"name":"quotes","nestedProperties":[{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"name":"text","tokenization":"word"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"name":"context","tokenization":"word"}]},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"moduleConfig":{"text2vec-cohere":{"skip":false,"vectorizePropertyName":true}},"name":"shortSummary","tokenization":"lowercase"},{"dataType":["date"],"indexFilterable":true,"indexRangeFilters":true,"indexSearchable":false,"moduleConfig":{"text2vec-cohere":{"skip":true,"vectorizePropertyName":true}},"name":"startTime"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"moduleConfig":{"text2vec-cohere":{"skip":false,"vectorizePropertyName":true}},"name":"title","tokenization":"lowercase"},{"dataType":["object[]"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"moduleConfig":{"text2vec-cohere":{"skip":true,"vectorizePropertyName":true}},"name":"transcriptions","nestedProperties":[{"dataType":["date"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"name":"created"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"name":"text","tokenization":"word"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"name":"speaker","tokenization":"word"},{"dataType":["number"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"name":"start"},{"dataType":["number"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"name":"end"},{"dataType":["number"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"name":"seq_number"},{"dataType":["date"],"description":"This nested property was generated by Weaviate's auto-schema feature on Wed Apr 30 08:42:50 2025","indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"name":"process_timestamp"},{"dataType":["text"],"description":"This nested property was generated by Weaviate's auto-schema feature on Wed Apr 30 08:42:50 2025","indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"name":"userId","tokenization":"word"},{"dataType":["date"],"description":"This nested property was generated by Weaviate's auto-schema feature on Wed Apr 30 08:42:50 2025","indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"name":"createdAt"},{"dataType":["date"],"description":"This nested property was generated by Weaviate's auto-schema feature on Wed Apr 30 08:42:50 2025","indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"name":"timestamp"}]}],"replicationConfig":{"asyncEnabled":true,"deletionStrategy":"NoAutomatedResolution","factor":1},"shardingConfig":{"actualCount":0,"actualVirtualCount":0,"desiredCount":0,"desiredVirtualCount":0,"function":"","key":"","strategy":"","virtualPerPhysical":0},"vectorIndexConfig":{"bq":{"enabled":false},"cleanupIntervalSeconds":300,"distance":"cosine","dynamicEfFactor":8,"dynamicEfMax":500,"dynamicEfMin":100,"ef":-1,"efConstruction":128,"filterStrategy":"sweeping","flatSearchCutoff":40000,"maxConnections":32,"multivector":{"aggregation":"maxSim","enabled":false},"pq":{"bitCompression":false,"centroids":256,"enabled":false,"encoder":{"distribution":"log-normal","type":"kmeans"},"segments":0,"trainingLimit":100000},"skip":false,"sq":{"enabled":false,"rescoreLimit":20,"trainingLimit":100000},"vectorCacheMaxObjects":1000000000000},"vectorIndexType":"hnsw","vectorizer":"text2vec-cohere"},{"class":"MemoryChunk","invertedIndexConfig":{"bm25":{"b":0.75,"k1":1.2},"cleanupIntervalSeconds":60,"stopwords":{"additions":null,"preset":"en","removals":null}},"moduleConfig":{"text2vec-cohere":{"baseUrl":"https://api.cohere.ai","model":"embed-multilingual-v3.0","truncate":"END","vectorizeClassName":true}},"multiTenancyConfig":{"autoTenantActivation":true,"autoTenantCreation":true,"enabled":true},"properties":[{"dataType":["Memory"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"moduleConfig":{"text2vec-cohere":{"skip":false,"vectorizePropertyName":false}},"name":"belongsToMemory"},{"dataType":["text"],"description":"200-token window of transcription text","indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"moduleConfig":{"text2vec-cohere":{"skip":false,"vectorizePropertyName":true}},"name":"chunk","tokenization":"lowercase"},{"dataType":["int"],"description":"Position of chunk in the original memory","indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"moduleConfig":{"text2vec-cohere":{"skip":true,"vectorizePropertyName":true}},"name":"chunkIndex"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"moduleConfig":{"text2vec-cohere":{"skip":true,"vectorizePropertyName":true}},"name":"memoryId","tokenization":"field"}],"replicationConfig":{"asyncEnabled":true,"deletionStrategy":"NoAutomatedResolution","factor":1},"shardingConfig":{"actualCount":0,"actualVirtualCount":0,"desiredCount":0,"desiredVirtualCount":0,"function":"","key":"","strategy":"","virtualPerPhysical":0},"vectorIndexConfig":{"bq":{"enabled":false},"cleanupIntervalSeconds":300,"distance":"cosine","dynamicEfFactor":8,"dynamicEfMax":500,"dynamicEfMin":100,"ef":-1,"efConstruction":128,"filterStrategy":"sweeping","flatSearchCutoff":40000,"maxConnections":32,"multivector":{"aggregation":"maxSim","enabled":false},"pq":{"bitCompression":false,"centroids":256,"enabled":false,"encoder":{"distribution":"log-normal","type":"kmeans"},"segments":0,"trainingLimit":100000},"skip":false,"sq":{"enabled":false,"rescoreLimit":20,"trainingLimit":100000},"vectorCacheMaxObjects":1000000000000},"vectorIndexType":"hnsw","vectorizer":"text2vec-cohere"},{"class":"SupportArticles","invertedIndexConfig":{"bm25":{"b":0.75,"k1":1.2},"cleanupIntervalSeconds":60,"stopwords":{"additions":null,"preset":"en","removals":null}},"moduleConfig":{"text2vec-cohere":{"baseUrl":"https://api.cohere.ai","model":"embed-multilingual-v3.0","truncate":"END","vectorizeClassName":true}},"multiTenancyConfig":{"autoTenantActivation":false,"autoTenantCreation":false,"enabled":false},"properties":[{"dataType":["int"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"moduleConfig":{"text2vec-cohere":{"skip":true,"vectorizePropertyName":true}},"name":"articleId"},{"dataType":["int"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"moduleConfig":{"text2vec-cohere":{"skip":true,"vectorizePropertyName":true}},"name":"parentId"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"moduleConfig":{"text2vec-cohere":{"skip":false,"vectorizePropertyName":true}},"name":"title","tokenization":"word"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"moduleConfig":{"text2vec-cohere":{"skip":false,"vectorizePropertyName":true}},"name":"description","tokenization":"word"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":true,"moduleConfig":{"text2vec-cohere":{"skip":false,"vectorizePropertyName":true}},"name":"body","tokenization":"lowercase"},{"dataType":["int"],"description":"date of update in Unix timestamp format","indexFilterable":true,"indexRangeFilters":true,"indexSearchable":false,"moduleConfig":{"text2vec-cohere":{"skip":true,"vectorizePropertyName":true}},"name":"updatedAt"},{"dataType":["text"],"indexFilterable":true,"indexRangeFilters":false,"indexSearchable":false,"moduleConfig":{"text2vec-cohere":{"skip":true,"vectorizePropertyName":true}},"name":"url","tokenization":"word"}],"replicationConfig":{"asyncEnabled":false,"deletionStrategy":"NoAutomatedResolution","factor":1},"shardingConfig":{"actualCount":3,"actualVirtualCount":384,"desiredCount":3,"desiredVirtualCount":384,"function":"murmur3","key":"_id","strategy":"hash","virtualPerPhysical":128},"vectorIndexConfig":{"bq":{"enabled":false},"cleanupIntervalSeconds":300,"distance":"cosine","dynamicEfFactor":8,"dynamicEfMax":500,"dynamicEfMin":100,"ef":-1,"efConstruction":128,"filterStrategy":"sweeping","flatSearchCutoff":40000,"maxConnections":32,"multivector":{"aggregation":"maxSim","enabled":false},"pq":{"bitCompression":false,"centroids":256,"enabled":false,"encoder":{"distribution":"log-normal","type":"kmeans"},"segments":0,"trainingLimit":100000},"skip":false,"sq":{"enabled":false,"rescoreLimit":20,"trainingLimit":100000},"vectorCacheMaxObjects":1000000000000},"vectorIndexType":"hnsw","vectorizer":"text2vec-cohere"}]}
- Full server logs around the error
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"53Qt1er3zwMY384EzXv6o1TC6np1\": status code: 500, error: local index \"Memory\" not found: deadline exceeded for waiting for update: version got=303302 want=305981\n: context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:02:19Z"}
{"action":"tombstone_cleanup_begin","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class":"Memory","level":"info","msg":"class Memory: shard gBeohcrkpaRwFuIja4k5sLm77x43: starting tombstone cleanup","shard":"gBeohcrkpaRwFuIja4k5sLm77x43","time":"2025-05-01T17:03:12Z","tombstones_in_cycle":1,"tombstones_total":1}
{"action":"tombstone_cleanup_complete","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class":"Memory","duration":4822380,"level":"info","msg":"class Memory: shard gBeohcrkpaRwFuIja4k5sLm77x43: completed tombstone cleanup in 4.82238ms","shard":"gBeohcrkpaRwFuIja4k5sLm77x43","time":"2025-05-01T17:03:12Z","tombstones_in_cycle":1,"tombstones_total":1}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"RBnWvJCBWqgvG4NSupLYsnbCiQ52\": status code: 500, error: local index \"Memory\" not found: deadline exceeded for waiting for update: version got=303303 want=305981\n: context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:03:23Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"OxB3oUOUaMULOtuEbauCquPBilF3\": status code: 500, error: local index \"Memory\" not found: deadline exceeded for waiting for update: version got=303311 want=305981\n: context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:06:07Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"8Jstbdt1KsY8ivbXMVvObNrHBuF2\": connect: Post \"http://10.40.2.40:7101/indices/Memory/shards/8Jstbdt1KsY8ivbXMVvObNrHBuF2/objects?schema_version=305981\": context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:06:49Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"TQlZtHSc2Fh9smblg73gqhIzpDu1\": connect: Post \"http://10.40.2.40:7101/indices/Memory/shards/TQlZtHSc2Fh9smblg73gqhIzpDu1/objects?schema_version=305981\": context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:07:01Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"OxB3oUOUaMULOtuEbauCquPBilF3\": connect: Post \"http://10.40.1.25:7101/indices/Memory/shards/OxB3oUOUaMULOtuEbauCquPBilF3/objects?schema_version=305981\": context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:07:10Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"hdeKk0g6HBNZZAyxlMv7tRwm2Nh2\": status code: 500, error: local index \"Memory\" not found: deadline exceeded for waiting for update: version got=290746 want=305981\n: context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:07:13Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"8Jstbdt1KsY8ivbXMVvObNrHBuF2\": status code: 500, error: local index \"Memory\" not found: deadline exceeded for waiting for update: version got=290748 want=305981\n: context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:07:55Z"}
{"action":"hnsw_prefill_cache_async","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","level":"info","msg":"not waiting for vector cache prefill, running in background","time":"2025-05-01T17:09:39Z","wait_for_cache_prefill":false}
{"build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","level":"info","msg":"Completed loading shard memory_DH8Xjz8THWWEq3SXgr7ieFzP68o1 in 72.78168ms","time":"2025-05-01T17:09:39Z"}
{"action":"hnsw_vector_cache_prefill","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","count":3000,"index_id":"main","level":"info","limit":1000000000000,"msg":"prefilled vector cache","time":"2025-05-01T17:09:39Z","took":15818395}
{"action":"hnsw_prefill_cache_async","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","level":"info","msg":"not waiting for vector cache prefill, running in background","time":"2025-05-01T17:09:40Z","wait_for_cache_prefill":false}
{"build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","level":"info","msg":"Completed loading shard memorychunk_DH8Xjz8THWWEq3SXgr7ieFzP68o1 in 41.999374ms","time":"2025-05-01T17:09:40Z"}
{"action":"hnsw_vector_cache_prefill","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","count":5000,"index_id":"main","level":"info","limit":1000000000000,"msg":"prefilled vector cache","time":"2025-05-01T17:09:40Z","took":91488716}
{"action":"tombstone_cleanup_begin","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class":"Memory","level":"info","msg":"class Memory: shard gBeohcrkpaRwFuIja4k5sLm77x43: starting tombstone cleanup","shard":"gBeohcrkpaRwFuIja4k5sLm77x43","time":"2025-05-01T17:13:12Z","tombstones_in_cycle":1,"tombstones_total":1}
{"action":"tombstone_cleanup_complete","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class":"Memory","duration":4669326,"level":"info","msg":"class Memory: shard gBeohcrkpaRwFuIja4k5sLm77x43: completed tombstone cleanup in 4.669326ms","shard":"gBeohcrkpaRwFuIja4k5sLm77x43","time":"2025-05-01T17:13:12Z","tombstones_in_cycle":1,"tombstones_total":1}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"EUbSwdjpxDgJThxU1xUnAJwkNWp2\": status code: 500, error: local index \"Memory\" not found: deadline exceeded for waiting for update: version got=303332 want=305981\n: context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:14:26Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"53Qt1er3zwMY384EzXv6o1TC6np1\": status code: 500, error: local index \"Memory\" not found: deadline exceeded for waiting for update: version got=303336 want=305981\n: context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:14:55Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"53Qt1er3zwMY384EzXv6o1TC6np1\": status code: 500, error: local index \"Memory\" not found: deadline exceeded for waiting for update: version got=303339 want=305981\n: context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:15:55Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"vLkNymwgRrMsnteQj5vOxQMyUP83\": connect: Post \"http://10.40.2.40:7101/indices/Memory/shards/vLkNymwgRrMsnteQj5vOxQMyUP83/objects?schema_version=305981\": context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:17:19Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"OrdRsvhAniVV97Mg0IJiQV5GsfH2\": connect: Post \"http://10.40.1.25:7101/indices/Memory/shards/OrdRsvhAniVV97Mg0IJiQV5GsfH2/objects?schema_version=305981\": context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:18:55Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"OrdRsvhAniVV97Mg0IJiQV5GsfH2\": connect: Post \"http://10.40.1.25:7101/indices/Memory/shards/OrdRsvhAniVV97Mg0IJiQV5GsfH2/objects?schema_version=305981\": context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:19:58Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"0msdQ552yLNi2BEabTWG7ja7HiQ2\": connect: Post \"http://10.40.1.25:7101/indices/Memory/shards/0msdQ552yLNi2BEabTWG7ja7HiQ2/objects?schema_version=305981\": context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:21:10Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"5sWm8zteK0YdPLWXzoutFH9T08R2\": status code: 500, error: local index \"Memory\" not found: deadline exceeded for waiting for update: version got=290789 want=305981\n: context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:21:57Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"RBnWvJCBWqgvG4NSupLYsnbCiQ52\": status code: 500, error: local index \"Memory\" not found: deadline exceeded for waiting for update: version got=303359 want=305981\n: context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:22:26Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"E75jYal5wjT5Tz7IgT5hsOQ1Ges2\": connect: Post \"http://10.40.1.25:7101/indices/Memory/shards/E75jYal5wjT5Tz7IgT5hsOQ1Ges2/objects?schema_version=305981\": context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:24:19Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"oeAOgkjDR6VXb6k6MRR53ypjZrE3\": connect: Post \"http://10.40.2.40:7101/indices/Memory/shards/oeAOgkjDR6VXb6k6MRR53ypjZrE3/objects?schema_version=305981\": context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:24:49Z"}
{"action":"requests_total","api":"rest","build_git_commit":"80dac5a","build_go_version":"go1.24.2","build_image_tag":"v1.30.2","build_wv_version":"1.30.2","class_name":"Memory","error":"put object: import into index memory: put remote object: shard=\"3VrR2LPfmNTSzJX2BoWt4Jddw442\": status code: 500, error: local index \"Memory\" not found: deadline exceeded for waiting for update: version got=290802 want=305981\n: context deadline exceeded","level":"error","msg":"unexpected error","query_type":"objects","time":"2025-05-01T17:27:03Z"}
-
The version of WeaviateDB and the client you’re using
version of WeaviateDB: 1.30.2
client: Python Client v4
weaviate.WeaviateAsyncClient(
connection_params=weaviate.connect.base.ConnectionParams.from_params(
http_host=cluster_url,
http_port=80,
http_secure=False,
grpc_host=cluster_url,
grpc_port=50051,
grpc_secure=False,
),
auth_client_secret=Auth.api_key(weaviate_api_key),
additional_config=AdditionalConfig(
connection=ConnectionConfig(
session_pool_connections=30,
session_pool_maxsize=200,
session_pool_max_retries=3,
),
timeout=Timeout(query=30, insert=120, init=2),
),
additional_headers={
“X-OpenAI-Api-Key”: openai_api_key,
“X-Cohere-Api-Key”: cohere_api_key,
},
skip_init_checks=True,
)
-
The specific operation that triggered the error (and any related code, if possible)
It started happening on it’s own, we tried restarting the pods, adding async replication nothing helped