Description
I am creating a collection as follows:
self.collections[self.test_case_collection_name] = self.client.collections.create(
name=self.test_case_collection_name,
vectorizer_config=Configure.Vectorizer.text2vec_openai(
model=self.embedding_model_name,
base_url=self.embedding_model_base_url),
properties=[
wvc.config.Property(
name="test_case_id",
data_type=wvc.config.DataType.TEXT,
skip_vectorization=True,
index_filterable=True,
),
wvc.config.Property(
name="test_case_content",
data_type=wvc.config.DataType.TEXT,
),
],
)
where embedding_model_base_url is a deployment endpoint for an inference service running on the same K8s cluster this Weaviate deployment is running on.
I then have the following function (serving as a wrapper for the built-in Weaviate insert functionality):
def insert_items(
self,
collection_type: CollectionType,
items: Union[str, List[str]],
item_ids: Union[str, List[str]],
item_intents: Union[str, List[str]] = None,
additional_properties: Dict[str, Any] = None
) -> None:
items = as_list_str(items)
item_ids = as_list_str(item_ids)
if item_intents is not None:
item_intents = as_list_str(item_intents)
if len(items) != len(item_intents):
logger.error(
f"Unable to insert items as the number of items differs from the number of item intents")
return
if len(items) != len(item_ids):
logger.error(
f"Unable to insert items as the number of items differs from the number of item IDs")
return
if collection_type == CollectionType.TEST_CASES:
collection_name = self.db_name + "_test_cases"
content_field = "test_case_content"
id_field = "test_case_id"
elif collection_type == CollectionType.TEST_PLANS:
collection_name = self.db_name + "_test_plans"
content_field = "test_plan_content"
id_field = "test_plan_id"
elif collection_type == CollectionType.TEST_SETS:
collection_name = self.db_name + "_test_sets"
content_field = "test_set_content"
id_field = "test_set_id"
intent_field = "test_set_intent"
else:
logger.error(f"Unknown collection type: {collection_type}")
return
collection = self.collections.get(collection_name)
if collection is None:
logger.error(f"Collection {collection_name} not found")
return
with collection.batch.dynamic() as batch:
for i, (item, item_id) in enumerate(zip(items, item_ids)):
try:
properties = {
id_field: item_id,
content_field: item
}
if collection_type == CollectionType.TEST_SETS and item_intents is not None:
properties[intent_field] = item_intents[i]
if additional_properties:
properties.update(additional_properties)
batch.add_object(properties=properties)
if batch.number_errors > 0:
raise Exception("Batch errors detected")
except Exception as e:
failed_objects = collection.batch.failed_objects
logger.error(f"Failed objects: {failed_objects}, Error: {str(e)}")
break
logger.info(f"{len(items)} item(s) added to {collection_name} collection")
I then try to use the insert_items function to add approximately 3000 records to the database. Upon initial startup of the Weaviate server, it appears that the insertion works when a fresh collection is instantiated. I can observe the external logs of the inference server and see that requests are being successfully processed.
However, at a certain point before the ~3000 records are inserted, the batches start to fail and I get the following error the client:
2025-04-28 17:11:59,690 - weaviate-client - ERROR - {'message': 'Failed to send all objects in a batch of 144', 'error': "WeaviateBatchError('Query call with protocol GRPC batch failed with message Deadline Exceeded.')"}
2025-04-28 17:11:59,691 - weaviate-client - ERROR - {'message': 'Failed to send 144 objects in a batch of 144. Please inspect client.batch.failed_objects or collection.batch.failed_objects for the failed objects.'}
2025-04-28 17:12:03,152 - weaviate-client - ERROR - {'message': 'Failed to send all objects in a batch of 192', 'error': "WeaviateBatchError('Query call with protocol GRPC batch failed with message Deadline Exceeded.')"}
2025-04-28 17:12:03,153 - weaviate-client - ERROR - {'message': 'Failed to send 192 objects in a batch of 192. Please inspect client.batch.failed_objects or collection.batch.failed_objects for the failed objects.'}
2025-04-28 17:12:03,168 - vectordb - ERROR - Failed objects: [], Error: Batch errors detected
2025-04-28 17:12:10,167 - weaviate-client - ERROR - {'message': 'Failed to send all objects in a batch of 192', 'error': "WeaviateBatchError('Query call with protocol GRPC batch failed with message Deadline Exceeded.')"}
2025-04-28 17:12:10,167 - weaviate-client - ERROR - {'message': 'Failed to send 192 objects in a batch of 192. Please inspect client.batch.failed_objects or collection.batch.failed_objects for the failed objects.'}
2025-04-28 17:15:03,181 - weaviate-client - ERROR - {'message': 'Failed to send all objects in a batch of 192', 'error': "WeaviateBatchError('Query call with protocol GRPC batch failed with message Deadline Exceeded.')"}
2025-04-28 17:15:03,183 - weaviate-client - ERROR - {'message': 'Failed to send 192 objects in a batch of 192. Please inspect client.batch.failed_objects or collection.batch.failed_objects for the failed objects.'}
2025-04-28 17:18:03,201 - weaviate-client - ERROR - {'message': 'Failed to send all objects in a batch of 48', 'error': "WeaviateBatchError('Query call with protocol GRPC batch failed with message Deadline Exceeded.')"}
2025-04-28 17:18:03,202 - weaviate-client - ERROR - {'message': 'Failed to send 48 objects in a batch of 48. Please inspect client.batch.failed_objects or collection.batch.failed_objects for the failed objects.'}
Note: When I uninstall Weaviate and re-install (using Helm and including deleting the underlying data volume), I can repeat the above process and add at least some items to the DB before getting an error again.
I see no particular errors in the server logs.
This is strange, because it is clearly not an issue with my inference server (as I can continue to make unrelated requests to the inference server after Weaviate stops working). So the issue is something along the lines of “the Weaviate server cannot reach (at all) the inference server via the provided endpoint.” This is quite unusual since the deployment begins seemingly fine, but only halts after some time. I think it is also worth mentioning that I don’t think this is a data size / bandwidth problem as A) the amount of data being vectorized is not larg, B) the node the server is deployed on has ample resources, and C) once the server stops working with inserts, I cannot insert any entries, not even a single one.
What I find even more strange, is that the following function (which also uses the vectorizer, of course) works after the insert function seizes to:
def query(self, query: str, collection_type: CollectionType, limit: int = 5, target_vector: str = "content") -> Dict[str, Any]:
if collection_type == CollectionType.TEST_CASES:
collection_name = self.test_case_collection_name
id_field = "test_case_id"
content_field = "test_case_content"
vector_name = None # No named vectors for test cases
elif collection_type == CollectionType.TEST_PLANS:
collection_name = self.test_plan_collection_name
id_field = "test_plan_id"
content_field = "test_plan_content"
vector_name = None # No named vectors for test plans
elif collection_type == CollectionType.TEST_SETS:
collection_name = self.test_set_collection_name
id_field = "test_set_id"
content_field = "test_set_content"
# Choose the appropriate named vector based on target_vector parameter
vector_name = "content_vector" if target_vector == "content" else "intent_vector"
intent_field = "test_set_intent"
else:
logger.error(f"Unknown collection type: {collection_type}")
return
group_by = wq.GroupBy(
prop=id_field,
objects_per_group=1,
number_of_groups=limit,
)
collection = self.collections.get(collection_name)
# For collections with named vectors, we need to specify which vector to use
if vector_name:
response = collection.query.hybrid(
query=query,
target_vector=vector_name,
group_by=group_by,
return_metadata=wq.MetadataQuery(score=True, distance=True),
)
else:
# For collections without named vectors, use the default query
response = collection.query.hybrid(
query=query,
group_by=group_by,
return_metadata=wq.MetadataQuery(score=True, distance=True),
)
output = {id_field: [], content_field: []}
# Add intent field to output if we're querying test sets
if collection_type == CollectionType.TEST_SETS:
output[intent_field] = []
for grp_name, grp_content in response.groups.items():
item_id = grp_content.objects[0].properties[id_field]
item_content = grp_content.objects[0].properties.get(content_field)
output[id_field].append(item_id)
output[content_field].append(item_content)
if collection_type == CollectionType.TEST_SETS:
item_intent = grp_content.objects[0].properties.get(intent_field)
output[intent_field].append(item_intent)
return output
Server Setup Information
- Weaviate Server Version: 1.30.0
- Deployment Method: k8s + Helm
- Multi Node? No Number of Running Nodes: 1
- Client Language and Version: go/1.22.0
- Multitenancy?: No
