[Question] search status code: 401

Support
1

Hi,

I’m running a 5-node Weaviate cluster on k8s. The node image version is semitechnologies/weaviate:1.26.3.

I have a collection called “Chunks” (see config below). After I’ve added a property to the collection I’m getting the following error from the cluster when I try to read or write data objects:

“msg:search index sources code:500 err:remote shard object search fdS9BqUGyiH9: status code: 401, error:”

This error occurs even when I create a new collection with a different schema and try writing data into it. Can you please suggest a solution?

Collection config -


{
    "class": "Chunks",
    "invertedIndexConfig": {
        "bm25": {
            "b": 0.75,
            "k1": 1.2
        },
        "cleanupIntervalSeconds": 60,
        "indexNullState": false,
        "indexPropertyLength": false,
        "indexTimestamps": false,
        "stopwords": {
            "preset": "en"
        }
    },
    "moduleConfig": {},
    "multiTenancyConfig": {
        "autoTenantActivation": false,
        "autoTenantCreation": false,
        "enabled": false
    },
    "properties": [
        {
            "dataType": [
                "uuid"
            ],
            "indexFilterable": true,
            "indexRangeFilters": false,
            "indexSearchable": false,
            "moduleConfig": {
                "none": {}
            },
            "name": "source_id",
            "tokenization": null
        },
        {
            "dataType": [
                "uuid"
            ],
            "indexFilterable": true,
            "indexRangeFilters": false,
            "indexSearchable": false,
            "moduleConfig": {
                "none": {}
            },
            "name": "groupspace",
            "tokenization": null
        },
        {
            "dataType": [
                "uuid"
            ],
            "indexFilterable": true,
            "indexRangeFilters": false,
            "indexSearchable": false,
            "moduleConfig": {
                "none": {}
            },
            "name": "source_chunk_id",
            "tokenization": null
        },
        {
            "dataType": [
                "uuid"
            ],
            "indexFilterable": true,
            "indexRangeFilters": false,
            "indexSearchable": false,
            "moduleConfig": {
                "none": {}
            },
            "name": "target_chunk_id",
            "tokenization": null
        },
        {
            "dataType": [
                "text"
            ],
            "indexFilterable": true,
            "indexRangeFilters": false,
            "indexSearchable": true,
            "moduleConfig": {
                "none": {}
            },
            "name": "entry_type",
            "tokenization": "word"
        },
        {
            "dataType": [
                "text"
            ],
            "indexFilterable": true,
            "indexRangeFilters": false,
            "indexSearchable": true,
            "moduleConfig": {
                "none": {}
            },
            "name": "chunk_index",
            "tokenization": "word"
        },
        {
            "dataType": [
                "uuid"
            ],
            "description": "This property was generated by Weaviate\u0027s auto-schema feature on Fri Oct 11 03:47:28 2024",
            "indexFilterable": true,
            "indexRangeFilters": false,
            "indexSearchable": false,
            "moduleConfig": {
                "none": {}
            },
            "name": "paper_id",
            "tokenization": null
        }
    ],
    "replicationConfig": {
        "asyncEnabled": false,
        "factor": 1
    },
    "shardingConfig": {
        "actualCount": 5,
        "actualVirtualCount": 640,
        "desiredCount": 5,
        "desiredVirtualCount": 640,
        "function": "murmur3",
        "key": "_id",
        "strategy": "hash",
        "virtualPerPhysical": 128
    },
    "vectorIndexConfig": {
        "distanceMetric": "cosine",
        "flat": {
            "distanceMetric": "cosine",
            "vectorCacheMaxObjects": 1000000000000
        },
        "hnsw": {
            "cleanupIntervalSeconds": 300,
            "distanceMetric": "cosine",
            "dynamicEfFactor": 8,
            "dynamicEfMax": 500,
            "dynamicEfMin": 100,
            "ef": -1,
            "efConstruction": 128,
            "flatSearchCutoff": 40000,
            "maxConnections": 32,
            "skip": false,
            "vectorCacheMaxObjects": 1000000000000
        },
        "threshold": 10000
    },
    "vectorIndexType": "dynamic",
    "vectorizer": "none"
}
2

Hi!

Where do you see this error?

What are the errors you get both from server and from client?

3

Hi,

I see these errors in pod logs and from Python client responses.
I have 5 pods, named weaviate-*.

This is the log from weaviate-0 pod:

{"action":"requests_total","api":"rest","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","class_name":"","error":"node: weaviate-3: unexpected status code 401 ()","level":"error","msg":"unexpected error","query_type":"nodes","time":"2024-10-15T05:09:37Z"}

When I look at weaviate-3 logs:

{"action":"requests_total","api":"rest","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","class_name":"","error":"list objects: search index sources: remote shard object search TRlmZZlax6LZ: status code: 401, error: ","level":"error","msg":"unexpected error","query_type":"objects","time":"2024-10-15T04:53:49Z"}

when I try to call the REST API /cluster/statistics I get:

{
  "error": [
    {
      "message": "node: weaviate-3: unexpected status code 401 ()"
    }
  ]
}