Shard assignment to Nodes not happening

Support
1

Hello Weaviate

  • Weaviate Server Version: 1.30.3
  • Deployment Method: Docker
  • Multi Node? Number of Running Nodes: 7
  • Client Language and Version: python v4
  • Multitenancy?: yes

Any additional Information

Docker-compose file

The shards of a particular collection are not assigned to any node.

root@ip-172-31-10-150:/weaviate# curl -X PUT http://localhost:8080/v1/schema/Sitemanager_c_kafka_crm_deal/shards/0aRG1lMubATV -H “Content-Type: application/json” -d ‘{“status”: “READY”, “node”: “node7”}’

{“status”:“READY”}

root@ip-172-31-10-150:/weaviate# curl -X PUT http://localhost:8080/v1/schema/Sitemanager_c_kafka_crm_deal/shards/yw51HyoaoXb7 -H “Content-Type: application/json” -d ‘{“status”: “READY”, “node”: “node6”}’

{“status”:“READY”}

root@ip-172-31-10-150:/weaviate# curl -X PUT http://localhost:8080/v1/schema/Sitemanager_c_kafka_crm_deal/shards/nabMpP3UP5eU -H “Content-Type: application/json” -d ‘{“status”: “READY”, “node”: “node5”}’

{“status”:“READY”}

root@ip-172-31-10-150:/weaviate# curl -X PUT http://localhost:8080/v1/schema/Sitemanager_c_kafka_crm_deal/shards/j42oexiqtoCg -H “Content-Type: application/json” -d ‘{“status”: “READY”, “node”: “node4”}’

{“status”:“READY”}

root@ip-172-31-10-150:/weaviate# curl -X PUT http://localhost:8080/v1/schema/Sitemanager_c_kafka_crm_deal/shards/cVrTYGYZo1AY -H “Content-Type: application/json” -d ‘{“status”: “READY”, “node”: “node3”}’

{“status”:“READY”}

root@ip-172-31-10-150:/weaviate# curl -X PUT http://localhost:8080/v1/schema/Sitemanager_c_kafka_crm_deal/shards/XzLgAO7dE3RP -H “Content-Type: application/json” -d ‘{“status”: “READY”, “node”: “node2”}’

{“status”:“READY”}

root@ip-172-31-10-150:/weaviate# curl -X PUT http://localhost:8080/v1/schema/Sitemanager_c_kafka_crm_deal/shards/W5C5Vo4Eo0A6 -H “Content-Type: application/json” -d ‘{“status”: “READY”, “node”: “node1”}’

{“status”:“READY”}

root@ip-172-31-10-150:/weaviate# curl http://localhost:8080/v1/schema/Sitemanager_c_kafka_crm_deal/shards | jq

% Total % Received % Xferd Average Speed Time Time Time Current

Dload Upload Total Spent Left Speed

100 429 100 429 0 0 112k 0 --:–:-- --:–:-- --:–:-- 139k

[

{

“name”****: “0aRG1lMubATV”,

“status”****: “READY”,

“vectorQueueSize”****: 0

},

{

“name”****: “W5C5Vo4Eo0A6”,

“status”****: “READY”,

“vectorQueueSize”****: 0

},

{

“name”****: “XzLgAO7dE3RP”,

“status”****: “READY”,

“vectorQueueSize”****: 0

},

{

“name”****: “cVrTYGYZo1AY”,

“status”****: “READY”,

“vectorQueueSize”****: 0

},

{

“name”****: “j42oexiqtoCg”,

“status”****: “READY”,

“vectorQueueSize”****: 0

},

{

“name”****: “nabMpP3UP5eU”,

“status”****: “READY”,

“vectorQueueSize”****: 0

},

{

“name”****: “yw51HyoaoXb7”,

“status”****: “READY”,

“vectorQueueSize”****: 0

}

]

This is how the Weaviate class is created.
" collection = await client.collections.create(
name=CLASS_NAME,
description=“Invoice data from Kafka”,
properties=properties,
vectorizer_config=None,
sharding_config=Configure.sharding(
desired_count=7
)
)"

version: '3.8'

services:
  weaviate-node-1:
    init: true
    command: [--host, 0.0.0.0, --port, '8080', --scheme, http]
    image: cr.weaviate.io/semitechnologies/weaviate:1.30.3
    ports:
      - 8080:8080
      - 6050:6060
      - 50051:50051
    restart: on-failure:0
    volumes:
      - weaviate-node-1-data:/var/lib/weaviate
    environment:
      ENABLE_BATCHING: "true"
      BATCH_MAX_SIZE: "100"
      BATCH_MAX_DELAY: "50ms"
      LOG_LEVEL: 'info'
      ASYNC_INDEXING: 'true'
      BATCH_ENABLED: 'true'
      BATCH_QUEUE_TIMEOUT_REPLACE_EXISTING_MS: 2000
      BATCH_QUEUE_TIMEOUT_DELETE_MS: 2000
      BATCH_QUEUE_TIMEOUT_OBJECT_MS: 2000
      BATCH_TOKENS_PER_SECOND: 40000
      BATCH_MAX_PARALLEL_BATCHES: 32
      ASYNC_INDEXING_QUEUE_SIZE: 100000
      QUERY_DEFAULTS_LIMIT: 1000
      QUERY_MAXIMUM_RESULTS: 100000
      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
      ENABLE_API_BASED_MODULES: 'false'
      ENABLE_MODULES: ''
      DISABLE_MODULES: 'text2vec-contextionary,text2vec-transformers,text2vec-openai,text2vec-huggingface'
      CLUSTER_HOSTNAME: 'node1'
      CLUSTER_GOSSIP_BIND_PORT: '7100'
      CLUSTER_DATA_BIND_PORT: '7101'
      RAFT_JOIN: 'node1,node2,node3,node4,node5,node6,node7'
      RAFT_BOOTSTRAP_EXPECT: 7
    deploy:
      resources:
        limits:
          cpus: '3.0'
          memory: '6G'
    networks:
      - weaviate-net

  weaviate-node-2:
    init: true
    command: [--host, 0.0.0.0, --port, '8080', --scheme, http]
    image: cr.weaviate.io/semitechnologies/weaviate:1.30.3
    ports:
      - 8081:8080
      - 6051:6060
      - 50052:50051
    restart: on-failure:0
    volumes:
      - weaviate-node-2-data:/var/lib/weaviate
    environment:
      ENABLE_BATCHING: "true"
      BATCH_MAX_SIZE: "100"
      BATCH_MAX_DELAY: "50ms"      
      LOG_LEVEL: 'info'
      ASYNC_INDEXING: 'true'
      BATCH_ENABLED: 'true'
      BATCH_QUEUE_TIMEOUT_REPLACE_EXISTING_MS: 2000
      BATCH_QUEUE_TIMEOUT_DELETE_MS: 2000
      BATCH_QUEUE_TIMEOUT_OBJECT_MS: 2000
      BATCH_TOKENS_PER_SECOND: 40000
      BATCH_MAX_PARALLEL_BATCHES: 32
      ASYNC_INDEXING_QUEUE_SIZE: 100000
      QUERY_DEFAULTS_LIMIT: 1000
      QUERY_MAXIMUM_RESULTS: 100000
      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
      ENABLE_API_BASED_MODULES: 'false'
      ENABLE_MODULES: ''
      DISABLE_MODULES: 'text2vec-contextionary,text2vec-transformers,text2vec-openai,text2vec-huggingface'
      CLUSTER_HOSTNAME: 'node2'
      CLUSTER_GOSSIP_BIND_PORT: '7102'
      CLUSTER_DATA_BIND_PORT: '7103'
      CLUSTER_JOIN: 'weaviate-node-1:7100'
      RAFT_JOIN: 'node1,node2,node3,node4,node5,node6,node7'
      RAFT_BOOTSTRAP_EXPECT: 7
    deploy:
      resources:
        limits:
          cpus: '3.0'
          memory: '6G'
    networks:
      - weaviate-net

  weaviate-node-3:
    init: true
    command: [--host, 0.0.0.0, --port, '8080', --scheme, http]
    image: cr.weaviate.io/semitechnologies/weaviate:1.30.3
    ports:
      - 8082:8080
      - 6052:6060
      - 50053:50051
    restart: on-failure:0
    volumes:
      - weaviate-node-3-data:/var/lib/weaviate
    environment:
      LOG_LEVEL: 'info'
      ENABLE_BATCHING: "true"
      BATCH_MAX_SIZE: "100"
      BATCH_MAX_DELAY: "50ms"
      ASYNC_INDEXING: 'true'
      BATCH_ENABLED: 'true'
      BATCH_QUEUE_TIMEOUT_REPLACE_EXISTING_MS: 2000
      BATCH_QUEUE_TIMEOUT_DELETE_MS: 2000
      BATCH_QUEUE_TIMEOUT_OBJECT_MS: 2000
      BATCH_TOKENS_PER_SECOND: 40000
      BATCH_MAX_PARALLEL_BATCHES: 32
      ASYNC_INDEXING_QUEUE_SIZE: 100000
      QUERY_DEFAULTS_LIMIT: 1000
      QUERY_MAXIMUM_RESULTS: 100000
      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
      ENABLE_API_BASED_MODULES: 'false'
      ENABLE_MODULES: ''
      DISABLE_MODULES: 'text2vec-contextionary,text2vec-transformers,text2vec-openai,text2vec-huggingface'
      CLUSTER_HOSTNAME: 'node3'
      CLUSTER_GOSSIP_BIND_PORT: '7104'
      CLUSTER_DATA_BIND_PORT: '7105'
      CLUSTER_JOIN: 'weaviate-node-1:7100'
      RAFT_JOIN: 'node1,node2,node3,node4,node5,node6,node7'
      RAFT_BOOTSTRAP_EXPECT: 7
    deploy:
      resources:
        limits:
          cpus: '3.0'
          memory: '6G'
    networks:
      - weaviate-net

  weaviate-node-4:
    init: true
    command: [--host, 0.0.0.0, --port, '8080', --scheme, http]
    image: cr.weaviate.io/semitechnologies/weaviate:1.30.3
    ports:
      - 8083:8080
      - 6053:6060
      - 50054:50051
    restart: on-failure:0
    volumes:
      - weaviate-node-4-data:/var/lib/weaviate
    environment:
      LOG_LEVEL: 'info'
      ENABLE_BATCHING: "true"
      BATCH_MAX_SIZE: "100"
      BATCH_MAX_DELAY: "50ms"
      ASYNC_INDEXING: 'true'
      BATCH_ENABLED: 'true'
      BATCH_QUEUE_TIMEOUT_REPLACE_EXISTING_MS: 2000
      BATCH_QUEUE_TIMEOUT_DELETE_MS: 2000
      BATCH_QUEUE_TIMEOUT_OBJECT_MS: 2000
      BATCH_TOKENS_PER_SECOND: 40000
      BATCH_MAX_PARALLEL_BATCHES: 32
      ASYNC_INDEXING_QUEUE_SIZE: 100000
      QUERY_DEFAULTS_LIMIT: 1000
      QUERY_MAXIMUM_RESULTS: 100000
      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
      ENABLE_API_BASED_MODULES: 'false'
      ENABLE_MODULES: ''
      DISABLE_MODULES: 'text2vec-contextionary,text2vec-transformers,text2vec-openai,text2vec-huggingface'
      CLUSTER_HOSTNAME: 'node4'
      CLUSTER_GOSSIP_BIND_PORT: '7106'
      CLUSTER_DATA_BIND_PORT: '7107'
      CLUSTER_JOIN: 'weaviate-node-1:7100'
      RAFT_JOIN: 'node1,node2,node3,node4,node5,node6,node7'
      RAFT_BOOTSTRAP_EXPECT: 7
    deploy:
      resources:
        limits:
          cpus: '3.0'
          memory: '6G'
    networks:
      - weaviate-net

  weaviate-node-5:
    init: true
    command: [--host, 0.0.0.0, --port, '8080', --scheme, http]
    image: cr.weaviate.io/semitechnologies/weaviate:1.30.3
    ports:
      - 8084:8080
      - 6054:6060
      - 50055:50051
    restart: on-failure:0
    volumes:
      - weaviate-node-5-data:/var/lib/weaviate
    environment:
      LOG_LEVEL: 'info'
      ENABLE_BATCHING: "true"
      BATCH_MAX_SIZE: "100"
      BATCH_MAX_DELAY: "50ms"
      ASYNC_INDEXING: 'true'
      BATCH_ENABLED: 'true'
      BATCH_QUEUE_TIMEOUT_REPLACE_EXISTING_MS: 2000
      BATCH_QUEUE_TIMEOUT_DELETE_MS: 2000
      BATCH_QUEUE_TIMEOUT_OBJECT_MS: 2000
      BATCH_TOKENS_PER_SECOND: 40000
      BATCH_MAX_PARALLEL_BATCHES: 32
      ASYNC_INDEXING_QUEUE_SIZE: 100000
      QUERY_DEFAULTS_LIMIT: 1000
      QUERY_MAXIMUM_RESULTS: 100000
      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
      ENABLE_API_BASED_MODULES: 'false'
      ENABLE_MODULES: ''
      DISABLE_MODULES: 'text2vec-contextionary,text2vec-transformers,text2vec-openai,text2vec-huggingface'
      CLUSTER_HOSTNAME: 'node5'
      CLUSTER_GOSSIP_BIND_PORT: '7108'
      CLUSTER_DATA_BIND_PORT: '7109'
      CLUSTER_JOIN: 'weaviate-node-1:7100'
      RAFT_JOIN: 'node1,node2,node3,node4,node5,node6,node7'
      RAFT_BOOTSTRAP_EXPECT: 7
    deploy:
      resources:
        limits:
          cpus: '3.0'
          memory: '6G'
    networks:
      - weaviate-net

  weaviate-node-6:
    init: true
    command: [--host, 0.0.0.0, --port, '8080', --scheme, http]
    image: cr.weaviate.io/semitechnologies/weaviate:1.30.3
    ports:
      - 8085:8080
      - 6055:6060
      - 50056:50051
    restart: on-failure:0
    volumes:
      - weaviate-node-6-data:/var/lib/weaviate
    environment:
      LOG_LEVEL: 'info'
      ENABLE_BATCHING: "true"
      BATCH_MAX_SIZE: "100"
      BATCH_MAX_DELAY: "50ms"
      ASYNC_INDEXING: 'true'
      BATCH_ENABLED: 'true'
      BATCH_QUEUE_TIMEOUT_REPLACE_EXISTING_MS: 2000
      BATCH_QUEUE_TIMEOUT_DELETE_MS: 2000
      BATCH_QUEUE_TIMEOUT_OBJECT_MS: 2000
      BATCH_TOKENS_PER_SECOND: 40000
      BATCH_MAX_PARALLEL_BATCHES: 32
      ASYNC_INDEXING_QUEUE_SIZE: 100000
      QUERY_DEFAULTS_LIMIT: 1000
      QUERY_MAXIMUM_RESULTS: 100000
      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
      ENABLE_API_BASED_MODULES: 'false'
      ENABLE_MODULES: ''
      DISABLE_MODULES: 'text2vec-contextionary,text2vec-transformers,text2vec-openai,text2vec-huggingface'
      CLUSTER_HOSTNAME: 'node6'
      CLUSTER_GOSSIP_BIND_PORT: '7110'
      CLUSTER_DATA_BIND_PORT: '7111'
      CLUSTER_JOIN: 'weaviate-node-1:7100'
      RAFT_JOIN: 'node1,node2,node3,node4,node5,node6,node7'
      RAFT_BOOTSTRAP_EXPECT: 7
    deploy:
      resources:
        limits:
          cpus: '3.0'
          memory: '6G'
    networks:
      - weaviate-net


  weaviate-node-7:
    init: true
    command: [--host, 0.0.0.0, --port, '8080', --scheme, http]
    image: cr.weaviate.io/semitechnologies/weaviate:1.30.3
    ports:
      - 8086:8080
      - 6056:6060
      - 50057:50051
    restart: on-failure:0
    volumes:
      - weaviate-node-7-data:/var/lib/weaviate
    environment:
      LOG_LEVEL: 'info'
      ENABLE_BATCHING: "true"
      BATCH_MAX_SIZE: "100"
      BATCH_MAX_DELAY: "50ms"
      ASYNC_INDEXING: 'true'
      BATCH_ENABLED: 'true'
      BATCH_QUEUE_TIMEOUT_REPLACE_EXISTING_MS: 2000
      BATCH_QUEUE_TIMEOUT_DELETE_MS: 2000
      BATCH_QUEUE_TIMEOUT_OBJECT_MS: 2000
      BATCH_TOKENS_PER_SECOND: 40000
      BATCH_MAX_PARALLEL_BATCHES: 32
      ASYNC_INDEXING_QUEUE_SIZE: 100000
      QUERY_DEFAULTS_LIMIT: 1000
      QUERY_MAXIMUM_RESULTS: 100000
      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
      ENABLE_API_BASED_MODULES: 'false'
      ENABLE_MODULES: ''
      DISABLE_MODULES: 'text2vec-contextionary,text2vec-transformers,text2vec-openai,text2vec-huggingface'
      CLUSTER_HOSTNAME: 'node7'
      CLUSTER_GOSSIP_BIND_PORT: '7112'
      CLUSTER_DATA_BIND_PORT: '7113'
      CLUSTER_JOIN: 'weaviate-node-1:7100'
      RAFT_JOIN: 'node1,node2,node3,node4,node5,node6,node7'
      RAFT_BOOTSTRAP_EXPECT: 7
    deploy:
      resources:
        limits:
          cpus: '3.0'
          memory: '6G'
    networks:
      - weaviate-net


volumes:
  weaviate-node-1-data:
    driver: local
    driver_opts:
      type: none
      device: /mnt/weaviate/pod1
      o: bind
  weaviate-node-2-data:
    driver: local
    driver_opts:
      type: none
      device: /mnt/weaviate/pod2
      o: bind
  weaviate-node-3-data:
    driver: local
    driver_opts:
      type: none
      device: /mnt/weaviate/pod3
      o: bind
  weaviate-node-4-data:
    driver: local
    driver_opts:
      type: none
      device: /mnt/weaviate/pod4
      o: bind
  weaviate-node-5-data:
    driver: local
    driver_opts:
      type: none
      device: /mnt/weaviate/pod5
      o: bind
  weaviate-node-6-data:
    driver: local
    driver_opts:
      type: none
      device: /mnt/weaviate/pod6
      o: bind
  weaviate-node-7-data:
    driver: local
    driver_opts:
      type: none
      device: /mnt/weaviate/pod7
      o: bind

networks:
  weaviate-net:
    driver: bridge

And we face errors like “{“data”:{“Get”:{“Sitemanager_c_kafka_crm_company”:null}},“errors”:[{“locations”:[{“column”:9,“line”:1}],“message”:“explorer: list class: search: object search at index sitemanager_c_kafka_crm_company: remote shard object search rVF4btvz2nAJ: status code: 422, error: local index "Sitemanager_c_kafka_crm_company" not found\n”,“path”:[“Get”,“Sitemanager_c_kafka_crm_company”]}]}”

2

hi!

what is the output of your nodes api? Are you able to see all nodes as part of the clusteR?

3

Hi @DudaNogueira , this is the output of Nodes API:

root@ip:/weaviate# curl http://localhost:8080/v1/nodes | jq
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   866  100   866    0     0   722k      0 --:--:-- --:--:-- --:--:--  845k
{
  "nodes": [
    {
      "batchStats": {
        "ratePerSecond": 0
      },
      "gitHash": "ca35d6b",
      "name": "node1",
      "shards": null,
      "status": "HEALTHY",
      "version": "1.30.3"
    },
    {
      "batchStats": {
        "ratePerSecond": 0
      },
      "gitHash": "ca35d6b",
      "name": "node2",
      "shards": null,
      "status": "HEALTHY",
      "version": "1.30.3"
    },
    {
      "batchStats": {
        "ratePerSecond": 0
      },
      "gitHash": "ca35d6b",
      "name": "node3",
      "shards": null,
      "status": "HEALTHY",
      "version": "1.30.3"
    },
    {
      "batchStats": {
        "ratePerSecond": 0
      },
      "gitHash": "ca35d6b",
      "name": "node4",
      "shards": null,
      "status": "HEALTHY",
      "version": "1.30.3"
    },
    {
      "batchStats": {
        "ratePerSecond": 0
      },
      "gitHash": "ca35d6b",
      "name": "node5",
      "shards": null,
      "status": "HEALTHY",
      "version": "1.30.3"
    },
    {
      "batchStats": {
        "ratePerSecond": 0
      },
      "gitHash": "ca35d6b",
      "name": "node6",
      "shards": null,
      "status": "HEALTHY",
      "version": "1.30.3"
    },
    {
      "batchStats": {
        "ratePerSecond": 0
      },
      "gitHash": "ca35d6b",
      "name": "node7",
      "shards": null,
      "status": "HEALTHY",
      "version": "1.30.3"
    }
  ]
}
4

@DudaNogueira This is another piece of info . Might be of help.

root@ip:/weaviate# curl -s http://localhost:8080/v1/schema | jq '.classes[] | {class: .class, shards: .shards}'
{
  "class": "Sitemanager_c_kafka_crm_contact",
  "shards": null
}
{
  "class": "Sitemanager_c_kafka_crm_invoice",
  "shards": null
}
{
  "class": "Sitemanager_c_kafka_crm_deal",
  "shards": null
}
{
  "class": "Sitemanager_b_crm_contact_company",
  "shards": null
}
{
  "class": "Sitemanager_c_kafka_crm_company",
  "shards": null
}
5

@DudaNogueira

Actually running 7 Weaviate docker containers on one EC2 instance as a Weaviate Cluster.