Hey @Mohamed_Shahin ,
In terms of configuration, we’ve deployed the latest (17.2.1) Weaviate helm chart to our k8s cluster (v1.29) running in GCP. The installation is fresh and the following is a configuration:
replicas: 3
env:
LOG_LEVEL: 'debug'
Currently in three-node configuration the following is the full log of one of the replicas between crashes:
{"action":"startup","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"created startup context, nothing done so far","startup_time_left":"59m59.99858618s","time":"2024-09-11T22:28:05Z"}
{"action":"config_load","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","config_file_path":"/weaviate-config/conf.yaml","level":"info","msg":"Usage of the weaviate.conf.json file is deprecated and will be removed in the future. Please use environment variables.","time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","deprecation":{"apiType":"Configuration","id":"config-files","locations":["--config-file=\"\""],"mitigation":"Configure Weaviate using environment variables.","msg":"use of deprecated command line argument --config-file","sinceTime":"2020-09-08T09:46:00.000Z","sinceVersion":"0.22.16","status":"deprecated"},"level":"warning","msg":"use of deprecated command line argument --config-file","time":"2024-09-11T22:28:05Z"}
{"action":"startup","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","default_vectorizer_module":"none","level":"info","msg":"the default vectorizer modules is set to \"none\", as a result all new schema classes without an explicit vectorizer setting, will use this vectorizer","time":"2024-09-11T22:28:05Z"}
{"action":"startup","auto_schema_enabled":true,"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"auto schema enabled setting is set to \"true\"","time":"2024-09-11T22:28:05Z"}
{"action":"startup","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"config loaded","startup_time_left":"59m59.99818449s","time":"2024-09-11T22:28:05Z"}
{"action":"startup","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"configured OIDC and anonymous access client","startup_time_left":"59m59.99815745s","time":"2024-09-11T22:28:05Z"}
{"action":"startup","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"initialized schema","startup_time_left":"59m59.99814683s","time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":" memberlist: Failed to join 172.16.17.159:7000: dial tcp 172.16.17.159:7000: connect: connection refused","time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":" memberlist: Failed to join 172.16.4.116:7000: dial tcp 172.16.4.116:7000: connect: connection refused","time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":" memberlist: Initiating push/pull sync with: 172.16.9.130:7000","time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":" memberlist: Stream connection from=172.16.9.130:41992","time":"2024-09-11T22:28:05Z"}
{"action":"startup","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"startup routine complete","time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"No resource limits set, weaviate will use all available memory and CPU. To limit resources, set LIMIT_RESOURCES=true","time":"2024-09-11T22:28:05Z"}
{"action":"startup","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"start registering modules","time":"2024-09-11T22:28:05Z"}
{"action":"startup","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"completed registering modules","time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"module offload-s3 is not enabled","time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"module offload-s3 is enabled","time":"2024-09-11T22:28:05Z"}
{"action":"startup","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"start initializing modules","time":"2024-09-11T22:28:05Z"}
{"action":"startup","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"finished initializing modules","time":"2024-09-11T22:28:05Z"}
{"action":"cluster_api_startup","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"serving cluster api on port 7001","port":7001,"time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"open cluster service","servers":{"weaviate-0":8300,"weaviate-1":8300,"weaviate-2":8300},"time":"2024-09-11T22:28:05Z"}
{"address":"172.16.9.130:8301","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"starting cloud rpc server ...","time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"starting raft sub-system ...","time":"2024-09-11T22:28:05Z"}
{"address":"172.16.9.130:8300","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"tcp transport","tcpMaxPool":3,"tcpTimeout":10000000000,"time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"loading local db","time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"local DB successfully loaded","time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"schema manager loaded","n":0,"time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","metadata_only_voters":false,"msg":"construct a new raft node","name":"weaviate-1","time":"2024-09-11T22:28:05Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","index":340,"level":"info","msg":"raft initial configuration","servers":"[[{Suffrage:Voter ID:weaviate-0 Address:172.16.4.110:8300} {Suffrage:Voter ID:weaviate-2 Address:172.16.17.144:8300} {Suffrage:Voter ID:weaviate-1 Address:172.16.9.51:8300}]]","time":"2024-09-11T22:28:05Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","last_snapshot_index":0,"last_store_applied_index_on_start":0,"level":"info","msg":"raft node constructed","raft_applied_index":0,"raft_last_index":346,"time":"2024-09-11T22:28:05Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","follower":{},"leader-address":"","leader-id":"","level":"info","msg":"raft entering follower state","time":"2024-09-11T22:28:05Z"}
{"action":"bootstrap","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"could not join a cluster from [172.16.9.130:8300]","level":"warning","msg":"failed to join cluster, will notify next if voter","servers":["172.16.9.130:8300"],"time":"2024-09-11T22:28:06Z","voter":true}
{"action":"bootstrap","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","expect":3,"got":{"weaviate-1":"172.16.9.130:8300"},"level":"debug","msg":"number of candidates lower than bootstrap expect param, stopping notify","time":"2024-09-11T22:28:06Z"}
{"action":"bootstrap","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"notified peers this node is ready to join as voter","servers":["172.16.9.130:8300"],"time":"2024-09-11T22:28:06Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","last-leader-addr":"","last-leader-id":"","level":"warning","msg":"raft heartbeat timeout reached, starting election","time":"2024-09-11T22:28:06Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"raft entering candidate state","node":{},"term":7112,"time":"2024-09-11T22:28:06Z"}
{"action":"raft","address":"172.16.4.110:8300","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","from":"weaviate-0","level":"debug","msg":"raft asking for vote","term":7112,"time":"2024-09-11T22:28:06Z"}
{"action":"raft","address":"172.16.17.144:8300","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","from":"weaviate-2","level":"debug","msg":"raft asking for vote","term":7112,"time":"2024-09-11T22:28:06Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","id":"weaviate-1","level":"debug","msg":"raft voting for self","term":7112,"time":"2024-09-11T22:28:06Z"}
{"action":"raft-net","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"could not resolve server id weaviate-0","fallback":"172.16.4.110:8300","id":"weaviate-0","level":"warning","msg":"raft-net unable to get address for server, using fallback address","time":"2024-09-11T22:28:06Z"}
{"action":"raft-net","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"could not resolve server id weaviate-2","fallback":"172.16.17.144:8300","id":"weaviate-2","level":"warning","msg":"raft-net unable to get address for server, using fallback address","time":"2024-09-11T22:28:06Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"raft calculated votes needed","needed":2,"term":7112,"time":"2024-09-11T22:28:06Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","from":"weaviate-1","level":"debug","msg":"raft vote granted","tally":1,"term":7112,"time":"2024-09-11T22:28:06Z"}
{"action":"inverted filter2search migration","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"migration skip flag set, skipping migration","time":"2024-09-11T22:28:07Z"}
{"action":"inverted filter2search migration","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"starting switching fallback mode","time":"2024-09-11T22:28:07Z"}
{"action":"inverted filter2search migration","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"no missing filterable indexes, fallback mode skipped","time":"2024-09-11T22:28:07Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","docker_image_tag":"1.26.3","level":"info","msg":"configured versions","server_version":"1.26.3","time":"2024-09-11T22:28:07Z"}
{"action":"grpc_startup","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"grpc server listening at [::]:50051","time":"2024-09-11T22:28:07Z"}
{"action":"restapi_management","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","docker_image_tag":"1.26.3","level":"info","msg":"Serving weaviate at http://[::]:8080","time":"2024-09-11T22:28:07Z"}
{"action":"bootstrap","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"could not join a cluster from [172.16.9.130:8300]","level":"warning","msg":"failed to join cluster, will notify next if voter","servers":["172.16.9.130:8300"],"time":"2024-09-11T22:28:07Z","voter":true}
{"action":"bootstrap","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","expect":3,"got":{"weaviate-1":"172.16.9.130:8300"},"level":"debug","msg":"number of candidates lower than bootstrap expect param, stopping notify","time":"2024-09-11T22:28:07Z"}
{"action":"bootstrap","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"notified peers this node is ready to join as voter","servers":["172.16.9.130:8300"],"time":"2024-09-11T22:28:07Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":" memberlist: Stream connection from=172.16.4.116:42192","time":"2024-09-11T22:28:07Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"warning","msg":"raft Election timeout reached, restarting election","time":"2024-09-11T22:28:08Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"raft entering candidate state","node":{},"term":7113,"time":"2024-09-11T22:28:08Z"}
{"action":"raft","address":"172.16.4.110:8300","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","from":"weaviate-0","level":"debug","msg":"raft asking for vote","term":7113,"time":"2024-09-11T22:28:08Z"}
{"action":"raft","address":"172.16.17.144:8300","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","from":"weaviate-2","level":"debug","msg":"raft asking for vote","term":7113,"time":"2024-09-11T22:28:08Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","id":"weaviate-1","level":"debug","msg":"raft voting for self","term":7113,"time":"2024-09-11T22:28:08Z"}
{"action":"raft-net","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"could not resolve server id weaviate-2","fallback":"172.16.17.144:8300","id":"weaviate-2","level":"warning","msg":"raft-net unable to get address for server, using fallback address","time":"2024-09-11T22:28:08Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","msg":"raft calculated votes needed","needed":2,"term":7113,"time":"2024-09-11T22:28:08Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","from":"weaviate-1","level":"debug","msg":"raft vote granted","tally":1,"term":7113,"time":"2024-09-11T22:28:08Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","from":"weaviate-0","level":"debug","msg":"raft vote granted","tally":2,"term":7113,"time":"2024-09-11T22:28:08Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"raft election won","tally":2,"term":7113,"time":"2024-09-11T22:28:08Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","leader":{},"level":"info","msg":"raft entering leader state","time":"2024-09-11T22:28:08Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"raft added peer, starting replication","peer":"weaviate-0","time":"2024-09-11T22:28:08Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"raft added peer, starting replication","peer":"weaviate-2","time":"2024-09-11T22:28:08Z"}
{"action":"raft-net","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"could not resolve server id weaviate-2","fallback":"172.16.17.144:8300","id":"weaviate-2","level":"warning","msg":"raft-net unable to get address for server, using fallback address","time":"2024-09-11T22:28:08Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"warning","msg":"raft appendEntries rejected, sending older logs","next":346,"peer":{"Suffrage":0,"ID":"weaviate-0","Address":"172.16.4.110:8300"},"time":"2024-09-11T22:28:08Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"raft pipelining replication","peer":{"Suffrage":0,"ID":"weaviate-0","Address":"172.16.4.110:8300"},"time":"2024-09-11T22:28:08Z"}
{"address":"172.16.9.130:8300","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"current Leader","time":"2024-09-11T22:28:08Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"starting migration from old schema","time":"2024-09-11T22:28:08Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"legacy schema is empty, nothing to migrate","time":"2024-09-11T22:28:08Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"migration from the old schema has been successfully completed","time":"2024-09-11T22:28:08Z"}
{"action":"raft-net","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"could not resolve server id weaviate-2","fallback":"172.16.17.144:8300","id":"weaviate-2","level":"warning","msg":"raft-net unable to get address for server, using fallback address","time":"2024-09-11T22:28:08Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":501887355,"level":"warning","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:08Z"}
{"action":"bootstrap","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"node reporting ready, node has probably recovered cluster from raft config. Exiting bootstrap process","time":"2024-09-11T22:28:08Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":970415071,"level":"warning","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:09Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":1415676468,"level":"warning","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:09Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":1849860337,"level":"debug","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:09Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":2298746284,"level":"debug","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:10Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":2734670652,"level":"debug","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:10Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":3233415067,"level":"debug","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:11Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":3716153072,"level":"debug","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:11Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":4190246908,"level":"debug","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:12Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":4684010383,"level":"debug","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:12Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":5120811571,"level":"debug","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:13Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":5592213807,"level":"debug","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:13Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":6086457392,"level":"debug","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:14Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":6563357968,"level":"debug","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:14Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","fields.time":7028038025,"level":"debug","msg":"raft failed to contact","server-id":"weaviate-2","time":"2024-09-11T22:28:15Z"}
{"action":"restapi_management","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","docker_image_tag":"1.26.3","level":"info","msg":"Shutting down... ","time":"2024-09-11T22:28:15Z"}
{"action":"restapi_management","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","docker_image_tag":"1.26.3","level":"info","msg":"Stopped serving weaviate at http://[::]:8080","time":"2024-09-11T22:28:15Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"closing raft FSM store ...","time":"2024-09-11T22:28:15Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"shutting down raft sub-system ...","time":"2024-09-11T22:28:15Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"transferring leadership to another server","time":"2024-09-11T22:28:15Z"}
{"action":"raft","address":null,"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","id":null,"level":"debug","msg":"raft starting leadership transfer","time":"2024-09-11T22:28:15Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"successfully transferred leadership to another server","time":"2024-09-11T22:28:15Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"raft aborting pipeline replication","peer":{"Suffrage":0,"ID":"weaviate-0","Address":"172.16.4.110:8300"},"time":"2024-09-11T22:28:15Z"}
{"action":"raft-net","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","local-address":"172.16.9.130:8300","msg":"raft-net accepted connection","remote-address":"172.16.4.116:46818","time":"2024-09-11T22:28:15Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"dial tcp 172.16.4.110:8300: i/o timeout","level":"error","msg":"raft failed to make requestVote RPC","target":{"Suffrage":0,"ID":"weaviate-0","Address":"172.16.4.110:8300"},"term":7112,"time":"2024-09-11T22:28:16Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"dial tcp 172.16.17.144:8300: i/o timeout","level":"error","msg":"raft failed to make requestVote RPC","target":{"Suffrage":0,"ID":"weaviate-2","Address":"172.16.17.144:8300"},"term":7112,"time":"2024-09-11T22:28:16Z"}
{"action":"raft-net","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","local-address":"172.16.9.130:8300","msg":"raft-net accepted connection","remote-address":"172.16.4.116:46832","time":"2024-09-11T22:28:17Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"dial tcp 172.16.17.144:8300: i/o timeout","level":"error","msg":"raft failed to make requestVote RPC","target":{"Suffrage":0,"ID":"weaviate-2","Address":"172.16.17.144:8300"},"term":7113,"time":"2024-09-11T22:28:18Z"}
{"action":"raft","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"dial tcp 172.16.17.144:8300: i/o timeout","level":"error","msg":"raft failed to appendEntries to","peer":{"Suffrage":0,"ID":"weaviate-2","Address":"172.16.17.144:8300"},"time":"2024-09-11T22:28:18Z"}
{"action":"raft-net","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"debug","local-address":"172.16.9.130:8300","msg":"raft-net accepted connection","remote-address":"172.16.4.116:46846","time":"2024-09-11T22:28:18Z"}
{"action":"raft","backoff time":10000000,"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"dial tcp 172.16.17.144:8300: i/o timeout","level":"error","msg":"raft failed to heartbeat to","peer":"172.16.17.144:8300","time":"2024-09-11T22:28:18Z"}
{"action":"raft-net","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"transport shutdown","level":"error","msg":"raft-net failed to decode incoming command","time":"2024-09-11T22:28:18Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"closing raft-net ...","time":"2024-09-11T22:28:18Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"closing log store ...","time":"2024-09-11T22:28:18Z"}
{"action":"raft-net","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"transport shutdown","level":"error","msg":"raft-net failed to decode incoming command","time":"2024-09-11T22:28:18Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"closing data store ...","time":"2024-09-11T22:28:18Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"closing loaded database ...","time":"2024-09-11T22:28:18Z"}
{"action":"raft-net","build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","error":"transport shutdown","level":"error","msg":"raft-net failed to decode incoming command","time":"2024-09-11T22:28:18Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"closing raft-rpc client ...","time":"2024-09-11T22:28:18Z"}
{"build_git_commit":"9a4ea6d","build_go_version":"go1.21.13","build_image_tag":"1.26.3","build_wv_version":"1.26.3","level":"info","msg":"closing raft-rpc server ...","time":"2024-09-11T22:28:18Z"}
Thanks for the help, mate!