Description
Seems like weaviate is not initializing properly. I’m unable to create a class as noted by the transaction-entry container log stated below Error creating schema class: status code: 422, error: {"error":[{"message":"leader not found"}]}
Weaviate is supposed be run as a single-node but it looks like it’s trying to join a cluster and look for a leader which there is none because I only have one instance using docker compose
Is there a way to force it to run as a single-node? Is something wrong with my configuration?
Also note that I’m using AWS EFS as my volume
Server Setup Information
- Weaviate Server Version: 1.29.0
- Deployment Method: AWS EC2 t2.micro as Docker host, Docker version 28.0.1, build 068a01e
- Multi Node? Number of Running Nodes: 1
- Client Language and Version: go version go1.23.2 linux/amd64
- Multitenancy?: no sure, probably not
Any additional Information
docker-compose.yml
services:
knomor-gateway:
build: ./knomor-gateway
container_name: knomor-gateway
ports:
- "80:80"
- "443:443"
networks:
- knomor-network
transaction-entry:
build: ./transaction-entry
container_name: transaction-entry
networks:
- knomor-network
expose:
- "9030"
depends_on:
- weaviate
weaviate:
command:
- --host
- 0.0.0.0
- --port
- '8080'
- --scheme
- http
image: cr.weaviate.io/semitechnologies/weaviate:1.29.0
container_name: weaviate
networks:
- knomor-network
expose:
- "8080"
- "50051"
volumes:
- /mnt/knomor-efs:/var/lib/weaviate
restart: on-failure:0
environment:
QUERY_DEFAULTS_LIMIT: 25
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: "true"
PERSISTENCE_DATA_PATH: "/var/lib/weaviate"
ENABLE_MODULES: "text2vec-openai"
RAFT_JOIN: "weaviate-0"
RAFT_BOOTSTRAP_EXPECT: 1
LIMIT_RESOURCES: true
OPENAI_API_KEY: <api-key>
networks:
knomor-network:
driver: bridge
main.go
func main() {
ctx := context.Background()
// init godotenv
err := godotenv.Load()
if err != nil {
log.Fatal("Error loading .env file")
}
// init db
var dbURL string
if os.Getenv("DATABASE_URL") != "" {
dbURL = os.Getenv("DATABASE_URL")
} else {
log.Fatal("Error: No DATABASE_URL environment variable found: main.go: run() -> os.Getenv(\"DATABASE_URL\")")
}
dbpool, err := pgxpool.New(ctx, dbURL)
if err != nil {
fmt.Fprintf(os.Stderr, "Unable to create connection pool: %v\n; main.go: run() -> pgxpool.New()", err)
os.Exit(1)
}
defer dbpool.Close()
queries := db.New(dbpool)
// init weaviate db
cfg := weaviate.Config{
Host: "weaviate:8080",
Scheme: "http",
}
client, err := weaviate.NewClient(cfg)
if err != nil {
fmt.Println(err)
}
// Check the weaviate db connection
ready, err := client.Misc().ReadyChecker().Do(context.Background())
if err != nil {
panic(err)
}
fmt.Println("vector store is ready: ", ready)
// Log weaviate configuration meta data
meta, err := client.Misc().MetaGetter().Do(context.Background())
if err != nil {
panic(err)
}
fmt.Println("weaviate config mata data: ", meta)
// Check if POPULATE_WEAVIATE mode is enabled in the .env file
if os.Getenv("POPULATE_WEAVIATE") == "true" {
standalones.PopulateWeaviate(ctx, queries, client)
return
}
...
}
populateWeaviate.go
func PopulateWeaviate(ctx context.Context, queries *db.Queries, client *weaviate.Client) {
// Define the schema for our class.
classObj := &models.Class{
Class: "LabeledTransaction",
VectorConfig: map[string]models.VectorConfig{
"description_vector": {
Vectorizer: map[string]interface{}{
"text2vec-openai": map[string]interface{}{
"sourceProperties": []string{"description"},
"model": "text-embedding-3-small",
"dimensions": 1536, // Optional (e.g. 1536, 512) for text-embedding-3-small
},
},
VectorIndexType: "hnsw",
},
},
Properties: []*models.Property{
{
DataType: []string{"text"},
Name: "description",
},
{
DataType: []string{"string[]"},
Name: "labels",
},
},
}
err := client.Schema().ClassCreator().WithClass(classObj).Do(ctx)
if err != nil {
fmt.Println("Error creating schema class:", err)
return
} else {
fmt.Println("Weaviate schema created successfully.")
}
....
}
transaction-entry container logs:
vector store is ready: false
weaviate config mata data: &{http://[::]:8080 map[text2vec-openai:map[documentationHref:https://platform.openai.com/docs/guides/embeddings/what-are-embeddings name:OpenAI Module]] 1.29.0}
Error creating schema class: status code: 422, error: {“error”:[{“message”:“leader not found”}]}
weaviate container logs:
{“action”:“startup”,“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“error”:“could not join raft join list: context deadline exceeded. Weaviate detected this node to have state stored. If the DB is still loading up we will hit this timeout. You can try increasing/setting RAFT_BOOTSTRAP_TIMEOUT env variable to a higher value.”,“level”:“fatal”,“msg”:“could not open cloud meta store”,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“warning”,“log_level_env”:“”,“msg”:“log level not recognized, defaulting to info”,“time”:“2025-03-11T08:39:34Z”}
{“action”:“startup”,“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“Feature flag LD integration disabled: could not locate WEAVIATE_LD_API_KEY env variable”,“time”:“2025-03-11T08:39:34Z”}
{“action”:“startup”,“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“default_vectorizer_module”:“none”,“level”:“info”,“msg”:“the default vectorizer modules is set to "none", as a result all new schema classes without an explicit vectorizer setting, will use this vectorizer”,“time”:“2025-03-11T08:39:34Z”}
{“action”:“startup”,“auto_schema_enabled”:true,“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“auto schema enabled setting is set to "true"”,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“Limiting resources: memory: 80%, cores: all but one”,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“limit”:0,“msg”:“Set memory limit”,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“module offload-s3 is enabled”,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“flag_key”:“collection-retrieval-strategy”,“level”:“info”,“msg”:“feature flag instantiated”,“time”:“2025-03-11T08:39:34Z”,“tool”:“feature_flag”,“value”:“LeaderOnly”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“warning”,“msg”:“Multiple vector spaces are present, GraphQL Explore and REST API list objects endpoint module include params has been disabled as a result.”,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“open cluster service”,“servers”:{“weaviate-0”:8300},“time”:“2025-03-11T08:39:34Z”}
{“address”:“172.18.0.5:8301”,“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“starting cloud rpc server …”,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“starting raft sub-system …”,“time”:“2025-03-11T08:39:34Z”}
{“address”:“172.18.0.5:8300”,“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“tcp transport”,“tcpMaxPool”:3,“tcpTimeout”:10000000000,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“loading local db”,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“local DB successfully loaded”,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“schema manager loaded”,“n”:0,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“metadata_only_voters”:false,“msg”:“construct a new raft node”,“name”:“4eafcef84a52”,“time”:“2025-03-11T08:39:34Z”}
{“action”:“raft”,“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“index”:1,“level”:“info”,“msg”:“initial configuration”,“servers”:“[[{Suffrage:Voter ID:4fec3d6ddd53 Address:172.18.0.7:8300}]]”,“time”:“2025-03-11T08:39:34Z”}
{“action”:“raft”,“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“follower”:{},“leader-address”:“”,“leader-id”:“”,“level”:“info”,“msg”:“entering follower state”,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“last_snapshot_index”:0,“last_store_applied_index_on_start”:0,“level”:“info”,“msg”:“raft node constructed”,“raft_applied_index”:0,“raft_last_index”:2,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“hasState”:true,“level”:“info”,“msg”:“raft init”,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“attempting to join”,“remoteNodes”:,“time”:“2025-03-11T08:39:34Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“attempting to join”,“remoteNodes”:,“time”:“2025-03-11T08:39:35Z”}
{“action”:“raft”,“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“warning”,“msg”:“heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election”,“time”:“2025-03-11T08:39:36Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“configured versions”,“server_version”:“1.29.0”,“time”:“2025-03-11T08:39:36Z”,“version”:“1.29.0”}
{“action”:“grpc_startup”,“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“grpc server listening at [::]:50051”,“time”:“2025-03-11T08:39:36Z”}
{“action”:“restapi_management”,“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“Serving weaviate at http://[::]:8080”,“time”:“2025-03-11T08:39:36Z”,“version”:“1.29.0”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“attempting to join”,“remoteNodes”:,“time”:“2025-03-11T08:39:36Z”}
{“action”:“telemetry_push”,“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“telemetry started”,“payload”:“\u0026{MachineID:b5e46904-afeb-4d66-b4a4-81326cc7e437 Type:INIT Version:1.29.0 ObjectsCount:0 OS:linux Arch:amd64 UsedModules: CollectionsCount:0}”,“time”:“2025-03-11T08:39:37Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“attempting to join”,“remoteNodes”:,“time”:“2025-03-11T08:39:37Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“attempting to join”,“remoteNodes”:,“time”:“2025-03-11T08:39:38Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“attempting to join”,“remoteNodes”:,“time”:“2025-03-11T08:39:39Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“attempting to join”,“remoteNodes”:,“time”:“2025-03-11T08:39:40Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“attempting to join”,“remoteNodes”:,“time”:“2025-03-11T08:39:41Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“attempting to join”,“remoteNodes”:,“time”:“2025-03-11T08:39:42Z”}
{“build_git_commit”:“35d800d”,“build_go_version”:“go1.22.12”,“build_image_tag”:“v1.29.0”,“build_wv_version”:“1.29.0”,“level”:“info”,“msg”:“attempting to join”,“remoteNodes”:,“time”:“2025-03-11T08:39:43Z”}