Random SSL/TLS Errors while vectorizing strings inside docker container

Support
1

Description

Hi guys,
I have these type of errors often inside a weaviate docker container:

explorer: get class: vectorize params: vectorize params: vectorize params: vectorize move to: vectorize move to: remote client vectorize: send POST request: Post “https://api.openai.com/v1/embeddings”: remote error: tls: bad record MAC (code: 0)

Server Setup Information

networks:
    lawxer:
        name: "lawxer"
        driver: bridge

services:
    traefik:
        image: "traefik:v3.1"
        container_name: "traefik"
        command:
            - "--ping=true"
            - "--log.level=DEBUG"
            - "--log.filePath=/var/log/traefik.log"
            - "--log.maxage=2"
            - "--accessLog.filePath=/var/log/access.log"
            - "--api.dashboard=true"
            - "--api.insecure=false"
            - "--providers.docker.endpoint=unix:///var/run/docker.sock"
            - "--providers.docker.exposedbydefault=false"
            - "--providers.docker.useBindPortIP=true"
            - "--entryPoints.web.address=:80"
            - "--entryPoints.web.http.redirections.entryPoint.to=websecure"
            - "--entryPoints.web.http.redirections.entryPoint.scheme=https"
            - "--entryPoints.websecure.address=:443"
            - "--entryPoints.websecure.http.tls.certResolver=myresolver"
            - "--certificatesresolvers.myresolver.acme.tlschallenge=true"
            - "--certificatesresolvers.myresolver.acme.email=EDITED"
            - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json"
        ports:
            - "80:80"
            - "443:443"
            - "8080:8080"
        volumes:
            - "/var/run/docker.sock:/var/run/docker.sock"
            - "./letsencrypt:/letsencrypt"
            - "./logs/traefik:/var/log"
            #- "./settings/traefik.yml:/etc/traefik/traefik.yml:ro"
        labels:
            - "traefik.enable=true"
            - "traefik.http.routers.traefik.entrypoints=websecure"
            - "traefik.http.routers.traefik.rule=Host(`apimanager.lawxer.ai`)"
            - "traefik.http.routers.traefik.tls.certresolver=myresolver"
            - "traefik.http.routers.traefik.service=api@internal"
            - "traefik.http.routers.traefik.middlewares=traefik-auth"
            - "traefik.http.middlewares.traefik-auth.basicauth.users=EDITED"
        restart: "unless-stopped"
        depends_on:
            - "weaviate"
            - "app"
        networks:
            - "lawxer"
        healthcheck:
            test: [ "CMD", "traefik", "healthcheck", "--ping" ]
            interval: 20s
            timeout: 5s
            retries: 2
            start_period: 20s

    cache:
        image: "redis:latest"
        container_name: "cache"
        ports:
            - "6379:6379"
        restart: "unless-stopped"
        networks:
            - "lawxer"
        labels:
            - "traefik.enable=false"
            - "traefik.docker.network=lawxer"
            - "traefik.http.services.cache.loadbalancer.server.port=6379"
        command:
            - "redis-server"
        healthcheck:
            test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
            interval: 20s
            timeout: 5s
            retries: 2

    weaviate:
        image: "cr.weaviate.io/semitechnologies/weaviate:1.25.7"
        container_name: "weaviate"
        command:
            - "--host"
            - "0.0.0.0"
            - "--port"
            - "8060"
            - "--scheme"
            - "http"
        ports:
            - "8060:8060"
            - "50051:50051"
        volumes:
            - "./weaviate:/var/lib/weaviate"
        restart: "unless-stopped"
        networks:
            - "lawxer"
        environment:
            QUERY_DEFAULTS_LIMIT: 25
            RECOUNT_PROPERTIES_AT_STARTUP: true
            AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'false'
            AUTHENTICATION_APIKEY_ENABLED: 'true'
            AUTHENTICATION_APIKEY_ALLOWED_KEYS: 'EDITED'
            AUTHENTICATION_APIKEY_USERS: 'lawxer-app'
            PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
            DEFAULT_VECTORIZER_MODULE: 'text2vec-openai'
            ENABLE_MODULES: 'text2vec-openai'
            ORIGIN: 'https://weaviate.lawxer.loc'
            CLUSTER_HOSTNAME: 'node1'
            LOG_LEVEL: 'debug'
        labels:
            - "traefik.enable=true"
            - "traefik.docker.network=lawxer"
            - "traefik.http.services.weaviate.loadbalancer.server.port=8060"
            - "traefik.http.routers.weaviate.entrypoints=websecure"
            - "traefik.http.routers.weaviate.rule=Host(`weaviate.lawxer.ai`)"
            - "traefik.http.routers.weaviate.tls.certresolver=myresolver"
        healthcheck:
            test: wget --no-verbose --tries=3 --spider http://localhost:8060/v1/.well-known/ready || exit 1
            interval: 20s
            timeout: 2s
            retries: 2
            start_period: 20s

    app:
        build:
            context: .
            dockerfile: Dockerfile
        image: "sergeliatko/lawxer-api${BUILD_TAG}"
        container_name: "app"
        volumes:
            - "./logs:/var/www/logs"
            - "./storage:/var/www/storage"
            - "./settings:/var/www/settings"
        ports:
            - "9501:9501"
        labels:
            - "traefik.enable=true"
            - "traefik.docker.network=lawxer"
            - "traefik.http.services.app.loadbalancer.server.port=9501"
            - "traefik.http.routers.app.entrypoints=websecure"
            - "traefik.http.routers.app.rule=Host(`api.lawxer.ai`)"
            - "traefik.http.routers.app.tls.certresolver=myresolver"
        depends_on:
            - "cache"
            - "weaviate"
        restart: "unless-stopped"
        networks:
            - "lawxer"
        healthcheck:
            test: [ "CMD", "curl", "-f", "--head", "http://localhost:9501/status" ]
            interval: 20s
            timeout: 5s
            retries: 2
            start_period: 20s

Any additional Information

Weaviate logs:

{"action":"startup","level":"debug","msg":"created startup context, nothing done so far","startup_time_left":"59m59.997979453s","time":"2024-07-23T16:10:29Z"}
{"action":"startup","default_vectorizer_module":"text2vec-openai","level":"info","msg":"the default vectorizer modules is set to \"text2vec-openai\", as a result all new schema classes without an explicit vectorizer setting, will use this vectorizer","time":"2024-07-23T16:10:29Z"}
{"action":"startup","auto_schema_enabled":true,"level":"info","msg":"auto schema enabled setting is set to \"true\"","time":"2024-07-23T16:10:29Z"}
{"action":"startup","level":"debug","msg":"config loaded","startup_time_left":"59m59.997753691s","time":"2024-07-23T16:10:29Z"}
{"action":"startup","level":"debug","msg":"configured OIDC and anonymous access client","startup_time_left":"59m59.997721555s","time":"2024-07-23T16:10:29Z"}
{"action":"startup","level":"debug","msg":"initialized schema","startup_time_left":"59m59.997704322s","time":"2024-07-23T16:10:29Z"}
{"action":"startup","level":"debug","msg":"startup routine complete","time":"2024-07-23T16:10:29Z"}
{"level":"info","msg":"No resource limits set, weaviate will use all available memory and CPU. To limit resources, set LIMIT_RESOURCES=true","time":"2024-07-23T16:10:29Z"}
{"action":"startup","level":"debug","msg":"start registering modules","time":"2024-07-23T16:10:29Z"}
{"action":"startup","level":"debug","module":"text2vec-openai","msg":"enabled module","time":"2024-07-23T16:10:29Z"}
{"action":"startup","level":"debug","msg":"completed registering modules","time":"2024-07-23T16:10:29Z"}
{"level":"info","msg":"open cluster service","servers":{"node1":8300},"time":"2024-07-23T16:10:29Z"}
{"address":"192.168.176.3:8301","level":"info","msg":"starting cloud rpc server ...","time":"2024-07-23T16:10:29Z"}
{"level":"info","msg":"starting raft sub-system ...","time":"2024-07-23T16:10:29Z"}
{"action":"cluster_api_startup","level":"debug","msg":"serving cluster api on port 7947","port":7947,"time":"2024-07-23T16:10:29Z"}
{"address":"192.168.176.3:8300","level":"info","msg":"tcp transport","tcpMaxPool":3,"tcpTimeout":10000000000,"time":"2024-07-23T16:10:29Z"}
{"level":"info","msg":"loading local db","time":"2024-07-23T16:10:29Z"}
{"level":"info","msg":"database has been successfully loaded","n":0,"time":"2024-07-23T16:10:29Z"}
{"level":"info","metadata_only_voters":false,"msg":"construct a new raft node","name":"node1","time":"2024-07-23T16:10:29Z"}
{"id":"2-3-1720451756875","last-index":3,"last-term":2,"level":"info","msg":"starting restore from snapshot","size-in-bytes":22160,"time":"2024-07-23T16:10:29Z"}
{"level":"info","msg":"restoring schema from snapshot","time":"2024-07-23T16:10:29Z"}
{"level":"info","msg":"successfully restored schema from snapshot","time":"2024-07-23T16:10:29Z"}
{"last_applied_index":0,"last_snapshot_index":3,"last_store_log_applied_index":0,"level":"info","msg":"load local db from snapshot","time":"2024-07-23T16:10:29Z"}
{"level":"info","msg":"reload local db: update schema ...","time":"2024-07-23T16:10:29Z"}
{"index":"Element","level":"info","msg":"reload local index","time":"2024-07-23T16:10:29Z"}
{"level":"info","msg":"successfully reloaded indexes from snapshot","n":1,"time":"2024-07-23T16:10:30Z"}
{"action":"","id":"2-3-1720451756875","last-index":3,"last-term":2,"level":"info","msg":"snapshot restore progress","percent-complete":"[100.00]%","read-bytes":22160,"size-in-bytes":22160,"time":"2024-07-23T16:10:30Z"}
{"id":"2-3-1720451756875","last-index":3,"last-term":2,"level":"info","msg":"restored from snapshot","size-in-bytes":22160,"time":"2024-07-23T16:10:30Z"}
{"action":"raft","index":4,"level":"info","msg":"raft initial configuration","servers":"[[{Suffrage:Voter ID:node1 Address:192.168.112.2:8300}]]","time":"2024-07-23T16:10:30Z"}
{"last_snapshot_index":3,"last_store_applied_index":3,"last_store_log_applied_index":0,"level":"info","msg":"raft node constructed","raft_applied_index":3,"raft_last_index":81,"time":"2024-07-23T16:10:30Z"}
{"action":"raft","follower":{},"leader-address":"","leader-id":"","level":"info","msg":"raft entering follower state","time":"2024-07-23T16:10:30Z"}
{"action":"hnsw_grow_index","level":"debug","msg":"index grown from 1000 to 3000, took 11.528µs\n","new_size":3000,"previous_size":1000,"time":"2024-07-23T16:10:31Z","took":11528}
{"action":"hnsw_grow_index","level":"debug","msg":"index grown from 3000 to 5000, took 11.42µs\n","new_size":5000,"previous_size":3000,"time":"2024-07-23T16:10:31Z","took":11420}
{"action":"hnsw_grow_index","level":"debug","msg":"index grown from 5000 to 7000, took 16.759µs\n","new_size":7000,"previous_size":5000,"time":"2024-07-23T16:10:31Z","took":16759}
{"action":"hnsw_grow_index","level":"debug","msg":"index grown from 7000 to 9000, took 18.096µs\n","new_size":9000,"previous_size":7000,"time":"2024-07-23T16:10:31Z","took":18096}
{"action":"hnsw_grow_index","level":"debug","msg":"index grown from 9000 to 11250, took 32.703µs\n","new_size":11250,"previous_size":9000,"time":"2024-07-23T16:10:31Z","took":32703}
{"action":"hnsw_deserialization","level":"debug","msg":"hnsw commit logger AddTombstone","ops":1007,"time":"2024-07-23T16:10:31Z"}
{"action":"hnsw_deserialization","level":"debug","msg":"hnsw commit logger ClearLinksAtLevel","ops":1675,"time":"2024-07-23T16:10:31Z"}
{"action":"hnsw_deserialization","level":"debug","msg":"hnsw commit logger DeleteNode","ops":1007,"time":"2024-07-23T16:10:31Z"}
{"action":"hnsw_deserialization","level":"debug","msg":"hnsw commit logger RemoveTombstone","ops":1007,"time":"2024-07-23T16:10:31Z"}
{"action":"hnsw_deserialization","level":"debug","msg":"hnsw commit logger SetEntryPointWithMaxLayer","ops":5,"time":"2024-07-23T16:10:31Z"}
{"action":"hnsw_deserialization","level":"debug","msg":"hnsw commit logger AddNode","ops":11243,"time":"2024-07-23T16:10:31Z"}
{"action":"hnsw_deserialization","level":"debug","msg":"hnsw commit logger ReplaceLinksAtLevel","ops":15753,"time":"2024-07-23T16:10:31Z"}
{"action":"hnsw_deserialization","level":"debug","msg":"hnsw commit logger AddLinkAtLevel","ops":353795,"time":"2024-07-23T16:10:31Z"}
{"action":"hnsw_prefill_cache_async","level":"info","msg":"not waiting for vector cache prefill, running in background","time":"2024-07-23T16:10:31Z","wait_for_cache_prefill":false}
{"action":"startup","level":"debug","msg":"shard=OSmYvNJv23qO is ready","time":"2024-07-23T16:10:31Z"}
{"level":"info","msg":"Completed loading shard element_OSmYvNJv23qO in 117.145438ms","time":"2024-07-23T16:10:31Z"}
{"action":"load_all_shards","level":"debug","msg":"finished loading all shards","time":"2024-07-23T16:10:31Z","took":"1.117810669s"}
{"action":"hnsw_vector_cache_prefill_level","count":1,"hnsw_level":3,"index_id":"main","level":"debug","msg":"prefilled level in vector cache","time":"2024-07-23T16:10:31Z","took":1397273}
{"action":"hnsw_vector_cache_prefill_level","count":4,"hnsw_level":2,"index_id":"main","level":"debug","msg":"prefilled level in vector cache","time":"2024-07-23T16:10:31Z","took":1576098}
{"action":"bootstrap","error":"could not join a cluster from [192.168.176.3:8300]","level":"warning","msg":"failed to join cluster, will notify next if voter","servers":["192.168.176.3:8300"],"time":"2024-07-23T16:10:31Z","voter":true}
{"action":"bootstrap","candidates":[{"Suffrage":0,"ID":"node1","Address":"192.168.176.3:8300"}],"level":"info","msg":"starting cluster bootstrapping","time":"2024-07-23T16:10:31Z"}
{"action":"bootstrap","error":"bootstrap only works on new clusters","level":"error","msg":"could not bootstrapping cluster","time":"2024-07-23T16:10:31Z"}
{"action":"bootstrap","level":"info","msg":"notified peers this node is ready to join as voter","servers":["192.168.176.3:8300"],"time":"2024-07-23T16:10:31Z"}
{"action":"hnsw_vector_cache_prefill_level","count":174,"hnsw_level":1,"index_id":"main","level":"debug","msg":"prefilled level in vector cache","time":"2024-07-23T16:10:31Z","took":28919685}
{"action":"hnsw_vector_cache_prefill_level","count":10057,"hnsw_level":0,"index_id":"main","level":"debug","msg":"prefilled level in vector cache","time":"2024-07-23T16:10:31Z","took":284769937}
{"action":"hnsw_vector_cache_prefill","count":13250,"index_id":"main","level":"info","limit":1000000000000,"msg":"prefilled vector cache","time":"2024-07-23T16:10:31Z","took":316856358}
{"action":"inverted filter2search migration","level":"debug","msg":"migration skip flag set, skipping migration","time":"2024-07-23T16:10:31Z"}
{"action":"inverted filter2search migration","level":"debug","msg":"starting switching fallback mode","time":"2024-07-23T16:10:31Z"}
{"action":"inverted filter2search migration","level":"debug","msg":"no missing filterable indexes, fallback mode skipped","time":"2024-07-23T16:10:31Z"}
{"action":"startup","level":"debug","msg":"start initializing modules","time":"2024-07-23T16:10:31Z"}
{"action":"startup","level":"debug","module":"text2vec-openai","msg":"initialized module","time":"2024-07-23T16:10:31Z"}
{"action":"startup","level":"debug","module":"text2vec-openai","msg":"initialized module extension","time":"2024-07-23T16:10:31Z"}
{"level":"warning","msg":"Multiple vector spaces are present, GraphQL Explore and REST API list objects endpoint module include params has been disabled as a result.","time":"2024-07-23T16:10:31Z"}
{"action":"startup","level":"debug","msg":"finished initializing modules","time":"2024-07-23T16:10:31Z"}
{"action":"graphql_rebuild","level":"debug","msg":"rebuilding the graphql schema","schema":{"Objects":{"classes":[{"class":"Element","description":"An element of a legal document.","invertedIndexConfig":{"bm25":{"b":0.75,"k1":1.2},"cleanupIntervalSeconds":60,"stopwords":{"additions":null,"preset":"en","removals":null}},"moduleConfig":{"text2vec-openai":{"baseURL":"https://api.openai.com","model":"text-embedding-3-small","type":"text","vectorizeClassName":false}},"multiTenancyConfig":{"autoTenantActivation":false,"autoTenantCreation":false,"enabled":false},"properties":[{"dataType":["text"],"description":"The ID of the legal document this element belongs to.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":true,"vectorizePropertyName":false}},"name":"document","tokenization":"word"},{"dataType":["int"],"description":"The order of the element within its parent element.","indexFilterable":true,"indexSearchable":false,"moduleConfig":{"text2vec-openai":{"skip":false,"vectorizePropertyName":false}},"name":"order"},{"dataType":["text"],"description":"The path of the element in the legal document.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":true,"vectorizePropertyName":false}},"name":"path","tokenization":"word"},{"dataType":["text"],"description":"The path of the parent element in the legal document.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":true,"vectorizePropertyName":false}},"name":"parentPath","tokenization":"word"},{"dataType":["text"],"description":"The text content of the element.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":false,"vectorizePropertyName":false}},"name":"content","tokenization":"word"},{"dataType":["text"],"description":"The title of the element.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":false,"vectorizePropertyName":false}},"name":"title","tokenization":"word"},{"dataType":["text"],"description":"The name of the element.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":false,"vectorizePropertyName":false}},"name":"name","tokenization":"word"},{"dataType":["text"],"description":"The name of the parent element.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":false,"vectorizePropertyName":false}},"name":"parentName","tokenization":"word"},{"dataType":["text"],"description":"The outline of the element.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":false,"vectorizePropertyName":false}},"name":"outline","tokenization":"word"},{"dataType":["text"],"description":"The type of the element.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":true,"vectorizePropertyName":false}},"name":"type","tokenization":"word"}],"replicationConfig":{"factor":1},"shardingConfig":{"virtualPerPhysical":128,"desiredCount":1,"actualCount":1,"desiredVirtualCount":128,"actualVirtualCount":128,"key":"_id","strategy":"hash","function":"murmur3"},"vectorIndexConfig":{"skip":false,"cleanupIntervalSeconds":300,"maxConnections":64,"efConstruction":128,"ef":-1,"dynamicEfMin":100,"dynamicEfMax":500,"dynamicEfFactor":8,"vectorCacheMaxObjects":1000000000000,"flatSearchCutoff":40000,"distance":"cosine","pq":{"enabled":false,"bitCompression":false,"segments":0,"centroids":256,"trainingLimit":100000,"encoder":{"type":"kmeans","distribution":"log-normal"}},"bq":{"enabled":false}},"vectorIndexType":"hnsw","vectorizer":"text2vec-openai"}]}},"time":"2024-07-23T16:10:31Z"}
{"action":"graphql_rebuild","level":"debug","msg":"successfully rebuild graphql schema","time":"2024-07-23T16:10:31Z"}
{"action":"recount","level":"info","msg":"Recounting properties, this may take a while","time":"2024-07-23T16:10:31Z"}
{"action":"raft","last-leader-addr":"","last-leader-id":"","level":"warning","msg":"raft heartbeat timeout reached, starting election","time":"2024-07-23T16:10:31Z"}
{"action":"raft","level":"info","msg":"raft entering candidate state","node":{},"term":80,"time":"2024-07-23T16:10:31Z"}
{"action":"raft","id":"node1","level":"debug","msg":"raft voting for self","term":80,"time":"2024-07-23T16:10:31Z"}
{"action":"raft","level":"debug","msg":"raft calculated votes needed","needed":1,"term":80,"time":"2024-07-23T16:10:31Z"}
{"action":"raft","from":"node1","level":"debug","msg":"raft vote granted","tally":1,"term":80,"time":"2024-07-23T16:10:31Z"}
{"action":"raft","level":"info","msg":"raft election won","tally":1,"term":80,"time":"2024-07-23T16:10:31Z"}
{"action":"raft","leader":{},"level":"info","msg":"raft entering leader state","time":"2024-07-23T16:10:31Z"}
{"address":"192.168.176.3:8300","level":"info","msg":"current Leader","time":"2024-07-23T16:10:32Z"}
{"action":"bootstrap","level":"info","msg":"node reporting ready, node has probably recovered cluster from raft config. Exiting bootstrap process","time":"2024-07-23T16:10:32Z"}
{"action":"recount","level":"warning","msg":"Recounted 10236 objects. Recounting properties complete. Please remove environment variable \tRECOUNT_PROPERTIES_AT_STARTUP before next startup","time":"2024-07-23T16:10:34Z"}
{"action":"grpc_startup","level":"info","msg":"grpc server listening at [::]:50051","time":"2024-07-23T16:10:34Z"}
{"action":"restapi_management","level":"info","msg":"Serving weaviate at http://[::]:8060","time":"2024-07-23T16:10:34Z"}
{"action":"telemetry_push","level":"info","msg":"telemetry started","payload":"\u0026{MachineID:1ab20893-e398-4179-82d2-c33abd4f1375 Type:INIT Version:1.25.7 NumObjects:0 OS:linux Arch:amd64 UsedModules:[text2vec-openai]}","time":"2024-07-23T16:10:34Z"}
{"action":"restapi_request","level":"debug","method":"GET","msg":"received HTTP request","time":"2024-07-23T16:10:35Z","url":{"Scheme":"","Opaque":"","User":null,"Host":"","Path":"/v1/schema/Element","RawPath":"","OmitHost":false,"ForceQuery":false,"RawQuery":"","Fragment":"","RawFragment":""}}
{"level":"debug","msg":"server.query","time":"2024-07-23T16:10:35Z","type":1}
{"action":"restapi_request","level":"debug","method":"GET","msg":"received HTTP request","time":"2024-07-23T16:10:35Z","url":{"Scheme":"","Opaque":"","User":null,"Host":"","Path":"/v1/schema/Vector","RawPath":"","OmitHost":false,"ForceQuery":false,"RawQuery":"","Fragment":"","RawFragment":""}}
{"level":"debug","msg":"server.query","time":"2024-07-23T16:10:35Z","type":1}
{"action":"restapi_request","level":"debug","method":"POST","msg":"received HTTP request","time":"2024-07-23T16:10:35Z","url":{"Scheme":"","Opaque":"","User":null,"Host":"","Path":"/v1/schema","RawPath":"","OmitHost":false,"ForceQuery":false,"RawQuery":"","Fragment":"","RawFragment":""}}
{"class":"Vector","level":"debug","msg":"server.execute","time":"2024-07-23T16:10:35Z","type":1}
{"level":"debug","log_index":83,"log_name":"LogCommand","log_type":0,"msg":"apply fsm store command","time":"2024-07-23T16:10:35Z"}
{"cmd_class":"Vector","cmd_schema_only":false,"cmd_type":1,"cmd_type_name":"TYPE_ADD_CLASS","level":"debug","log_index":83,"log_name":"LogCommand","log_type":0,"msg":"server.apply","time":"2024-07-23T16:10:35Z"}
{"action":"graphql_rebuild","level":"debug","msg":"rebuilding the graphql schema","schema":{"Objects":{"classes":[{"class":"Element","description":"An element of a legal document.","invertedIndexConfig":{"bm25":{"b":0.75,"k1":1.2},"cleanupIntervalSeconds":60,"stopwords":{"additions":null,"preset":"en","removals":null}},"moduleConfig":{"text2vec-openai":{"baseURL":"https://api.openai.com","model":"text-embedding-3-small","type":"text","vectorizeClassName":false}},"multiTenancyConfig":{"autoTenantActivation":false,"autoTenantCreation":false,"enabled":false},"properties":[{"dataType":["text"],"description":"The ID of the legal document this element belongs to.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":true,"vectorizePropertyName":false}},"name":"document","tokenization":"word"},{"dataType":["int"],"description":"The order of the element within its parent element.","indexFilterable":true,"indexSearchable":false,"moduleConfig":{"text2vec-openai":{"skip":false,"vectorizePropertyName":false}},"name":"order"},{"dataType":["text"],"description":"The path of the element in the legal document.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":true,"vectorizePropertyName":false}},"name":"path","tokenization":"word"},{"dataType":["text"],"description":"The path of the parent element in the legal document.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":true,"vectorizePropertyName":false}},"name":"parentPath","tokenization":"word"},{"dataType":["text"],"description":"The text content of the element.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":false,"vectorizePropertyName":false}},"name":"content","tokenization":"word"},{"dataType":["text"],"description":"The title of the element.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":false,"vectorizePropertyName":false}},"name":"title","tokenization":"word"},{"dataType":["text"],"description":"The name of the element.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":false,"vectorizePropertyName":false}},"name":"name","tokenization":"word"},{"dataType":["text"],"description":"The name of the parent element.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":false,"vectorizePropertyName":false}},"name":"parentName","tokenization":"word"},{"dataType":["text"],"description":"The outline of the element.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":false,"vectorizePropertyName":false}},"name":"outline","tokenization":"word"},{"dataType":["text"],"description":"The type of the element.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":true,"vectorizePropertyName":false}},"name":"type","tokenization":"word"}],"replicationConfig":{"factor":1},"shardingConfig":{"virtualPerPhysical":128,"desiredCount":1,"actualCount":1,"desiredVirtualCount":128,"actualVirtualCount":128,"key":"_id","strategy":"hash","function":"murmur3"},"vectorIndexConfig":{"skip":false,"cleanupIntervalSeconds":300,"maxConnections":64,"efConstruction":128,"ef":-1,"dynamicEfMin":100,"dynamicEfMax":500,"dynamicEfFactor":8,"vectorCacheMaxObjects":1000000000000,"flatSearchCutoff":40000,"distance":"cosine","pq":{"enabled":false,"bitCompression":false,"segments":0,"centroids":256,"trainingLimit":100000,"encoder":{"type":"kmeans","distribution":"log-normal"}},"bq":{"enabled":false}},"vectorIndexType":"hnsw","vectorizer":"text2vec-openai"},{"class":"Vector","description":"A vector of a string.","invertedIndexConfig":{"bm25":{"b":0.75,"k1":1.2},"cleanupIntervalSeconds":60,"stopwords":{"additions":null,"preset":"en","removals":null}},"moduleConfig":{"text2vec-openai":{"baseURL":"https://api.openai.com","model":"text-embedding-3-small","type":"text","vectorizeClassName":false}},"multiTenancyConfig":{"autoTenantActivation":false,"autoTenantCreation":false,"enabled":false},"properties":[{"dataType":["text"],"description":"The content value of the string.","indexFilterable":true,"indexSearchable":true,"moduleConfig":{"text2vec-openai":{"skip":false,"vectorizePropertyName":false}},"name":"value","tokenization":"word"}],"replicationConfig":{"factor":1},"shardingConfig":{"virtualPerPhysical":128,"desiredCount":1,"actualCount":1,"desiredVirtualCount":128,"actualVirtualCount":128,"key":"_id","strategy":"hash","function":"murmur3"},"vectorIndexConfig":{"skip":false,"cleanupIntervalSeconds":300,"maxConnections":64,"efConstruction":128,"ef":-1,"dynamicEfMin":100,"dynamicEfMax":500,"dynamicEfFactor":8,"vectorCacheMaxObjects":1000000000000,"flatSearchCutoff":40000,"distance":"cosine","pq":{"enabled":false,"bitCompression":false,"segments":0,"centroids":256,"trainingLimit":100000,"encoder":{"type":"kmeans","distribution":"log-normal"}},"bq":{"enabled":false}},"vectorIndexType":"hnsw","vectorizer":"text2vec-openai"}]}},"time":"2024-07-23T16:10:35Z"}
{"action":"graphql_rebuild","level":"debug","msg":"successfully rebuild graphql schema","time":"2024-07-23T16:10:35Z"}
{"level":"warning","msg":"prop len tracker file /var/lib/weaviate/vector/LkXsmhPzfkEj/proplengths does not exist, creating new tracker","time":"2024-07-23T16:10:36Z"}
{"action":"hnsw_prefill_cache_async","level":"info","msg":"not waiting for vector cache prefill, running in background","time":"2024-07-23T16:10:36Z","wait_for_cache_prefill":false}
{"action":"startup","level":"debug","msg":"shard=LkXsmhPzfkEj is ready","time":"2024-07-23T16:10:36Z"}
{"level":"info","msg":"Created shard vector_LkXsmhPzfkEj in 1.241324ms","time":"2024-07-23T16:10:36Z"}
{"action":"load_all_shards","level":"debug","msg":"finished loading all shards","time":"2024-07-23T16:10:36Z","took":"1.00201509s"}
{"action":"hnsw_vector_cache_prefill_level","count":0,"hnsw_level":0,"index_id":"main","level":"debug","msg":"prefilled level in vector cache","time":"2024-07-23T16:10:36Z","took":110475}
{"action":"hnsw_vector_cache_prefill","count":1000,"index_id":"main","level":"info","limit":1000000000000,"msg":"prefilled vector cache","time":"2024-07-23T16:10:36Z","took":159470}

The query example:

{
  Get {
    Element(
      nearText: {
        concepts: ["Does the contract specify the duration?"],
        moveTo: {
          concepts: ["duration of the contract","this document validity has a duration of"],
          force: 0.63
        },
        certainty: 0.689
      },
      where: {
        path: ["document"],
        operator: Equal,
        valueText: "absdef-16354621354"
      },
      autocut: 3,
      limit: 15,
    ) {
      content,
      title,
      name,
      outline,
      path,
      parentPath,
      parentName,
      document,
      order,
      type,
      _additional {
        id,
        certainty
      }
    }
  }
}

The errors appear at about 5% of queries

Is there any way to point me into the right direction to debug this thing?

2

hi @SergeLiatko !!

Welcome to our community :hugs: !!

Logs point to something on OpenAi side while vectorizing the moveTo.

The error appearing on 5% of the queries too, as this may be a load balancer delivering that request to a faulty node.

I am afraid there isn’t much we can do from Weaviate’s perspective. :grimacing:

One thing to explore is if Azure OpenAi can get you better results.

Let me know if this helps

Thanks!

3

@DudaNogueira

Hi, thanks for the update. I’ve spent a week fighting with this and finally ended up with a custom retry strategy for this (at least in 1-3 extra attempts the vectorization works).

If the delays will pose a problem in the future, I’ll move to your cloud services (already using it in another project, never had issues)

Thank you for your help.

1 Like
4

Great! Thanks for using Weaviate :slight_smile:

1 Like