An issue with the vectorizer module text2vec-transformers: 1024-dim vectors in 768-dim model

Support
1

Hi everyone! I have an issue with the vectorizer module text2vec-transformers . When doing some local tests with docker compose:

  text2vector-transformers:
    image: semitechnologies/transformers-inference:sentence-transformers-gtr-t5-large
    environment:
      ENABLE_CUDA: '0'

this leads to 1024-dim vectors, although sentence-transformers/gtr-t5-large · Hugging Face is 768-dim. When I generate vectors manually with sentence_transformers, the vectors are 768-dim. What do I miss here?

Schema:

{
    "class": "Document",
    "properties": [
        ...
    ],
    "vectorizer": "text2vec-transformers",
    "moduleConfig": {
        "text2vec-transformers": {
            "vectorizeClassName": false
        },
        "reranker-transformers": {}
    },
    "vectorIndexConfig": {
        "distance": "cosine"
    },
    "replicationConfig": {
        "factor": 3
    },
    "vectorIndexType": "hnsw",
    "multiTenancyConfig": {
        "enabled": true
    }
}

Here is what /v1/meta shows:

{
  "hostname": "http://[::]:8080",
  "modules": {
    "text2vec-transformers": {
      "model": {
        "_name_or_path": "./models/model",
        "add_cross_attention": false,
        "architectures": [
          "T5EncoderModel"
        ],
        "bad_words_ids": null,
        "begin_suppress_tokens": null,
        "bos_token_id": null,
        "chunk_size_feed_forward": 0,
        "cross_attention_hidden_size": null,
        "d_ff": 4096,
        "d_kv": 64,
        "d_model": 1024,
        "decoder_start_token_id": 0,
        "dense_act_fn": "relu",
        "diversity_penalty": 0,
        "do_sample": false,
        "dropout_rate": 0.1,
        "early_stopping": false,
        "encoder_no_repeat_ngram_size": 0,
        "eos_token_id": 1,
        "exponential_decay_length_penalty": null,
        "feed_forward_proj": "relu",
        "finetuning_task": null,
        "forced_bos_token_id": null,
        "forced_eos_token_id": null,
        "id2label": {
          "0": "LABEL_0",
          "1": "LABEL_1"
        },
        "initializer_factor": 1,
        "is_decoder": false,
        "is_encoder_decoder": true,
        "is_gated_act": false,
        "label2id": {
          "LABEL_0": 0,
          "LABEL_1": 1
        },
        "layer_norm_epsilon": 0.000001,
        "length_penalty": 1,
        "max_length": 20,
        "min_length": 0,
        "model_type": "t5",
        "n_positions": 512,
        "no_repeat_ngram_size": 0,
        "num_beam_groups": 1,
        "num_beams": 1,
        "num_decoder_layers": 24,
        "num_heads": 16,
        "num_layers": 24,
        "num_return_sequences": 1,
        "output_attentions": false,
        "output_hidden_states": false,
        "output_past": true,
        "output_scores": false,
        "pad_token_id": 0,
        "prefix": null,
        "problem_type": null,
        "pruned_heads": {},
        "relative_attention_max_distance": 128,
        "relative_attention_num_buckets": 32,
        "remove_invalid_values": false,
        "repetition_penalty": 1,
        "return_dict": true,
        "return_dict_in_generate": false,
        "sep_token_id": null,
        "suppress_tokens": null,
        "task_specific_params": {
          "summarization": {
            "early_stopping": true,
            "length_penalty": 2,
            "max_length": 200,
            "min_length": 30,
            "no_repeat_ngram_size": 3,
            "num_beams": 4,
            "prefix": "summarize: "
          },
          "translation_en_to_de": {
            "early_stopping": true,
            "max_length": 300,
            "num_beams": 4,
            "prefix": "translate English to German: "
          },
          "translation_en_to_fr": {
            "early_stopping": true,
            "max_length": 300,
            "num_beams": 4,
            "prefix": "translate English to French: "
          },
          "translation_en_to_ro": {
            "early_stopping": true,
            "max_length": 300,
            "num_beams": 4,
            "prefix": "translate English to Romanian: "
          }
        },
        "temperature": 1,
        "tf_legacy_loss": false,
        "tie_encoder_decoder": false,
        "tie_word_embeddings": true,
        "tokenizer_class": null,
        "top_k": 50,
        "top_p": 1,
        "torch_dtype": "float32",
        "torchscript": false,
        "transformers_version": "4.29.2",
        "typical_p": 1,
        "use_bfloat16": false,
        "use_cache": true,
        "vocab_size": 32128
      }
    }
  },
  "version": "1.21.5"
}
2

Hi @Timur ! Welcome to our community :hugs:

I have checked with our core engineers (that’s the reason of the delay here) and indeed we do have a bug.

They are working on it already, and we should have a fix in the comming days.

With your help and the help from our community we are able to constantly improve Weaviate.

Thanks a lot for this report. We really appreaciate it!

3

Hello @Timur our newest 1.8.5 release fixes this vector dimensions discrepancy. Now vector embeddings generated by T5 models are having proper vector dimensions.

Thanks for reporting this issue!