{
  "$schema": "https://truvec.dev/blog/schema/benchmark.schema.json",
  "schema_version": 1,
  "sample": false,
  "source": {
    "benchmark_id": 15,
    "exported_at": "2026-11-14T09:38:59.971Z",
    "app_url": "https://app.truvec.dev"
  },
  "name": "Multilingual embedding models on a 5-language help center",
  "description": null,
  "run": {
    "top_k": 5,
    "created_at": "2026-11-14T09:32:50.615960Z",
    "completed_at": "2026-11-14T09:38:29.357118Z",
    "ranking": "MRR, then recall@k"
  },
  "corpus": {
    "name": "MediaWiki help pages in English, French, Spanish, German and Arabic",
    "description": "33 help pages from mediawiki.org (CC0), each in five languages: English, French, Spanish, German and Arabic.",
    "url": "https://www.mediawiki.org/wiki/Help:Contents",
    "documents": 165,
    "chunks": 2050,
    "chunking": {
      "strategy": "recursive",
      "separator": "\n\n",
      "chunk_size": 1000,
      "chunk_overlap": 200
    }
  },
  "golden_set": {
    "name": "Generated golden set, reviewed",
    "questions": 100,
    "reviewed": true,
    "generated_by": "gpt-5.6-luna"
  },
  "notes": [
    "The corpus is 33 help pages from mediawiki.org (Help namespace, public domain under CC0), each taken in English, French, Spanish, German and Arabic: 165 documents, fetched as plain text on 14 November 2026. Pages were picked at random among the 106 help pages translated into all five languages, keeping those between 2,500 and 40,000 characters whose Arabic version is mostly in Arabic.",
    "The 100 questions cover the five languages: 33 in English, 21 in French, 16 in German, 15 in Spanish and 15 in Arabic. The generator wrote every question in English, so during review we translated 67 of them into the language of the passage they were written from.",
    "Each question expects the passage in its own language. The same passage exists in the four other languages: a model that returns one of those instead gets no credit for it, even though it answers the question. Scores here measure same-language retrieval inside a mixed-language corpus, and are lower than they would be on five separate corpora.",
    "BGE Base EN v1.5 is English-only: it is included as a baseline to show what a monolingual model loses. It reads at most 380 tokens per chunk in Truvec and Cohere Embed Multilingual v3 at most 512, so both saw some of the longer chunks cut short.",
    "With 100 questions, a gap of a couple of points between two models is within the noise, and each language has only 15 to 33 questions: per-language differences need more questions than this."
  ],
  "models": [
    {
      "rank": 1,
      "name": "OpenAI text-embedding-3-large",
      "provider": "openai",
      "model_name": "text-embedding-3-large",
      "dimension": 3072,
      "price_per_million_tokens": 0.13,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.97,
        "recall": 0.965,
        "precision": 0.19599999999999965,
        "f1_score": 0.3252380952380948,
        "mrr": 0.8480000000000001,
        "ndcg": 0.8757838497373254
      },
      "correct_answers": 97,
      "hit_rate_at": {
        "1": 0.78,
        "2": 0.85,
        "3": 0.91,
        "4": 0.93,
        "5": 0.97
      },
      "rank_histogram": {
        "1": 78,
        "2": 7,
        "3": 6,
        "4": 2,
        "5": 4,
        "miss": 3
      },
      "scores": {
        "mean_relevant_score": 0.6601908563917528,
        "mean_top_distractor_score": 0.6214110841000001,
        "score_gap": 0.038816627628865986
      },
      "latency": {
        "embedding_ms_per_query": 8.095544809475541,
        "search_ms_p50": 23.103229235857725,
        "search_ms_p95": 54.03342763893305
      },
      "cost": {
        "corpus_tokens": 435753,
        "index_cost_usd": 0.05664789,
        "cost_per_1k_queries_usd": 0.0038857
      },
      "vector_storage_bytes": 25190400
    },
    {
      "rank": 2,
      "name": "Cohere Embed Multilingual v3",
      "provider": "cohere",
      "model_name": "embed-multilingual-v3.0",
      "dimension": 1024,
      "price_per_million_tokens": 0.1,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.93,
        "recall": 0.925,
        "precision": 0.1879999999999997,
        "f1_score": 0.3119047619047615,
        "mrr": 0.8106666666666666,
        "ndcg": 0.8387724143983349
      },
      "correct_answers": 93,
      "hit_rate_at": {
        "1": 0.74,
        "2": 0.82,
        "3": 0.87,
        "4": 0.91,
        "5": 0.93
      },
      "rank_histogram": {
        "1": 74,
        "2": 8,
        "3": 5,
        "4": 4,
        "5": 2,
        "miss": 7
      },
      "scores": {
        "mean_relevant_score": 0.7302731831182792,
        "mean_top_distractor_score": 0.6959106396000001,
        "score_gap": 0.03475769494623655
      },
      "latency": {
        "embedding_ms_per_query": 7.813271810300648,
        "search_ms_p50": 11.661910451948643,
        "search_ms_p95": 48.64237825386226
      },
      "cost": {
        "corpus_tokens": 421590,
        "index_cost_usd": 0.042159,
        "cost_per_1k_queries_usd": 0.002989
      },
      "vector_storage_bytes": 8396800
    },
    {
      "rank": 3,
      "name": "Cohere Embed v4",
      "provider": "cohere",
      "model_name": "embed-v4.0",
      "dimension": 1536,
      "price_per_million_tokens": 0.12,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.95,
        "recall": 0.945,
        "precision": 0.19199999999999967,
        "f1_score": 0.3185714285714282,
        "mrr": 0.7933333333333334,
        "ndcg": 0.8306240921422392
      },
      "correct_answers": 95,
      "hit_rate_at": {
        "1": 0.7,
        "2": 0.81,
        "3": 0.88,
        "4": 0.9,
        "5": 0.95
      },
      "rank_histogram": {
        "1": 70,
        "2": 11,
        "3": 7,
        "4": 2,
        "5": 5,
        "miss": 5
      },
      "scores": {
        "mean_relevant_score": 0.6277015112631579,
        "mean_top_distractor_score": 0.5989986843,
        "score_gap": 0.026879957263157905
      },
      "latency": {
        "embedding_ms_per_query": 10.747332191094756,
        "search_ms_p50": 11.429332429543138,
        "search_ms_p95": 30.54544867482031
      },
      "cost": {
        "corpus_tokens": 435753,
        "index_cost_usd": 0.05229036,
        "cost_per_1k_queries_usd": 0.0035868
      },
      "vector_storage_bytes": 12595200
    },
    {
      "rank": 4,
      "name": "Gemini Embedding 2",
      "provider": "google",
      "model_name": "gemini-embedding-2",
      "dimension": 3072,
      "price_per_million_tokens": 0.2,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.93,
        "recall": 0.925,
        "precision": 0.18599999999999967,
        "f1_score": 0.3095238095238091,
        "mrr": 0.7814999999999999,
        "ndcg": 0.8150418190117332
      },
      "correct_answers": 93,
      "hit_rate_at": {
        "1": 0.68,
        "2": 0.83,
        "3": 0.86,
        "4": 0.91,
        "5": 0.93
      },
      "rank_histogram": {
        "1": 68,
        "2": 15,
        "3": 3,
        "4": 5,
        "5": 2,
        "miss": 7
      },
      "scores": {
        "mean_relevant_score": 0.8127070596774194,
        "mean_top_distractor_score": 0.7995070170999997,
        "score_gap": 0.012374232580645166
      },
      "latency": {
        "embedding_ms_per_query": 21.875918279401958,
        "search_ms_p50": 21.97854802943766,
        "search_ms_p95": 64.97346842661496
      },
      "cost": {
        "corpus_tokens": 435753,
        "index_cost_usd": 0.08715060000000001,
        "cost_per_1k_queries_usd": 0.005978
      },
      "vector_storage_bytes": 25190400
    },
    {
      "rank": 5,
      "name": "Voyage 4 Large",
      "provider": "voyage",
      "model_name": "voyage-4-large",
      "dimension": 1024,
      "price_per_million_tokens": 0.12,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.95,
        "recall": 0.95,
        "precision": 0.19199999999999967,
        "f1_score": 0.31904761904761864,
        "mrr": 0.7805,
        "ndcg": 0.8232343950159638
      },
      "correct_answers": 95,
      "hit_rate_at": {
        "1": 0.68,
        "2": 0.79,
        "3": 0.88,
        "4": 0.91,
        "5": 0.95
      },
      "rank_histogram": {
        "1": 68,
        "2": 11,
        "3": 9,
        "4": 3,
        "5": 4,
        "miss": 5
      },
      "scores": {
        "mean_relevant_score": 0.6161680103157894,
        "mean_top_distractor_score": 0.6024025509999998,
        "score_gap": 0.012950714105263168
      },
      "latency": {
        "embedding_ms_per_query": 16.970451488159597,
        "search_ms_p50": 13.293205527588725,
        "search_ms_p95": 32.658246904611566
      },
      "cost": {
        "corpus_tokens": 435753,
        "index_cost_usd": 0.05229036,
        "cost_per_1k_queries_usd": 0.0035868
      },
      "vector_storage_bytes": 8396800
    },
    {
      "rank": 6,
      "name": "Qwen3 Embedding 8B",
      "provider": "fireworks",
      "model_name": "fireworks/qwen3-embedding-8b",
      "dimension": 4096,
      "price_per_million_tokens": 0.1,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.9,
        "recall": 0.895,
        "precision": 0.1799999999999997,
        "f1_score": 0.29952380952380914,
        "mrr": 0.7488333333333334,
        "ndcg": 0.7847903104954149
      },
      "correct_answers": 90,
      "hit_rate_at": {
        "1": 0.65,
        "2": 0.79,
        "3": 0.83,
        "4": 0.86,
        "5": 0.9
      },
      "rank_histogram": {
        "1": 65,
        "2": 14,
        "3": 4,
        "4": 3,
        "5": 4,
        "miss": 10
      },
      "scores": {
        "mean_relevant_score": 0.8050934795555557,
        "mean_top_distractor_score": 0.7840421682999998,
        "score_gap": 0.019258548333333347
      },
      "latency": {
        "embedding_ms_per_query": 20.668854592368007,
        "search_ms_p50": 21.47119538858533,
        "search_ms_p95": 55.60884154401718
      },
      "cost": {
        "corpus_tokens": 435753,
        "index_cost_usd": 0.043575300000000004,
        "cost_per_1k_queries_usd": 0.002989
      },
      "vector_storage_bytes": 33587200
    },
    {
      "rank": 7,
      "name": "OpenAI text-embedding-3-small",
      "provider": "openai",
      "model_name": "text-embedding-3-small",
      "dimension": 1536,
      "price_per_million_tokens": 0.02,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.88,
        "recall": 0.88,
        "precision": 0.17999999999999972,
        "f1_score": 0.2980952380952378,
        "mrr": 0.7338333333333334,
        "ndcg": 0.7681931052455198
      },
      "correct_answers": 88,
      "hit_rate_at": {
        "1": 0.64,
        "2": 0.76,
        "3": 0.83,
        "4": 0.84,
        "5": 0.88
      },
      "rank_histogram": {
        "1": 64,
        "2": 12,
        "3": 7,
        "4": 1,
        "5": 4,
        "miss": 12
      },
      "scores": {
        "mean_relevant_score": 0.6925301263636368,
        "mean_top_distractor_score": 0.6477477822000001,
        "score_gap": 0.03790518954545455
      },
      "latency": {
        "embedding_ms_per_query": 5.6850570673123,
        "search_ms_p50": 11.035828851163387,
        "search_ms_p95": 52.30583320371806
      },
      "cost": {
        "corpus_tokens": 435753,
        "index_cost_usd": 0.00871506,
        "cost_per_1k_queries_usd": 0.0005978000000000001
      },
      "vector_storage_bytes": 12595200
    },
    {
      "rank": 8,
      "name": "BGE Base EN v1.5",
      "provider": "fireworks",
      "model_name": "BAAI/bge-base-en-v1.5",
      "dimension": 768,
      "price_per_million_tokens": 0.008,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.7,
        "recall": 0.7,
        "precision": 0.14399999999999985,
        "f1_score": 0.238095238095238,
        "mrr": 0.583,
        "ndcg": 0.610369706417667
      },
      "correct_answers": 70,
      "hit_rate_at": {
        "1": 0.52,
        "2": 0.59,
        "3": 0.62,
        "4": 0.66,
        "5": 0.7
      },
      "rank_histogram": {
        "1": 52,
        "2": 7,
        "3": 3,
        "4": 4,
        "5": 4,
        "miss": 30
      },
      "scores": {
        "mean_relevant_score": 0.7613404055714287,
        "mean_top_distractor_score": 0.7275720819,
        "score_gap": 0.03292720114285715
      },
      "latency": {
        "embedding_ms_per_query": 17.630654950626194,
        "search_ms_p50": 10.648825904354453,
        "search_ms_p95": 24.978168751113113
      },
      "cost": {
        "corpus_tokens": 396366,
        "index_cost_usd": 0.003170928,
        "cost_per_1k_queries_usd": 0.00023912
      },
      "vector_storage_bytes": 6297600
    }
  ],
  "questions": [
    {
      "question": "¿Cómo decide WebFonts si usa una fuente de mi dispositivo, la descarga o la recupera de la caché?",
      "expected_excerpt": "## Uso Cuando la extensión está instalada y habilitada, tras el inicio de sesión, las preferencias de enlace en la página wiki, aparecerá un menú para seleccionar una fuente de la página. De forma predeterminada, la primera fuente de dicho menú se aplica a la wiki. Un usuario pue",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 1,
        "Cohere Embed Multilingual v3": 1,
        "Cohere Embed v4": 1,
        "Gemini Embedding 2": 4,
        "Voyage 4 Large": 3,
        "Qwen3 Embedding 8B": 5,
        "OpenAI text-embedding-3-small": 2,
        "BGE Base EN v1.5": null
      }
    },
    {
      "question": "¿Qué permiso se necesita para crear listas de envío de MassMessage, y dónde se puede crear una lista nueva?",
      "expected_excerpt": "Note that creating delivery lists requires the user to have the createwithcontentmodel user right (see phab:T92795). On most Wikimedia projects, only logged-in users have createwithcontentmodel rights. MassMessage includes a special page for managing delivery lists. To create a n",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 5,
        "Cohere Embed Multilingual v3": null,
        "Cohere Embed v4": 4,
        "Gemini Embedding 2": 3,
        "Voyage 4 Large": 3,
        "Qwen3 Embedding 8B": 2,
        "OpenAI text-embedding-3-small": 3,
        "BGE Base EN v1.5": null
      }
    },
    {
      "question": "Quelles sont les étapes pour commencer une nouvelle traduction avec l'outil de traduction de contenu ?",
      "expected_excerpt": "Deuxième manière: cliquez sur \"Contributions\", puis sur le bouton \"Traduction\" en haut de la page. Troisième méthode: chercher la page à traduire sur le wiki dans la langue d'origine, et cliquez sur la langue de destination dans la liste des liens inter-langage de la barre vertic",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 3,
        "Cohere Embed Multilingual v3": 3,
        "Cohere Embed v4": null,
        "Gemini Embedding 2": 2,
        "Voyage 4 Large": 1,
        "Qwen3 Embedding 8B": 5,
        "OpenAI text-embedding-3-small": 5,
        "BGE Base EN v1.5": null
      }
    },
    {
      "question": "كيف أشترك في موضوع نقاش ثم أوقف تلقي الإشعارات عنه لاحقًا؟",
      "expected_excerpt": "preloadtitle – لو ذكرت، ستستخدم القيمة في صفة سطر موضوع الرسالة على صفحة النقاش preload - لو ذكرت، ستستخدم الصفحة المذكورة في القيمة في التحميل المسبق لمحتوى الرسالة على صفحة النقاش preloadparams[] – لو ذكرت، ستستخدم القيمة لاستبدال المعطيات الموجودة في قالب التحميل المسبق؛ كل مع",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 4,
        "Cohere Embed Multilingual v3": 2,
        "Cohere Embed v4": 2,
        "Gemini Embedding 2": null,
        "Voyage 4 Large": 5,
        "Qwen3 Embedding 8B": 4,
        "OpenAI text-embedding-3-small": 3,
        "BGE Base EN v1.5": null
      }
    },
    {
      "question": "Wie setze ich mein Wiki-Passwort mit meinem Benutzernamen oder meiner E-Mail-Adresse zurück?",
      "expected_excerpt": "## Muss ich meine E-Mail-Adresse bestätigen, um das Zurücksetzen zu ermöglichen? Nein, du musst deine E-Mail-Adresse nicht bestätigen, um Passwort-Rücksetzungsmails zu erzeugen. Nichtsdestotrotz wird dringend empfohlen, deine E-Mail-Adresse zu bestätigen. Dadurch kannst du sicher",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 1,
        "Cohere Embed Multilingual v3": 3,
        "Cohere Embed v4": 3,
        "Gemini Embedding 2": 1,
        "Voyage 4 Large": 2,
        "Qwen3 Embedding 8B": 5,
        "OpenAI text-embedding-3-small": 2,
        "BGE Base EN v1.5": null
      }
    },
    {
      "question": "What are the main navigation areas available on a MediaWiki page?",
      "expected_excerpt": "The sidebar gives you access to important pages in the wiki such as recent changes or upload file. MediaWiki requires you to log in before seeing all of the sidebar options. Links (often called tabs) which relate to the page currently displayed: its associated discussion page, th",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 3,
        "Cohere Embed Multilingual v3": null,
        "Cohere Embed v4": 5,
        "Gemini Embedding 2": 3,
        "Voyage 4 Large": 1,
        "Qwen3 Embedding 8B": null,
        "OpenAI text-embedding-3-small": null,
        "BGE Base EN v1.5": 2
      }
    },
    {
      "question": "¿Durante cuánto tiempo se conserva la dirección IP asociada a una edición hecha con una cuenta temporal?",
      "expected_excerpt": "Restringimos el acceso a los datos de direcciones IP de las cuentas temporales a los usuarios que requieren de este acceso con fines de moderación antiabuso. Los criterios de acceso a estos datos se articulan en una política legal llamada Acceso a las direcciones IP de las cuenta",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 3,
        "Cohere Embed Multilingual v3": 1,
        "Cohere Embed v4": 1,
        "Gemini Embedding 2": 1,
        "Voyage 4 Large": 1,
        "Qwen3 Embedding 8B": 5,
        "OpenAI text-embedding-3-small": 2,
        "BGE Base EN v1.5": 4
      }
    },
    {
      "question": "ما المعلومات التي تتلقاها خدمات الترجمة الخارجية، وما القيد المتعلق بالبيانات الشخصية عند ربطها؟",
      "expected_excerpt": "### توسيع الدعم اللغوي بخدمات الترجمة الجديدة تم تصميم ترجمة المحتوى كمنصة قابلة للتوسيع. لذلك من الممكن تطوير عملاء جدد لدمج خدمات الترجمة الإضافية. بعض الاعتبارات حول طريقة تكامل خدمات الترجمة: يتم نشر الترجمات الآلية وتصحيحات المستخدم التي تم إجراؤها كجزء من بيانات الترجمات ال",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 1,
        "Cohere Embed Multilingual v3": 2,
        "Cohere Embed v4": 3,
        "Gemini Embedding 2": 4,
        "Voyage 4 Large": 4,
        "Qwen3 Embedding 8B": 3,
        "OpenAI text-embedding-3-small": null,
        "BGE Base EN v1.5": null
      }
    }
  ]
}
