{
  "$schema": "https://truvec.dev/blog/schema/benchmark.schema.json",
  "schema_version": 1,
  "sample": false,
  "source": {
    "benchmark_id": 13,
    "exported_at": "2026-11-14T09:24:44.833Z",
    "app_url": "https://app.truvec.dev"
  },
  "name": "Embedding models on French administrative guides and help pages",
  "description": null,
  "run": {
    "top_k": 5,
    "created_at": "2026-11-14T09:15:51.543125Z",
    "completed_at": "2026-11-14T09:24:08.623389Z",
    "ranking": "MRR, then recall@k"
  },
  "corpus": {
    "name": "service-public.fr practical guides and French Wikipedia help pages",
    "description": "230 practical guides for individuals from service-public.fr (Licence Ouverte 2.0) and 70 help pages from French Wikipedia (CC BY-SA 4.0).",
    "url": "https://www.service-public.gouv.fr",
    "documents": 300,
    "chunks": 4345,
    "chunking": {
      "strategy": "recursive",
      "separator": "\n\n",
      "chunk_size": 1000,
      "chunk_overlap": 200
    }
  },
  "golden_set": {
    "name": "Generated golden set, reviewed",
    "questions": 100,
    "reviewed": true,
    "generated_by": "gpt-5.6-luna"
  },
  "notes": [
    "The corpus is 230 practical guides (fiches pratiques) for individuals from the service-public.fr open data, published under Licence Ouverte 2.0, and 70 help pages from French Wikipedia (Aide namespace, CC BY-SA 4.0), fetched on 14 November 2026. Guides were picked at random among those between 5,000 and 30,000 characters, spread across themes (justice, family, housing, money and taxes, work, transport, health, foreigners, papers and citizenship).",
    "All 100 questions are written in French. 98 are about the administrative guides and 2 about the help pages: questions are drawn from chunks at random, and the guides make up most of the text.",
    "BGE Base EN v1.5 is English-only: it is included as a baseline.",
    "With 100 questions, a gap of a couple of points between two models is within the noise."
  ],
  "models": [
    {
      "rank": 1,
      "name": "Qwen3 Embedding 8B",
      "provider": "fireworks",
      "model_name": "fireworks/qwen3-embedding-8b",
      "dimension": 4096,
      "price_per_million_tokens": 0.1,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.98,
        "recall": 0.98,
        "precision": 0.19599999999999962,
        "f1_score": 0.3266666666666662,
        "mrr": 0.909,
        "ndcg": 0.9269021388946926
      },
      "correct_answers": 98,
      "hit_rate_at": {
        "1": 0.86,
        "2": 0.93,
        "3": 0.96,
        "4": 0.96,
        "5": 0.98
      },
      "rank_histogram": {
        "1": 86,
        "2": 7,
        "3": 3,
        "4": 0,
        "5": 2,
        "miss": 2
      },
      "scores": {
        "mean_relevant_score": 0.8294601224489799,
        "mean_top_distractor_score": 0.7803000854999997,
        "score_gap": 0.050218991020408145
      },
      "latency": {
        "embedding_ms_per_query": 10.426567969843745,
        "search_ms_p50": 20.755560137331486,
        "search_ms_p95": 59.65659643989056
      },
      "cost": {
        "corpus_tokens": 958116,
        "index_cost_usd": 0.09581160000000001,
        "cost_per_1k_queries_usd": 0.003085
      },
      "vector_storage_bytes": 71188480
    },
    {
      "rank": 2,
      "name": "Cohere Embed v4",
      "provider": "cohere",
      "model_name": "embed-v4.0",
      "dimension": 1536,
      "price_per_million_tokens": 0.12,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.99,
        "recall": 0.99,
        "precision": 0.19799999999999962,
        "f1_score": 0.32999999999999957,
        "mrr": 0.8898333333333334,
        "ndcg": 0.9151367970825703
      },
      "correct_answers": 99,
      "hit_rate_at": {
        "1": 0.82,
        "2": 0.92,
        "3": 0.96,
        "4": 0.97,
        "5": 0.99
      },
      "rank_histogram": {
        "1": 82,
        "2": 10,
        "3": 4,
        "4": 1,
        "5": 2,
        "miss": 1
      },
      "scores": {
        "mean_relevant_score": 0.6722431061616162,
        "mean_top_distractor_score": 0.6163258069000002,
        "score_gap": 0.05666846888888889
      },
      "latency": {
        "embedding_ms_per_query": 19.794600210152566,
        "search_ms_p50": 18.71430641040206,
        "search_ms_p95": 68.87485235929489
      },
      "cost": {
        "corpus_tokens": 958116,
        "index_cost_usd": 0.11497392,
        "cost_per_1k_queries_usd": 0.003702
      },
      "vector_storage_bytes": 26695680
    },
    {
      "rank": 3,
      "name": "Cohere Embed Multilingual v3",
      "provider": "cohere",
      "model_name": "embed-multilingual-v3.0",
      "dimension": 1024,
      "price_per_million_tokens": 0.1,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.99,
        "recall": 0.99,
        "precision": 0.19799999999999962,
        "f1_score": 0.32999999999999957,
        "mrr": 0.8745,
        "ndcg": 0.9038868640816539
      },
      "correct_answers": 99,
      "hit_rate_at": {
        "1": 0.79,
        "2": 0.91,
        "3": 0.97,
        "4": 0.98,
        "5": 0.99
      },
      "rank_histogram": {
        "1": 79,
        "2": 12,
        "3": 6,
        "4": 1,
        "5": 1,
        "miss": 1
      },
      "scores": {
        "mean_relevant_score": 0.7961759142424242,
        "mean_top_distractor_score": 0.7558563881000001,
        "score_gap": 0.040874701010100996
      },
      "latency": {
        "embedding_ms_per_query": 8.792499471455812,
        "search_ms_p50": 21.248674020171165,
        "search_ms_p95": 51.87701133545487
      },
      "cost": {
        "corpus_tokens": 958116,
        "index_cost_usd": 0.09581160000000001,
        "cost_per_1k_queries_usd": 0.003085
      },
      "vector_storage_bytes": 17797120
    },
    {
      "rank": 4,
      "name": "Voyage 4 Large",
      "provider": "voyage",
      "model_name": "voyage-4-large",
      "dimension": 1024,
      "price_per_million_tokens": 0.12,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.97,
        "recall": 0.97,
        "precision": 0.19399999999999962,
        "f1_score": 0.32333333333333286,
        "mrr": 0.8648333333333332,
        "ndcg": 0.8914460946182848
      },
      "correct_answers": 97,
      "hit_rate_at": {
        "1": 0.79,
        "2": 0.9,
        "3": 0.94,
        "4": 0.95,
        "5": 0.97
      },
      "rank_histogram": {
        "1": 79,
        "2": 11,
        "3": 4,
        "4": 1,
        "5": 2,
        "miss": 3
      },
      "scores": {
        "mean_relevant_score": 0.636743885154639,
        "mean_top_distractor_score": 0.5956296681,
        "score_gap": 0.042190507216494845
      },
      "latency": {
        "embedding_ms_per_query": 14.706519739702344,
        "search_ms_p50": 19.841571571305394,
        "search_ms_p95": 57.62487957254052
      },
      "cost": {
        "corpus_tokens": 958116,
        "index_cost_usd": 0.11497392,
        "cost_per_1k_queries_usd": 0.003702
      },
      "vector_storage_bytes": 17797120
    },
    {
      "rank": 5,
      "name": "OpenAI text-embedding-3-large",
      "provider": "openai",
      "model_name": "text-embedding-3-large",
      "dimension": 3072,
      "price_per_million_tokens": 0.13,
      "status": "completed",
      "metrics": {
        "hit_rate": 1,
        "recall": 1,
        "precision": 0.19999999999999962,
        "f1_score": 0.3333333333333329,
        "mrr": 0.8468333333333333,
        "ndcg": 0.8854746415306708
      },
      "correct_answers": 100,
      "hit_rate_at": {
        "1": 0.74,
        "2": 0.9,
        "3": 0.94,
        "4": 0.97,
        "5": 1
      },
      "rank_histogram": {
        "1": 74,
        "2": 16,
        "3": 4,
        "4": 3,
        "5": 3,
        "miss": 0
      },
      "scores": {
        "mean_relevant_score": 0.7292252361999998,
        "mean_top_distractor_score": 0.6890854402999999,
        "score_gap": 0.04013979590000002
      },
      "latency": {
        "embedding_ms_per_query": 8.990385113283992,
        "search_ms_p50": 24.289949564263225,
        "search_ms_p95": 90.01334689091891
      },
      "cost": {
        "corpus_tokens": 958116,
        "index_cost_usd": 0.12455508,
        "cost_per_1k_queries_usd": 0.0040105
      },
      "vector_storage_bytes": 53391360
    },
    {
      "rank": 6,
      "name": "Gemini Embedding 2",
      "provider": "google",
      "model_name": "gemini-embedding-2",
      "dimension": 3072,
      "price_per_million_tokens": 0.2,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.99,
        "recall": 0.99,
        "precision": 0.19799999999999962,
        "f1_score": 0.32999999999999957,
        "mrr": 0.8366666666666667,
        "ndcg": 0.8752914009331598
      },
      "correct_answers": 99,
      "hit_rate_at": {
        "1": 0.73,
        "2": 0.89,
        "3": 0.94,
        "4": 0.94,
        "5": 0.99
      },
      "rank_histogram": {
        "1": 73,
        "2": 16,
        "3": 5,
        "4": 0,
        "5": 5,
        "miss": 1
      },
      "scores": {
        "mean_relevant_score": 0.8236193387878785,
        "mean_top_distractor_score": 0.7981466708000003,
        "score_gap": 0.025192068282828287
      },
      "latency": {
        "embedding_ms_per_query": 19.764893860556185,
        "search_ms_p50": 20.678936270996928,
        "search_ms_p95": 59.31852294597774
      },
      "cost": {
        "corpus_tokens": 958116,
        "index_cost_usd": 0.19162320000000002,
        "cost_per_1k_queries_usd": 0.00617
      },
      "vector_storage_bytes": 53391360
    },
    {
      "rank": 7,
      "name": "OpenAI text-embedding-3-small",
      "provider": "openai",
      "model_name": "text-embedding-3-small",
      "dimension": 1536,
      "price_per_million_tokens": 0.02,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.97,
        "recall": 0.97,
        "precision": 0.19399999999999962,
        "f1_score": 0.32333333333333286,
        "mrr": 0.8184999999999999,
        "ndcg": 0.8563095587806693
      },
      "correct_answers": 97,
      "hit_rate_at": {
        "1": 0.73,
        "2": 0.82,
        "3": 0.91,
        "4": 0.94,
        "5": 0.97
      },
      "rank_histogram": {
        "1": 73,
        "2": 9,
        "3": 9,
        "4": 3,
        "5": 3,
        "miss": 3
      },
      "scores": {
        "mean_relevant_score": 0.7703049252577322,
        "mean_top_distractor_score": 0.7364492909999999,
        "score_gap": 0.03338188298969073
      },
      "latency": {
        "embedding_ms_per_query": 7.136375978589058,
        "search_ms_p50": 13.201437890529633,
        "search_ms_p95": 44.96410493738947
      },
      "cost": {
        "corpus_tokens": 958116,
        "index_cost_usd": 0.01916232,
        "cost_per_1k_queries_usd": 0.000617
      },
      "vector_storage_bytes": 26695680
    },
    {
      "rank": 8,
      "name": "BGE Base EN v1.5",
      "provider": "fireworks",
      "model_name": "BAAI/bge-base-en-v1.5",
      "dimension": 768,
      "price_per_million_tokens": 0.008,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.75,
        "recall": 0.75,
        "precision": 0.1499999999999998,
        "f1_score": 0.24999999999999978,
        "mrr": 0.6116666666666667,
        "ndcg": 0.6471808868043303
      },
      "correct_answers": 75,
      "hit_rate_at": {
        "1": 0.5,
        "2": 0.68,
        "3": 0.73,
        "4": 0.75,
        "5": 0.75
      },
      "rank_histogram": {
        "1": 50,
        "2": 18,
        "3": 5,
        "4": 2,
        "5": 0,
        "miss": 25
      },
      "scores": {
        "mean_relevant_score": 0.7634406577333332,
        "mean_top_distractor_score": 0.742848721,
        "score_gap": 0.018503617066666665
      },
      "latency": {
        "embedding_ms_per_query": 19.411734379827976,
        "search_ms_p50": 13.38830403983593,
        "search_ms_p95": 50.561792100779705
      },
      "cost": {
        "corpus_tokens": 957862,
        "index_cost_usd": 0.007662896,
        "cost_per_1k_queries_usd": 0.00024680000000000004
      },
      "vector_storage_bytes": 13347840
    }
  ],
  "questions": [
    {
      "question": "Après la validation de la demande en ligne, quel délai faut-il généralement prévoir pour recevoir son permis de conduire ?",
      "expected_excerpt": "À partir de la réception du SMS vous informant de la validation de votre demande en ligne, il faut compter environ 15 jours pour recevoir votre permis de conduire (titre de conduite). À la fin de l’étape de fabrication de votre titre de conduite, vous recevez un mail contenant un",
      "first_relevant_rank": {
        "Qwen3 Embedding 8B": 1,
        "Cohere Embed v4": null,
        "Cohere Embed Multilingual v3": 1,
        "Voyage 4 Large": null,
        "OpenAI text-embedding-3-large": 2,
        "Gemini Embedding 2": 5,
        "OpenAI text-embedding-3-small": 4,
        "BGE Base EN v1.5": 3
      }
    },
    {
      "question": "Quelles sont les conditions pour utiliser un sas Parafe à l’arrivée en France ?",
      "expected_excerpt": "Vous n'êtes pas autorisé à utiliser un sas Parafe pour quitter le territoire. ## Qui peut utiliser le dispositif Parafe ? Votre passeport doit être biométrique et en cours de validité. Attention : Si vous vivez en France et voyagez seul à l'étranger, vous devez avoir une autorisa",
      "first_relevant_rank": {
        "Qwen3 Embedding 8B": 5,
        "Cohere Embed v4": 4,
        "Cohere Embed Multilingual v3": 2,
        "Voyage 4 Large": null,
        "OpenAI text-embedding-3-large": 1,
        "Gemini Embedding 2": 5,
        "OpenAI text-embedding-3-small": 3,
        "BGE Base EN v1.5": 1
      }
    },
    {
      "question": "Quels territoires sont concernés par une interdiction de retour lorsqu’une IRTF est prononcée en France ?",
      "expected_excerpt": "À savoir : Si vous êtes resté en France malgré l'OQTF, l'IRTF peut être prolongée de 2 ans au maximum dans la limite maximum de 5 ans. ## Quelles sont les conséquences de l'IRTF ? L'IRTF vous interdit de revenir non seulement en France, mais aussi dans tout pays de l'espace Schen",
      "first_relevant_rank": {
        "Qwen3 Embedding 8B": 5,
        "Cohere Embed v4": 5,
        "Cohere Embed Multilingual v3": 3,
        "Voyage 4 Large": 2,
        "OpenAI text-embedding-3-large": 5,
        "Gemini Embedding 2": 5,
        "OpenAI text-embedding-3-small": null,
        "BGE Base EN v1.5": 4
      }
    },
    {
      "question": "Que dois-je faire auprès de ma caisse de Sécurité sociale si ma situation professionnelle change, par exemple si je pars à la retraite ou change d'activité ?",
      "expected_excerpt": "En cas de changement de situation professionnelle(retraite, changement d'activité, etc.), contactez votre organisme de Sécurité sociale pour connaître les démarches à effectuer. En cas de changement d'organisme assurant votre prise en charge des frais de santé, l'ancien organisme",
      "first_relevant_rank": {
        "Qwen3 Embedding 8B": 1,
        "Cohere Embed v4": 5,
        "Cohere Embed Multilingual v3": 4,
        "Voyage 4 Large": 4,
        "OpenAI text-embedding-3-large": 4,
        "Gemini Embedding 2": 3,
        "OpenAI text-embedding-3-small": 3,
        "BGE Base EN v1.5": null
      }
    },
    {
      "question": "Où une victime peut-elle déposer plainte, et quels documents peut-elle obtenir à la fin de son audition ?",
      "expected_excerpt": "### Sur place Vous devez vous rendre dans un commissariat de police ou à la gendarmerie de votre choix. À savoir : Les services de police ou de gendarmerie sont obligés d'enregistrer votre plainte si vous êtes victime d'une infraction. Les officiers et agents de police judiciaire",
      "first_relevant_rank": {
        "Qwen3 Embedding 8B": 3,
        "Cohere Embed v4": 2,
        "Cohere Embed Multilingual v3": 2,
        "Voyage 4 Large": 3,
        "OpenAI text-embedding-3-large": 2,
        "Gemini Embedding 2": 5,
        "OpenAI text-embedding-3-small": null,
        "BGE Base EN v1.5": null
      }
    },
    {
      "question": "Comment faire enregistrer une modification de convention de Pacs et quelles pièces faut-il présenter en mairie ?",
      "expected_excerpt": "Ce document indique notamment vos identités et les références de votre Pacs initial. Vous devez utiliser le formulaire cerfa suivant (n° 15790) : ## Comment faire enregistrer la convention modificative de Pacs ? Vous devez faire enregistrer votre convention modificative de Pacs. ",
      "first_relevant_rank": {
        "Qwen3 Embedding 8B": 1,
        "Cohere Embed v4": 1,
        "Cohere Embed Multilingual v3": 2,
        "Voyage 4 Large": 2,
        "OpenAI text-embedding-3-large": 3,
        "Gemini Embedding 2": 3,
        "OpenAI text-embedding-3-small": 2,
        "BGE Base EN v1.5": null
      }
    },
    {
      "question": "Comment un locataire peut-il contester un complément de loyer, et dans quel délai ?",
      "expected_excerpt": "Le montant du complément de loyer et les caractéristiques du logement qui le justifient doivent être indiqués dans le bail. Attention : Pour un bail signé depuis le 18 août 2022, le complément de loyer est interdit si le logement a au moins 1 de ces caractéristiques suivantes : s",
      "first_relevant_rank": {
        "Qwen3 Embedding 8B": 2,
        "Cohere Embed v4": 2,
        "Cohere Embed Multilingual v3": 3,
        "Voyage 4 Large": 3,
        "OpenAI text-embedding-3-large": 1,
        "Gemini Embedding 2": 2,
        "OpenAI text-embedding-3-small": 1,
        "BGE Base EN v1.5": null
      }
    },
    {
      "question": "Dans quelles situations un véhicule d’occasion peut-il circuler avec un certificat W garage ?",
      "expected_excerpt": "Leur apposition est obligatoire. Le numéro d’immatriculation est reproduit sur la plaque en caractères noirs sur fond rose. L'original du certificat doit être à bord du véhicule en circulation. Il doit être présenté en cas de contrôle. À noter : En cas d'utilisation du certificat",
      "first_relevant_rank": {
        "Qwen3 Embedding 8B": 2,
        "Cohere Embed v4": 1,
        "Cohere Embed Multilingual v3": 3,
        "Voyage 4 Large": 1,
        "OpenAI text-embedding-3-large": 1,
        "Gemini Embedding 2": 3,
        "OpenAI text-embedding-3-small": 3,
        "BGE Base EN v1.5": null
      }
    }
  ]
}
