{
  "$schema": "https://truvec.dev/blog/schema/benchmark.schema.json",
  "schema_version": 1,
  "sample": false,
  "source": {
    "benchmark_id": 14,
    "exported_at": "2026-11-14T09:28:44.998Z",
    "app_url": "https://app.truvec.dev"
  },
  "name": "Embedding models on contracts and regulations",
  "description": null,
  "run": {
    "top_k": 5,
    "created_at": "2026-11-14T09:26:08.463704Z",
    "completed_at": "2026-11-14T09:28:35.002567Z",
    "ranking": "MRR, then recall@k"
  },
  "corpus": {
    "name": "Commercial contracts (CUAD), GitHub terms and policies, and three EU regulations",
    "description": "36 commercial contracts from SEC filings (CUAD dataset, CC BY 4.0), 19 GitHub terms, agreements and policies (CC0), and the GDPR, the Digital Services Act and the AI Act.",
    "url": "https://www.atticusprojectai.org/cuad",
    "documents": 58,
    "chunks": 5024,
    "chunking": {
      "strategy": "recursive",
      "separator": "\n\n",
      "chunk_size": 1000,
      "chunk_overlap": 200
    }
  },
  "golden_set": {
    "name": "Generated golden set, reviewed",
    "questions": 100,
    "reviewed": true,
    "generated_by": "gpt-5.6-luna"
  },
  "notes": [
    "The corpus has three parts. 36 commercial contracts from the CUAD dataset (The Atticus Project, CC BY 4.0, text extracted from SEC EDGAR filings), picked across contract types among those between 25,000 and 90,000 characters. 19 terms, agreements and policies from github.com/github/site-policy (CC0). The English texts of Regulation (EU) 2016/679 (GDPR), Regulation (EU) 2022/2065 (Digital Services Act) and Regulation (EU) 2024/1689 (AI Act), from the Publications Office of the EU.",
    "The contract texts are used as published in CUAD: page numbers, redaction marks and layout left over from the filings are still in the chunks.",
    "Reviewing the questions, we rewrote 27 that did not say which document they were about (\"Which state’s law governs the agreement?\") to name the contract or its parties, the way a lawyer would ask. The name is often absent from the chunk that holds the clause, which is part of what makes legal retrieval hard.",
    "With 100 questions, a gap of a couple of points between two models is within the noise."
  ],
  "models": [
    {
      "rank": 1,
      "name": "Voyage 4 Large",
      "provider": "voyage",
      "model_name": "voyage-4-large",
      "dimension": 1024,
      "price_per_million_tokens": 0.12,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.94,
        "recall": 0.94,
        "precision": 0.19199999999999967,
        "f1_score": 0.3180952380952377,
        "mrr": 0.8281666666666666,
        "ndcg": 0.855566131206949
      },
      "correct_answers": 94,
      "hit_rate_at": {
        "1": 0.75,
        "2": 0.87,
        "3": 0.89,
        "4": 0.92,
        "5": 0.94
      },
      "rank_histogram": {
        "1": 75,
        "2": 12,
        "3": 2,
        "4": 3,
        "5": 2,
        "miss": 6
      },
      "scores": {
        "mean_relevant_score": 0.5761365370212767,
        "mean_top_distractor_score": 0.5063598658999998,
        "score_gap": 0.06822147308510637
      },
      "latency": {
        "embedding_ms_per_query": 12.705510361120105,
        "search_ms_p50": 13.072900474071503,
        "search_ms_p95": 51.6180741135031
      },
      "cost": {
        "corpus_tokens": 772490,
        "index_cost_usd": 0.0926988,
        "cost_per_1k_queries_usd": 0.0025572
      },
      "vector_storage_bytes": 20578304
    },
    {
      "rank": 2,
      "name": "Voyage 4",
      "provider": "voyage",
      "model_name": "voyage-4",
      "dimension": 1024,
      "price_per_million_tokens": 0.06,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.92,
        "recall": 0.915,
        "precision": 0.1859999999999997,
        "f1_score": 0.30857142857142816,
        "mrr": 0.8148333333333335,
        "ndcg": 0.8378147566887975
      },
      "correct_answers": 92,
      "hit_rate_at": {
        "1": 0.73,
        "2": 0.88,
        "3": 0.89,
        "4": 0.9,
        "5": 0.92
      },
      "rank_histogram": {
        "1": 73,
        "2": 15,
        "3": 1,
        "4": 1,
        "5": 2,
        "miss": 8
      },
      "scores": {
        "mean_relevant_score": 0.628285026195652,
        "mean_top_distractor_score": 0.5607417101000002,
        "score_gap": 0.06627111532608695
      },
      "latency": {
        "embedding_ms_per_query": 19.468766106292605,
        "search_ms_p50": 11.977208079770207,
        "search_ms_p95": 23.461418156512078
      },
      "cost": {
        "corpus_tokens": 772490,
        "index_cost_usd": 0.0463494,
        "cost_per_1k_queries_usd": 0.0012786
      },
      "vector_storage_bytes": 20578304
    },
    {
      "rank": 3,
      "name": "Gemini Embedding 2",
      "provider": "google",
      "model_name": "gemini-embedding-2",
      "dimension": 3072,
      "price_per_million_tokens": 0.2,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.88,
        "recall": 0.875,
        "precision": 0.17799999999999971,
        "f1_score": 0.2952380952380949,
        "mrr": 0.7418333333333332,
        "ndcg": 0.7727553921539996
      },
      "correct_answers": 88,
      "hit_rate_at": {
        "1": 0.65,
        "2": 0.77,
        "3": 0.84,
        "4": 0.85,
        "5": 0.88
      },
      "rank_histogram": {
        "1": 65,
        "2": 12,
        "3": 7,
        "4": 1,
        "5": 3,
        "miss": 12
      },
      "scores": {
        "mean_relevant_score": 0.807052164204545,
        "mean_top_distractor_score": 0.7828139927000001,
        "score_gap": 0.023523973068181804
      },
      "latency": {
        "embedding_ms_per_query": 18.926669349893928,
        "search_ms_p50": 17.954323906451464,
        "search_ms_p95": 44.387387158349156
      },
      "cost": {
        "corpus_tokens": 772490,
        "index_cost_usd": 0.154498,
        "cost_per_1k_queries_usd": 0.004262
      },
      "vector_storage_bytes": 61734912
    },
    {
      "rank": 4,
      "name": "Cohere Embed v4",
      "provider": "cohere",
      "model_name": "embed-v4.0",
      "dimension": 1536,
      "price_per_million_tokens": 0.12,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.84,
        "recall": 0.835,
        "precision": 0.16999999999999976,
        "f1_score": 0.2819047619047616,
        "mrr": 0.7343333333333334,
        "ndcg": 0.7568249675918753
      },
      "correct_answers": 84,
      "hit_rate_at": {
        "1": 0.67,
        "2": 0.75,
        "3": 0.79,
        "4": 0.81,
        "5": 0.84
      },
      "rank_histogram": {
        "1": 67,
        "2": 8,
        "3": 4,
        "4": 2,
        "5": 3,
        "miss": 16
      },
      "scores": {
        "mean_relevant_score": 0.6675076711904762,
        "mean_top_distractor_score": 0.6210744639999999,
        "score_gap": 0.04732003726190477
      },
      "latency": {
        "embedding_ms_per_query": 13.126984206028283,
        "search_ms_p50": 14.64542606845498,
        "search_ms_p95": 26.878292253240907
      },
      "cost": {
        "corpus_tokens": 772490,
        "index_cost_usd": 0.0926988,
        "cost_per_1k_queries_usd": 0.0025572
      },
      "vector_storage_bytes": 30867456
    },
    {
      "rank": 5,
      "name": "Qwen3 Embedding 8B",
      "provider": "fireworks",
      "model_name": "fireworks/qwen3-embedding-8b",
      "dimension": 4096,
      "price_per_million_tokens": 0.1,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.86,
        "recall": 0.86,
        "precision": 0.17599999999999974,
        "f1_score": 0.29142857142857115,
        "mrr": 0.7053333333333335,
        "ndcg": 0.7441885657524269
      },
      "correct_answers": 86,
      "hit_rate_at": {
        "1": 0.61,
        "2": 0.71,
        "3": 0.81,
        "4": 0.85,
        "5": 0.86
      },
      "rank_histogram": {
        "1": 61,
        "2": 10,
        "3": 10,
        "4": 4,
        "5": 1,
        "miss": 14
      },
      "scores": {
        "mean_relevant_score": 0.7814191648837212,
        "mean_top_distractor_score": 0.7462907775,
        "score_gap": 0.0330743131395349
      },
      "latency": {
        "embedding_ms_per_query": 165.63688127789646,
        "search_ms_p50": 19.21003614552319,
        "search_ms_p95": 72.04674545209855
      },
      "cost": {
        "corpus_tokens": 772490,
        "index_cost_usd": 0.077249,
        "cost_per_1k_queries_usd": 0.002131
      },
      "vector_storage_bytes": 82313216
    },
    {
      "rank": 6,
      "name": "OpenAI text-embedding-3-large",
      "provider": "openai",
      "model_name": "text-embedding-3-large",
      "dimension": 3072,
      "price_per_million_tokens": 0.13,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.83,
        "recall": 0.83,
        "precision": 0.16999999999999976,
        "f1_score": 0.2814285714285712,
        "mrr": 0.6970000000000002,
        "ndcg": 0.7292385024863473
      },
      "correct_answers": 83,
      "hit_rate_at": {
        "1": 0.6,
        "2": 0.75,
        "3": 0.78,
        "4": 0.82,
        "5": 0.83
      },
      "rank_histogram": {
        "1": 60,
        "2": 15,
        "3": 3,
        "4": 4,
        "5": 1,
        "miss": 17
      },
      "scores": {
        "mean_relevant_score": 0.679818439879518,
        "mean_top_distractor_score": 0.634239764,
        "score_gap": 0.04929822578313251
      },
      "latency": {
        "embedding_ms_per_query": 6.649633636698127,
        "search_ms_p50": 14.759794808924198,
        "search_ms_p95": 20.021873526275158
      },
      "cost": {
        "corpus_tokens": 772490,
        "index_cost_usd": 0.10042369999999999,
        "cost_per_1k_queries_usd": 0.0027703000000000003
      },
      "vector_storage_bytes": 61734912
    },
    {
      "rank": 7,
      "name": "OpenAI text-embedding-3-small",
      "provider": "openai",
      "model_name": "text-embedding-3-small",
      "dimension": 1536,
      "price_per_million_tokens": 0.02,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.77,
        "recall": 0.765,
        "precision": 0.15599999999999978,
        "f1_score": 0.25857142857142834,
        "mrr": 0.6558333333333333,
        "ndcg": 0.6802698445256276
      },
      "correct_answers": 77,
      "hit_rate_at": {
        "1": 0.57,
        "2": 0.7,
        "3": 0.74,
        "4": 0.77,
        "5": 0.77
      },
      "rank_histogram": {
        "1": 57,
        "2": 13,
        "3": 4,
        "4": 3,
        "5": 0,
        "miss": 23
      },
      "scores": {
        "mean_relevant_score": 0.7120548236363634,
        "mean_top_distractor_score": 0.6713952465000003,
        "score_gap": 0.03969142558441558
      },
      "latency": {
        "embedding_ms_per_query": 5.974293285980821,
        "search_ms_p50": 15.036932192742825,
        "search_ms_p95": 31.871584826149043
      },
      "cost": {
        "corpus_tokens": 772490,
        "index_cost_usd": 0.015449800000000001,
        "cost_per_1k_queries_usd": 0.0004262
      },
      "vector_storage_bytes": 30867456
    },
    {
      "rank": 8,
      "name": "BGE Base EN v1.5",
      "provider": "fireworks",
      "model_name": "BAAI/bge-base-en-v1.5",
      "dimension": 768,
      "price_per_million_tokens": 0.008,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.78,
        "recall": 0.775,
        "precision": 0.15799999999999978,
        "f1_score": 0.2619047619047617,
        "mrr": 0.6185,
        "ndcg": 0.6541024910655996
      },
      "correct_answers": 78,
      "hit_rate_at": {
        "1": 0.52,
        "2": 0.65,
        "3": 0.68,
        "4": 0.75,
        "5": 0.78
      },
      "rank_histogram": {
        "1": 52,
        "2": 13,
        "3": 3,
        "4": 7,
        "5": 3,
        "miss": 22
      },
      "scores": {
        "mean_relevant_score": 0.7608597666666668,
        "mean_top_distractor_score": 0.7441305093999999,
        "score_gap": 0.019667465384615383
      },
      "latency": {
        "embedding_ms_per_query": 13.646149430423975,
        "search_ms_p50": 11.526880087330937,
        "search_ms_p95": 23.071898357011378
      },
      "cost": {
        "corpus_tokens": 772490,
        "index_cost_usd": 0.00617992,
        "cost_per_1k_queries_usd": 0.00017047999999999998
      },
      "vector_storage_bytes": 15433728
    }
  ],
  "questions": [
    {
      "question": "Under the Whitesmoke promotion and distribution agreement, what restrictions apply to the Distributor's treatment of products already installed by end users after the agreement ends?",
      "expected_excerpt": "3.8 [ * ]. During the Term and for a period of [ * ] following the expiration or termination of this Agreement, Distributor shall not, and shall not engage any third party to: (a) restrict, modify, or reconfigure in any manner any of the Products that have been installed by End U",
      "first_relevant_rank": {
        "Voyage 4 Large": 1,
        "Voyage 4": 1,
        "Gemini Embedding 2": 2,
        "Cohere Embed v4": 3,
        "Qwen3 Embedding 8B": 2,
        "OpenAI text-embedding-3-large": null,
        "OpenAI text-embedding-3-small": 4,
        "BGE Base EN v1.5": 4
      }
    },
    {
      "question": "What responsibilities apply to anyone collecting personal information through the GitHub Service?",
      "expected_excerpt": "Your use of information from the Service must comply with the GitHub Privacy Statement. ## 8. Privacy Misuse of personal information is prohibited. Any person, entity, or service collecting data from the Service must comply with the GitHub Privacy Statement, particularly in regar",
      "first_relevant_rank": {
        "Voyage 4 Large": 2,
        "Voyage 4": 2,
        "Gemini Embedding 2": 3,
        "Cohere Embed v4": 1,
        "Qwen3 Embedding 8B": 2,
        "OpenAI text-embedding-3-large": null,
        "OpenAI text-embedding-3-small": null,
        "BGE Base EN v1.5": 4
      }
    },
    {
      "question": "Under the video-on-demand content license between Rogers and EuroMedia, when may a party disclose the other's confidential information to comply with legal requirements, and what must it do beforehand?",
      "expected_excerpt": "to the greatest extent possible. (b) Each party agrees to use the other party's confidential information solely for the purpose of fulfilling its obligations under this Agreement, to hold the other party's confidential information in confidence and to protect the confidentiality ",
      "first_relevant_rank": {
        "Voyage 4 Large": 5,
        "Voyage 4": 2,
        "Gemini Embedding 2": 4,
        "Cohere Embed v4": 5,
        "Qwen3 Embedding 8B": 3,
        "OpenAI text-embedding-3-large": null,
        "OpenAI text-embedding-3-small": 4,
        "BGE Base EN v1.5": 4
      }
    },
    {
      "question": "Under GitHub's Corporate Terms of Service, when can GitHub assign the agreement without the customer's consent?",
      "expected_excerpt": "### 3. Non-Assignability Neither Party may assign or otherwise transfer this Agreement, in whole or in part, without the other Party's prior written consent, such consent not to be unreasonably withheld, and any attempt to do so will be null and void, except that GitHub may assig",
      "first_relevant_rank": {
        "Voyage 4 Large": 1,
        "Voyage 4": 1,
        "Gemini Embedding 2": 1,
        "Cohere Embed v4": 1,
        "Qwen3 Embedding 8B": 3,
        "OpenAI text-embedding-3-large": 2,
        "OpenAI text-embedding-3-small": null,
        "BGE Base EN v1.5": 4
      }
    },
    {
      "question": "What kinds of conduct are prohibited at the GitHub event?",
      "expected_excerpt": "Unacceptable behavior includes, but is not limited to: * The use of sexualized language or imagery * Inappropriate physical contact, sexual attention, or advances * Trolling, insulting or derogatory comments or behavior * Personal or political attacks toward Event Participants * ",
      "first_relevant_rank": {
        "Voyage 4 Large": 1,
        "Voyage 4": 1,
        "Gemini Embedding 2": 2,
        "Cohere Embed v4": 2,
        "Qwen3 Embedding 8B": 3,
        "OpenAI text-embedding-3-large": 1,
        "OpenAI text-embedding-3-small": 1,
        "BGE Base EN v1.5": 5
      }
    },
    {
      "question": "What extra charges does a sponsor pay under the program, including when using a credit card?",
      "expected_excerpt": "### 2.2. Fees. An additional service fee (\"Service Fee\") of 3% per transaction will be deducted from Sponsor's account to cover GitHub's administrative costs associated with offering the Program. If payment is made via credit card, an additional 3% fee will be applied. ### 2.3. T",
      "first_relevant_rank": {
        "Voyage 4 Large": 1,
        "Voyage 4": 1,
        "Gemini Embedding 2": 1,
        "Cohere Embed v4": 1,
        "Qwen3 Embedding 8B": null,
        "OpenAI text-embedding-3-large": 4,
        "OpenAI text-embedding-3-small": null,
        "BGE Base EN v1.5": 3
      }
    },
    {
      "question": "What uses of GitHub-hosted Actions are not allowed?",
      "expected_excerpt": "* The provision of a stand-alone or integrated application or service offering the Actions product or service, or any elements of the Actions product or service, for commercial purposes; * Any activity that places a burden on our servers, where that burden is disproportionate to ",
      "first_relevant_rank": {
        "Voyage 4 Large": 1,
        "Voyage 4": 1,
        "Gemini Embedding 2": 1,
        "Cohere Embed v4": 2,
        "Qwen3 Embedding 8B": 3,
        "OpenAI text-embedding-3-large": 2,
        "OpenAI text-embedding-3-small": 2,
        "BGE Base EN v1.5": null
      }
    },
    {
      "question": "Do I need GitHub's permission to use its trademarks or logos?",
      "expected_excerpt": "## GitHub's Logos The license grant included with the Software is not for GitHub's trademarks, which include the Software logo designs. GitHub reserves all trademark and copyright rights in and to all GitHub trademarks. GitHub's logos include, for instance, the stylized designs t",
      "first_relevant_rank": {
        "Voyage 4 Large": 1,
        "Voyage 4": 1,
        "Gemini Embedding 2": 3,
        "Cohere Embed v4": 1,
        "Qwen3 Embedding 8B": 4,
        "OpenAI text-embedding-3-large": 2,
        "OpenAI text-embedding-3-small": 1,
        "BGE Base EN v1.5": 1
      }
    }
  ]
}
