{
  "$schema": "https://truvec.dev/blog/schema/benchmark.schema.json",
  "schema_version": 1,
  "sample": false,
  "source": {
    "benchmark_id": 8,
    "exported_at": "2026-11-14T00:56:57.403Z",
    "app_url": "https://app.truvec.dev"
  },
  "name": "OpenAI vs Voyage AI on a technical knowledge base",
  "description": null,
  "run": {
    "top_k": 5,
    "created_at": "2026-11-14T00:55:38.089392Z",
    "completed_at": "2026-11-14T00:56:15.703224Z",
    "ranking": "MRR, then recall@k"
  },
  "corpus": {
    "name": "Docker documentation: manuals, FAQs, billing and support pages",
    "description": "261 pages of the public Docker documentation: Docker Desktop, Docker Hub, Compose, accounts, security, subscription and billing, FAQs, support and getting-started guides.",
    "url": "https://github.com/docker/docs",
    "documents": 261,
    "chunks": 1944,
    "chunking": {
      "strategy": "recursive",
      "separator": "\n\n",
      "chunk_size": 1000,
      "chunk_overlap": 200
    }
  },
  "golden_set": {
    "name": "Generated golden set, reviewed",
    "questions": 100,
    "reviewed": true,
    "generated_by": "gpt-5.6-luna"
  },
  "notes": [
    "The corpus is a subset of the public Docker documentation (Apache 2.0), taken from github.com/docker/docs at commit 553482c: the Desktop, Docker Hub, Compose, accounts, security and subscription and billing manuals, the FAQs, the support pages and the getting-started guides. Release notes and pages shorter than 600 characters were left out, and pages were cut at 40,000 characters.",
    "Several posts on this blog use this corpus and the same 100 questions, each with its own selection of models. A model can differ by a question between two of these runs: questions are embedded again at each run, and vector search is approximate.",
    "With 100 questions, a gap of a couple of points between two models is within the noise."
  ],
  "models": [
    {
      "rank": 1,
      "name": "Voyage 4 Large",
      "provider": "voyage",
      "model_name": "voyage-4-large",
      "dimension": 1024,
      "price_per_million_tokens": 0.12,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.99,
        "recall": 0.975,
        "precision": 0.20599999999999966,
        "f1_score": 0.3380952380952379,
        "mrr": 0.9216666666666665,
        "ndcg": 0.9243010127312083
      },
      "correct_answers": 99,
      "hit_rate_at": {
        "1": 0.87,
        "2": 0.94,
        "3": 0.99,
        "4": 0.99,
        "5": 0.99
      },
      "rank_histogram": {
        "1": 87,
        "2": 7,
        "3": 5,
        "4": 0,
        "5": 0,
        "miss": 1
      },
      "scores": {
        "mean_relevant_score": 0.6531894676767677,
        "mean_top_distractor_score": 0.6068937536,
        "score_gap": 0.046386561010101024
      },
      "latency": {
        "embedding_ms_per_query": 17.869800459593534,
        "search_ms_p50": 21.78719500079751,
        "search_ms_p95": 60.986425285227554
      },
      "cost": {
        "corpus_tokens": 336360,
        "index_cost_usd": 0.040363199999999995,
        "cost_per_1k_queries_usd": 0.002376
      },
      "vector_storage_bytes": 7962624
    },
    {
      "rank": 2,
      "name": "Voyage 4",
      "provider": "voyage",
      "model_name": "voyage-4",
      "dimension": 1024,
      "price_per_million_tokens": 0.06,
      "status": "completed",
      "metrics": {
        "hit_rate": 1,
        "recall": 0.995,
        "precision": 0.21199999999999963,
        "f1_score": 0.3471428571428571,
        "mrr": 0.9153333333333333,
        "ndcg": 0.9285378503730927
      },
      "correct_answers": 100,
      "hit_rate_at": {
        "1": 0.85,
        "2": 0.95,
        "3": 0.99,
        "4": 0.99,
        "5": 1
      },
      "rank_histogram": {
        "1": 85,
        "2": 10,
        "3": 4,
        "4": 0,
        "5": 1,
        "miss": 0
      },
      "scores": {
        "mean_relevant_score": 0.6923111795000003,
        "mean_top_distractor_score": 0.6462398736999999,
        "score_gap": 0.04607130580000002
      },
      "latency": {
        "embedding_ms_per_query": 44.48924149852246,
        "search_ms_p50": 18.194705480709672,
        "search_ms_p95": 55.89457424357532
      },
      "cost": {
        "corpus_tokens": 336360,
        "index_cost_usd": 0.020181599999999997,
        "cost_per_1k_queries_usd": 0.001188
      },
      "vector_storage_bytes": 7962624
    },
    {
      "rank": 3,
      "name": "Voyage 4 Lite",
      "provider": "voyage",
      "model_name": "voyage-4-lite",
      "dimension": 1024,
      "price_per_million_tokens": 0.02,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.99,
        "recall": 0.975,
        "precision": 0.20599999999999966,
        "f1_score": 0.3380952380952379,
        "mrr": 0.8933333333333333,
        "ndcg": 0.9067466674372643
      },
      "correct_answers": 99,
      "hit_rate_at": {
        "1": 0.81,
        "2": 0.95,
        "3": 0.99,
        "4": 0.99,
        "5": 0.99
      },
      "rank_histogram": {
        "1": 81,
        "2": 14,
        "3": 4,
        "4": 0,
        "5": 0,
        "miss": 1
      },
      "scores": {
        "mean_relevant_score": 0.6653339821212121,
        "mean_top_distractor_score": 0.6216108191,
        "score_gap": 0.04393168232323233
      },
      "latency": {
        "embedding_ms_per_query": 14.506500638090074,
        "search_ms_p50": 12.447818415239453,
        "search_ms_p95": 24.89382554776965
      },
      "cost": {
        "corpus_tokens": 336360,
        "index_cost_usd": 0.0067272,
        "cost_per_1k_queries_usd": 0.000396
      },
      "vector_storage_bytes": 7962624
    },
    {
      "rank": 4,
      "name": "OpenAI text-embedding-3-large",
      "provider": "openai",
      "model_name": "text-embedding-3-large",
      "dimension": 3072,
      "price_per_million_tokens": 0.13,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.99,
        "recall": 0.975,
        "precision": 0.20599999999999963,
        "f1_score": 0.3380952380952379,
        "mrr": 0.867,
        "ndcg": 0.8848806698984153
      },
      "correct_answers": 99,
      "hit_rate_at": {
        "1": 0.79,
        "2": 0.88,
        "3": 0.94,
        "4": 0.98,
        "5": 0.99
      },
      "rank_histogram": {
        "1": 79,
        "2": 9,
        "3": 6,
        "4": 4,
        "5": 1,
        "miss": 1
      },
      "scores": {
        "mean_relevant_score": 0.6865508511111111,
        "mean_top_distractor_score": 0.6498379860000004,
        "score_gap": 0.03667220191919192
      },
      "latency": {
        "embedding_ms_per_query": 12.707279929891229,
        "search_ms_p50": 27.47622667811811,
        "search_ms_p95": 123.9184321137145
      },
      "cost": {
        "corpus_tokens": 336360,
        "index_cost_usd": 0.0437268,
        "cost_per_1k_queries_usd": 0.002574
      },
      "vector_storage_bytes": 23887872
    },
    {
      "rank": 5,
      "name": "OpenAI text-embedding-3-small",
      "provider": "openai",
      "model_name": "text-embedding-3-small",
      "dimension": 1536,
      "price_per_million_tokens": 0.02,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.95,
        "recall": 0.935,
        "precision": 0.19799999999999965,
        "f1_score": 0.3247619047619045,
        "mrr": 0.7686666666666667,
        "ndcg": 0.8067558320662286
      },
      "correct_answers": 95,
      "hit_rate_at": {
        "1": 0.63,
        "2": 0.84,
        "3": 0.92,
        "4": 0.94,
        "5": 0.95
      },
      "rank_histogram": {
        "1": 63,
        "2": 21,
        "3": 8,
        "4": 2,
        "5": 1,
        "miss": 5
      },
      "scores": {
        "mean_relevant_score": 0.7154965138947369,
        "mean_top_distractor_score": 0.6871834310999999,
        "score_gap": 0.02744412694736841
      },
      "latency": {
        "embedding_ms_per_query": 4.785056002438068,
        "search_ms_p50": 12.046341551467776,
        "search_ms_p95": 32.14051823597401
      },
      "cost": {
        "corpus_tokens": 336360,
        "index_cost_usd": 0.0067272,
        "cost_per_1k_queries_usd": 0.000396
      },
      "vector_storage_bytes": 11943936
    }
  ],
  "questions": [
    {
      "question": "When both JIT and SCIM are enabled, how are organization roles and account deactivation handled?",
      "expected_excerpt": "Roles don't move back and forth. JIT sets the organization role only when it first adds the user. A later SCIM update can change that role, and the next sign-in leaves the SCIM role in place. When a user isn't assigned to the Docker application in the IdP, the next synchronizatio",
      "first_relevant_rank": {
        "Voyage 4 Large": 1,
        "Voyage 4": 1,
        "Voyage 4 Lite": 2,
        "OpenAI text-embedding-3-large": 5,
        "OpenAI text-embedding-3-small": null
      }
    },
    {
      "question": "When multiple Compose files are supplied with `-f`, how are their service settings combined, and which file takes precedence?",
      "expected_excerpt": "Compose merges files in the order they're specified on the command line. Subsequent files may merge, override, or add to their predecessors. For example: ```console $ docker compose -f compose.yaml -f compose.admin.yaml run backup_db ``` The `compose.yaml` file might specify a `w",
      "first_relevant_rank": {
        "Voyage 4 Large": 1,
        "Voyage 4": 2,
        "Voyage 4 Lite": 3,
        "OpenAI text-embedding-3-large": 1,
        "OpenAI text-embedding-3-small": 4
      }
    },
    {
      "question": "What command builds the Docker image for the Spring Boot hello world app, and how can I check the size of the resulting image?",
      "expected_excerpt": "```dockerfile CMD [\"./mvnw\", \"spring-boot:run\"] ``` And with that, you should have the following Dockerfile: ```dockerfile FROM eclipse-temurin:21.0.8_9-jdk-jammy WORKDIR /app COPY .mvn/ .mvn COPY mvnw pom.xml ./ RUN ./mvnw dependency:go-offline COPY src ./src CMD [\"./mvnw\", \"spr",
      "first_relevant_rank": {
        "Voyage 4 Large": 2,
        "Voyage 4": 3,
        "Voyage 4 Lite": 2,
        "OpenAI text-embedding-3-large": 1,
        "OpenAI text-embedding-3-small": 5
      }
    },
    {
      "question": "How can I combine multiple Compose files, and what issue might arise as the configuration becomes more complex?",
      "expected_excerpt": "# Use multiple Compose files This section contains information on the ways you can work with multiple Compose files. Using multiple Compose files lets you customize a Compose application for different environments or workflows. This is useful for large applications that may use d",
      "first_relevant_rank": {
        "Voyage 4 Large": 2,
        "Voyage 4": 3,
        "Voyage 4 Lite": 2,
        "OpenAI text-embedding-3-large": 1,
        "OpenAI text-embedding-3-small": 1
      }
    },
    {
      "question": "How do I push an image to Docker Hub, and what tagging restriction applies?",
      "expected_excerpt": "## Push an image to Docker Hub Select the image from the list, select the **More options** button and select **Push to Hub**. > [!NOTE] > > You can only push an image to Docker Hub if the image belongs to your Docker ID or your organization. That is, the image must contain the co",
      "first_relevant_rank": {
        "Voyage 4 Large": null,
        "Voyage 4": 5,
        "Voyage 4 Lite": null,
        "OpenAI text-embedding-3-large": null,
        "OpenAI text-embedding-3-small": 1
      }
    },
    {
      "question": "Which organizational roles and subscription plans are needed to create organization access tokens in Docker?",
      "expected_excerpt": "> [!WARNING] > > OATs don't work with Docker Desktop or Image Access Management. For > those features, use > PATs instead. ## Best practices - Set an expiration date and rotate tokens on a schedule - Grant only the repositories and scopes each job needs - Review the **Last used**",
      "first_relevant_rank": {
        "Voyage 4 Large": 1,
        "Voyage 4": 1,
        "Voyage 4 Lite": 1,
        "OpenAI text-embedding-3-large": 4,
        "OpenAI text-embedding-3-small": null
      }
    },
    {
      "question": "Which Docker Desktop versions are supported under the Pro, Team, and Business plans?",
      "expected_excerpt": "### Supported versions Docker Desktop support is scoped by plan: | Plan | Supported versions | Machines per subscription | | :--- | :--- | :--- | | Docker Pro | Latest only | One | | Docker Team | Latest only | One per subscription seat | | Docker Business | Latest plus versions ",
      "first_relevant_rank": {
        "Voyage 4 Large": 1,
        "Voyage 4": 1,
        "Voyage 4 Lite": 1,
        "OpenAI text-embedding-3-large": 2,
        "OpenAI text-embedding-3-small": 3
      }
    },
    {
      "question": "What tasks can Docker Compose perform for a multi-container application besides starting its services?",
      "expected_excerpt": "# Docker Compose Docker Compose is a tool for defining and running multi-container applications. It is the key to unlocking a streamlined and efficient development and deployment experience. Compose simplifies the control of your entire application stack, making it easy to manage",
      "first_relevant_rank": {
        "Voyage 4 Large": 2,
        "Voyage 4": 2,
        "Voyage 4 Lite": 3,
        "OpenAI text-embedding-3-large": 1,
        "OpenAI text-embedding-3-small": 1
      }
    }
  ]
}
