{
  "$schema": "https://truvec.dev/blog/schema/benchmark.schema.json",
  "schema_version": 1,
  "sample": false,
  "source": {
    "benchmark_id": 6,
    "exported_at": "2026-11-14T00:56:55.829Z",
    "app_url": "https://app.truvec.dev"
  },
  "name": "BGE Base vs OpenAI on English documentation",
  "description": null,
  "run": {
    "top_k": 5,
    "created_at": "2026-11-14T00:55:36.697868Z",
    "completed_at": "2026-11-14T00:55:59.042990Z",
    "ranking": "MRR, then recall@k"
  },
  "corpus": {
    "name": "Docker documentation: manuals, FAQs, billing and support pages",
    "description": "261 pages of the public Docker documentation: Docker Desktop, Docker Hub, Compose, accounts, security, subscription and billing, FAQs, support and getting-started guides.",
    "url": "https://github.com/docker/docs",
    "documents": 261,
    "chunks": 1944,
    "chunking": {
      "strategy": "recursive",
      "separator": "\n\n",
      "chunk_size": 1000,
      "chunk_overlap": 200
    }
  },
  "golden_set": {
    "name": "Generated golden set, reviewed",
    "questions": 100,
    "reviewed": true,
    "generated_by": "gpt-5.6-luna"
  },
  "notes": [
    "The corpus is a subset of the public Docker documentation (Apache 2.0), taken from github.com/docker/docs at commit 553482c: the Desktop, Docker Hub, Compose, accounts, security and subscription and billing manuals, the FAQs, the support pages and the getting-started guides. Release notes and pages shorter than 600 characters were left out, and pages were cut at 40,000 characters.",
    "Several posts on this blog use this corpus and the same 100 questions, each with its own selection of models. A model can differ by a question between two of these runs: questions are embedded again at each run, and vector search is approximate.",
    "With 100 questions, a gap of a couple of points between two models is within the noise.",
    "Chunks average about 170 tokens, well under the 380 tokens BGE Base EN v1.5 reads per chunk in Truvec (model limit 512), so BGE saw the chunks in full."
  ],
  "models": [
    {
      "rank": 1,
      "name": "OpenAI text-embedding-3-large",
      "provider": "openai",
      "model_name": "text-embedding-3-large",
      "dimension": 3072,
      "price_per_million_tokens": 0.13,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.99,
        "recall": 0.975,
        "precision": 0.20599999999999963,
        "f1_score": 0.3380952380952379,
        "mrr": 0.867,
        "ndcg": 0.8848806698984153
      },
      "correct_answers": 99,
      "hit_rate_at": {
        "1": 0.79,
        "2": 0.88,
        "3": 0.94,
        "4": 0.98,
        "5": 0.99
      },
      "rank_histogram": {
        "1": 79,
        "2": 9,
        "3": 6,
        "4": 4,
        "5": 1,
        "miss": 1
      },
      "scores": {
        "mean_relevant_score": 0.6865887438383838,
        "mean_top_distractor_score": 0.6498463346000001,
        "score_gap": 0.03670184656565658
      },
      "latency": {
        "embedding_ms_per_query": 11.642887769266963,
        "search_ms_p50": 23.970378329977393,
        "search_ms_p95": 56.62247028667479
      },
      "cost": {
        "corpus_tokens": 336360,
        "index_cost_usd": 0.0437268,
        "cost_per_1k_queries_usd": 0.002574
      },
      "vector_storage_bytes": 23887872
    },
    {
      "rank": 2,
      "name": "BGE Base EN v1.5",
      "provider": "fireworks",
      "model_name": "BAAI/bge-base-en-v1.5",
      "dimension": 768,
      "price_per_million_tokens": 0.008,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.95,
        "recall": 0.925,
        "precision": 0.19399999999999967,
        "f1_score": 0.31904761904761864,
        "mrr": 0.7726666666666668,
        "ndcg": 0.802102666667776
      },
      "correct_answers": 95,
      "hit_rate_at": {
        "1": 0.64,
        "2": 0.85,
        "3": 0.9,
        "4": 0.92,
        "5": 0.95
      },
      "rank_histogram": {
        "1": 64,
        "2": 21,
        "3": 5,
        "4": 2,
        "5": 3,
        "miss": 5
      },
      "scores": {
        "mean_relevant_score": 0.8059181671578949,
        "mean_top_distractor_score": 0.7818177453999999,
        "score_gap": 0.023495881473684213
      },
      "latency": {
        "embedding_ms_per_query": 22.01987592037767,
        "search_ms_p50": 17.799114109948277,
        "search_ms_p95": 40.36950685549527
      },
      "cost": {
        "corpus_tokens": 336360,
        "index_cost_usd": 0.00269088,
        "cost_per_1k_queries_usd": 0.0001584
      },
      "vector_storage_bytes": 5971968
    },
    {
      "rank": 3,
      "name": "OpenAI text-embedding-3-small",
      "provider": "openai",
      "model_name": "text-embedding-3-small",
      "dimension": 1536,
      "price_per_million_tokens": 0.02,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.96,
        "recall": 0.945,
        "precision": 0.19999999999999965,
        "f1_score": 0.32809523809523783,
        "mrr": 0.7706666666666666,
        "ndcg": 0.810624360138574
      },
      "correct_answers": 96,
      "hit_rate_at": {
        "1": 0.63,
        "2": 0.84,
        "3": 0.92,
        "4": 0.94,
        "5": 0.96
      },
      "rank_histogram": {
        "1": 63,
        "2": 21,
        "3": 8,
        "4": 2,
        "5": 2,
        "miss": 4
      },
      "scores": {
        "mean_relevant_score": 0.7148567318749999,
        "mean_top_distractor_score": 0.6872026004999998,
        "score_gap": 0.026699465208333325
      },
      "latency": {
        "embedding_ms_per_query": 6.286913021467626,
        "search_ms_p50": 18.24641670100391,
        "search_ms_p95": 47.94218142051248
      },
      "cost": {
        "corpus_tokens": 336360,
        "index_cost_usd": 0.0067272,
        "cost_per_1k_queries_usd": 0.000396
      },
      "vector_storage_bytes": 11943936
    }
  ],
  "questions": [
    {
      "question": "How do I push an image to Docker Hub, and what tagging restriction applies?",
      "expected_excerpt": "## Push an image to Docker Hub Select the image from the list, select the **More options** button and select **Push to Hub**. > [!NOTE] > > You can only push an image to Docker Hub if the image belongs to your Docker ID or your organization. That is, the image must contain the co",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": null,
        "BGE Base EN v1.5": 3,
        "OpenAI text-embedding-3-small": 1
      }
    },
    {
      "question": "How can I import a custom MCP catalog into Docker Desktop, and where does it appear afterward?",
      "expected_excerpt": "### Custom catalogs with Dynamic MCP Custom catalogs work particularly well with Dynamic MCP, where agents discover and add MCP servers on-demand during conversations. When you run the gateway with a custom catalog, the `mcp-find` tool searches only within that catalog. If your c",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 1,
        "BGE Base EN v1.5": 2,
        "OpenAI text-embedding-3-small": 3
      }
    },
    {
      "question": "When both JIT and SCIM are enabled, how are organization roles and account deactivation handled?",
      "expected_excerpt": "Roles don't move back and forth. JIT sets the organization role only when it first adds the user. A later SCIM update can change that role, and the next sign-in leaves the SCIM role in place. When a user isn't assigned to the Docker application in the IdP, the next synchronizatio",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 5,
        "BGE Base EN v1.5": 3,
        "OpenAI text-embedding-3-small": null
      }
    },
    {
      "question": "Which Docker Desktop versions are supported under the Pro, Team, and Business plans?",
      "expected_excerpt": "### Supported versions Docker Desktop support is scoped by plan: | Plan | Supported versions | Machines per subscription | | :--- | :--- | :--- | | Docker Pro | Latest only | One | | Docker Team | Latest only | One per subscription seat | | Docker Business | Latest plus versions ",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 2,
        "BGE Base EN v1.5": 1,
        "OpenAI text-embedding-3-small": 3
      }
    },
    {
      "question": "Where can I inspect all Docker Desktop settings that have been applied, including settings not visible in the GUI?",
      "expected_excerpt": "As a workaround, you can check the `settings-store.json` file to view all applied settings: - Mac: `~/Library/Application Support/Docker/settings-store.json` - Windows: `%APPDATA%\\Docker\\settings-store.json` - Linux: `~/.docker/desktop/settings-store.json` The `settings-store.jso",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 1,
        "BGE Base EN v1.5": 4,
        "OpenAI text-embedding-3-small": 2
      }
    },
    {
      "question": "What must an organization’s only owner do before leaving, and where can they initiate the departure?",
      "expected_excerpt": "If the account is the only owner of an organization or company, assign the owner role to another member first. Leave after that member can administer the organization or company. To leave an organization and its teams: 1. Sign in to Docker Home and select the organization. 1. Sel",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 3,
        "BGE Base EN v1.5": 1,
        "OpenAI text-embedding-3-small": 2
      }
    },
    {
      "question": "How can I use a value from a `.env` file inside a Docker Compose service's environment settings?",
      "expected_excerpt": "Basic example: ```console $ cat .env ## define COMPOSE_DEBUG based on DEV_MODE, defaults to false COMPOSE_DEBUG=${DEV_MODE:-false} $ cat compose.yaml services: webapp: image: my-webapp-image environment: - DEBUG=${COMPOSE_DEBUG} $ DEV_MODE=true docker compose config services: web",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 1,
        "BGE Base EN v1.5": 5,
        "OpenAI text-embedding-3-small": 2
      }
    },
    {
      "question": "What file and service configuration are required for Docker Hub automated tests, and how is the test result determined?",
      "expected_excerpt": "## Set up automated test files To set up your automated tests, create a `docker-compose.test.yml` file which defines a `sut` service that lists the tests to be run. The `docker-compose.test.yml` file should be located in the same directory that contains the Dockerfile used to bui",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 2,
        "BGE Base EN v1.5": 5,
        "OpenAI text-embedding-3-small": null
      }
    }
  ]
}
