{
  "$schema": "https://truvec.dev/blog/schema/benchmark.schema.json",
  "schema_version": 1,
  "sample": false,
  "source": {
    "benchmark_id": 9,
    "exported_at": "2026-11-14T00:56:57.791Z",
    "app_url": "https://app.truvec.dev"
  },
  "name": "Qwen3 Embedding 8B vs OpenAI on a technical corpus",
  "description": null,
  "run": {
    "top_k": 5,
    "created_at": "2026-11-14T00:55:39.911001Z",
    "completed_at": "2026-11-14T00:55:58.153645Z",
    "ranking": "MRR, then recall@k"
  },
  "corpus": {
    "name": "Docker documentation: manuals, FAQs, billing and support pages",
    "description": "261 pages of the public Docker documentation: Docker Desktop, Docker Hub, Compose, accounts, security, subscription and billing, FAQs, support and getting-started guides.",
    "url": "https://github.com/docker/docs",
    "documents": 261,
    "chunks": 1944,
    "chunking": {
      "strategy": "recursive",
      "separator": "\n\n",
      "chunk_size": 1000,
      "chunk_overlap": 200
    }
  },
  "golden_set": {
    "name": "Generated golden set, reviewed",
    "questions": 100,
    "reviewed": true,
    "generated_by": "gpt-5.6-luna"
  },
  "notes": [
    "The corpus is a subset of the public Docker documentation (Apache 2.0), taken from github.com/docker/docs at commit 553482c: the Desktop, Docker Hub, Compose, accounts, security and subscription and billing manuals, the FAQs, the support pages and the getting-started guides. Release notes and pages shorter than 600 characters were left out, and pages were cut at 40,000 characters.",
    "Several posts on this blog use this corpus and the same 100 questions, each with its own selection of models. A model can differ by a question between two of these runs: questions are embedded again at each run, and vector search is approximate.",
    "With 100 questions, a gap of a couple of points between two models is within the noise."
  ],
  "models": [
    {
      "rank": 1,
      "name": "OpenAI text-embedding-3-large",
      "provider": "openai",
      "model_name": "text-embedding-3-large",
      "dimension": 3072,
      "price_per_million_tokens": 0.13,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.99,
        "recall": 0.975,
        "precision": 0.20599999999999963,
        "f1_score": 0.3380952380952379,
        "mrr": 0.867,
        "ndcg": 0.8848806698984153
      },
      "correct_answers": 99,
      "hit_rate_at": {
        "1": 0.79,
        "2": 0.88,
        "3": 0.94,
        "4": 0.98,
        "5": 0.99
      },
      "rank_histogram": {
        "1": 79,
        "2": 9,
        "3": 6,
        "4": 4,
        "5": 1,
        "miss": 1
      },
      "scores": {
        "mean_relevant_score": 0.6865914127272729,
        "mean_top_distractor_score": 0.6498421786000002,
        "score_gap": 0.03670871343434342
      },
      "latency": {
        "embedding_ms_per_query": 8.302818080410361,
        "search_ms_p50": 19.48544243350625,
        "search_ms_p95": 49.76345510222017
      },
      "cost": {
        "corpus_tokens": 336360,
        "index_cost_usd": 0.0437268,
        "cost_per_1k_queries_usd": 0.002574
      },
      "vector_storage_bytes": 23887872
    },
    {
      "rank": 2,
      "name": "Qwen3 Embedding 8B",
      "provider": "fireworks",
      "model_name": "fireworks/qwen3-embedding-8b",
      "dimension": 4096,
      "price_per_million_tokens": 0.1,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.99,
        "recall": 0.98,
        "precision": 0.20799999999999966,
        "f1_score": 0.3409523809523808,
        "mrr": 0.8583333333333334,
        "ndcg": 0.8829243197079457
      },
      "correct_answers": 99,
      "hit_rate_at": {
        "1": 0.77,
        "2": 0.88,
        "3": 0.95,
        "4": 0.99,
        "5": 0.99
      },
      "rank_histogram": {
        "1": 77,
        "2": 11,
        "3": 7,
        "4": 4,
        "5": 0,
        "miss": 1
      },
      "scores": {
        "mean_relevant_score": 0.8267609705050505,
        "mean_top_distractor_score": 0.7848626868999997,
        "score_gap": 0.04199380222222221
      },
      "latency": {
        "embedding_ms_per_query": 13.22240942157805,
        "search_ms_p50": 20.213348558172584,
        "search_ms_p95": 43.46081991679966
      },
      "cost": {
        "corpus_tokens": 336360,
        "index_cost_usd": 0.033636,
        "cost_per_1k_queries_usd": 0.00198
      },
      "vector_storage_bytes": 31850496
    },
    {
      "rank": 3,
      "name": "OpenAI text-embedding-3-small",
      "provider": "openai",
      "model_name": "text-embedding-3-small",
      "dimension": 1536,
      "price_per_million_tokens": 0.02,
      "status": "completed",
      "metrics": {
        "hit_rate": 0.95,
        "recall": 0.935,
        "precision": 0.19799999999999965,
        "f1_score": 0.3247619047619045,
        "mrr": 0.7686666666666667,
        "ndcg": 0.8067558320662286
      },
      "correct_answers": 95,
      "hit_rate_at": {
        "1": 0.63,
        "2": 0.84,
        "3": 0.92,
        "4": 0.94,
        "5": 0.95
      },
      "rank_histogram": {
        "1": 63,
        "2": 21,
        "3": 8,
        "4": 2,
        "5": 1,
        "miss": 5
      },
      "scores": {
        "mean_relevant_score": 0.715495250105263,
        "mean_top_distractor_score": 0.6871847976999997,
        "score_gap": 0.02744142463157894
      },
      "latency": {
        "embedding_ms_per_query": 5.249810619279742,
        "search_ms_p50": 17.093863571062684,
        "search_ms_p95": 38.204202079214156
      },
      "cost": {
        "corpus_tokens": 336360,
        "index_cost_usd": 0.0067272,
        "cost_per_1k_queries_usd": 0.000396
      },
      "vector_storage_bytes": 11943936
    }
  ],
  "questions": [
    {
      "question": "How do I push an image to Docker Hub, and what tagging restriction applies?",
      "expected_excerpt": "## Push an image to Docker Hub Select the image from the list, select the **More options** button and select **Push to Hub**. > [!NOTE] > > You can only push an image to Docker Hub if the image belongs to your Docker ID or your organization. That is, the image must contain the co",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": null,
        "Qwen3 Embedding 8B": 4,
        "OpenAI text-embedding-3-small": 1
      }
    },
    {
      "question": "What environment variables does Compose generate when a service uses models with the short syntax?",
      "expected_excerpt": "```yaml services: app: image: my-app models: - llm - embedding-model models: llm: model: ai/smollm2 embedding-model: model: ai/all-minilm ``` With short syntax, the platform automatically generates environment variables based on the model name: - `LLM_URL` - URL to access the LLM",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 1,
        "Qwen3 Embedding 8B": 3,
        "OpenAI text-embedding-3-small": null
      }
    },
    {
      "question": "Which organizational roles and subscription plans are needed to create organization access tokens in Docker?",
      "expected_excerpt": "> [!WARNING] > > OATs don't work with Docker Desktop or Image Access Management. For > those features, use > PATs instead. ## Best practices - Set an expiration date and rotate tokens on a schedule - Grant only the repositories and scopes each job needs - Review the **Last used**",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 4,
        "Qwen3 Embedding 8B": 2,
        "OpenAI text-embedding-3-small": null
      }
    },
    {
      "question": "Which Docker Desktop versions are supported under the Pro, Team, and Business plans?",
      "expected_excerpt": "### Supported versions Docker Desktop support is scoped by plan: | Plan | Supported versions | Machines per subscription | | :--- | :--- | :--- | | Docker Pro | Latest only | One | | Docker Team | Latest only | One per subscription seat | | Docker Business | Latest plus versions ",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 2,
        "Qwen3 Embedding 8B": 1,
        "OpenAI text-embedding-3-small": 3
      }
    },
    {
      "question": "How do I create, publish, and share a custom MCP catalog with my team?",
      "expected_excerpt": "Create a catalog and specify which servers to include: ```console $ docker mcp catalog create registry.example.com/mcp/data-tools:latest \\ --title \"Data Analysis Tools\" \\ --server catalog://mcp/docker-mcp-catalog/sequentialthinking \\ --server catalog://mcp/docker-mcp-catalog/brav",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 1,
        "Qwen3 Embedding 8B": 3,
        "OpenAI text-embedding-3-small": 2
      }
    },
    {
      "question": "Where can I inspect all Docker Desktop settings that have been applied, including settings not visible in the GUI?",
      "expected_excerpt": "As a workaround, you can check the `settings-store.json` file to view all applied settings: - Mac: `~/Library/Application Support/Docker/settings-store.json` - Windows: `%APPDATA%\\Docker\\settings-store.json` - Linux: `~/.docker/desktop/settings-store.json` The `settings-store.jso",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 1,
        "Qwen3 Embedding 8B": 3,
        "OpenAI text-embedding-3-small": 2
      }
    },
    {
      "question": "How do I search Docker Desktop logs, move between matching entries, and clear the terminal?",
      "expected_excerpt": "### Logs Select **Logs** to view output from the container in real time. While viewing logs, you can: - Use `Cmd + f`/`Ctrl + f` to open the search bar and find specific entries. Search matches are highlighted in yellow. - Press `Enter` or `Shift + Enter` to jump to the next or p",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 4,
        "Qwen3 Embedding 8B": 3,
        "OpenAI text-embedding-3-small": 1
      }
    },
    {
      "question": "How are the Containers proxy and Docker Desktop proxy different when Docker Desktop accesses the network?",
      "expected_excerpt": "Host firewalls can permit or deny inbound connections by filtering on `com.docker.backend`. ## Using Docker Desktop with a proxy Docker Desktop can use your system’s default proxy settings or custom settings that you configure with Docker Desktop’s proxy setting. Docker Desktop r",
      "first_relevant_rank": {
        "OpenAI text-embedding-3-large": 3,
        "Qwen3 Embedding 8B": 1,
        "OpenAI text-embedding-3-small": 2
      }
    }
  ]
}
