docugraph 0.1.0__tar.gz → 0.2.0__tar.gz

docugraph-0.2.0/.github/workflows/bench.yml

@@ -0,0 +1,79 @@
+# The retrieval benchmark, run against the real embedder, weekly and on demand.
+#
+# Fast CI proves the harness with a deterministic fake embedder
+# (tests/integration/test_bench_runner.py) and never downloads weights. What
+# that leaves unexercised is the thing the benchmark exists for: the numbers
+# on the shipped model. This job runs `docugraph bench run` on the real
+# bge-small-en-v1.5 (~130 MB, cached between runs) and holds the result
+# against the committed baseline with `docugraph bench compare`, which exits 1
+# when nDCG@10 -- overall or in any single failure-mode category -- drops by
+# more than the tolerance.
+#
+# The baseline under benchmarks/retrieval/baselines/ is committed, per
+# embedding model, and regenerated deliberately: when the dataset or the
+# retrieval stack changes on purpose, a human reruns the benchmark, reads the
+# delta table, and commits the new baseline with the change that moved it.
+# Until a baseline exists for the model, this job fails -- a comparison
+# against nothing proving nothing would be green that means "checked nothing"
+# (the live-reranker header has the longer form of this argument).
+#
+# Reranking arms are not measured here: the cross-encoder is ~2.3 GB and has
+# its own opt-in workflow; run `docugraph bench run --rerank` locally when a
+# reranker change needs numbers.
+name: Retrieval benchmark
+
+on:
+  # Manual: Actions -> Retrieval benchmark -> Run workflow.
+  workflow_dispatch:
+  schedule:
+    # Weekly, Monday 06:17 UTC. An off-hour minute, deliberately: scheduled
+    # workflows on :00 queue behind everyone else's.
+    - cron: "17 6 * * 1"
+
+env:
+  FASTEMBED_CACHE_PATH: /home/runner/.cache/fastembed
+
+jobs:
+  bench:
+    name: bench run · compare to baseline
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Sync dependencies
+        run: uv sync --all-extras
+
+      - name: Cache embedding model
+        uses: actions/cache@v4
+        with:
+          path: /home/runner/.cache/fastembed
+          key: fastembed-bge-small-en-v1.5
+
+      - name: Run benchmark
+        run: uv run docugraph bench run --json bench-report.json
+
+      - name: Compare against committed baseline
+        run: |
+          baseline="benchmarks/retrieval/baselines/bge-small-en-v1.5.json"
+          if [ ! -f "$baseline" ]; then
+            echo "::error::No committed baseline at $baseline. Run 'docugraph bench run --json' locally and commit the report as the baseline; a compare against nothing is not a gate."
+            exit 1
+          fi
+          uv run docugraph bench compare "$baseline" bench-report.json
+
+      - name: Upload report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: bench-report
+          path: bench-report.json
+          if-no-files-found: ignore

{docugraph-0.1.0 → docugraph-0.2.0}/.github/workflows/ci.yml

@@ -165,7 +165,7 @@ jobs:
           python3 - <<'PY'
           import json, pathlib, sys
 
-          EXPECTED = "0.1.0"  # the version this repo publishes; pinned deliberately
+          EXPECTED = "0.1.1"  # the version this repo publishes; pinned deliberately
           raw = pathlib.Path("/tmp/handshake.jsonl").read_text()
 
           # stdout carries the JSON-RPC stream and nothing else. A stray line

{docugraph-0.1.0 → docugraph-0.2.0}/.github/workflows/docker-publish.yml

@@ -12,7 +12,6 @@ on:
 
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
 
 jobs:
   build-and-push:
@@ -25,6 +24,15 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      # ghcr.io requires a lowercase repository path, and `github.repository`
+      # keeps the owner's case. Expressions have no lowercase function, so it
+      # is done in the shell. `docker/metadata-action` lowercases the `images:`
+      # input itself, but the tags built by hand below are not, so they all read
+      # the same resolved value.
+      - name: Resolve image name
+        id: image
+        run: echo "name=${REGISTRY}/${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT"
+
       # The arm64 half of the multi-platform build runs under emulation on the
       # amd64 runners, which requires QEMU registered with binfmt. Without this
       # step the arm64 build fails.
@@ -45,7 +53,7 @@ jobs:
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: ${{ steps.image.outputs.name }}
           tags: |
             # The tag as pushed (v1.2.3)
             type=ref,event=tag
@@ -78,7 +86,7 @@ jobs:
           file: ./docker/Dockerfile
           target: development
           push: true
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
+          tags: ${{ steps.image.outputs.name }}:dev
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max

{docugraph-0.1.0 → docugraph-0.2.0}/.github/workflows/release.yml

@@ -80,12 +80,18 @@ jobs:
 
     env:
       REGISTRY: ghcr.io
-      IMAGE_NAME: ${{ github.repository }}
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      # ghcr.io requires a lowercase repository path, and `github.repository`
+      # keeps the owner's case. Expressions have no lowercase function, so it
+      # is done in the shell.
+      - name: Resolve image name
+        id: image
+        run: echo "name=${REGISTRY}/${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT"
+
       # linux/amd64 only. Building arm64 means running it under emulation, which
       # needs a QEMU step registered with binfmt -- the step that used to sit
       # here. It is a post-release item: add both back once an arm64 image has
@@ -117,8 +123,8 @@ jobs:
           target: production
           push: true
           tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.version.outputs.version }}
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ steps.image.outputs.name }}:${{ steps.version.outputs.version }}
+            ${{ steps.image.outputs.name }}:latest
           cache-from: type=gha
           cache-to: type=gha,mode=max
           platforms: linux/amd64

{docugraph-0.1.0 → docugraph-0.2.0}/PKG-INFO

@@ -1,7 +1,11 @@
 Metadata-Version: 2.5
 Name: docugraph
-Version: 0.1.0
+Version: 0.2.0
 Summary: Documentation RAG for coding agents: vector search, knowledge graph, agent memory, and an MCP server — torch-free local embeddings via fastembed
+Project-URL: Homepage, https://github.com/CJ-coding-apps/docugraph
+Project-URL: Repository, https://github.com/CJ-coding-apps/docugraph
+Project-URL: Issues, https://github.com/CJ-coding-apps/docugraph/issues
+Project-URL: Changelog, https://github.com/CJ-coding-apps/docugraph/releases
 Author-email: Jim Cuttler <jimcuttler@gmail.com>
 License: Apache-2.0
 License-File: LICENSE
@@ -74,6 +78,7 @@ Code and other MCP clients. The CLI is for out-of-band index bootstrapping
 - **Agent memory**: persistent, repository/branch-scoped key-value store
 - **Optional reranking**: local multilingual cross-encoder (BAAI/bge-reranker-v2-m3), opt-in
 - **Torch-free**: default embeddings and reranking run on ONNX — no PyTorch in the dependency tree
+- **Measured retrieval**: a committed adversarial benchmark scores every search mode per failure category, with a regression gate (`docugraph bench`)
 
 ## Quick start
 
@@ -304,6 +309,42 @@ required**. However, Graphiti performs entity extraction with an **LLM**, so
 OpenAI/Anthropic key). Without one, the graph tools degrade gracefully with a
 clear message; vector and keyword search are unaffected.
 
+## Retrieval benchmark
+
+Retrieval quality is measured, not asserted. `benchmarks/retrieval/` commits a
+hand-authored documentation corpus for a fictional SDK plus graded relevance
+judgments, built adversarially around seven failure modes: exact-token lookups
+(error codes, parameter names), paraphrase queries with no lexical overlap,
+keyword-stuffed distractor pages, near-sibling semantic distractors, version-
+pinned questions, acronym/synonym gaps, and "X without Y" constraint queries.
+Each query records the trap it sets in a `design_note`, and a manifest pins the
+dataset by hash — editing it is a deliberate act that invalidates baselines.
+
+```bash
+# Score every arm (vector-only, BM25-only, LanceDB-native hybrid, RRF, weighted)
+# against a temporary store built from the committed corpus:
+docugraph bench run --json report.json
+
+# Hold a report against a committed baseline; exits 1 when nDCG@10 drops by
+# more than the tolerance, overall or in any single category:
+docugraph bench compare benchmarks/retrieval/baselines/bge-small-en-v1.5.json report.json
+```
+
+Reported metrics are recall@1/5/10, MRR@10, and nDCG@10 (graded, linear gain),
+macro-averaged overall and per category. The per-category table is the point:
+a fusion change that wins on average while losing exact-code lookup shows up
+as the regression it is. Baselines are per embedding model and live in
+`benchmarks/retrieval/baselines/`; the `Retrieval benchmark` workflow re-runs
+the suite weekly against the committed baseline.
+
+Honest scope: the corpus is small and purpose-built, so scores are comparable
+across docugraph changes — not with BEIR or any external leaderboard. The
+first thing this benchmark caught predates it: before v0.2.0 no FTS index was
+ever created, so "keyword" search silently returned vector results and
+RRF fused the vector ranking with itself. The benchmark's `fts` arm and the
+hybrid arm's refuse-to-score-on-fallback check exist so that class of silent
+degradation cannot come back unnoticed.
+
 ### KuzuDB version
 
 `kuzu` is pinned to `==0.11.3` in `pyproject.toml` (reached through

{docugraph-0.1.0 → docugraph-0.2.0}/README.md

@@ -16,6 +16,7 @@ Code and other MCP clients. The CLI is for out-of-band index bootstrapping
 - **Agent memory**: persistent, repository/branch-scoped key-value store
 - **Optional reranking**: local multilingual cross-encoder (BAAI/bge-reranker-v2-m3), opt-in
 - **Torch-free**: default embeddings and reranking run on ONNX — no PyTorch in the dependency tree
+- **Measured retrieval**: a committed adversarial benchmark scores every search mode per failure category, with a regression gate (`docugraph bench`)
 
 ## Quick start
 
@@ -246,6 +247,42 @@ required**. However, Graphiti performs entity extraction with an **LLM**, so
 OpenAI/Anthropic key). Without one, the graph tools degrade gracefully with a
 clear message; vector and keyword search are unaffected.
 
+## Retrieval benchmark
+
+Retrieval quality is measured, not asserted. `benchmarks/retrieval/` commits a
+hand-authored documentation corpus for a fictional SDK plus graded relevance
+judgments, built adversarially around seven failure modes: exact-token lookups
+(error codes, parameter names), paraphrase queries with no lexical overlap,
+keyword-stuffed distractor pages, near-sibling semantic distractors, version-
+pinned questions, acronym/synonym gaps, and "X without Y" constraint queries.
+Each query records the trap it sets in a `design_note`, and a manifest pins the
+dataset by hash — editing it is a deliberate act that invalidates baselines.
+
+```bash
+# Score every arm (vector-only, BM25-only, LanceDB-native hybrid, RRF, weighted)
+# against a temporary store built from the committed corpus:
+docugraph bench run --json report.json
+
+# Hold a report against a committed baseline; exits 1 when nDCG@10 drops by
+# more than the tolerance, overall or in any single category:
+docugraph bench compare benchmarks/retrieval/baselines/bge-small-en-v1.5.json report.json
+```
+
+Reported metrics are recall@1/5/10, MRR@10, and nDCG@10 (graded, linear gain),
+macro-averaged overall and per category. The per-category table is the point:
+a fusion change that wins on average while losing exact-code lookup shows up
+as the regression it is. Baselines are per embedding model and live in
+`benchmarks/retrieval/baselines/`; the `Retrieval benchmark` workflow re-runs
+the suite weekly against the committed baseline.
+
+Honest scope: the corpus is small and purpose-built, so scores are comparable
+across docugraph changes — not with BEIR or any external leaderboard. The
+first thing this benchmark caught predates it: before v0.2.0 no FTS index was
+ever created, so "keyword" search silently returned vector results and
+RRF fused the vector ranking with itself. The benchmark's `fts` arm and the
+hybrid arm's refuse-to-score-on-fallback check exist so that class of silent
+degradation cannot come back unnoticed.
+
 ### KuzuDB version
 
 `kuzu` is pinned to `==0.11.3` in `pyproject.toml` (reached through

docugraph-0.2.0/benchmarks/retrieval/MANIFEST.json

@@ -0,0 +1,13 @@
+{
+  "dataset": "docugraph-retrieval-bench",
+  "version": "0.1.0-pilot",
+  "description": "Hand-authored adversarial documentation corpus (fictional 'Quorva' SDK) with graded relevance judgments across seven retrieval failure modes. Pilot scale; see the benchmark README section for authoring rules.",
+  "counts": {
+    "corpus": 36,
+    "queries": 21
+  },
+  "sha256": {
+    "corpus.jsonl": "6318b464c188eccd62a4f6eea8c457e7c63780d1d3b7d3e6a0728795955ea5ca",
+    "queries.jsonl": "afcc08a10c73d350244fd13c2006a047aa82d18a4be4fbfc2323cb3741979344"
+  }
+}