errlore 0.1.0__tar.gz

errlore-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,26 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: pip install -e .[dev]
+      - name: Ruff
+        run: ruff check .
+      - name: Mypy
+        run: mypy
+      - name: Tests
+        run: pytest --cov=errlore --cov-fail-under=80

errlore-0.1.0/.gitignore

@@ -0,0 +1,27 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+venv/
+
+# Tooling
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+
+# Data — runtime data never belongs in git
+*.jsonl
+*.idx
+*.lock
+agent_memory/
+
+# OS / editors
+.DS_Store
+.idea/
+.vscode/
+dist/

errlore-0.1.0/CHANGELOG.md

@@ -0,0 +1,25 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - 2026-07-05
+
+### Added
+
+- `AgentMemory` facade with four core methods: `log_error`, `resolve`, `inject_for`, `report_outcome`.
+- Lesson store with JSONL persistence, deduplication (exact + 85% word overlap), confidence tracking, and automatic decay of unused lessons.
+- Per-model, per-task-type error tracking and known-issue warning injection.
+- `TrustEngine` with Bayesian cold-start blending, adaptive learning rate, volatility damping, domain-specific EMA bias, entropy enforcement, and temporal decay.
+- `best_model(domain)` convenience method for model routing based on trust weights.
+- Closed reinforcement loop: `report_outcome` reinforces/decays lessons and updates trust weights; idempotent (no double-reinforcement).
+- Injection handle persistence (`injections.jsonl`) for cross-restart outcome reporting.
+- Optional semantic retrieval via FastEmbed embeddings (`pip install errlore[embeddings]`), with automatic fallback to word-overlap.
+- Lesson text sanitization (rejects raw JSON, code-only, too-short content).
+- Atomic file rewrites for lesson/error updates (no data loss on crash).
+- Thread-safe API (all public methods are safe to call from multiple threads).
+- Integration examples for OpenAI, Anthropic, and LangChain (all runnable offline).
+- Retrieval benchmark (`benchmarks/bench_retrieval.py`) with adversarial gold set.
+- 155 tests, mypy strict, ruff linting.

errlore-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ma4etaSS
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

errlore-0.1.0/PKG-INFO

@@ -0,0 +1,214 @@
+Metadata-Version: 2.4
+Name: errlore
+Version: 0.1.0
+Summary: Memory for AI agents that learns from failures: lessons, known-issues injection, and per-model trust — embedded, file-based, no server.
+Project-URL: Homepage, https://github.com/Ma4etaSS/errlore
+Author: Ma4etaSS
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agents,error-learning,lessons,llm,memory,trust
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.12
+Requires-Dist: filelock>=3.13
+Provides-Extra: dev
+Requires-Dist: fastembed<0.9,>=0.6; extra == 'dev'
+Requires-Dist: mypy>=1.11; extra == 'dev'
+Requires-Dist: numpy>=1.26; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: embeddings
+Requires-Dist: fastembed<0.9,>=0.6; extra == 'embeddings'
+Requires-Dist: numpy>=1.26; extra == 'embeddings'
+Description-Content-Type: text/markdown
+
+# errlore
+
+**Memory for AI agents that learns from failures.**
+
+[![CI](https://github.com/Ma4etaSS/errlore/actions/workflows/ci.yml/badge.svg)](https://github.com/Ma4etaSS/errlore/actions/workflows/ci.yml)
+[![Python 3.12+](https://img.shields.io/badge/python-3.12%2B-blue)](https://python.org)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+
+Extracted from a 324K LOC production multi-LLM orchestration system, keeping the one part that demonstrably worked: the error-memory loop that made agents stop repeating mistakes.
+
+Your agent keeps making the same mistakes. errlore fixes that:
+
+- **Lessons** -- every resolved failure becomes a lesson; relevant lessons are injected
+  into the prompt for similar future tasks.
+- **Known issues** -- per-model weakness tracking ("gpt-5.5 keeps hallucinating dates in
+  extraction tasks") injected as warnings.
+- **Trust** -- Bayesian per-model, per-domain trust weights: know which model to pick
+  for which job, based on observed outcomes.
+- **Closed loop** -- errlore tracks whether an injected lesson actually helped and
+  reinforces or decays it automatically.
+
+Embedded, file-based (JSONL), no server, no database, no API keys required.
+Works fully offline. Your data never leaves your machine.
+
+## Quickstart (< 5 minutes)
+
+```bash
+pip install errlore
+```
+
+```python
+from errlore import AgentMemory
+
+mem = AgentMemory("./agent_memory")
+
+# 1. Agent failed -- record it
+err_id = mem.log_error("gpt-5.5", "extraction", error="hallucinated dates")
+
+# 2. You fixed it -- extract a lesson
+mem.resolve(err_id, "Added date format validation",
+            lesson="For date extraction, demand ISO-8601 and verify against source")
+
+# 3. Next similar task -- lessons + known issues injected automatically
+inj = mem.inject_for("extract dates from contract", model="gpt-5.5",
+                      task_type="extraction")
+prompt = f"Your task: extract dates\n{inj.text}"
+print(prompt)
+
+# 4. Close the loop -- did the lesson help?
+mem.report_outcome(inj, success=True)
+
+# 5. Check stats
+print(mem.stats())
+# {'errors_total': 1, 'errors_resolved': 1, 'errors_unresolved': 0,
+#  'lessons_total': 1, 'lessons_applied': 1, 'pending_injections': 0,
+#  'trust': {'gpt-5.5': 0.55}}
+```
+
+No API keys needed. errlore itself never calls any LLM -- it manages local
+JSONL files and does text matching. LLM calls are yours to make (or not).
+
+## How it works
+
+errlore runs three reinforcement loops around your agent:
+
+### 1. Lesson loop
+
+```
+Agent fails  -->  log_error()  -->  resolve() + lesson
+                                        |
+Agent runs   <--  inject_for() <--------+
+     |
+     +--> report_outcome(success=True)  -->  lesson confidence +0.1
+     +--> report_outcome(success=False) -->  lesson confidence -0.1
+```
+
+Lessons with high confidence surface first. Unused lessons decay over time.
+
+### 2. Known-issue loop
+
+Per-model, per-task-type error tracking. When a model has failed on a task
+type before, `inject_for` adds a warning block to the prompt. Separate from
+lessons: lessons are *solutions*, known issues are *warnings*.
+
+### 3. Trust loop
+
+Bayesian per-model weights with adaptive learning rate, cold-start blending,
+entropy enforcement, and temporal decay.  After enough observations, call
+`mem.best_model("code_generation")` to pick the model that historically
+performs best on that domain.
+
+## Semantic retrieval (optional)
+
+By default, errlore finds relevant lessons via word overlap (zero
+dependencies). For higher recall on paraphrased queries, enable embedding
+search:
+
+```bash
+pip install errlore[embeddings]   # installs fastembed + numpy
+```
+
+```python
+mem = AgentMemory("./agent_memory", embeddings=True)
+```
+
+### Benchmark (adversarial paraphrasing)
+
+Tested on 40 lessons with adversarially paraphrased queries
+(`benchmarks/bench_retrieval.py`):
+
+| Metric    | word-overlap | embeddings |
+|-----------|-------------|------------|
+| recall@1  | 0.000       | 0.375      |
+| recall@3  | 0.000       | 0.575      |
+| recall@5  | 0.000       | 0.675      |
+| MRR       | 0.000       | 0.290      |
+
+The gold set is intentionally adversarial (queries share few literal words
+with the lesson text), which is why word-overlap scores zero.  On natural
+queries with shared vocabulary, word-overlap works fine.
+
+## Integrations
+
+errlore is framework-agnostic. It produces a text block; you put it in the
+system prompt.
+
+| Provider   | Example                                              |
+|------------|------------------------------------------------------|
+| OpenAI     | [examples/openai_agent.py](examples/openai_agent.py) |
+| Anthropic  | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
+| LangChain  | [examples/langchain_agent.py](examples/langchain_agent.py) |
+
+All examples run offline with `python examples/<name>.py` (mock responses,
+no API keys). Set `use_api=True` to call real models.
+
+## API overview
+
+The main entry point is `AgentMemory`. All other classes are internal --
+you only need them for advanced use.
+
+| Method / Property             | Description                                    |
+|-------------------------------|------------------------------------------------|
+| `log_error(model, task_type, error)` | Record an error. Returns error ID.      |
+| `resolve(err_id, resolution, lesson)` | Mark error fixed, extract a lesson.    |
+| `inject_for(task, model)`     | Build prompt injection (lessons + warnings).   |
+| `report_outcome(inj, success)` | Close the loop: reinforce lessons, update trust.|
+| `add_lesson(pattern, solution)` | Add a lesson directly (sanitized).            |
+| `lessons(limit)`              | List all lessons (sorted by confidence).       |
+| `best_model(domain)`          | Model with the highest trust weight.           |
+| `model_penalty(model, task_type)` | Error-history penalty `[0, 1]`.            |
+| `pending_injections()`        | Injections not yet reported.                   |
+| `stats()`                     | Aggregate counts + trust weights.              |
+| `.trust`                      | Access the underlying `TrustEngine` (or None). |
+
+### Supporting classes (advanced)
+
+| Class             | Purpose                                    |
+|-------------------|--------------------------------------------|
+| `LessonStore`     | Low-level lesson CRUD + search.            |
+| `TrustEngine`     | Bayesian trust weights with persistence.   |
+| `FeedbackSignal`  | Typed quality signal for trust updates.    |
+| `Injection`       | Dataclass returned by `inject_for`.        |
+
+## Data & privacy
+
+- All data is stored in local JSONL files in the directory you specify.
+- Nothing is sent to any server.  errlore itself makes zero network calls.
+- Works fully offline -- no API keys, no accounts, no telemetry.
+- Files: `errors.jsonl`, `lessons.jsonl`, `injections.jsonl`, `trust.json`,
+  `model_accuracy.jsonl`.
+- Sidecar files (auto-managed): `*.idx` (byte-offset index), `*.lock`
+  (filelock), `vectors.npy` (embedding vectors), `vector_meta.json`
+  (embedding metadata), `trust.json` (trust engine state).
+
+## Roadmap
+
+- [ ] Log compaction for injections journal
+- [ ] Async API (`alog_error`, `ainject_for`, etc.)
+- [ ] Multi-agent shared memory (multiple agents, one lesson store)
+- [ ] Lesson clustering and auto-summarization
+- [ ] Dashboard / CLI for browsing lessons and trust weights
+- [ ] Export/import for lesson sharing between projects
+
+## License
+
+MIT

errlore-0.1.0/README.md

@@ -0,0 +1,186 @@
+# errlore
+
+**Memory for AI agents that learns from failures.**
+
+[![CI](https://github.com/Ma4etaSS/errlore/actions/workflows/ci.yml/badge.svg)](https://github.com/Ma4etaSS/errlore/actions/workflows/ci.yml)
+[![Python 3.12+](https://img.shields.io/badge/python-3.12%2B-blue)](https://python.org)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+
+Extracted from a 324K LOC production multi-LLM orchestration system, keeping the one part that demonstrably worked: the error-memory loop that made agents stop repeating mistakes.
+
+Your agent keeps making the same mistakes. errlore fixes that:
+
+- **Lessons** -- every resolved failure becomes a lesson; relevant lessons are injected
+  into the prompt for similar future tasks.
+- **Known issues** -- per-model weakness tracking ("gpt-5.5 keeps hallucinating dates in
+  extraction tasks") injected as warnings.
+- **Trust** -- Bayesian per-model, per-domain trust weights: know which model to pick
+  for which job, based on observed outcomes.
+- **Closed loop** -- errlore tracks whether an injected lesson actually helped and
+  reinforces or decays it automatically.
+
+Embedded, file-based (JSONL), no server, no database, no API keys required.
+Works fully offline. Your data never leaves your machine.
+
+## Quickstart (< 5 minutes)
+
+```bash
+pip install errlore
+```
+
+```python
+from errlore import AgentMemory
+
+mem = AgentMemory("./agent_memory")
+
+# 1. Agent failed -- record it
+err_id = mem.log_error("gpt-5.5", "extraction", error="hallucinated dates")
+
+# 2. You fixed it -- extract a lesson
+mem.resolve(err_id, "Added date format validation",
+            lesson="For date extraction, demand ISO-8601 and verify against source")
+
+# 3. Next similar task -- lessons + known issues injected automatically
+inj = mem.inject_for("extract dates from contract", model="gpt-5.5",
+                      task_type="extraction")
+prompt = f"Your task: extract dates\n{inj.text}"
+print(prompt)
+
+# 4. Close the loop -- did the lesson help?
+mem.report_outcome(inj, success=True)
+
+# 5. Check stats
+print(mem.stats())
+# {'errors_total': 1, 'errors_resolved': 1, 'errors_unresolved': 0,
+#  'lessons_total': 1, 'lessons_applied': 1, 'pending_injections': 0,
+#  'trust': {'gpt-5.5': 0.55}}
+```
+
+No API keys needed. errlore itself never calls any LLM -- it manages local
+JSONL files and does text matching. LLM calls are yours to make (or not).
+
+## How it works
+
+errlore runs three reinforcement loops around your agent:
+
+### 1. Lesson loop
+
+```
+Agent fails  -->  log_error()  -->  resolve() + lesson
+                                        |
+Agent runs   <--  inject_for() <--------+
+     |
+     +--> report_outcome(success=True)  -->  lesson confidence +0.1
+     +--> report_outcome(success=False) -->  lesson confidence -0.1
+```
+
+Lessons with high confidence surface first. Unused lessons decay over time.
+
+### 2. Known-issue loop
+
+Per-model, per-task-type error tracking. When a model has failed on a task
+type before, `inject_for` adds a warning block to the prompt. Separate from
+lessons: lessons are *solutions*, known issues are *warnings*.
+
+### 3. Trust loop
+
+Bayesian per-model weights with adaptive learning rate, cold-start blending,
+entropy enforcement, and temporal decay.  After enough observations, call
+`mem.best_model("code_generation")` to pick the model that historically
+performs best on that domain.
+
+## Semantic retrieval (optional)
+
+By default, errlore finds relevant lessons via word overlap (zero
+dependencies). For higher recall on paraphrased queries, enable embedding
+search:
+
+```bash
+pip install errlore[embeddings]   # installs fastembed + numpy
+```
+
+```python
+mem = AgentMemory("./agent_memory", embeddings=True)
+```
+
+### Benchmark (adversarial paraphrasing)
+
+Tested on 40 lessons with adversarially paraphrased queries
+(`benchmarks/bench_retrieval.py`):
+
+| Metric    | word-overlap | embeddings |
+|-----------|-------------|------------|
+| recall@1  | 0.000       | 0.375      |
+| recall@3  | 0.000       | 0.575      |
+| recall@5  | 0.000       | 0.675      |
+| MRR       | 0.000       | 0.290      |
+
+The gold set is intentionally adversarial (queries share few literal words
+with the lesson text), which is why word-overlap scores zero.  On natural
+queries with shared vocabulary, word-overlap works fine.
+
+## Integrations
+
+errlore is framework-agnostic. It produces a text block; you put it in the
+system prompt.
+
+| Provider   | Example                                              |
+|------------|------------------------------------------------------|
+| OpenAI     | [examples/openai_agent.py](examples/openai_agent.py) |
+| Anthropic  | [examples/anthropic_agent.py](examples/anthropic_agent.py) |
+| LangChain  | [examples/langchain_agent.py](examples/langchain_agent.py) |
+
+All examples run offline with `python examples/<name>.py` (mock responses,
+no API keys). Set `use_api=True` to call real models.
+
+## API overview
+
+The main entry point is `AgentMemory`. All other classes are internal --
+you only need them for advanced use.
+
+| Method / Property             | Description                                    |
+|-------------------------------|------------------------------------------------|
+| `log_error(model, task_type, error)` | Record an error. Returns error ID.      |
+| `resolve(err_id, resolution, lesson)` | Mark error fixed, extract a lesson.    |
+| `inject_for(task, model)`     | Build prompt injection (lessons + warnings).   |
+| `report_outcome(inj, success)` | Close the loop: reinforce lessons, update trust.|
+| `add_lesson(pattern, solution)` | Add a lesson directly (sanitized).            |
+| `lessons(limit)`              | List all lessons (sorted by confidence).       |
+| `best_model(domain)`          | Model with the highest trust weight.           |
+| `model_penalty(model, task_type)` | Error-history penalty `[0, 1]`.            |
+| `pending_injections()`        | Injections not yet reported.                   |
+| `stats()`                     | Aggregate counts + trust weights.              |
+| `.trust`                      | Access the underlying `TrustEngine` (or None). |
+
+### Supporting classes (advanced)
+
+| Class             | Purpose                                    |
+|-------------------|--------------------------------------------|
+| `LessonStore`     | Low-level lesson CRUD + search.            |
+| `TrustEngine`     | Bayesian trust weights with persistence.   |
+| `FeedbackSignal`  | Typed quality signal for trust updates.    |
+| `Injection`       | Dataclass returned by `inject_for`.        |
+
+## Data & privacy
+
+- All data is stored in local JSONL files in the directory you specify.
+- Nothing is sent to any server.  errlore itself makes zero network calls.
+- Works fully offline -- no API keys, no accounts, no telemetry.
+- Files: `errors.jsonl`, `lessons.jsonl`, `injections.jsonl`, `trust.json`,
+  `model_accuracy.jsonl`.
+- Sidecar files (auto-managed): `*.idx` (byte-offset index), `*.lock`
+  (filelock), `vectors.npy` (embedding vectors), `vector_meta.json`
+  (embedding metadata), `trust.json` (trust engine state).
+
+## Roadmap
+
+- [ ] Log compaction for injections journal
+- [ ] Async API (`alog_error`, `ainject_for`, etc.)
+- [ ] Multi-agent shared memory (multiple agents, one lesson store)
+- [ ] Lesson clustering and auto-summarization
+- [ ] Dashboard / CLI for browsing lessons and trust weights
+- [ ] Export/import for lesson sharing between projects
+
+## License
+
+MIT

errlore-0.1.0/benchmarks/bench_retrieval.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""Benchmark: word-overlap vs embedding retrieval on a gold dataset.
+
+Loads ``tests/gold/retrieval_gold.jsonl``, indexes all 40 lessons in a
+:class:`~errlore.lessons.store.LessonStore`, and measures recall@k / MRR
+for both retrieval strategies.
+
+Exit code:
+    0 -- embeddings recall@5 > word-overlap recall@5  (gate PASS)
+    1 -- embeddings did NOT beat word-overlap          (gate FAIL)
+
+Usage::
+
+    .venv/bin/python benchmarks/bench_retrieval.py
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Gold data loader
+# ---------------------------------------------------------------------------
+
+_GOLD_PATH = Path(__file__).resolve().parent.parent / "tests" / "gold" / "retrieval_gold.jsonl"
+
+
+def _load_gold() -> list[dict[str, str]]:
+    """Read gold entries from JSONL."""
+    entries: list[dict[str, str]] = []
+    with open(_GOLD_PATH, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                entries.append(json.loads(line))
+    return entries
+
+
+# ---------------------------------------------------------------------------
+# Metrics
+# ---------------------------------------------------------------------------
+
+
+def _compute_metrics(
+    gold: list[dict[str, str]],
+    id_map: dict[int, str],
+    search_fn: Any,  # Callable[[str], list[Lesson]]
+    max_k: int = 5,
+) -> dict[str, float]:
+    """Compute recall@1, recall@3, recall@5, and MRR."""
+    hits: dict[int, int] = {k: 0 for k in (1, 3, 5)}
+    reciprocal_ranks: list[float] = []
+
+    for idx, entry in enumerate(gold):
+        target_id = id_map[idx]
+        results = search_fn(entry["query"])
+        result_ids = [le.id for le in results]
+
+        # Reciprocal rank
+        rr = 0.0
+        for rank, rid in enumerate(result_ids[:max_k], 1):
+            if rid == target_id:
+                rr = 1.0 / rank
+                break
+        reciprocal_ranks.append(rr)
+
+        for k in hits:
+            if target_id in result_ids[:k]:
+                hits[k] += 1
+
+    n = len(gold)
+    return {
+        "recall@1": hits[1] / n,
+        "recall@3": hits[3] / n,
+        "recall@5": hits[5] / n,
+        "MRR": sum(reciprocal_ranks) / n,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Benchmark runners
+# ---------------------------------------------------------------------------
+
+
+def _run_word_overlap(gold: list[dict[str, str]]) -> tuple[dict[str, float], dict[int, str]]:
+    """Run benchmark with word-overlap retrieval (no retriever)."""
+    from errlore.lessons.store import LessonStore
+
+    with tempfile.TemporaryDirectory() as tmp:
+        store = LessonStore(Path(tmp))
+        id_map: dict[int, str] = {}
+
+        for idx, entry in enumerate(gold):
+            lid = store.log_lesson(
+                pattern=entry["lesson_pattern"],
+                solution=entry["lesson_solution"],
+            )
+            id_map[idx] = lid
+
+        def search(query: str) -> list[Any]:
+            return store.search_lessons(query=query, limit=5)
+
+        metrics = _compute_metrics(gold, id_map, search)
+    return metrics, id_map
+
+
+def _run_embeddings(gold: list[dict[str, str]]) -> tuple[dict[str, float], dict[int, str]]:
+    """Run benchmark with embedding retrieval (FastEmbedBackend + VectorIndex)."""
+    from errlore.lessons.store import LessonStore
+    from errlore.retrieval.backend import FastEmbedBackend
+    from errlore.retrieval.index import VectorIndex
+
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp_path = Path(tmp)
+        backend = FastEmbedBackend()
+        index = VectorIndex(tmp_path, backend)
+        store = LessonStore(tmp_path, retriever=index)
+        id_map: dict[int, str] = {}
+
+        for idx, entry in enumerate(gold):
+            lid = store.log_lesson(
+                pattern=entry["lesson_pattern"],
+                solution=entry["lesson_solution"],
+            )
+            id_map[idx] = lid
+
+        def search(query: str) -> list[Any]:
+            return store.search_lessons(query=query, limit=5)
+
+        metrics = _compute_metrics(gold, id_map, search)
+    return metrics, id_map
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+
+def main() -> int:
+    """Run both benchmarks, print results, return exit code."""
+    gold = _load_gold()
+    print(f"Gold dataset: {len(gold)} entries from {_GOLD_PATH.name}")
+    print()
+
+    print("Running word-overlap benchmark...")
+    wo_metrics, _ = _run_word_overlap(gold)
+
+    print("Running embeddings benchmark...")
+    emb_metrics, _ = _run_embeddings(gold)
+
+    # -- Pretty table -----------------------------------------------------
+    header = f"{'metric':<16} {'word-overlap':>14} {'embeddings':>14}"
+    sep = "-" * len(header)
+    print()
+    print(header)
+    print(sep)
+    for key in ("recall@1", "recall@3", "recall@5", "MRR"):
+        wo_val = wo_metrics[key]
+        emb_val = emb_metrics[key]
+        print(f"{key:<16} {wo_val:>14.3f} {emb_val:>14.3f}")
+    print(sep)
+    print()
+
+    # -- Gate check -------------------------------------------------------
+    wo_r5 = wo_metrics["recall@5"]
+    emb_r5 = emb_metrics["recall@5"]
+    passed = emb_r5 > wo_r5
+
+    status = "PASS" if passed else "FAIL"
+    print(
+        f"Gate: embeddings recall@5 ({emb_r5:.3f})"
+        f" > word-overlap recall@5 ({wo_r5:.3f}): {status}",
+    )
+
+    return 0 if passed else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())