contextos-memory-runtime 1.0.0rc2__py3-none-any.whl

contextos/benchmarks/optimization.py

@@ -0,0 +1,305 @@
+"""Offline Phase 5 token-selection evaluation and scale smoke test."""
+
+from __future__ import annotations
+
+import time
+from collections import Counter
+from dataclasses import dataclass
+from uuid import UUID
+
+from contextos.core.enums import MemoryStatus, MemoryType, OptimizationStrategy
+from contextos.core.models import ContextBudget, Memory, ScoredMemory, SelectionResult
+from contextos.services.optimization import (
+    MemoryContextOptimizer,
+    information_tokens,
+    redundancy_similarity,
+)
+from contextos.services.token_counter import DeterministicWordTokenCounter
+
+
+SMOKE_QUERY = "Which local model should I use given my machine and previous attempts?"
+INFORMATION_UNITS = {"runtime", "failed_large", "successful_small", "hardware_limit"}
+UNIT_WEIGHTS = {
+    "runtime": 1.0,
+    "failed_large": 1.2,
+    "successful_small": 1.0,
+    "hardware_limit": 1.2,
+}
+STRATEGY_LABELS = {
+    OptimizationStrategy.TOP_RANK_STOP: "top_rank_stop",
+    OptimizationStrategy.TOP_RANK_SKIP: "top_rank_skip",
+    OptimizationStrategy.GREEDY: "greedy",
+    OptimizationStrategy.CONTEXTOS: "contextos",
+}
+
+
+@dataclass(frozen=True)
+class OptimizationMetrics:
+    tokens: int
+    utilization: float
+    coverage: float
+    weighted_coverage: float
+    relevance_retained: float
+    redundancy: float
+    efficiency: float
+
+
+def information_unit_coverage(
+    selected_ids: list[str],
+    annotations: dict[str, set[str]],
+    required_units: set[str],
+) -> float:
+    if not required_units:
+        return 0.0
+    covered = set().union(*(annotations.get(identifier, set()) for identifier in selected_ids))
+    return len(covered & required_units) / len(required_units)
+
+
+def weighted_information_unit_coverage(
+    selected_ids: list[str],
+    annotations: dict[str, set[str]],
+    unit_weights: dict[str, float],
+) -> float:
+    total = sum(unit_weights.values())
+    if total <= 0:
+        return 0.0
+    covered = set().union(*(annotations.get(identifier, set()) for identifier in selected_ids))
+    return sum(weight for unit, weight in unit_weights.items() if unit in covered) / total
+
+
+def retained_relevance(
+    selected: list[ScoredMemory], candidates: list[ScoredMemory]
+) -> float:
+    total = sum(item.final_score for item in candidates)
+    return sum(item.final_score for item in selected) / total if total else 0.0
+
+
+def redundancy_rate(selected: list[ScoredMemory]) -> float:
+    if len(selected) < 2:
+        return 0.0
+    similarities: list[float] = []
+    for index, left in enumerate(selected):
+        for right in selected[index + 1:]:
+            similarities.append(redundancy_similarity(
+                information_tokens(left.memory.content),
+                information_tokens(right.memory.content),
+            ))
+    return sum(similarities) / len(similarities)
+
+
+def budget_utilization(tokens: int, available_tokens: int) -> float:
+    return tokens / available_tokens if available_tokens > 0 else 0.0
+
+
+def coverage_efficiency(coverage: float, tokens: int) -> float:
+    return coverage / tokens if tokens > 0 else 0.0
+
+
+def _padded(base: str, target: int, repeated_token: str) -> str:
+    counter = DeterministicWordTokenCounter()
+    current = counter.count(base)
+    if current > target:
+        raise ValueError("Base content already exceeds target")
+    return base + (" " + repeated_token) * (target - current)
+
+
+def smoke_candidates() -> tuple[list[ScoredMemory], dict[str, set[str]]]:
+    specs = [
+        (1, _padded("User uses Ollama.", 12, "Ollama"), 0.95, {"runtime"}),
+        (
+            2,
+            _padded(
+                "Qwen 30B failed due to insufficient available memory.",
+                20,
+                "memory",
+            ),
+            0.93,
+            {"failed_large"},
+        ),
+        (
+            3,
+            _padded("User successfully runs Qwen 9B locally.", 18, "Qwen"),
+            0.90,
+            {"successful_small"},
+        ),
+        (
+            4,
+            _padded(
+                "Historically Qwen 30B failed due to insufficient available memory.",
+                180,
+                "memory",
+            ),
+            0.92,
+            {"failed_large"},
+        ),
+        (
+            5,
+            _padded("User prefers concise responses.", 10, "responses"),
+            0.10,
+            set(),
+        ),
+        (
+            6,
+            _padded("User uses Ollama for local model inference.", 16, "Ollama"),
+            0.88,
+            {"runtime"},
+        ),
+        (
+            7,
+            _padded(
+                "User's machine has limited memory for very large models.",
+                17,
+                "memory",
+            ),
+            0.87,
+            {"hardware_limit"},
+        ),
+    ]
+    candidates: list[ScoredMemory] = []
+    annotations: dict[str, set[str]] = {}
+    for rank, (number, content, score, units) in enumerate(specs, 1):
+        identifier = UUID(f"10000000-0000-0000-0000-{number:012d}")
+        status = MemoryStatus.HISTORICAL if number == 4 else MemoryStatus.ACTIVE
+        scored = ScoredMemory(
+            memory=Memory(
+                id=identifier,
+                content=content,
+                status=status,
+                type=MemoryType.FACT,
+                importance=0.8,
+                confidence=0.9,
+            ),
+            final_score=score,
+            rank=rank,
+            retrieval_sources=["lexical", "dense"],
+        )
+        candidates.append(scored)
+        annotations[str(identifier)] = units
+    return candidates, annotations
+
+
+def measure(
+    result: SelectionResult,
+    candidates: list[ScoredMemory],
+    annotations: dict[str, set[str]],
+) -> OptimizationMetrics:
+    identifiers = [str(item.memory.id) for item in result.selected_memories]
+    coverage = information_unit_coverage(identifiers, annotations, INFORMATION_UNITS)
+    return OptimizationMetrics(
+        tokens=result.total_tokens,
+        utilization=result.utilization,
+        coverage=coverage,
+        weighted_coverage=weighted_information_unit_coverage(
+            identifiers, annotations, UNIT_WEIGHTS
+        ),
+        relevance_retained=retained_relevance(result.selected_memories, candidates),
+        redundancy=redundancy_rate(result.selected_memories),
+        efficiency=coverage_efficiency(coverage, result.total_tokens),
+    )
+
+
+def synthetic_candidates(count: int = 1_000) -> list[ScoredMemory]:
+    return [
+        ScoredMemory(
+            memory=Memory(
+                id=UUID(f"20000000-0000-0000-0000-{index:012d}"),
+                content=(
+                    f"Memory topic{index} records constraint group{index % 37} "
+                    f"tool{index % 83} outcome{index % 19}."
+                ),
+                status=MemoryStatus.ACTIVE,
+                importance=(index % 10) / 10,
+                confidence=0.8,
+            ),
+            final_score=1.0 / (1 + index % 100),
+            rank=index + 1,
+            retrieval_sources=["dense"] if index % 2 else ["lexical", "dense"],
+        )
+        for index in range(count)
+    ]
+
+
+def main() -> None:
+    counter = DeterministicWordTokenCounter()
+    optimizer = MemoryContextOptimizer(token_counter=counter)
+    candidates, annotations = smoke_candidates()
+    aggregate: dict[OptimizationStrategy, list[OptimizationMetrics]] = {
+        strategy: [] for strategy in OptimizationStrategy
+    }
+    print("Smoke scenario")
+    print(
+        "Budget  Strategy       IDs                 Tokens  Util   Coverage  "
+        "Weighted  Relevance  Redundancy  Efficiency  Exclusions"
+    )
+    for budget_size in (40, 60, 100, 250):
+        for strategy in OptimizationStrategy:
+            result = optimizer.optimize(
+                SMOKE_QUERY,
+                candidates,
+                ContextBudget(max_tokens=budget_size),
+                strategy,
+            )
+            metrics = measure(result, candidates, annotations)
+            aggregate[strategy].append(metrics)
+            ids = ",".join(str(item.memory.id)[-2:] for item in result.selected_memories) or "-"
+            exclusions = Counter(
+                decision.exclusion_reason.value
+                for decision in result.trace.decisions
+                if decision.exclusion_reason is not None
+            )
+            exclusion_text = ",".join(
+                f"{reason}:{count}" for reason, count in sorted(exclusions.items())
+            ) or "-"
+            print(
+                f"{budget_size:>6}  {STRATEGY_LABELS[strategy]:<14} {ids:<19} "
+                f"{metrics.tokens:>6}  {metrics.utilization:>5.3f}  "
+                f"{metrics.coverage:>8.3f}  {metrics.weighted_coverage:>8.3f}  "
+                f"{metrics.relevance_retained:>9.3f}  "
+                f"{metrics.redundancy:>10.3f}  {metrics.efficiency:>10.4f}  "
+                f"{exclusion_text}"
+            )
+
+    print("\nAverage evaluation metrics across budgets")
+    print(
+        "Strategy    Tokens  Utilization  Coverage  Weighted  Relevance  "
+        "Redundancy  Efficiency"
+    )
+    for strategy, rows in aggregate.items():
+        count = len(rows)
+        print(
+            f"{STRATEGY_LABELS[strategy]:<13} "
+            f"{sum(row.tokens for row in rows) / count:>6.1f}  "
+            f"{sum(row.utilization for row in rows) / count:>11.3f}  "
+            f"{sum(row.coverage for row in rows) / count:>8.3f}  "
+            f"{sum(row.weighted_coverage for row in rows) / count:>8.3f}  "
+            f"{sum(row.relevance_retained for row in rows) / count:>9.3f}  "
+            f"{sum(row.redundancy for row in rows) / count:>10.3f}  "
+            f"{sum(row.efficiency for row in rows) / count:>10.4f}"
+        )
+
+    scale = synthetic_candidates()
+    started = time.perf_counter()
+    first = optimizer.optimize(
+        "Find machine tools and constraints",
+        scale,
+        ContextBudget(max_tokens=500),
+    )
+    elapsed_ms = (time.perf_counter() - started) * 1000
+    second = optimizer.optimize(
+        "Find machine tools and constraints",
+        scale,
+        ContextBudget(max_tokens=500),
+    )
+    deterministic = [
+        item.memory.id for item in first.selected_memories
+    ] == [item.memory.id for item in second.selected_memories]
+    print("\nSynthetic scale")
+    print(
+        f"candidates=1000 selected={len(first.selected_memories)} "
+        f"tokens={first.total_tokens}/500 latency_ms={elapsed_ms:.3f} "
+        f"deterministic={deterministic}"
+    )
+
+
+if __name__ == "__main__":
+    main()

contextos/benchmarks/rescue_integration.py

@@ -0,0 +1,127 @@
+"""Deterministic Phase 4 -> 5 -> 6 oversized-rescue smoke scenario."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import tempfile
+from pathlib import Path
+from uuid import UUID
+
+from contextos.core.enums import MemoryStatus, MemoryType
+from contextos.core.models import CompilationConfig, ContextBudget, Memory, RetrievalQuery
+from contextos.embedding.deterministic import DeterministicEmbedding
+from contextos.services.compilation import QueryAwareContextCompiler
+from contextos.services.optimization import MemoryContextOptimizer
+from contextos.services.retrieval import HybridRetrievalEngine
+from contextos.services.retrieval_index import RetrievalIndexSynchronizer
+from contextos.services.token_counter import DeterministicWordTokenCounter
+from contextos.storage.database import Database
+from contextos.storage.lexical.bm25 import BM25Index
+from contextos.storage.memory_repo import SqliteMemoryRepository
+from contextos.storage.vector.in_memory import InMemoryVectorStore
+
+
+def _memory(number: int, content: str, status: MemoryStatus = MemoryStatus.ACTIVE) -> Memory:
+    return Memory(
+        id=UUID(f"80000000-0000-0000-0000-{number:012d}"),
+        content=content,
+        status=status,
+        type=MemoryType.FACT,
+        provenance_event_id=UUID(f"81000000-0000-0000-0000-{number:012d}"),
+        confidence=0.95,
+        importance=0.9,
+    )
+
+
+async def run_smoke() -> dict[str, object]:
+    padding = " ".join(f"diagnostic{index}" for index in range(320)) + "."
+    unrelated = " ".join(f"gardening{index}" for index in range(320)) + "."
+    memories = [
+        _memory(1, "Ollama is the selected local model runner."),
+        _memory(
+            2,
+            "Qwen30B failed on the local machine because available VRAM was insufficient. "
+            + padding,
+        ),
+        _memory(3, "Qwen9B works on the local machine."),
+        _memory(4, "A historical garden watering archive. " + unrelated, MemoryStatus.HISTORICAL),
+    ]
+    with tempfile.TemporaryDirectory(prefix="contextos-rescue-") as directory:
+        database_path = Path(directory) / "contextos.db"
+        database = Database(database_path)
+        await database.initialize()
+        try:
+            repository = SqliteMemoryRepository(database.connection())
+            for memory in memories:
+                await repository.create(memory)
+
+            embedding = DeterministicEmbedding(64)
+            lexical = BM25Index()
+            vector = InMemoryVectorStore(embedding.dimension)
+            synchronizer = RetrievalIndexSynchronizer(
+                memory_repo=repository,
+                lexical_index=lexical,
+                vector_store=vector,
+                embedding_service=embedding,
+            )
+            retrieval = HybridRetrievalEngine(
+                memory_repo=repository,
+                lexical_index=lexical,
+                vector_store=vector,
+                embedding_service=embedding,
+                index_synchronizer=synchronizer,
+            )
+            query = "local model machine attempts"
+            retrieved = await retrieval.retrieve(RetrievalQuery(text=query, k=10))
+            counter = DeterministicWordTokenCounter()
+            optimizer = MemoryContextOptimizer(token_counter=counter)
+            selection = optimizer.optimize(
+                query, retrieved.memories, ContextBudget(max_tokens=24)
+            )
+            compiler = QueryAwareContextCompiler(token_counter=counter)
+            compiled = await compiler.compile(
+                query, selection, CompilationConfig(budget=48)
+            )
+            return {
+                "database_on_disk": database_path.exists(),
+                "retrieved_ids": [str(value.memory.id) for value in retrieved.memories],
+                "selected_ids": [
+                    str(value.memory.id) for value in selection.selected_memories
+                ],
+                "rescue_ids": [
+                    str(value.memory.id)
+                    for value in selection.compiler_rescue_candidates
+                ],
+                "emitted_facts": [
+                    {
+                        "text": fact.text,
+                        "input_kind": fact.input_kind.value,
+                        "source_memory_ids": [
+                            str(value) for value in fact.source_memory_ids
+                        ],
+                        "provenance_event_ids": [
+                            str(value) for value in fact.provenance_event_ids
+                        ],
+                    }
+                    for fact in compiled.facts
+                ],
+                "tokens": compiled.total_tokens,
+                "budget": compiled.budget,
+                "oversized_rescue_inputs": compiled.trace.oversized_rescue_inputs,
+                "rescued_facts_included": compiled.trace.rescued_facts_included,
+                "unsupported_fact_rate": compiled.unsupported_fact_rate,
+                "historical_unrelated_rescued": memories[3].id in {
+                    value.memory.id for value in selection.compiler_rescue_candidates
+                },
+            }
+        finally:
+            await database.close()
+
+
+def main() -> None:
+    print(json.dumps(asyncio.run(run_smoke()), indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

contextos/benchmarks/retrieval.py

@@ -0,0 +1,266 @@
+"""Deterministic, offline Phase 4 retrieval evaluation.
+
+Run with: python -m contextos.benchmarks.retrieval
+"""
+
+from __future__ import annotations
+
+import asyncio
+import math
+import tempfile
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from contextos.core.enums import MemoryStatus, MemoryType, RetrievalMode, TemporalScope
+from contextos.core.models import Memory, RetrievalQuery
+from contextos.embedding.deterministic import DeterministicEmbedding
+from contextos.services.retrieval import HybridRetrievalEngine
+from contextos.services.retrieval_index import RetrievalIndexSynchronizer
+from contextos.storage.database import Database
+from contextos.storage.lexical.bm25 import BM25Index
+from contextos.storage.memory_repo import SqliteMemoryRepository
+from contextos.storage.vector.in_memory import InMemoryVectorStore
+
+
+@dataclass(frozen=True)
+class EvaluationCase:
+    query_id: str
+    text: str
+    qrels: dict[str, int]
+    temporal_scope: TemporalScope = TemporalScope.CURRENT
+    memory_types: set[MemoryType] | None = None
+
+
+@dataclass
+class EvaluationSummary:
+    recall: float
+    precision: float
+    hit_rate: float
+    mrr: float
+    ndcg: float
+    average_latency_ms: float
+    stage_latency_ms: dict[str, float] = field(default_factory=dict)
+    rankings: dict[str, list[str]] = field(default_factory=dict)
+
+
+def recall_at_k(retrieved: list[str], qrels: dict[str, int], k: int) -> float:
+    relevant = {identifier for identifier, grade in qrels.items() if grade > 0}
+    if not relevant:
+        return 0.0
+    return len(set(retrieved[:k]) & relevant) / len(relevant)
+
+
+def precision_at_k(retrieved: list[str], qrels: dict[str, int], k: int) -> float:
+    if k <= 0:
+        return 0.0
+    relevant = {identifier for identifier, grade in qrels.items() if grade > 0}
+    return len(set(retrieved[:k]) & relevant) / k
+
+
+def hit_rate_at_k(retrieved: list[str], qrels: dict[str, int], k: int) -> float:
+    return float(any(qrels.get(identifier, 0) > 0 for identifier in retrieved[:k]))
+
+
+def reciprocal_rank(retrieved: list[str], qrels: dict[str, int]) -> float:
+    for rank, identifier in enumerate(retrieved, 1):
+        if qrels.get(identifier, 0) > 0:
+            return 1.0 / rank
+    return 0.0
+
+
+def ndcg_at_k(retrieved: list[str], qrels: dict[str, int], k: int) -> float:
+    def dcg(grades: list[int]) -> float:
+        return sum((2**grade - 1) / math.log2(rank + 1) for rank, grade in enumerate(grades, 1))
+
+    actual = dcg([qrels.get(identifier, 0) for identifier in retrieved[:k]])
+    ideal = dcg(sorted(qrels.values(), reverse=True)[:k])
+    return actual / ideal if ideal else 0.0
+
+
+def evaluation_memories() -> list[Memory]:
+    specs = [
+        ("m01", "User currently uses Ollama for local model inference.",
+         MemoryStatus.ACTIVE, MemoryType.FACT),
+        ("m02", "User previously focused primarily on Python.",
+         MemoryStatus.HISTORICAL, MemoryType.SKILL),
+        ("m03", "User currently focuses on C++17 for systems interviews.",
+         MemoryStatus.ACTIVE, MemoryType.GOAL),
+        ("m04", "User prefers concise technical responses.",
+         MemoryStatus.ACTIVE, MemoryType.PREFERENCE),
+        ("m05", "User enjoys biryani.", MemoryStatus.ACTIVE, MemoryType.PREFERENCE),
+        (
+            "m06",
+            "Qwen 30B, the much larger model, failed locally because available "
+            "memory was insufficient.",
+            MemoryStatus.ACTIVE,
+            MemoryType.FACT,
+        ),
+        ("m07", "User has successfully run Qwen 9B locally.",
+         MemoryStatus.ACTIVE, MemoryType.FACT),
+        ("m08", "The Atlas project uses PostgreSQL for durable storage.",
+         MemoryStatus.ACTIVE, MemoryType.PROJECT),
+        ("m09", "User tests Python services with pytest.",
+         MemoryStatus.ACTIVE, MemoryType.PROCEDURE),
+        ("m10", "User likes detailed travel stories.",
+         MemoryStatus.ACTIVE, MemoryType.PREFERENCE),
+        ("m11", "A deleted note claimed the runtime was Docker.",
+         MemoryStatus.DELETED, MemoryType.FACT),
+        ("m12", "An expired goal was to study Java.",
+         MemoryStatus.EXPIRED, MemoryType.GOAL),
+        ("m13", "User builds local search tools with SQLite.",
+         MemoryStatus.ACTIVE, MemoryType.PROJECT),
+        ("m14", "User previously used llama.cpp for inference.",
+         MemoryStatus.SUPERSEDED, MemoryType.FACT),
+    ]
+    from uuid import UUID
+
+    return [
+        Memory(
+            id=UUID(f"00000000-0000-0000-0000-{index:012d}"),
+            content=content,
+            status=status,
+            type=memory_type,
+            source_type="benchmark",
+            confidence=0.9,
+            importance=0.7,
+            tags=[short_id],
+        )
+        for index, (short_id, content, status, memory_type) in enumerate(specs, 1)
+    ]
+
+
+def evaluation_cases() -> list[EvaluationCase]:
+    def uid(number: int) -> str:
+        return f"00000000-0000-0000-0000-{number:012d}"
+
+    return [
+        EvaluationCase(
+            "q1", "Which local AI runtime does the user use?", {uid(1): 2, uid(7): 1}
+        ),
+        EvaluationCase("q2", "What programming language is the user focusing on now?", {uid(3): 2}),
+        EvaluationCase(
+            "q3", "What language did the user focus on previously?", {uid(2): 2},
+            TemporalScope.HISTORICAL,
+        ),
+        EvaluationCase("q4", "What response style does the user prefer?", {uid(4): 2}),
+        EvaluationCase(
+            "q5",
+            "What happened when the user tried a much larger Qwen model?",
+            {uid(6): 2},
+        ),
+        EvaluationCase("q6", "Which database backs the Atlas project?", {uid(8): 2}),
+        EvaluationCase(
+            "q7", "What active career learning objective involves systems?", {uid(3): 2},
+            memory_types={MemoryType.GOAL},
+        ),
+    ]
+
+
+async def build_engine(path: Path) -> tuple[Database, HybridRetrievalEngine]:
+    database = Database(path)
+    await database.initialize()
+    repository = SqliteMemoryRepository(database.connection())
+    for memory in evaluation_memories():
+        await repository.create(memory)
+    embedding = DeterministicEmbedding()
+    lexical = BM25Index()
+    vector = InMemoryVectorStore(embedding.dimension)
+    synchronizer = RetrievalIndexSynchronizer(
+        memory_repo=repository,
+        lexical_index=lexical,
+        vector_store=vector,
+        embedding_service=embedding,
+    )
+    engine = HybridRetrievalEngine(
+        memory_repo=repository,
+        lexical_index=lexical,
+        vector_store=vector,
+        embedding_service=embedding,
+        index_synchronizer=synchronizer,
+    )
+    return database, engine
+
+
+async def evaluate(
+    engine: HybridRetrievalEngine, mode: RetrievalMode, *, k: int = 5
+) -> EvaluationSummary:
+    totals = {"recall": 0.0, "precision": 0.0, "hit": 0.0, "mrr": 0.0, "ndcg": 0.0}
+    latencies: list[float] = []
+    stage_totals: dict[str, float] = {}
+    stage_counts: dict[str, int] = {}
+    rankings: dict[str, list[str]] = {}
+    cases = evaluation_cases()
+    for case in cases:
+        started = time.perf_counter()
+        result = await engine.retrieve(RetrievalQuery(
+            text=case.text,
+            k=k,
+            mode=mode,
+            temporal_scope=case.temporal_scope,
+            allowed_memory_types=case.memory_types,
+        ))
+        latencies.append((time.perf_counter() - started) * 1000)
+        for stage in result.trace.stages:
+            stage_totals[stage.stage_name] = (
+                stage_totals.get(stage.stage_name, 0.0) + stage.latency_ms
+            )
+            stage_counts[stage.stage_name] = stage_counts.get(stage.stage_name, 0) + 1
+        identifiers = [str(item.memory.id) for item in result.memories]
+        rankings[case.query_id] = [item.memory.content for item in result.memories]
+        totals["recall"] += recall_at_k(identifiers, case.qrels, k)
+        totals["precision"] += precision_at_k(identifiers, case.qrels, k)
+        totals["hit"] += hit_rate_at_k(identifiers, case.qrels, k)
+        totals["mrr"] += reciprocal_rank(identifiers, case.qrels)
+        totals["ndcg"] += ndcg_at_k(identifiers, case.qrels, k)
+    count = len(cases)
+    return EvaluationSummary(
+        recall=totals["recall"] / count,
+        precision=totals["precision"] / count,
+        hit_rate=totals["hit"] / count,
+        mrr=totals["mrr"] / count,
+        ndcg=totals["ndcg"] / count,
+        average_latency_ms=sum(latencies) / count,
+        stage_latency_ms={
+            name: total / stage_counts[name] for name, total in stage_totals.items()
+        },
+        rankings=rankings,
+    )
+
+
+async def _main() -> None:
+    with tempfile.TemporaryDirectory(prefix="contextos-retrieval-") as directory:
+        database, engine = await build_engine(Path(directory) / "benchmark.db")
+        try:
+            print("Mode       Recall@5  Precision@5  HitRate@5  MRR     NDCG@5  Avg ms")
+            summaries: dict[RetrievalMode, EvaluationSummary] = {}
+            for mode in RetrievalMode:
+                summaries[mode] = await evaluate(engine, mode)
+                item = summaries[mode]
+                print(
+                    f"{mode.value:<10} {item.recall:>8.3f}  {item.precision:>11.3f}  "
+                    f"{item.hit_rate:>9.3f}  {item.mrr:>6.3f}  {item.ndcg:>6.3f}  "
+                    f"{item.average_latency_ms:>6.3f}"
+                )
+            print("\nAverage stage latency (ms):")
+            print("Mode       Lexical   Dense   Fusion   Total")
+            for mode, item in summaries.items():
+                print(
+                    f"{mode.value:<10} "
+                    f"{item.stage_latency_ms.get('lexical_search', 0.0):>7.3f} "
+                    f"{item.stage_latency_ms.get('dense_search', 0.0):>7.3f} "
+                    f"{item.stage_latency_ms.get('fusion_rerank', 0.0):>7.3f} "
+                    f"{item.average_latency_ms:>7.3f}"
+                )
+            print("\nSmoke rankings (top 3):")
+            for case in evaluation_cases()[:5]:
+                print(f"\n{case.query_id}: {case.text}")
+                for mode in RetrievalMode:
+                    ranking = summaries[mode].rankings[case.query_id][:3]
+                    print(f"  {mode.value:<8} " + " | ".join(ranking))
+        finally:
+            await database.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(_main())