secondbrain-py 0.2.1__py3-none-any.whl

brain/eval/answer_eval.py

@@ -0,0 +1,281 @@
+"""Answer-quality eval for `brain ask` (Plan 06, Phase 3).
+
+Measures synthesized answers against a golden fact-set with deterministic,
+local-only scoring — substring + significant-token overlap, NO LLM judge in the
+loop — so the harness stays reproducible and never requires Ollama to *score*
+(the live ``ask_fn`` that produces answers may, but that is the caller's
+concern). Mirrors the hybrid-search eval package's shape (``corpus`` loader +
+dataclass report) so the two read consistently.
+"""
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import yaml
+
+from .errors import EvalCorpusError
+
+if TYPE_CHECKING:
+    from ..ask import AskResult
+
+# Default answer-corpus path: eval/ -> brain/ -> src/ -> repo root, then into
+# tests/eval/. Committed (unlike the hybrid golden corpus) because it is fully
+# synthetic — no real doc IDs, names, or employers (CLAUDE.md rule 15).
+_DEFAULT_ANSWER_CORPUS_PATH: Path = (
+    Path(__file__).resolve().parent.parent.parent.parent
+    / "tests"
+    / "eval"
+    / "answer_corpus.yaml"
+)
+
+_VALID_ANSWER_CATEGORIES: frozenset[str] = frozenset(
+    ("synthesis", "multi-hop", "person", "timeline")
+)
+
+_REQUIRED_ANSWER_FIELDS: frozenset[str] = frozenset(
+    ("question", "expected_facts", "category")
+)
+_ALLOWED_ANSWER_FIELDS: frozenset[str] = frozenset(
+    ("question", "expected_facts", "category", "notes")
+)
+_ANSWER_CORPUS_VERSION = 1
+
+# A fact counts as covered when at least this fraction of its significant
+# (non-stopword) tokens appear in the answer — a majority by default. Single
+# distinctive tokens (ratio 1.0) and exact phrase substrings always pass.
+_FACT_COVERAGE_THRESHOLD = 0.5
+
+# Minimal English stopword set for significant-token extraction. Deliberately
+# small — only the highest-frequency function words — so distinctive content
+# words (the signal we score on) survive.
+_STOPWORDS: frozenset[str] = frozenset(
+    (
+        "the", "a", "an", "of", "to", "and", "or", "in", "on", "for", "with",
+        "is", "are", "was", "were", "be", "been", "my", "i", "we", "you",
+        "that", "this", "it", "as", "at", "by", "from", "about", "into",
+    )
+)
+
+_TOKEN_RE = re.compile(r"[a-z0-9]+")
+
+
+@dataclass(frozen=True)
+class AnswerEvalCase:
+    """One golden Q&A case from the answer corpus."""
+
+    question: str
+    expected_facts: list[str]  # short synthetic phrases; all should appear/be implied
+    category: str  # one of _VALID_ANSWER_CATEGORIES
+    notes: str = ""  # human-readable rationale, not graded
+
+
+@dataclass(frozen=True)
+class AnswerScore:
+    """Per-case answer-quality score."""
+
+    question: str
+    category: str
+    fact_recall: float  # fraction of expected_facts covered by the answer
+    citation_count: int  # number of distinct citations the answer produced
+
+
+@dataclass(frozen=True)
+class AnswerEvalReport:
+    """Aggregate answer-eval result over a corpus run."""
+
+    scores: list[AnswerScore]
+    mean_fact_recall: float
+    mean_citation_count: float
+    timestamp: str  # ISO-8601
+
+
+def _significant_tokens(text: str) -> list[str]:
+    """Lowercase content tokens with stopwords removed."""
+    return [t for t in _TOKEN_RE.findall(text.lower()) if t not in _STOPWORDS]
+
+
+def _fact_covered(answer_lower: str, answer_tokens: set[str], fact: str) -> bool:
+    """Return True if ``fact`` is covered by the answer.
+
+    Covered when the exact (lowercased) fact phrase is a substring of the
+    answer, OR at least :data:`_FACT_COVERAGE_THRESHOLD` of the fact's
+    significant tokens appear in the answer's token set.
+    """
+    if fact.strip().lower() in answer_lower:
+        return True
+    fact_tokens = _significant_tokens(fact)
+    if not fact_tokens:
+        # No content words to compare — fall back to the substring result above.
+        return False
+    present = sum(1 for t in fact_tokens if t in answer_tokens)
+    return present / len(fact_tokens) >= _FACT_COVERAGE_THRESHOLD
+
+
+def score_answer(answer: str, expected_facts: list[str]) -> float:
+    """Fraction of ``expected_facts`` covered by ``answer`` in ``[0.0, 1.0]``.
+
+    Deterministic substring + significant-token-overlap matching (no LLM). An
+    empty ``expected_facts`` list scores ``1.0`` (nothing to miss).
+    """
+    if not expected_facts:
+        return 1.0
+    answer_lower = answer.lower()
+    answer_tokens = set(_significant_tokens(answer))
+    covered = sum(
+        1
+        for fact in expected_facts
+        if _fact_covered(answer_lower, answer_tokens, fact)
+    )
+    return covered / len(expected_facts)
+
+
+def run_answer_eval(
+    cases: list[AnswerEvalCase],
+    ask_fn: Callable[[str], AskResult],
+    *,
+    now: datetime | None = None,
+) -> AnswerEvalReport:
+    """Run ``ask_fn`` over every case and score the answers.
+
+    ``ask_fn`` maps a question to an :class:`~brain.ask.AskResult` (typically a
+    closure over ``ask_no_loop`` against the live brain). ``now`` overrides the
+    report timestamp (defaults to the current UTC time) so tests stay
+    deterministic.
+    """
+    scores: list[AnswerScore] = []
+    for case in cases:
+        result = ask_fn(case.question)
+        scores.append(
+            AnswerScore(
+                question=case.question,
+                category=case.category,
+                fact_recall=score_answer(result.answer, case.expected_facts),
+                citation_count=len(result.citations),
+            )
+        )
+    if scores:
+        mean_recall = sum(s.fact_recall for s in scores) / len(scores)
+        mean_citations = sum(s.citation_count for s in scores) / len(scores)
+    else:
+        mean_recall = 0.0
+        mean_citations = 0.0
+    stamp = (now or datetime.now(UTC)).isoformat()
+    return AnswerEvalReport(
+        scores=scores,
+        mean_fact_recall=mean_recall,
+        mean_citation_count=mean_citations,
+        timestamp=stamp,
+    )
+
+
+def load_answer_corpus(path: Path | None = None) -> list[AnswerEvalCase]:
+    """Load + validate the answer-eval corpus YAML.
+
+    Raises:
+        EvalCorpusError: missing/malformed file, version mismatch, unknown
+            category, missing required field, or empty ``expected_facts``.
+    """
+    if path is None:
+        path = _DEFAULT_ANSWER_CORPUS_PATH
+
+    if not path.exists():
+        raise EvalCorpusError(f"answer corpus file not found: {path}")
+
+    try:
+        raw = yaml.safe_load(path.read_text(encoding="utf-8"))
+    except yaml.YAMLError as exc:
+        raise EvalCorpusError(
+            f"answer corpus YAML parse error in {path}: {exc}"
+        ) from exc
+
+    if not isinstance(raw, dict):
+        raise EvalCorpusError(
+            f"answer corpus file must be a YAML mapping, got {type(raw).__name__}"
+        )
+
+    version = raw.get("version")
+    if version != _ANSWER_CORPUS_VERSION:
+        raise EvalCorpusError(
+            f"answer corpus version mismatch: expected {_ANSWER_CORPUS_VERSION}, "
+            f"got {version!r}."
+        )
+
+    raw_cases = raw.get("cases")
+    if not isinstance(raw_cases, list):
+        raise EvalCorpusError("answer corpus file must have a 'cases' list")
+
+    cases: list[AnswerEvalCase] = []
+    for i, entry in enumerate(raw_cases):
+        if not isinstance(entry, dict):
+            raise EvalCorpusError(
+                f"answer corpus case #{i + 1} must be a YAML mapping, "
+                f"got {type(entry).__name__}"
+            )
+
+        missing = _REQUIRED_ANSWER_FIELDS - entry.keys()
+        if missing:
+            raise EvalCorpusError(
+                f"answer corpus case #{i + 1} is missing required field(s): "
+                f"{', '.join(sorted(missing))}"
+            )
+
+        unknown = entry.keys() - _ALLOWED_ANSWER_FIELDS
+        if unknown:
+            raise EvalCorpusError(
+                f"answer corpus case #{i + 1} has unknown field(s): "
+                f"{', '.join(sorted(unknown))}. "
+                f"Allowed: {', '.join(sorted(_ALLOWED_ANSWER_FIELDS))}"
+            )
+
+        category = entry["category"]
+        if category not in _VALID_ANSWER_CATEGORIES:
+            raise EvalCorpusError(
+                f"answer corpus case #{i + 1} has unknown category {category!r}. "
+                f"Valid: {', '.join(sorted(_VALID_ANSWER_CATEGORIES))}"
+            )
+
+        expected_facts = entry["expected_facts"]
+        if not isinstance(expected_facts, list):
+            raise EvalCorpusError(
+                f"answer corpus case #{i + 1} 'expected_facts' must be a list, "
+                f"got {type(expected_facts).__name__}"
+            )
+        if not expected_facts:
+            raise EvalCorpusError(
+                f"answer corpus case #{i + 1} ({entry['question']!r}) has empty "
+                f"'expected_facts'"
+            )
+
+        cases.append(
+            AnswerEvalCase(
+                question=str(entry["question"]),
+                expected_facts=[str(f) for f in expected_facts],
+                category=category,
+                notes=str(entry.get("notes", "")),
+            )
+        )
+
+    return cases
+
+
+def answer_eval_report_to_dict(report: AnswerEvalReport) -> dict[str, Any]:
+    """JSON projection of an :class:`AnswerEvalReport` (CLI ``--json``)."""
+    return {
+        "mean_fact_recall": report.mean_fact_recall,
+        "mean_citation_count": report.mean_citation_count,
+        "timestamp": report.timestamp,
+        "scores": [
+            {
+                "question": s.question,
+                "category": s.category,
+                "fact_recall": s.fact_recall,
+                "citation_count": s.citation_count,
+            }
+            for s in report.scores
+        ],
+    }

brain/eval/baseline.py

@@ -0,0 +1,265 @@
+"""Baseline save/load and diff helpers for eval reports.
+
+Baselines are stored as JSON files (sorted keys, floats rounded to 4 decimals
+for byte-stable diffs) via an atomic write so a crash can never leave a
+half-written file.  Baseline names are validated against a strict allowlist
+pattern to prevent path traversal.
+"""
+
+import dataclasses
+import json
+import re
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from ..vault._atomic import atomic_write_text
+from .errors import EvalBaselineError
+from .runner import CategorySummary, EvalReport, EvalResult
+
+_BASELINE_NAME_RE = re.compile(r"^[A-Za-z0-9_-]+$")
+
+
+def _assert_baseline_name(name: str) -> None:
+    """Validate that *name* is safe to use as a filename component.
+
+    Raises:
+        EvalBaselineError: When *name* contains characters outside
+            ``[A-Za-z0-9_-]`` (e.g. slashes, dots, spaces).
+    """
+    if not _BASELINE_NAME_RE.fullmatch(name):
+        raise EvalBaselineError(
+            f"baseline name {name!r} is invalid; "
+            f"use only letters, digits, hyphens, and underscores"
+        )
+
+
+def _round_floats(obj: Any, decimals: int = 4) -> Any:
+    """Recursively round all floats in a JSON-compatible structure."""
+    if isinstance(obj, float):
+        return round(obj, decimals)
+    if isinstance(obj, dict):
+        return {k: _round_floats(v, decimals) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_round_floats(v, decimals) for v in obj]
+    return obj
+
+
+def save_baseline(report: EvalReport, *, path: Path) -> None:
+    """Write *report* to *path* as JSON, atomically.
+
+    Keys are sorted and floats rounded to 4 decimal places so repeated
+    runs over the same corpus produce byte-stable diffs.  The parent
+    directory is created if it does not exist.
+
+    Args:
+        report: The eval report to persist.
+        path: Destination file path (typically under ``tests/eval/baselines/``).
+    """
+    path.parent.mkdir(parents=True, exist_ok=True)
+    raw: dict[str, Any] = dataclasses.asdict(report)
+    raw = _round_floats(raw)
+    # datetime → ISO-8601 with Z suffix for UTC.
+    raw["generated_at"] = report.generated_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+    text = json.dumps(raw, sort_keys=True, default=str, indent=2) + "\n"
+    atomic_write_text(path, text)
+
+
+def load_baseline(path: Path) -> EvalReport:
+    """Load an :class:`~brain.eval.runner.EvalReport` from a JSON baseline file.
+
+    Args:
+        path: Path to the baseline JSON file.
+
+    Raises:
+        EvalBaselineError: When the file is missing, contains invalid JSON,
+            or has an unexpected structure.
+    """
+    if not path.exists():
+        raise EvalBaselineError(f"baseline file not found: {path}")
+    try:
+        data: dict[str, Any] = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise EvalBaselineError(
+            f"baseline file contains invalid JSON: {path}: {exc}"
+        ) from exc
+    try:
+        results = [
+            EvalResult(
+                query=r["query"],
+                category=r["category"],
+                expected_doc_ids=list(r["expected_doc_ids"]),
+                actual_doc_ids=list(r["actual_doc_ids"]),
+                ndcg_at_5=float(r["ndcg_at_5"]),
+                mrr=float(r["mrr"]),
+                recall_at_20=float(r["recall_at_20"]),
+            )
+            for r in data["results"]
+        ]
+        per_category = {
+            k: CategorySummary(
+                category=v["category"],
+                count=int(v["count"]),
+                mean_ndcg_at_5=float(v["mean_ndcg_at_5"]),
+                mean_mrr=float(v["mean_mrr"]),
+                mean_recall_at_20=float(v["mean_recall_at_20"]),
+            )
+            for k, v in data["per_category"].items()
+        }
+        generated_at_raw: str = data["generated_at"]
+        # Python 3.11+ accepts the "Z" suffix in fromisoformat.
+        generated_at = datetime.fromisoformat(generated_at_raw)
+        if generated_at.tzinfo is None:
+            generated_at = generated_at.replace(tzinfo=UTC)
+        return EvalReport(
+            results=results,
+            mean_ndcg_at_5=float(data["mean_ndcg_at_5"]),
+            mean_mrr=float(data["mean_mrr"]),
+            mean_recall_at_20=float(data["mean_recall_at_20"]),
+            per_category=per_category,
+            config_signature=dict(data["config_signature"]),
+            generated_at=generated_at,
+        )
+    except (KeyError, ValueError, TypeError) as exc:
+        raise EvalBaselineError(
+            f"baseline file has unexpected structure: {path}: {exc}"
+        ) from exc
+
+
+# ---------------------------------------------------------------------------
+# Diff helpers
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class QueryDiff:
+    """Per-query metric delta between a baseline and a current run."""
+
+    query: str
+    category: str
+    ndcg_at_5_delta: float  # current - baseline; negative = regression
+    mrr_delta: float
+    recall_at_20_delta: float
+
+
+@dataclass(frozen=True)
+class BaselineDiff:
+    """Aggregate diff between a baseline :class:`EvalReport` and a current run.
+
+    Per-query deltas and the aggregate means cover the INTERSECTION of the two
+    runs' query sets (queries present in BOTH). Queries present in only one run
+    are surfaced separately in :attr:`added_queries` / :attr:`removed_queries`
+    and excluded from the aggregate, so a growing or shrinking query set never
+    dilutes a real regression toward 0 — the aggregate always compares
+    like-for-like. This is the input contract for the ``--fail-below`` gate.
+    """
+
+    per_query: list[QueryDiff]
+    mean_ndcg_at_5_delta: float
+    mean_mrr_delta: float
+    mean_recall_at_20_delta: float
+    config_signature_changed: bool
+    baseline_signature: dict[str, Any]
+    current_signature: dict[str, Any]
+    # Query strings present in ``current`` but not ``baseline`` (added), and in
+    # ``baseline`` but not ``current`` (removed). Reported separately so the
+    # aggregate stays intersection-only; deterministic source order.
+    added_queries: list[str]
+    removed_queries: list[str]
+
+
+def diff_reports(baseline: EvalReport, current: EvalReport) -> BaselineDiff:
+    """Compute the delta between *baseline* and *current*.
+
+    Queries are matched by query string. Per-query deltas AND the aggregate mean
+    deltas are computed over the INTERSECTION of the two query sets (queries
+    present in BOTH reports), in the baseline's result order. Queries present in
+    only one report are surfaced separately as
+    :attr:`~BaselineDiff.added_queries` (in *current* only) /
+    :attr:`~BaselineDiff.removed_queries` (in *baseline* only) and excluded from
+    the aggregate — so a growing or shrinking query set never dilutes a real
+    regression toward 0 (an earlier union-with-0-fill averaged each added query's
+    ``current - 0`` positive delta into the mean, masking regressions on the
+    shared queries).
+
+    Args:
+        baseline: The reference (earlier) eval report.
+        current: The new eval report to compare.
+
+    Returns:
+        A :class:`BaselineDiff` whose per-query and aggregate deltas cover the
+        shared queries, plus the added/removed query id lists.
+    """
+    # Build lookup dicts keyed by query string.
+    base_by_q = {r.query: r for r in baseline.results}
+    curr_by_q = {r.query: r for r in current.results}
+
+    # Per-query deltas over the SHARED queries only, in baseline result order.
+    per_query: list[QueryDiff] = []
+    for q_str, base_r in base_by_q.items():
+        curr_r = curr_by_q.get(q_str)
+        if curr_r is None:
+            continue  # removed query — reported separately, not in the aggregate
+        per_query.append(
+            QueryDiff(
+                query=q_str,
+                category=curr_r.category,
+                ndcg_at_5_delta=curr_r.ndcg_at_5 - base_r.ndcg_at_5,
+                mrr_delta=curr_r.mrr - base_r.mrr,
+                recall_at_20_delta=curr_r.recall_at_20 - base_r.recall_at_20,
+            )
+        )
+
+    # Query-set changes surfaced separately (deterministic source order): never
+    # folded into the aggregate.
+    added_queries = [q for q in curr_by_q if q not in base_by_q]
+    removed_queries = [q for q in base_by_q if q not in curr_by_q]
+
+    n = len(per_query)
+    mean_ndcg_delta = sum(d.ndcg_at_5_delta for d in per_query) / n if n else 0.0
+    mean_mrr_delta = sum(d.mrr_delta for d in per_query) / n if n else 0.0
+    mean_recall_delta = sum(d.recall_at_20_delta for d in per_query) / n if n else 0.0
+
+    return BaselineDiff(
+        per_query=per_query,
+        mean_ndcg_at_5_delta=mean_ndcg_delta,
+        mean_mrr_delta=mean_mrr_delta,
+        mean_recall_at_20_delta=mean_recall_delta,
+        config_signature_changed=baseline.config_signature != current.config_signature,
+        baseline_signature=dict(baseline.config_signature),
+        current_signature=dict(current.config_signature),
+        added_queries=added_queries,
+        removed_queries=removed_queries,
+    )
+
+
+# ---------------------------------------------------------------------------
+# ``--fail-below`` regression gate (Task 5.5)
+# ---------------------------------------------------------------------------
+
+# A mean metric "regresses" when its 4-decimal-ROUNDED delta is more negative
+# than one unit at the baseline's 4-decimal serialization precision (``-1e-4``).
+# Comparing the ROUNDED delta — not the raw float — keeps the boundary
+# constructible: a delta of exactly ``-0.0001`` is NOT a regression (it equals
+# the threshold), whereas ``-0.0002`` is. A raw-float compare would be
+# IEEE-754-fragile right at that boundary. Uniform across all three metrics
+# (nDCG@5 / MRR / Recall@20) — no per-metric overrides (CLAUDE.md decision).
+_FAIL_BELOW_THRESHOLD: float = -1e-4
+
+
+def mean_metrics_regressed(diff: BaselineDiff) -> bool:
+    """True iff any mean metric delta regressed past the ``--fail-below`` gate.
+
+    Checks ``round(delta, 4) < -1e-4`` for each of nDCG@5 / MRR / Recall@20 (the
+    aggregate over the shared query set — :func:`diff_reports` already excludes
+    added/removed queries so a changed corpus never dilutes a real regression).
+    """
+    return any(
+        round(delta, 4) < _FAIL_BELOW_THRESHOLD
+        for delta in (
+            diff.mean_ndcg_at_5_delta,
+            diff.mean_mrr_delta,
+            diff.mean_recall_at_20_delta,
+        )
+    )