rememberstack 0.1.0__py3-none-any.whl

rememberstack/eval/resolution.py

@@ -0,0 +1,255 @@
+"""The ER golden-set suite (WP-2.1, D17/D22): per-type P/R over golden pairs.
+
+Runs the cascade's decision function over every human-adjudicated pair,
+computes precision/recall per entity type, records the curves on the
+`resolver_versions` row (the acceptance home) and the run in `eval_runs`.
+No threshold ships as final without these curves; the floors here are
+starting points to tighten as the golden set grows.
+"""
+
+from typing import Final
+from uuid import UUID
+from uuid import uuid4
+from uuid import uuid5
+
+from sqlalchemy import bindparam
+from sqlalchemy import JSON
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
+
+from rememberstack.spine.resolver import CascadeResolver
+
+PRECISION_FLOOR: Final = 0.90
+"""Suite-blocking precision floor per type (starting point, D22)."""
+
+RECALL_FLOOR: Final = 0.80
+"""Suite-blocking recall floor per type (starting point, D22)."""
+
+_PAIR_NAMESPACE: Final = UUID("601de77a-0000-4000-8000-000000000000")
+
+SYNTHETIC_GOLDEN_PAIRS: Final[tuple[dict[str, object], ...]] = (
+    # exact / near-exact strata
+    {
+        "entity_type": "Organization",
+        "surface_a": "Acme Corporation",
+        "surface_b": "Acme Corp",
+        "label": "match",
+        "hardness": "easy",
+        "expected_blocking_tier": "T1",
+        "context_a": "Acme Corporation, the industrial supplier.",
+        "context_b": "Acme Corp announced quarterly results.",
+    },
+    {
+        "entity_type": "Organization",
+        "surface_a": "Acme Corporation",
+        "surface_b": "Zenith Industries",
+        "label": "no_match",
+        "hardness": "easy",
+        "expected_blocking_tier": None,
+        "context_a": None,
+        "context_b": None,
+    },
+    # the Czech slice (registries §5): diacritics, inflection, family names
+    {
+        "entity_type": "Person",
+        "surface_a": "Pavel Kovář",
+        "surface_b": "Pavel Kovar",
+        "label": "match",
+        "hardness": "easy",
+        "expected_blocking_tier": "T0",  # unaccent folds the diacritic
+        "context_a": "Pavel Kovář of the Brno office.",
+        "context_b": "an email signed Pavel Kovar, Brno office",
+    },
+    {
+        "entity_type": "Person",
+        "surface_a": "Jan Novák",
+        "surface_b": "Jana Nováková",
+        "label": "no_match",  # feminine surname: typically a different person
+        "hardness": "hard_negative",
+        "expected_blocking_tier": "T1",
+        "context_a": "Jan Novák, the finance director.",
+        "context_b": "Jana Nováková from the legal team.",
+    },
+    {
+        "entity_type": "Person",
+        "surface_a": "Petr Svoboda",
+        "surface_b": "Petra Svobodu",  # accusative inflection of a NAME variant
+        "label": "no_match",
+        "hardness": "hard_negative",
+        "expected_blocking_tier": "T1",
+        "context_a": "Petr Svoboda leads the platform team.",
+        "context_b": "the committee appointed Petra Svobodu",
+    },
+    {
+        "entity_type": "Person",
+        "surface_a": "Karel Dvořák",
+        "surface_b": "Karel Dvorzak",  # phonetic spelling drift
+        "label": "match",
+        "hardness": "hard_positive",
+        "expected_blocking_tier": "T2",
+        "context_a": "Karel Dvořák, the composer's namesake in sales.",
+        "context_b": "meeting notes mention Karel Dvorzak from sales",
+    },
+)
+
+
+def seed_synthetic_golden_pairs(*, engine: Engine, deployment_id: UUID) -> None:
+    """Insert or refresh the synthetic starter pairs (stable ids).
+
+    These bootstrap the machinery and the Czech slice; real deployments grow
+    the set through human adjudication (WP-0.6 tooling) — synthetic pairs
+    stay marked `is_synthetic` so measured curves can be stratified.
+    """
+    with engine.begin() as connection:
+        for pair in SYNTHETIC_GOLDEN_PAIRS:
+            connection.execute(
+                _UPSERT_PAIR,
+                {
+                    "pair_id": uuid5(
+                        _PAIR_NAMESPACE,
+                        f"{deployment_id}:{pair['surface_a']}|{pair['surface_b']}",
+                    ),
+                    "deployment_id": deployment_id,
+                    **pair,
+                },
+            )
+
+
+def run_resolution_suite(
+    *,
+    engine: Engine,
+    resolver: CascadeResolver,
+    deployment_id: UUID,
+    component_version: str,
+) -> dict[str, object]:
+    """Judge every golden pair, record curves + the run, return the report.
+
+    Passing means every measured type meets the precision and recall floors.
+    The curves land on the resolver_versions row (notes) — the D22 record the
+    exit criterion names — and the run in eval_runs.
+    """
+    with engine.connect() as connection:
+        pairs = (
+            connection.execute(_SELECT_PAIRS, {"deployment_id": deployment_id})
+            .mappings()
+            .all()
+        )
+    by_type: dict[str, dict[str, int]] = {}
+    for pair in pairs:
+        matched, tier = resolver.judge_pair(
+            surface_a=pair["surface_a"],
+            surface_b=pair["surface_b"],
+            entity_type=pair["entity_type"],
+            context_a=pair["context_a"],
+            context_b=pair["context_b"],
+        )
+        counts = by_type.setdefault(
+            pair["entity_type"], {"tp": 0, "fp": 0, "fn": 0, "tn": 0}
+        )
+        actual = pair["label"] == "match"
+        if matched and actual:
+            counts["tp"] += 1
+        elif matched and not actual:
+            counts["fp"] += 1
+        elif not matched and actual:
+            counts["fn"] += 1
+        else:
+            counts["tn"] += 1
+        del tier  # per-pair deciding tier; stratified curves arrive with WP-0.6
+    curves = {
+        entity_type: {
+            "precision": _ratio(counts["tp"], counts["tp"] + counts["fp"]),
+            "recall": _ratio(counts["tp"], counts["tp"] + counts["fn"]),
+            "pairs": sum(counts.values()),
+        }
+        for entity_type, counts in by_type.items()
+    }
+    # an UNDEFINED metric (no positive pairs, or no predicted positives) is
+    # an unmeasured stratum and BLOCKS the suite — 0/0 never counts as
+    # perfect, so thresholds cannot be approved for a type the golden set
+    # does not actually measure (Codex review):
+    passed = bool(curves) and all(
+        curve["precision"] is not None
+        and curve["recall"] is not None
+        and curve["precision"] >= PRECISION_FLOOR
+        and curve["recall"] >= RECALL_FLOOR
+        for curve in curves.values()
+    )
+    with engine.begin() as connection:
+        connection.execute(
+            _RECORD_RUN,
+            {
+                "eval_run_id": uuid4(),
+                "deployment_id": deployment_id,
+                "component_version": component_version,
+                "metrics": {
+                    "curves": curves,
+                    "floors": {"precision": PRECISION_FLOOR, "recall": RECALL_FLOOR},
+                },
+                "passed": passed,
+            },
+        )
+        connection.execute(
+            _RECORD_CURVES,
+            {
+                "deployment_id": deployment_id,
+                "resolver_version": component_version,
+                "notes": {"curves": curves},
+            },
+        )
+    return {"curves": curves, "passed": passed}
+
+
+def _ratio(numerator: int, denominator: int) -> float | None:
+    """A ratio that is honestly None when its denominator is unmeasured."""
+    return numerator / denominator if denominator else None
+
+
+_UPSERT_PAIR = text(
+    """
+    INSERT INTO golden_pairs (
+        pair_id, deployment_id, entity_type, surface_a, surface_b,
+        context_a, context_b, label, hardness, expected_blocking_tier,
+        is_synthetic, adjudicated_by
+    ) VALUES (
+        :pair_id, :deployment_id, :entity_type, :surface_a, :surface_b,
+        :context_a, :context_b, :label, :hardness, :expected_blocking_tier,
+        true, 'synthetic-starter'
+    )
+    ON CONFLICT (pair_id) DO UPDATE
+        SET label = EXCLUDED.label,
+            hardness = EXCLUDED.hardness,
+            context_a = EXCLUDED.context_a,
+            context_b = EXCLUDED.context_b,
+            expected_blocking_tier = EXCLUDED.expected_blocking_tier
+    """
+)
+
+_SELECT_PAIRS = text(
+    """
+    SELECT entity_type, surface_a, surface_b, context_a, context_b, label
+    FROM golden_pairs
+    WHERE deployment_id = :deployment_id
+    ORDER BY entity_type, pair_id
+    """
+)
+
+_RECORD_RUN = text(
+    """
+    INSERT INTO eval_runs (
+        eval_run_id, deployment_id, suite, component_version, metrics, passed
+    ) VALUES (
+        :eval_run_id, :deployment_id, 'resolution', :component_version,
+        :metrics, :passed
+    )
+    """
+).bindparams(bindparam("metrics", type_=JSON))
+
+_RECORD_CURVES = text(
+    """
+    UPDATE resolver_versions
+    SET notes = CAST(:notes AS jsonb)::text
+    WHERE deployment_id = :deployment_id
+      AND resolver_version = :resolver_version
+    """
+).bindparams(bindparam("notes", type_=JSON))

rememberstack/eval/retrieval_spikes.py

@@ -0,0 +1,50 @@
+"""Persistence boundary for the WP-5.6 retrieval spike battery."""
+
+from typing import Final
+from uuid import UUID
+from uuid import uuid4
+
+from sqlalchemy import bindparam
+from sqlalchemy import JSON
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
+
+from rememberstack.model import RetrievalSpikeReport
+
+RETRIEVAL_SPIKE_VERSION: Final = "retrieval-spikes-2026.07b"
+"""Version stamped on every complete six-spike measurement record."""
+
+
+def record_retrieval_spike_report(
+    *,
+    engine: Engine,
+    deployment_id: UUID,
+    report: RetrievalSpikeReport,
+    component_version: str = RETRIEVAL_SPIKE_VERSION,
+) -> UUID:
+    """Append the complete spike report to D22's ``eval_runs`` history."""
+    eval_run_id = uuid4()
+    with engine.begin() as connection:
+        connection.execute(
+            _INSERT_RUN,
+            {
+                "eval_run_id": eval_run_id,
+                "deployment_id": deployment_id,
+                "component_version": component_version,
+                "metrics": report.model_dump(mode="json"),
+                "passed": report.passed,
+            },
+        )
+    return eval_run_id
+
+
+_INSERT_RUN = text(
+    """
+    INSERT INTO eval_runs (
+        eval_run_id, deployment_id, suite, component_version, metrics, passed
+    ) VALUES (
+        :eval_run_id, :deployment_id, 'retrieval', :component_version,
+        :metrics, :passed
+    )
+    """
+).bindparams(bindparam("metrics", type_=JSON))

rememberstack/eval/skeleton.py

@@ -0,0 +1,231 @@
+"""The walking-skeleton eval pack (WP-1.7, D22): the S-subset as canaries.
+
+The scenario battery is the retrieval golden set's skeleton (retrieval §11),
+so these cases live in the `retrieval` suite: S1 (current fact via resolve +
+lookup), S2 (semantic observation), S5 (the hydration chain), S39 (typed
+negatives), plus the grain contract (claims answers are evidence grain, never
+current-fact). Seeding writes the canaries; the evaluator replays each
+scenario against a composed QueryEngine and judges the envelope.
+"""
+
+from typing import Final
+from uuid import UUID
+from uuid import uuid5
+
+from sqlalchemy import bindparam
+from sqlalchemy import JSON
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
+
+from rememberstack.model import CanaryCase
+from rememberstack.model import Envelope
+from rememberstack.model import Grain
+from rememberstack.model import NegativeKind
+from rememberstack.surfaces.query_engine import QueryEngine
+
+_CANARY_NAMESPACE: Final = UUID("5ce1e701-0000-4000-8000-000000000000")
+
+SKELETON_CANARIES: Final[tuple[dict[str, object], ...]] = (
+    {
+        "description": "S1: current employer via resolve + live works_for lookup",
+        "input": {"scenario": "s1", "name": "Alice Novak", "predicate": "works_for"},
+        "expected": {"label": "Alice Novak works for Acme.", "min_evidence": 1},
+    },
+    {
+        "description": "S2: semantic observation lookup (headcount)",
+        "input": {"scenario": "s2", "name": "Acme", "property_query": "headcount"},
+        "expected": {"label_contains": "600"},
+    },
+    {
+        "description": "S5: hydration chain down to spans and sources",
+        "input": {"scenario": "s5", "name": "Alice Novak", "predicate": "works_for"},
+        "expected": {"min_evidence": 2, "min_sources": 1},
+    },
+    {
+        "description": "S39: unknown entity vs known-empty are typed differently",
+        "input": {"scenario": "s39", "unknown_name": "Contoso", "known_name": "Acme"},
+        "expected": {},
+    },
+    {
+        "description": "grain contract: claims answers are evidence grain",
+        "input": {"scenario": "grain_contract", "query": "Alice Novak employer"},
+        "expected": {"min_evidence": 1},
+    },
+)
+
+
+def seed_skeleton_canaries(*, engine: Engine, deployment_id: UUID) -> None:
+    """Insert or refresh the skeleton pack (stable per-deployment ids).
+
+    This function is the canonical definition: re-seeding updates an existing
+    canary in place, so a changed case can never silently keep evaluating its
+    stale stored form (Codex review).
+    """
+    with engine.begin() as connection:
+        for canary in SKELETON_CANARIES:
+            connection.execute(
+                _INSERT_CANARY,
+                {
+                    "canary_id": uuid5(
+                        _CANARY_NAMESPACE, f"{deployment_id}:{canary['description']}"
+                    ),
+                    "deployment_id": deployment_id,
+                    "description": canary["description"],
+                    "input": canary["input"],
+                    "expected": canary["expected"],
+                },
+            )
+
+
+def make_skeleton_evaluator(*, query_engine: QueryEngine, deployment_id: UUID):
+    """Build the retrieval-suite evaluator over one composed QueryEngine."""
+
+    def evaluate(case: CanaryCase) -> bool:
+        """Replay one scenario and judge its envelope against expectations."""
+        scenario = case.input.get("scenario")
+        if scenario == "s1":
+            return _s1(query_engine, deployment_id, case)
+        if scenario == "s2":
+            return _s2(query_engine, deployment_id, case)
+        if scenario == "s5":
+            return _s5(query_engine, deployment_id, case)
+        if scenario == "s39":
+            return _s39(query_engine, deployment_id, case)
+        if scenario == "grain_contract":
+            return _grain_contract(query_engine, deployment_id, case)
+        return False  # an unknown scenario never silently passes
+
+    return evaluate
+
+
+def _s1(engine: QueryEngine, deployment_id: UUID, case: CanaryCase) -> bool:
+    """Resolve the person; the live relation answers with its label."""
+    entity = _resolve_one(engine, deployment_id, str(case.input["name"]))
+    if entity is None:
+        return False
+    answer = engine.lookup_relations(
+        deployment_id=deployment_id,
+        subject_entity_id=entity,
+        predicate=str(case.input["predicate"]),
+    )
+    return (
+        answer.grain is Grain.FACT
+        and len(answer.facts) == 1
+        and answer.facts[0].label == case.expected["label"]
+        and answer.facts[0].evidence_count >= int(case.expected["min_evidence"])  # type: ignore[call-overload]
+        and answer.facts[0].validity.invalidated_at is None
+    )
+
+
+def _s2(engine: QueryEngine, deployment_id: UUID, case: CanaryCase) -> bool:
+    """Semantic property match over the entity's observation statements."""
+    entity = _resolve_one(engine, deployment_id, str(case.input["name"]))
+    if entity is None:
+        return False
+    answer = engine.lookup_observations(
+        deployment_id=deployment_id,
+        entity_id=entity,
+        property_query=str(case.input["property_query"]),
+    )
+    # discrimination is bounded by the corpus: with richer golden corpora
+    # (WP-0.6 tooling) this tightens to "only the matching property returns";
+    # here the match must exist, rank in the top results, and drop nothing:
+    return (
+        answer.grain is Grain.FACT
+        and answer.dropped_by_hydration == 0
+        and any(
+            str(case.expected["label_contains"]) in fact.label
+            for fact in answer.facts[:3]
+        )
+    )
+
+
+def _s5(engine: QueryEngine, deployment_id: UUID, case: CanaryCase) -> bool:
+    """The full chain: relation → evidence spans → source handles."""
+    entity = _resolve_one(engine, deployment_id, str(case.input["name"]))
+    if entity is None:
+        return False
+    relations = engine.lookup_relations(
+        deployment_id=deployment_id,
+        subject_entity_id=entity,
+        predicate=str(case.input["predicate"]),
+    )
+    if not relations.facts:
+        return False
+    hydrated = engine.hydrate_relation(
+        deployment_id=deployment_id, relation_id=relations.facts[0].fact_id
+    )
+    source_docs = {source.doc_id for source in hydrated.sources}
+    return (
+        hydrated.grain is Grain.COMPOSITE
+        and len(hydrated.evidence) >= int(case.expected["min_evidence"])  # type: ignore[call-overload]
+        and len(hydrated.sources) >= int(case.expected["min_sources"])  # type: ignore[call-overload]
+        # provenance linkage (Codex review): every claim's offsets are
+        # span-coherent and its document is among the returned sources:
+        and all(
+            claim.char_end - claim.char_start == len(claim.source_span)
+            and claim.doc_id in source_docs
+            for claim in hydrated.evidence
+        )
+    )
+
+
+def _s39(engine: QueryEngine, deployment_id: UUID, case: CanaryCase) -> bool:
+    """Unknown entity and known-empty carry their distinct typed negatives."""
+    unknown = engine.resolve(
+        deployment_id=deployment_id, name=str(case.input["unknown_name"])
+    )
+    if unknown.negative is None or unknown.negative.kind is not (
+        NegativeKind.UNKNOWN_ENTITY
+    ):
+        return False
+    known = _resolve_one(engine, deployment_id, str(case.input["known_name"]))
+    if known is None:
+        return False
+    empty = engine.lookup_relations(
+        deployment_id=deployment_id, subject_entity_id=known, predicate="reports_to"
+    )
+    return (
+        empty.negative is not None and empty.negative.kind is NegativeKind.KNOWN_EMPTY
+    )
+
+
+def _grain_contract(engine: QueryEngine, deployment_id: UUID, case: CanaryCase) -> bool:
+    """Claims answers are evidence grain — never a current-fact answer.
+
+    Substance is required (Codex review): an implementation returning empty
+    evidence for every query must fail, and every returned claim must be
+    current testimony (the default channel's contract).
+    """
+    answer: Envelope = engine.search_claims(
+        deployment_id=deployment_id, query=str(case.input["query"])
+    )
+    return (
+        answer.grain is Grain.EVIDENCE
+        and not answer.facts
+        and len(answer.evidence) >= int(case.expected["min_evidence"])  # type: ignore[call-overload]
+        and all(claim.is_current_testimony for claim in answer.evidence)
+    )
+
+
+def _resolve_one(engine: QueryEngine, deployment_id: UUID, name: str) -> UUID | None:
+    """Resolve to exactly one current entity, or None."""
+    resolved = engine.resolve(deployment_id=deployment_id, name=name)
+    if len(resolved.entities) != 1:
+        return None
+    return resolved.entities[0].entity_id
+
+
+_INSERT_CANARY = text(
+    """
+    INSERT INTO canary_cases (
+        canary_id, deployment_id, suite, description, input, expected
+    ) VALUES (
+        :canary_id, :deployment_id, 'retrieval', :description, :input, :expected
+    )
+    ON CONFLICT (canary_id) DO UPDATE
+        SET description = EXCLUDED.description,
+            input = EXCLUDED.input,
+            expected = EXCLUDED.expected
+    """
+).bindparams(bindparam("input", type_=JSON), bindparam("expected", type_=JSON))

rememberstack/llm/__init__.py

@@ -0,0 +1 @@
+"""Programmatic LLM package."""