rememberstack 0.1.0__py3-none-any.whl

rememberstack/eval/consumption.py

@@ -0,0 +1,141 @@
+"""Repeatable S58 cold-harness protocol for the rendered consumption skill."""
+
+from typing import Final
+from uuid import UUID
+from uuid import uuid5
+
+from sqlalchemy import bindparam
+from sqlalchemy import JSON
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
+
+from rememberstack.eval.harness import CaseEvaluator
+from rememberstack.eval.skeleton import make_skeleton_evaluator
+from rememberstack.model import CanaryCase
+from rememberstack.model import ModelRequest
+from rememberstack.model import RenderedConsumptionSkill
+from rememberstack.model import S58Answer
+from rememberstack.ports.model_provider import ModelProviderPort
+from rememberstack.surfaces.query_engine import QueryEngine
+
+_CANARY_NAMESPACE: Final = UUID("55800000-0000-4000-8000-000000000000")
+
+S58_CANARIES: Final[tuple[dict[str, object], ...]] = (
+    {
+        "description": "S58: a cold agent plans a grain-safe memory answer",
+        "input": {
+            "scenario": "s58",
+            "task": (
+                "Brief me on Acme, determine whether Alice currently works for "
+                "Acme, and show what the sources said. Plane K may be empty. "
+                "One candidate fact has withdrawn support and a live "
+                "contradiction group. Choose how to orient, verify, and audit."
+            ),
+        },
+        "expected": {
+            "orientation": "knowledge",
+            "empty_knowledge": "fallback_p3_or_search",
+            "current_truth": "fact_lookup",
+            "grain_handling": "separate",
+            "withdrawn_support": "caveat_and_transcript",
+            "claims_as_of": "assertion_history_only",
+            "contradictions": "report_co_members",
+            "readable_content": "prefer_mounts",
+            "audit": "hydrate_to_sources",
+        },
+    },
+)
+
+
+def seed_s58_canaries(*, engine: Engine, deployment_id: UUID) -> None:
+    """Insert or refresh the stable S58 retrieval canary for one deployment."""
+    with engine.begin() as connection:
+        for canary in S58_CANARIES:
+            connection.execute(
+                _INSERT_CANARY,
+                {
+                    "canary_id": uuid5(
+                        _CANARY_NAMESPACE, f"{deployment_id}:{canary['description']}"
+                    ),
+                    "deployment_id": deployment_id,
+                    "description": canary["description"],
+                    "input": canary["input"],
+                    "expected": canary["expected"],
+                },
+            )
+
+
+def make_s58_evaluator(
+    *, model_provider: ModelProviderPort, model: str, skill: RenderedConsumptionSkill
+) -> CaseEvaluator:
+    """Build the S58 evaluator whose only system context is the rendered skill."""
+
+    def evaluate(case: CanaryCase) -> bool:
+        """Ask one cold model for a plan and compare its structured decisions."""
+        if case.input.get("scenario") != "s58":
+            return False
+        task = case.input.get("task")
+        if not isinstance(task, str) or not task.strip():
+            return False
+        expected = S58Answer.model_validate(case.expected)
+        answer = model_provider.generate(
+            request=ModelRequest(model=model, prompt=_prompt(skill=skill, task=task)),
+            response_type=S58Answer,
+        )
+        return answer.output == expected
+
+    return evaluate
+
+
+def make_retrieval_evaluator(
+    *,
+    query_engine: QueryEngine,
+    deployment_id: UUID,
+    model_provider: ModelProviderPort,
+    model: str,
+    skill: RenderedConsumptionSkill,
+) -> CaseEvaluator:
+    """Compose the walking-skeleton and S58 cases under one retrieval suite."""
+    skeleton = make_skeleton_evaluator(
+        query_engine=query_engine, deployment_id=deployment_id
+    )
+    s58 = make_s58_evaluator(model_provider=model_provider, model=model, skill=skill)
+
+    def evaluate(case: CanaryCase) -> bool:
+        """Dispatch S58 to the cold harness and earlier cases to the query engine."""
+        if case.input.get("scenario") == "s58":
+            return s58(case)
+        return skeleton(case)
+
+    return evaluate
+
+
+def _prompt(*, skill: RenderedConsumptionSkill, task: str) -> str:
+    """Build a cold prompt containing no project context beyond skill + task."""
+    return (
+        "You are a cold agent evaluating a memory-consumption skill. You have "
+        "never seen this memory system. Treat the skill below as your only "
+        "knowledge of it; do not rely on outside conventions or guess missing "
+        "capabilities. Select the best structured action for every field.\n\n"
+        "<consumption-skill>\n"
+        f"{skill.content}"
+        "</consumption-skill>\n\n"
+        "<task>\n"
+        f"{task}\n"
+        "</task>"
+    )
+
+
+_INSERT_CANARY = text(
+    """
+    INSERT INTO canary_cases (
+        canary_id, deployment_id, suite, description, input, expected
+    ) VALUES (
+        :canary_id, :deployment_id, 'retrieval', :description, :input, :expected
+    )
+    ON CONFLICT (canary_id) DO UPDATE
+        SET description = EXCLUDED.description,
+            input = EXCLUDED.input,
+            expected = EXCLUDED.expected
+    """
+).bindparams(bindparam("input", type_=JSON), bindparam("expected", type_=JSON))

rememberstack/eval/contradiction.py

@@ -0,0 +1,184 @@
+"""The D43 contradiction eval gate (WP-2.5): the SHIPPING criterion.
+
+"Never silently resolve" is policy enforced in E3 + eval, not a schema
+invariant — so the adjudicator ships only behind this gate: contradiction
+precision/recall over golden statement pairs, recorded in `eval_runs`
+(suite `contradiction`) and blocking below the floors.
+"""
+
+from typing import Final
+from uuid import UUID
+from uuid import uuid4
+from uuid import uuid5
+
+from sqlalchemy import bindparam
+from sqlalchemy import JSON
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
+
+from rememberstack.model import ObservationOutcome
+from rememberstack.spine.observation_adjudication import ObservationAdjudicator
+
+CONTRADICTION_PRECISION_FLOOR: Final = 0.90
+"""Gate floor: flagged contradictions must be real (starting point, D22)."""
+
+CONTRADICTION_RECALL_FLOOR: Final = 0.80
+"""Gate floor: real contradictions must be flagged (starting point, D22)."""
+
+_CASE_NAMESPACE: Final = UUID("c0217ad1-0000-4000-8000-000000000000")
+
+SYNTHETIC_CONTRADICTION_CASES: Final[tuple[dict[str, object], ...]] = (
+    {
+        "description": "same period, incompatible revenue figures",
+        "existing": "Acme's FY2023 revenue was $5M.",
+        "new": "Acme's FY2023 revenue was $7M.",
+        "expected_contradiction": True,
+    },
+    {
+        "description": "same period, incompatible headcount figures",
+        "existing": "Acme's headcount at year-end 2023 was 500.",
+        "new": "Acme's headcount at year-end 2023 was 800.",
+        "expected_contradiction": True,
+    },
+    {
+        "description": "different property is never a contradiction",
+        "existing": "Acme's FY2023 revenue was $5M.",
+        "new": "Acme's FY2023 profit was $1M.",
+        "expected_contradiction": False,
+    },
+    {
+        "description": "different period is never a contradiction",
+        "existing": "Acme's FY2023 revenue was $5M.",
+        "new": "Acme's Q1-2023 revenue was $2M.",
+        "expected_contradiction": False,
+    },
+    {
+        "description": "a changing state moving on is supersession, not conflict",
+        "existing": "Acme's headcount is 500.",
+        "new": "Acme's headcount is 600 as of 2025.",
+        "expected_contradiction": False,
+    },
+)
+
+
+def seed_contradiction_cases(*, engine: Engine, deployment_id: UUID) -> None:
+    """Insert or refresh the golden pairs (stable per-deployment ids)."""
+    with engine.begin() as connection:
+        for case in SYNTHETIC_CONTRADICTION_CASES:
+            connection.execute(
+                _UPSERT_CASE,
+                {
+                    "canary_id": uuid5(
+                        _CASE_NAMESPACE, f"{deployment_id}:{case['description']}"
+                    ),
+                    "deployment_id": deployment_id,
+                    "description": case["description"],
+                    "input": {"existing": case["existing"], "new": case["new"]},
+                    "expected": {"contradiction": case["expected_contradiction"]},
+                },
+            )
+
+
+def run_contradiction_suite(
+    *,
+    engine: Engine,
+    adjudicator: ObservationAdjudicator,
+    deployment_id: UUID,
+    component_version: str,
+) -> dict[str, object]:
+    """Judge every golden pair; record P/R; block below the floors.
+
+    Precision: of the pairs the adjudicator flags contradict, how many are
+    real. Recall: of the real contradictions, how many are flagged. An empty
+    or one-sided golden set never passes (0/0 blocks, D22).
+    """
+    with engine.connect() as connection:
+        cases = (
+            connection.execute(_SELECT_CASES, {"deployment_id": deployment_id})
+            .mappings()
+            .all()
+        )
+    tp = fp = fn = tn = 0
+    for case in cases:
+        outcome, _confidence = adjudicator.judge_statements(
+            existing=str(case["input"]["existing"]), new=str(case["input"]["new"])
+        )
+        flagged = outcome is ObservationOutcome.CONTRADICT
+        actual = bool(case["expected"]["contradiction"])
+        if flagged and actual:
+            tp += 1
+        elif flagged and not actual:
+            fp += 1
+        elif not flagged and actual:
+            fn += 1
+        else:
+            tn += 1
+    precision = tp / (tp + fp) if (tp + fp) else None
+    recall = tp / (tp + fn) if (tp + fn) else None
+    # a one-sided golden set never passes (Codex review): both real
+    # contradictions AND real non-contradictions must be measured, or the
+    # gate cannot see false positives / false negatives at all.
+    passed = (
+        precision is not None
+        and recall is not None
+        and (tp + fn) > 0
+        and (fp + tn) > 0
+        and precision >= CONTRADICTION_PRECISION_FLOOR
+        and recall >= CONTRADICTION_RECALL_FLOOR
+    )
+    metrics = {
+        "precision": precision,
+        "recall": recall,
+        "cases": tp + fp + fn + tn,
+        "floors": {
+            "precision": CONTRADICTION_PRECISION_FLOOR,
+            "recall": CONTRADICTION_RECALL_FLOOR,
+        },
+    }
+    with engine.begin() as connection:
+        connection.execute(
+            _RECORD_RUN,
+            {
+                "eval_run_id": uuid4(),
+                "deployment_id": deployment_id,
+                "component_version": component_version,
+                "metrics": metrics,
+                "passed": passed,
+            },
+        )
+    return {**metrics, "passed": passed}
+
+
+_UPSERT_CASE = text(
+    """
+    INSERT INTO canary_cases (
+        canary_id, deployment_id, suite, description, input, expected
+    ) VALUES (
+        :canary_id, :deployment_id, 'contradiction', :description,
+        :input, :expected
+    )
+    ON CONFLICT (canary_id) DO UPDATE
+        SET description = EXCLUDED.description,
+            input = EXCLUDED.input,
+            expected = EXCLUDED.expected
+    """
+).bindparams(bindparam("input", type_=JSON), bindparam("expected", type_=JSON))
+
+_SELECT_CASES = text(
+    """
+    SELECT description, input, expected FROM canary_cases
+    WHERE deployment_id = :deployment_id AND suite = 'contradiction'
+    ORDER BY canary_id
+    """
+)
+
+_RECORD_RUN = text(
+    """
+    INSERT INTO eval_runs (
+        eval_run_id, deployment_id, suite, component_version, metrics, passed
+    ) VALUES (
+        :eval_run_id, :deployment_id, 'contradiction', :component_version,
+        :metrics, :passed
+    )
+    """
+).bindparams(bindparam("metrics", type_=JSON))

rememberstack/eval/harness.py

@@ -0,0 +1,136 @@
+"""The evaluation-harness skeleton (WP-0.5, D22): suites over golden canaries.
+
+The harness loads a suite's canary cases from the spine, evaluates each with
+the suite's registered evaluator, records the run in `eval_runs`, and returns
+a report CI gates on. A suite with cases but no registered evaluator fails
+those cases — absence of measurement is never compliance. Real evaluators
+arrive with their phases; this skeleton owns loading, reporting, and the
+CI-blocking contract.
+"""
+
+from collections.abc import Callable
+from uuid import UUID
+from uuid import uuid4
+
+from sqlalchemy import bindparam
+from sqlalchemy import JSON
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
+
+from rememberstack.model import CanaryCase
+from rememberstack.model import CaseFailure
+from rememberstack.model import EvalSuite
+from rememberstack.model import SuiteReport
+
+CaseEvaluator = Callable[[CanaryCase], bool]
+"""Evaluates one canary: True = the guarded behavior holds."""
+
+
+class EvalHarness:
+    """Run evaluation suites over the golden canaries and record the history."""
+
+    def __init__(self, *, engine: Engine) -> None:
+        """Bind the harness to the spine; evaluators register per suite."""
+        self._engine = engine
+        self._evaluators: dict[EvalSuite, CaseEvaluator] = {}
+
+    def register_evaluator(self, *, suite: EvalSuite, evaluator: CaseEvaluator) -> None:
+        """Bind a suite's evaluator (phases plug their real logic in here)."""
+        self._evaluators[suite] = evaluator
+
+    def run_suite(
+        self, *, deployment_id: UUID, suite: EvalSuite, component_version: str
+    ) -> SuiteReport:
+        """Evaluate every canary in the suite and persist the run's verdict."""
+        cases = self._load_cases(deployment_id=deployment_id, suite=suite)
+        failures = tuple(
+            failure
+            for case in cases
+            if (failure := self._evaluate(case=case)) is not None
+        )
+        report = SuiteReport(suite=suite, total_cases=len(cases), failures=failures)
+        self._record_run(
+            deployment_id=deployment_id,
+            report=report,
+            component_version=component_version,
+        )
+        return report
+
+    def _evaluate(self, *, case: CanaryCase) -> CaseFailure | None:
+        """Run one canary; no registered evaluator is itself a failure."""
+        evaluator = self._evaluators.get(case.suite)
+        if evaluator is None:
+            return CaseFailure(
+                canary_id=case.canary_id,
+                description=case.description,
+                reason=f"no evaluator registered for suite {case.suite}",
+            )
+        if evaluator(case):
+            return None
+        return CaseFailure(
+            canary_id=case.canary_id,
+            description=case.description,
+            reason="guarded behavior does not hold",
+        )
+
+    def _load_cases(
+        self, *, deployment_id: UUID, suite: EvalSuite
+    ) -> tuple[CanaryCase, ...]:
+        """Load the suite's canaries from the spine."""
+        with self._engine.connect() as connection:
+            rows = connection.execute(
+                _SELECT_CANARIES, {"deployment_id": deployment_id, "suite": suite}
+            ).mappings()
+            return tuple(
+                CanaryCase(
+                    canary_id=row["canary_id"],
+                    suite=EvalSuite(row["suite"]),
+                    description=row["description"],
+                    input=row["input"],
+                    expected=row["expected"],
+                )
+                for row in rows
+            )
+
+    def _record_run(
+        self, *, deployment_id: UUID, report: SuiteReport, component_version: str
+    ) -> None:
+        """Append the run to eval_runs — the D22 measurement history."""
+        with self._engine.begin() as connection:
+            connection.execute(
+                _INSERT_RUN,
+                {
+                    "eval_run_id": uuid4(),
+                    "deployment_id": deployment_id,
+                    "suite": report.suite,
+                    "component_version": component_version,
+                    "metrics": {
+                        "total_cases": report.total_cases,
+                        "failures": [
+                            failure.model_dump(mode="json")
+                            for failure in report.failures
+                        ],
+                    },
+                    "passed": report.passed,
+                },
+            )
+
+
+_SELECT_CANARIES = text(
+    """
+    SELECT canary_id, suite, description, input, expected
+    FROM canary_cases
+    WHERE deployment_id = :deployment_id AND suite = :suite
+    ORDER BY created_at, canary_id
+    """
+)
+
+_INSERT_RUN = text(
+    """
+    INSERT INTO eval_runs (
+        eval_run_id, deployment_id, suite, component_version, metrics, passed
+    ) VALUES (
+        :eval_run_id, :deployment_id, :suite, :component_version, :metrics, :passed
+    )
+    """
+).bindparams(bindparam("metrics", type_=JSON))