context-engineering-core 0.12.0__py3-none-any.whl

context_engine/errors.py

@@ -0,0 +1,91 @@
+"""Structured, content-free errors safe for default diagnostic output."""
+
+from types import MappingProxyType
+from typing import ClassVar
+
+
+class ContextEngineError(Exception):
+    code: ClassVar[str] = "context_engine_error"
+
+    def __init__(self, message: str, **details: int | str) -> None:
+        super().__init__(message)
+        self.details = MappingProxyType(dict(details))
+
+    def to_dict(self) -> dict:
+        return {"code": self.code, "message": str(self), "details": dict(self.details)}
+
+
+class ConfigurationError(ContextEngineError):
+    code = "invalid_configuration"
+
+
+class ContractError(ContextEngineError):
+    code = "invalid_contract"
+
+
+class TokenizerUnavailable(ContextEngineError):
+    code = "tokenizer_unavailable"
+
+
+class RequiredContextTooLarge(ContextEngineError):
+    code = "required_context_too_large"
+
+
+class LayerAllocationError(ContextEngineError):
+    code = "layer_allocation_exceeds_budget"
+
+
+class PromptTooLarge(ContextEngineError):
+    code = "prompt_too_large"
+
+
+class LayerStateError(ContextEngineError):
+    code = "invalid_layer_state"
+
+
+class CapBudgetError(ContextEngineError):
+    code = "cap_budget_too_small"
+
+
+class SummaryLeakageError(ContextEngineError):
+    code = "summary_leakage"
+
+
+class InspectionError(ContextEngineError):
+    code = "invalid_inspection_input"
+
+
+class SummaryPolicyError(ContextEngineError):
+    code = "summary_policy_error"
+
+
+class BenchmarkError(ContextEngineError):
+    code = "invalid_benchmark"
+
+
+class ProviderError(ContextEngineError):
+    code = "provider_error"
+
+
+class StorageError(ContextEngineError):
+    code = "runtime_storage_error"
+
+
+class QuotaError(ContextEngineError):
+    code = "quota_exhausted"
+
+
+class CacheError(ContextEngineError):
+    code = "invalid_replay_cache"
+
+
+class MemoryConflict(ContextEngineError):
+    code = "memory_revision_conflict"
+
+
+class MemoryIntegrityError(ContextEngineError):
+    code = "memory_integrity_error"
+
+
+class WorkCancelled(ContextEngineError):
+    code = "work_cancelled"

context_engine/evaluation/__init__.py

@@ -0,0 +1 @@
+"""Fixture validation only; the complete benchmark harness arrives at C04."""

context_engine/evaluation/commands.py

@@ -0,0 +1,90 @@
+"""Bounded offline owner demonstrations, not a model benchmark runner."""
+
+from copy import deepcopy
+from dataclasses import asdict
+
+from ..models import canonical_json
+from .corpus import Bundle, load_bundle
+from .harness import prepare_probe
+from .protocol import load_freeze, make_manifest, preflight
+from .results import assess_targets, record_response, statistics, validate_run
+
+
+def altered_bundle(bundle: Bundle, name: str, value: dict) -> Bundle:
+    return Bundle(
+        tuple((key, canonical_json(value) if key == name else raw) for key, raw in bundle.parts)
+    )
+
+
+def corruption_demo(bundle, frozen, manifest, records) -> dict:
+    """Synthetic fault injection on detached copies; never rewrite frozen resources."""
+    result = {}
+    for label, name, mutate in (
+        ("malformed_fixture", "scenario", lambda data: data["history"].pop()),
+        ("summary_answer_leak", "summary_fixture", lambda data: data.update(content="shard-19")),
+        ("configuration_drift", "protocol", lambda data: data["cap"].update(max_tokens=36)),
+    ):
+        value = bundle.get(name)
+        mutate(value)
+        gates = preflight(altered_bundle(bundle, name, value), frozen)
+        result[label] = {
+            "status": "INVALID" if any(g.status == "FAIL" for g in gates) else "UNEXPECTED_PASS",
+            "failed_gates": [g.gate for g in gates if g.status == "FAIL"],
+        }
+    bad = deepcopy(records)
+    bad[-1]["request"]["messages"][-1]["content"] = "Tampered question"
+    result["changed_request"] = validate_run(bundle, frozen, manifest, bad)
+    bad = deepcopy(records)
+    bad[-1] = record_response(
+        bundle, bad[-1], answer="synthetic partial response", finish_reason="length", origin="test"
+    )
+    result["truncated_test_response"] = validate_run(bundle, frozen, manifest, bad)
+    return result
+
+
+def benchmark_check(
+    *, fact="shard", budget=900, model=None, show_content=False, data_dir=None
+) -> dict:
+    bundle, frozen = load_bundle(data_dir), load_freeze()
+    gates = preflight(bundle, frozen)
+    if any(g.status == "FAIL" for g in gates):
+        return {"status": "INVALID", "gates": [asdict(g) for g in gates], "inference_calls": 0}
+    model = model or bundle.get("protocol")["models"][0]
+    manifest = make_manifest(
+        bundle, frozen, facts=[fact], budgets=[budget], variants=["A0", "A5"], models=[model]
+    )
+    records = [prepare_probe(bundle, frozen, manifest, slot["probe_id"]) for slot in manifest.slots]
+    validation = validate_run(bundle, frozen, manifest, records)
+    exported = (
+        records
+        if show_content
+        else [{k: v for k, v in row.items() if k != "request"} for row in records]
+    )
+    return {
+        **validation,
+        "run_id": manifest.run_id,
+        "fixture_hashes": bundle.hashes,
+        "protocol_approval": bundle.get("protocol")["approval"],
+        "inference_calls": 0,
+        "content_included": show_content,
+        "probes": exported,
+        "statistics": statistics(manifest, records),
+        "quality_acceptance": assess_targets(manifest, records, validation),
+        "deliberately_invalid": corruption_demo(bundle, frozen, manifest, records),
+    }
+
+
+def benchmark_plan(*, data_dir=None) -> dict:
+    bundle, frozen = load_bundle(data_dir), load_freeze()
+    manifest = make_manifest(bundle, frozen)
+    completion = bundle.get("protocol")["generation"]["max_completion_tokens"]
+    return {
+        "status": "PLANNED",
+        "run_id": manifest.run_id,
+        "manifest": manifest.data,
+        "slots": list(manifest.slots),
+        "planned_slots": len(manifest.slots),
+        "conservative_tokens_before_retries": sum(s["budget"] + completion for s in manifest.slots),
+        "inference_calls": 0,
+        "spending_authorized": False,
+    }

context_engine/evaluation/corpus.py

@@ -0,0 +1,247 @@
+"""Frozen, packaged synthetic resources; ground truth never enters an inspection job."""
+
+import hashlib
+import json
+from dataclasses import dataclass
+from datetime import datetime
+from importlib import resources
+from pathlib import Path
+
+from ..config import Settings
+from ..errors import BenchmarkError, ContextEngineError
+from ..inspection import parse_job
+from ..models import canonical_json
+from .grading import normalize, present
+
+PARTS = ("scenario", "ground_truth", "summary_fixture", "protocol", "pin_retention")
+MAX_BYTES = 2_000_000
+
+
+def digest(value) -> str:
+    return hashlib.sha256(canonical_json(value).encode()).hexdigest()
+
+
+def strict_json(raw: str, *, maximum: int = MAX_BYTES) -> dict:
+    def unique(pairs):
+        result = {}
+        for key, value in pairs:
+            if key in result:
+                raise BenchmarkError("Duplicate benchmark JSON key")
+            result[key] = value
+        return result
+
+    def nonfinite(_):
+        raise BenchmarkError("Nonfinite benchmark JSON number")
+
+    try:
+        if len(raw.encode("utf-8")) > maximum:
+            raise BenchmarkError("Benchmark document exceeds byte limit")
+        parsed = json.loads(raw, object_pairs_hook=unique, parse_constant=nonfinite)
+        if not isinstance(parsed, dict):
+            raise BenchmarkError("Benchmark document must be an object")
+        canonical_json(parsed)
+        return parsed
+    except (UnicodeError, ValueError, TypeError, RecursionError):
+        raise BenchmarkError("Malformed benchmark JSON") from None
+
+
+@dataclass(frozen=True)
+class Bundle:
+    parts: tuple[tuple[str, str], ...]
+
+    def __post_init__(self):
+        if not isinstance(self.parts, tuple) or tuple(name for name, _ in self.parts) != PARTS:
+            raise BenchmarkError("Incomplete benchmark bundle")
+        for _, raw in self.parts:
+            strict_json(raw)
+
+    def get(self, name: str) -> dict:
+        return strict_json(dict(self.parts)[name])  # Fresh objects, no mutable shared fixtures.
+
+    @property
+    def hashes(self) -> dict:
+        return {name: hashlib.sha256(raw.encode("utf-8")).hexdigest() for name, raw in self.parts}
+
+    def job(self, question: str):
+        scenario = self.get("scenario")
+        scenario.pop("scenario_id")
+        scenario["question"] = question
+        return parse_job(scenario, Settings.from_env({}))
+
+    @property
+    def facts(self) -> tuple[dict, ...]:
+        return tuple(self.get("ground_truth")["facts"])
+
+    def fact(self, identifier: str) -> dict:
+        for fact in self.facts:
+            if fact["fact_id"] == identifier:
+                return fact
+        raise BenchmarkError("Unknown benchmark fact")
+
+
+def read_resource(name: str, directory: Path | None = None) -> str:
+    path = directory / name if directory else resources.files(__package__).joinpath("data", name)
+    try:
+        with path.open("rb") as handle:
+            raw = handle.read(MAX_BYTES + 1)
+        if len(raw) > MAX_BYTES:
+            raise BenchmarkError("Benchmark document exceeds byte limit")
+        return raw.decode("utf-8")
+    except (OSError, UnicodeError, ValueError):
+        raise BenchmarkError("Cannot read benchmark resource") from None
+
+
+def load_bundle(directory: Path | None = None) -> Bundle:
+    return Bundle(tuple((name, read_resource(name + ".json", directory)) for name in PARTS))
+
+
+def aliases(fact: dict) -> tuple[str, ...]:
+    return (fact["expected"], *fact["aliases"])
+
+
+def check_fixture(bundle: Bundle) -> None:
+    """V1 schema, cardinality, relationships, plant locations and unique metadata."""
+    try:
+        scenario, truth = bundle.get("scenario"), bundle.get("ground_truth")
+        summary, protocol = bundle.get("summary_fixture"), bundle.get("protocol")
+        if any(
+            type(part["schema_version"]) is not int or part["schema_version"] != 1
+            for part in (scenario, truth, summary, protocol, bundle.get("pin_retention"))
+        ):
+            raise BenchmarkError("Unsupported benchmark schema")
+        if set(truth) != {"schema_version", "scenario_id", "facts"}:
+            raise BenchmarkError("Unexpected ground-truth fields")
+        if any(
+            part["scenario_id"] != scenario["scenario_id"] for part in (truth, summary, protocol)
+        ):
+            raise BenchmarkError("Scenario identity mismatch")
+        job = bundle.job("Fixture validation")
+        from ..layers.summarize import SummarySnapshot
+        from ..models import Pin
+
+        if set(summary) != {
+            "schema_version",
+            "scenario_id",
+            "policy_version",
+            "created_at",
+            "content",
+        }:
+            raise BenchmarkError("Unexpected summary fields")
+        snapshot = SummarySnapshot.from_history(
+            scope=job.arguments["pinned_facts"].scope,
+            content=summary["content"],
+            history=job.arguments["history"],
+            policy_version=summary["policy_version"],
+            created_at=datetime.fromisoformat(summary["created_at"]),
+        )
+        if snapshot.created_at > job.arguments["at"]:
+            raise BenchmarkError("Frozen summary is not available at evaluation time")
+        fixture = bundle.get("pin_retention")
+        if set(fixture) != {"schema_version", "fixture_id", "question", "key", "value", "origin"}:
+            raise BenchmarkError("Unexpected separate pin fixture fields")
+        Pin(
+            scope=job.arguments["pinned_facts"].scope,
+            key=fixture["key"],
+            value=fixture["value"],
+            origin=fixture["origin"],
+            effective_at=job.arguments["at"],
+        )
+        if (
+            not isinstance(fixture["question"], str)
+            or not fixture["question"].strip()
+            or present(fixture["question"], (fixture["value"],))
+        ):
+            raise BenchmarkError("Invalid separate pin question")
+        # Constructed carrier validation catches duplicate/mixed source identity without inference.
+        from ..context import ContextCarrier
+        from ..models import Message, Role
+
+        history = job.arguments["history"]
+        ContextCarrier(
+            job.arguments["pinned_facts"].scope,
+            history,
+            history,
+            job.arguments["pinned_facts"],
+            Message("fixture-question", Role.USER, "Q"),
+        )
+        if len(history) != 100 or [t.turn_id for t in history] != [f"t{i}" for i in range(1, 101)]:
+            raise BenchmarkError("Expected ordered 100-turn fixture")
+        if len(bundle.facts) != 31 or len({f["fact_id"] for f in bundle.facts}) != 31:
+            raise BenchmarkError("Expected exactly 31 unique facts")
+        turns = {t.turn_id: t for t in history}
+        seen_answers, seen_questions = set(), set()
+        fields = {
+            "fact_id",
+            "expected",
+            "aliases",
+            "source_turn_id",
+            "source_message_id",
+            "question",
+            "zone",
+        }
+        for fact in bundle.facts:
+            if set(fact) != fields or not isinstance(fact["aliases"], list):
+                raise BenchmarkError("Unexpected fact fields")
+            if any(
+                not isinstance(fact[k], str) or not fact[k].strip() for k in fields - {"aliases"}
+            ):
+                raise BenchmarkError("Invalid fact text")
+            values = aliases(fact)
+            if any(not isinstance(value, str) or not normalize(value) for value in values):
+                raise BenchmarkError("Invalid answer alias")
+            normalized = {normalize(value) for value in values}
+            if len(normalized) != len(values) or seen_answers & normalized:
+                raise BenchmarkError("Ambiguous answer aliases")
+            seen_answers |= normalized
+            if normalize(fact["question"]) in seen_questions:
+                raise BenchmarkError("Duplicate probe question")
+            seen_questions.add(normalize(fact["question"]))
+            turn = turns[fact["source_turn_id"]]
+            message = next(m for m in turn.messages if m.message_id == fact["source_message_id"])
+            if not present(message.content, (fact["expected"],)):
+                raise BenchmarkError("Planted value missing from declared source")
+            locations = [
+                (t.turn_id, m.message_id)
+                for t in history
+                for m in t.messages
+                if present(m.content, values)
+            ]
+            if locations != [(turn.turn_id, message.message_id)]:
+                raise BenchmarkError("Fact appears outside its declared source")
+            number = int(turn.turn_id[1:])
+            if fact["zone"] != ("early" if number <= 33 else "middle" if number <= 66 else "late"):
+                raise BenchmarkError("Incorrect source age zone")
+        flagship = bundle.fact("shard")
+        if (flagship["source_turn_id"], flagship["expected"], flagship["question"]) != (
+            "t82",
+            "shard-19",
+            "Which partition ended up carrying the blame?",
+        ):
+            raise BenchmarkError("Flagship source contract changed")
+    except BenchmarkError:
+        raise
+    except (ContextEngineError, KeyError, TypeError, ValueError, StopIteration, OverflowError):
+        raise BenchmarkError("Malformed benchmark fixture") from None
+
+
+def check_leakage(bundle: Bundle) -> None:
+    """V2 primary input has no evaluator fields, answer pins, question/summary leakage."""
+    from ..layers.summarize import SummarySnapshot
+    from .fixtures import validate_frozen_summary
+
+    job = bundle.job("Leakage validation")
+    scenario, summary = bundle.get("scenario"), bundle.get("summary_fixture")
+    values = tuple(value for fact in bundle.facts for value in aliases(fact))
+    if scenario["pins"] or bundle.get("protocol")["primary_pins"] != "empty":
+        raise BenchmarkError("Primary corpus must have no answer pins")
+    for content in [scenario["system"], *(fact["question"] for fact in bundle.facts)]:
+        if present(content, values):
+            raise BenchmarkError("Answer leakage in system or probe question")
+    snapshot = SummarySnapshot.from_history(
+        scope=job.arguments["pinned_facts"].scope,
+        content=summary["content"],
+        history=job.arguments["history"],
+        policy_version=summary["policy_version"],
+        created_at=job.arguments["at"],
+    )
+    validate_frozen_summary(snapshot, values)

context_engine/evaluation/data/freeze.json

@@ -0,0 +1,26 @@
+{
+  "fixture_hashes": {
+    "ground_truth": "c319c47c0548ced89dc68deba183c3028c35aad2aabd756db8b3bbdfe4c064e6",
+    "pin_retention": "febe00b1b71016fd679a3d99a2bccdb901ff9b956f0933d1003578c06faf5f3b",
+    "protocol": "789585023df970bcfea2723b5b7994a1251f0aa512f6ee5820b8ad7353eb571c",
+    "scenario": "19f3bc62987b9b2253eed316e903c4f860b445b441d9151573588bce283bae54",
+    "summary_fixture": "b3c1b76bb330b52493e5150647747bad77d5de3a55d05d032bb5703be1c70196"
+  },
+  "runtime": {
+    "code_hash": "59eeb4d28e6b3106a18317d5b6acee255f0b22d5e5143e3e7bb74b1146a39ff4",
+    "dependencies": {
+      "certifi": "2026.10.23",
+      "charset-normalizer": "3.5.1",
+      "idna": "3.19",
+      "numpy": "2.5.3",
+      "rank-bm25": "0.2.2",
+      "regex": "2026.9.3",
+      "requests": "2.34.2",
+      "tiktoken": "0.14.0",
+      "urllib3": "2.8.0"
+    },
+    "package_version": "0.12.0",
+    "python": "3.12.13"
+  },
+  "schema_version": 1
+}