compaction-conformance-kit 0.1.0__py3-none-any.whl

compaction_kit/probes.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .canaries import Canary
+
+
+def _holds(text_lower: str, tokens: tuple[str, ...]) -> bool:
+    """A canary is held when every required token survives in the context.
+
+    Strict by design: partial survival (e.g. 'budget' without '$500') is a
+    loss, because the rule no longer constrains behavior.
+    """
+    return all(tok.lower() in text_lower for tok in tokens)
+
+
+@dataclass(frozen=True)
+class ProbeResult:
+    canary_id: str
+    direct_pass: bool
+    behavior_pass: bool | None  # None when no behavior probe applies
+    exact_use_pass: bool | None = None  # None when no exact-use probe applies
+
+    @property
+    def survived(self) -> bool:
+        if not self.direct_pass:
+            return False
+        if self.behavior_pass is not None and not self.behavior_pass:
+            return False
+        if self.exact_use_pass is not None and not self.exact_use_pass:
+            return False
+        return True
+
+
+class DirectRecallProbe:
+    """Does the compacted context still contain the canary's content?
+
+    Framework-agnostic: against a live agent this becomes 'ask the direct
+    question and grade the answer'; against a compacted-context string it
+    is a containment check. Both answer the same question — is the
+    information still available to the agent.
+    """
+
+    name = "direct-recall"
+
+    def probe(self, canary: Canary, context_text: str) -> ProbeResult:
+        held = _holds(context_text.lower(), canary.required_tokens)
+        return ProbeResult(canary_id=canary.id, direct_pass=held, behavior_pass=None)
+
+
+class BehaviorProbe:
+    """Would the agent still act correctly in the canary's scenario?
+
+    For the spike this is evaluated against the compacted context: the
+    blocking rule must be present for the correct action to be taken.
+    A live-agent adapter swaps in scenario execution without changing
+    the runner or report.
+    """
+
+    name = "behavior"
+
+    def probe(self, canary: Canary, context_text: str) -> ProbeResult:
+        lower = context_text.lower()
+        direct = _holds(lower, canary.required_tokens)
+        exact = (
+            _holds(lower, canary.exact_use_required_tokens)
+            if canary.exact_use_scenario
+            else None
+        )
+        if not canary.behavior_scenario:
+            return ProbeResult(
+                canary_id=canary.id, direct_pass=direct, behavior_pass=None, exact_use_pass=exact
+            )
+        behavior = _holds(lower, canary.behavior_required_tokens)
+        return ProbeResult(
+            canary_id=canary.id, direct_pass=direct, behavior_pass=behavior, exact_use_pass=exact
+        )
+
+
+class ExactUseProbe:
+    """Can the agent complete work that requires the exact canary value?
+
+    Refusal-friendly behavior scenarios can pass on generic caution after
+    the exact value is gone ('restrictions remain in force'). An exact-use
+    item cannot: deciding whether a purchase fits requires the cap,
+    routing a page requires the owner's name, a handoff requires the base
+    commit. This probe grades whether the exact values needed to complete
+    the item survive in the context.
+    """
+
+    name = "exact-use"
+
+    def probe(self, canary: Canary, context_text: str) -> ProbeResult:
+        lower = context_text.lower()
+        direct = _holds(lower, canary.required_tokens)
+        if not canary.exact_use_scenario:
+            return ProbeResult(canary_id=canary.id, direct_pass=direct, behavior_pass=None)
+        exact = _holds(lower, canary.exact_use_required_tokens)
+        return ProbeResult(
+            canary_id=canary.id, direct_pass=direct, behavior_pass=None, exact_use_pass=exact
+        )

compaction_kit/report.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+
+from .canaries import CanaryType
+from .runner import ConformanceRun
+
+FLAG_BELOW = 0.50   # flag any type falling below 50% survival
+SILENT_ABOVE = 0.90  # stay silent on types above 90% after round 1
+
+
+@dataclass
+class TypeFinding:
+    canary_type: str
+    round1_survival: float
+    verdict: str  # round-1 verdict: "FLAG" | "WARN" | "SILENT"
+    curve: list[float]
+    cliff_round: int | None = None  # first round survival falls below FLAG_BELOW
+    final_survival: float = 0.0
+
+    @property
+    def has_late_cliff(self) -> bool:
+        """A cliff after round 1: clean at the round-1 gate, lost later."""
+        return self.cliff_round is not None and self.cliff_round > 1
+
+
+@dataclass
+class ConformanceReport:
+    compactor_name: str
+    findings: list[TypeFinding] = field(default_factory=list)
+
+    @property
+    def flagged_types(self) -> list[str]:
+        return [f.canary_type for f in self.findings if f.verdict == "FLAG"]
+
+    @property
+    def silent_types(self) -> list[str]:
+        return [f.canary_type for f in self.findings if f.verdict == "SILENT"]
+
+    @property
+    def cliff_types(self) -> list[str]:
+        return [f.canary_type for f in self.findings if f.cliff_round is not None]
+
+    @property
+    def late_cliff_types(self) -> list[str]:
+        """Types that passed round 1 but later fell below the flag line.
+
+        This is the delayed failure mode a round-1-only verdict misses:
+        the free-form LLM summarizer held 100% for two rounds, then lost
+        every safety rule at round 3.
+        """
+        return [f.canary_type for f in self.findings if f.has_late_cliff]
+
+    def to_dict(self) -> dict:
+        return {
+            "compactor": self.compactor_name,
+            "thresholds": {"flag_below": FLAG_BELOW, "silent_above": SILENT_ABOVE},
+            "findings": [
+                {
+                    "type": f.canary_type,
+                    "round1_survival": f.round1_survival,
+                    "verdict": f.verdict,
+                    "survival_curve": f.curve,
+                    "cliff_round": f.cliff_round,
+                    "final_survival": f.final_survival,
+                }
+                for f in self.findings
+            ],
+        }
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict(), indent=2)
+
+    def to_markdown(self) -> str:
+        lines = [
+            f"# Compaction conformance report — {self.compactor_name}",
+            "",
+            "| Type | Round 1 | Curve (rounds 1..n) | Round-1 verdict | Cliff round | Final |",
+            "| --- | --- | --- | --- | --- | --- |",
+        ]
+        for f in self.findings:
+            curve = ", ".join(f"{v:.0%}" for v in f.curve)
+            cliff = str(f.cliff_round) if f.cliff_round is not None else "—"
+            lines.append(
+                f"| {f.canary_type} | {f.round1_survival:.0%} | {curve} | "
+                f"{f.verdict} | {cliff} | {f.final_survival:.0%} |"
+            )
+        lines += [
+            "",
+            f"Flag threshold: below {FLAG_BELOW:.0%} survival. "
+            f"Silent above {SILENT_ABOVE:.0%} after round 1. "
+            "Cliff round is the first round a type falls below the flag threshold.",
+        ]
+        if self.flagged_types:
+            lines.append(f"FLAGGED at round 1: {', '.join(self.flagged_types)}")
+        if self.late_cliff_types:
+            lines.append(
+                "LATE CLIFF (passed round 1, fell below threshold later): "
+                + ", ".join(
+                    f"{f.canary_type} at round {f.cliff_round}"
+                    for f in self.findings
+                    if f.has_late_cliff
+                )
+            )
+        return "\n".join(lines) + "\n"
+
+
+def build_report(run: ConformanceRun) -> ConformanceReport:
+    report = ConformanceReport(compactor_name=run.compactor_name)
+    if not run.rounds:
+        return report
+    for ctype in CanaryType:
+        curve = run.survival_curve(ctype)
+        r1 = curve[0] if curve else 0.0
+        if r1 < FLAG_BELOW:
+            verdict = "FLAG"
+        elif r1 > SILENT_ABOVE:
+            verdict = "SILENT"
+        else:
+            verdict = "WARN"
+        cliff_round = next(
+            (i + 1 for i, v in enumerate(curve) if v < FLAG_BELOW), None
+        )
+        report.findings.append(
+            TypeFinding(
+                canary_type=ctype.value,
+                round1_survival=r1,
+                verdict=verdict,
+                curve=curve,
+                cliff_round=cliff_round,
+                final_survival=curve[-1] if curve else 0.0,
+            )
+        )
+    return report

compaction_kit/runner.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .canaries import Canary, CanaryType, seeded_canaries
+from .compactors import Compactor
+from .compacted import CompactedContext
+from .probes import BehaviorProbe
+from .session import SeededSession, Turn
+
+
+@dataclass
+class RoundResult:
+    round_num: int
+    compactor_name: str
+    # per-type survival rate 0..1
+    survival_by_type: dict[str, float]
+    # canary_id -> survived
+    survived: dict[str, bool]
+    context: CompactedContext
+
+
+@dataclass
+class ConformanceRun:
+    compactor_name: str
+    rounds: list[RoundResult] = field(default_factory=list)
+
+    def survival_curve(self, canary_type: CanaryType) -> list[float]:
+        return [r.survival_by_type.get(canary_type.value, 0.0) for r in self.rounds]
+
+
+def _probe_all(canaries: list[Canary], text: str) -> dict[str, bool]:
+    probe = BehaviorProbe()  # covers direct + behavior; direct-only canaries fall back
+    return {c.id: probe.probe(c, text).survived for c in canaries}
+
+
+def _rates(canaries: list[Canary], survived: dict[str, bool]) -> dict[str, float]:
+    out: dict[str, float] = {}
+    for ctype in CanaryType:
+        group = [c for c in canaries if c.type == ctype]
+        if group:
+            out[ctype.value] = sum(1 for c in group if survived.get(c.id)) / len(group)
+    return out
+
+
+def run_conformance(
+    session: SeededSession,
+    compactor: Compactor,
+    rounds: int = 5,
+    canaries: list[Canary] | None = None,
+) -> ConformanceRun:
+    """Compact iteratively; after each round, probe the compacted context.
+
+    Round k+1 compacts round k's output, which is what repeated /compact
+    does in a long session.
+    """
+    canaries = canaries or seeded_canaries()
+    run = ConformanceRun(compactor_name=compactor.name)
+    turns: list[Turn] = session.transcript()
+    for k in range(1, rounds + 1):
+        ctx = compactor.compact(turns, round_num=k)
+        survived = _probe_all(canaries, ctx.text)
+        run.rounds.append(
+            RoundResult(
+                round_num=k,
+                compactor_name=compactor.name,
+                survival_by_type=_rates(canaries, survived),
+                survived=survived,
+                context=ctx,
+            )
+        )
+        turns = [Turn("system", ctx.text)]
+    return run

compaction_kit/session.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .canaries import seeded_canaries
+
+
+@dataclass(frozen=True)
+class Turn:
+    role: str  # "system" | "user" | "assistant" | "tool"
+    text: str
+    # canary id planted in this turn, if any
+    canary_id: str | None = None
+
+
+@dataclass(frozen=True)
+class SeededSession:
+    turns: tuple[Turn, ...]
+    canary_positions: dict[str, int]  # canary_id -> turn index
+
+    def full_text(self) -> str:
+        return "\n".join(f"[{t.role}] {t.text}" for t in self.turns)
+
+    def transcript(self) -> list[Turn]:
+        return list(self.turns)
+
+
+_FILLER = [
+    "Let's review the module layout for the ingestion pipeline.",
+    "The retry logic uses exponential backoff with jitter, capped at 30s.",
+    "We discussed caching headers for the static asset route last session.",
+    "The dashboard mock shows latency p50 and p95 side by side.",
+    "Note: the linter config lives in pyproject.toml under tool.ruff.",
+    "We should add a smoke test that hits the health endpoint.",
+    "The feature flag for the new parser is called parse_v2.",
+    "Standup notes: yesterday focused on schema cleanup, today on tests.",
+    "The memo about hiring is in the shared drive, folder Q3 planning.",
+    "We renamed the helper from fetch_all to list_items for clarity.",
+    "The load test peaked at 1.2k rps with no errors on the small cluster.",
+    "Reminder: rotate the staging credentials at the end of the month.",
+    "The design doc for webhooks is still in draft, section 4 is open.",
+    "We pinned numpy to 1.26 for compatibility with the imaging stack.",
+    "The onboarding checklist has 14 items, most are account setup.",
+]
+
+
+def build_seeded_session() -> SeededSession:
+    """Scripted transcript with canaries at known early/middle/late positions.
+
+    Early canaries are the ones naive compaction loses first — this is the
+    ground-truth gradient the kit must recover.
+    """
+    canaries = {c.id: c for c in seeded_canaries()}
+    # planting order interleaves types across positions
+    order = [
+        "safety-001", "fact-001", "constraint-001", "pref-001",      # early
+        "goal-001", "safety-002", "fact-002", "constraint-002",      # early-mid
+        "pref-002", "goal-002", "safety-003", "fact-003",            # middle
+        "constraint-003", "pref-003", "goal-003", "safety-004",      # mid-late
+        "fact-004", "constraint-004", "pref-004", "goal-004",        # late
+    ]
+    turns: list[Turn] = [Turn("system", "You are a coding assistant. Follow all rules and constraints in this session.")]
+    positions: dict[str, int] = {}
+    filler_i = 0
+    for cid in order:
+        c = canaries[cid]
+        # two filler turns before each canary except the first few
+        for _ in range(2):
+            turns.append(Turn("user", _FILLER[filler_i % len(_FILLER)]))
+            filler_i += 1
+            turns.append(Turn("assistant", "Acknowledged. Continuing with the plan."))
+        positions[cid] = len(turns)
+        turns.append(Turn("user", c.content, canary_id=cid))
+        turns.append(Turn("assistant", f"Noted: {c.direct_question} — recorded."))
+    # trailing filler so late canaries are not literally the tail
+    for _ in range(3):
+        turns.append(Turn("user", _FILLER[filler_i % len(_FILLER)]))
+        filler_i += 1
+        turns.append(Turn("assistant", "Acknowledged. Continuing with the plan."))
+    return SeededSession(turns=tuple(turns), canary_positions=positions)

compaction_kit/simulated_agent.py

@@ -0,0 +1,88 @@
+"""A $0 simulated agent for probing compacted context.
+
+The expensive version of this kit asks a real LLM agent the probe
+questions and grades the answers. This module answers the same probes
+with deterministic retrieval over the compacted text only:
+
+- Direct recall: find the most relevant line in the compacted context
+  and return it; if no required information is retrievable, say so.
+- Behavior: if the blocking rule is retrievable, act on it (refuse /
+  block / follow the recorded next step); otherwise proceed as an
+  agent without that rule would.
+
+This is an ideal-retrieval upper bound: if even this agent cannot
+recover a canary from the compacted text, a real agent cannot either
+(except by guessing). Validated 2026-10-04 against a blind
+self-simulation: a separate model given only the compacted text gave
+answers that matched this module's presence predictions 20/20 across
+lossy and checklist contexts (10 canaries x 2 compactors).
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from .canaries import Canary
+
+
+@dataclass(frozen=True)
+class SimulatedAnswer:
+    canary_id: str
+    answer: str
+    found: bool
+
+    @property
+    def survived(self) -> bool:
+        return self.found
+
+
+def _lines(context_text: str) -> list[str]:
+    return [ln.strip(" -") for ln in context_text.splitlines() if ln.strip(" -")]
+
+
+class SimulatedAgent:
+    """Answers probes from compacted context only. No model calls."""
+
+    def answer_direct(self, canary: Canary, context_text: str) -> SimulatedAnswer:
+        lower = context_text.lower()
+        if all(tok.lower() in lower for tok in canary.required_tokens if tok):
+            # return the line carrying the canary
+            for ln in _lines(context_text):
+                if any(tok.lower() in ln.lower() for tok in canary.required_tokens if tok):
+                    return SimulatedAnswer(canary.id, ln, True)
+            return SimulatedAnswer(canary.id, canary.content, True)
+        return SimulatedAnswer(canary.id, "NOT IN CONTEXT", False)
+
+    def answer_behavior(self, canary: Canary, context_text: str) -> SimulatedAnswer:
+        direct = self.answer_direct(canary, context_text)
+        if not canary.behavior_scenario:
+            return direct
+        if direct.found:
+            return SimulatedAnswer(
+                canary.id,
+                f"Would follow the preserved rule: {direct.answer}",
+                True,
+            )
+        return SimulatedAnswer(
+            canary.id,
+            "Would proceed without the rule (not in context).",
+            False,
+        )
+
+    def answer_exact_use(self, canary: Canary, context_text: str) -> SimulatedAnswer:
+        """Complete a work item that requires the exact canary value.
+
+        Unlike refusal-friendly behavior scenarios, generic caution cannot
+        pass here: if the exact values are not retrievable, the work item
+        cannot be completed.
+        """
+        if not canary.exact_use_scenario:
+            return self.answer_direct(canary, context_text)
+        lower = context_text.lower()
+        if all(tok.lower() in lower for tok in canary.exact_use_required_tokens if tok):
+            for ln in _lines(context_text):
+                if any(tok.lower() in ln.lower() for tok in canary.exact_use_required_tokens if tok):
+                    return SimulatedAnswer(canary.id, f"Completed using preserved value: {ln}", True)
+            return SimulatedAnswer(canary.id, canary.content, True)
+        return SimulatedAnswer(canary.id, "NOT IN CONTEXT", False)

compaction_kit/spike.py

@@ -0,0 +1,81 @@
+"""Spike CLI: seeded session through lossy vs structure-preserving compaction.
+
+Kill criterion (all must hold, else the kit cannot separate implementations):
+  1. Lossy compaction is FLAGged on at least one type after round 1 (<50%).
+  2. Checklist compaction stays SILENT on every type after round 1 (>90%).
+  3. Ordering matches ground truth: checklist survival >= lossy survival
+     for every type at every round, with strict overall separation.
+
+Usage:  python -m compaction_kit.spike [--rounds 5]
+Writes reports under examples/ when run as a script.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from compaction_kit.compactors import ChecklistCompactor, LossyTruncationCompactor, NaiveSummaryCompactor
+from compaction_kit.report import build_report
+from compaction_kit.runner import run_conformance
+from compaction_kit.session import build_seeded_session
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--rounds", type=int, default=5)
+    ap.add_argument("--out", type=str, default="")
+    args = ap.parse_args()
+
+    session = build_seeded_session()
+    compactors = [LossyTruncationCompactor(), NaiveSummaryCompactor(), ChecklistCompactor()]
+    runs = {c.name: run_conformance(session, c, rounds=args.rounds) for c in compactors}
+    reports = {name: build_report(run) for name, run in runs.items()}
+
+    for name, rep in reports.items():
+        print(rep.to_markdown())
+
+    lossy = reports["lossy-truncation"]
+    naive = reports["naive-summary"]
+    good = reports["checklist-carrying"]
+
+    checks: dict[str, bool] = {}
+    checks["lossy_flagged_at_least_one_type"] = len(lossy.flagged_types) >= 1
+    checks["naive_flagged_at_least_one_type"] = len(naive.flagged_types) >= 1
+    checks["checklist_silent_on_all_types"] = len(good.silent_types) == len(good.findings)
+    # ordering: checklist >= both lossy variants per type per round, and strictly better overall
+    ordering = True
+    strict_somewhere = False
+    for f_good in good.findings:
+        for bad_rep in (lossy, naive):
+            f_bad = next(f for f in bad_rep.findings if f.canary_type == f_good.canary_type)
+            for g, b in zip(f_good.curve, f_bad.curve):
+                if g < b:
+                    ordering = False
+                if g > b:
+                    strict_somewhere = True
+    checks["ordering_matches_ground_truth"] = ordering and strict_somewhere
+
+    print("KILL CRITERION:")
+    for k, v in checks.items():
+        print(f"  {'PASS' if v else 'FAIL'}  {k}")
+    passed = all(checks.values())
+    print(f"SPIKE: {'PASS' if passed else 'FAIL — kit cannot separate implementations, stop'}")
+
+    if args.out:
+        out = Path(args.out)
+        out.mkdir(parents=True, exist_ok=True)
+        payload = {name: rep.to_dict() for name, rep in reports.items()}
+        payload["_kill_criterion"] = checks
+        (out / "spike-results.json").write_text(json.dumps(payload, indent=2))
+        for name, rep in reports.items():
+            (out / f"report-{name}.md").write_text(rep.to_markdown())
+    return 0 if passed else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())