tensorcode 0.1.0a1__py3-none-any.whl

tensorcode/chunking.py

@@ -0,0 +1,195 @@
+"""Automatization: a sequence done the same way often enough stops being deliberated.
+
+The deliberate-to-automatic transition is the most reliable finding in skill learning, and a
+procedure interpreter has none of it: a learned skill walks its steps, evaluates every guard and
+records every decision, on the thousandth run exactly as on the first. That is not carefulness,
+it is an inability to learn *how* it does something as opposed to *that* it works.
+
+A chunk here is the compiled form of a sequence that has run identically :attr:`Chunks.repeats`
+times: the steps actually taken, the guards whose outcome was the same every time, and nothing
+else. Running a chunk skips the deliberation, not the acts — a click is still a click. What it
+skips is evaluating guards whose answer has never varied and recording a decision per step.
+
+Automatization trades adaptability for speed, so the trade is made explicit:
+
+* the expanded form is never discarded; a chunk is an *index* into it;
+* a chunk is retired the moment a step's outcome diverges from what the recorded runs saw
+  (:func:`divergent`), and the walk continues expanded from that point — the fallback is the
+  whole reason this is safe;
+* every chunk carries how often it was used and how often it fell back, so a chunk that keeps
+  breaking is visible rather than quietly wrong.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from typing import Iterable, Mapping, Sequence
+
+#: bindings that mean a body step did not do what the recorded runs saw it do.
+#: Deliberately *not* ``error_summary``: the assistant's own ``output_facts`` always fills that
+#: in, falling back to the first line of a perfectly good output, so reading it as trouble would
+#: retire every chunk on its first use. ``ok`` and ``errors`` are the marks that mean trouble.
+TROUBLE = ("error", "problem", "timed_out", "unavailable")
+
+
+@dataclass(frozen=True)
+class Trace:
+    """One run of a procedure: which steps ran, which were skipped, and how it ended."""
+
+    procedure: str
+    taken: tuple[int, ...]  # program counters whose step executed, in order
+    skipped: tuple[int, ...]  # program counters whose guard did not hold
+    status: str = "done"
+    #: (program counter, binding that showed trouble) for the steps that showed any. Some steps
+    #: are *meant* to fail — "stat the folder before creating it" expects "no such file" — so a
+    #: chunk has to know what normal looked like rather than treating any error as a surprise.
+    marks: tuple[tuple[int, str], ...] = ()
+
+    @property
+    def shape(self) -> str:
+        """Identity of the *path* through the procedure, which is what can be compiled."""
+        return hashlib.sha256(json.dumps([self.procedure, list(self.taken), sorted(self.skipped),
+                                          sorted(self.marks)]).encode()).hexdigest()[:12]
+
+
+@dataclass
+class Chunk:
+    """A compiled path: run these steps in this order, without re-deciding."""
+
+    procedure: str
+    taken: tuple[int, ...]
+    skipped: frozenset[int]
+    shape: str
+    from_runs: int
+    expected: Mapping[int, str] = field(default_factory=dict)  # pc -> the trouble every run saw there
+    uses: int = 0
+    fallbacks: int = 0
+    retired: bool = False
+    retired_because: str = ""
+
+    def assumes_skipped(self, pc: int) -> bool:
+        return pc in self.skipped
+
+    def expects(self, pc: int) -> str:
+        """The trouble mark every recorded run saw at this step ("" if they all saw none)."""
+        return self.expected.get(pc, "")
+
+    def describe(self) -> str:
+        state = "retired" if self.retired else "live"
+        return (f"chunk {self.procedure}/{self.shape} ({state}): {len(self.taken)} steps, "
+                f"{len(self.skipped)} guards assumed, from {self.from_runs} runs, "
+                f"used {self.uses}, fell back {self.fallbacks}")
+
+
+def _marked(bindings: Mapping[str, object], name: str):
+    """Values a step bound under ``name``, including under a step's ``as:`` prefix.
+
+    A step that names its results (``"as": "sib"``) binds ``sib_ok`` and ``sib_errors``, so a
+    check that only looked for ``ok`` would never see trouble in exactly the procedures that
+    keep several commands' results apart.
+    """
+    for key, value in bindings.items():
+        if key == name or key.endswith(f"_{name}"):
+            yield key, value
+
+
+def mark(bindings: Mapping[str, object]) -> str:
+    """The name of the binding that says this step had trouble, or ``""`` if none did.
+
+    Deliberately shallow: it reads only the marks the procedures already set. A chunk that skips
+    guards cannot notice a subtle difference, which is exactly the known cost of automatization —
+    so what is watched is trouble, and the *name* is what is compared, because the text of an
+    error varies while its kind does not.
+    """
+    for name in TROUBLE:
+        for key, value in _marked(bindings, name):
+            if isinstance(value, bool) and value:
+                return key
+            if isinstance(value, str) and value.strip():
+                return key
+    for key, value in _marked(bindings, "errors"):
+        if isinstance(value, (list, tuple)) and value:
+            return key
+    for key, value in _marked(bindings, "ok"):
+        if value is False:
+            return key
+    return ""
+
+
+def divergent(bindings: Mapping[str, object], expected: str = "") -> str:
+    """Why this step differs from the runs a chunk was compiled from ("" when it does not).
+
+    Both directions count. Trouble where there was none is the obvious case; *no* trouble where
+    every recorded run had some is equally a divergence, because a guard that branched on it —
+    and is now being skipped — would have gone the other way.
+    """
+    found = mark(bindings)
+    if found == expected:
+        return ""
+    if found and expected:
+        return f"{found} where the recorded runs had {expected}"
+    if found:
+        return f"{found}, which the recorded runs never had"
+    return f"no {expected}, which every recorded run had"
+
+
+@dataclass
+class Chunks:
+    """What this mind has automatized, and how well each chunk is holding up."""
+
+    repeats: int = 3  # identical paths before a chunk is compiled
+    enabled: bool = True
+    runs: dict[str, list[str]] = field(default_factory=dict)  # procedure -> recent path shapes
+    traces: dict[str, Trace] = field(default_factory=dict)  # shape -> the path it stands for
+    compiled: dict[str, Chunk] = field(default_factory=dict)  # procedure -> live chunk
+    history: list[str] = field(default_factory=list)  # compile/retire events, in order
+
+    def record(self, trace: Trace) -> Chunk | None:
+        """Note how a run went; compile a chunk once the same path has repeated enough."""
+        if trace.status != "done":
+            self.runs.setdefault(trace.procedure, []).clear()
+            return None
+        shapes = self.runs.setdefault(trace.procedure, [])
+        shapes.append(trace.shape)
+        self.traces[trace.shape] = trace
+        recent = shapes[-self.repeats:]
+        if len(recent) < self.repeats or len(set(recent)) != 1:
+            return None
+        live = self.compiled.get(trace.procedure)
+        if live is not None and live.shape == trace.shape and not live.retired:
+            return live
+        chunk = Chunk(trace.procedure, trace.taken, frozenset(trace.skipped), trace.shape, self.repeats,
+                      dict(trace.marks))
+        self.compiled[trace.procedure] = chunk
+        self.history.append(f"compiled {trace.procedure}/{trace.shape} after {self.repeats} identical runs")
+        return chunk
+
+    def chunk_for(self, procedure: str | None) -> Chunk | None:
+        if not self.enabled or procedure is None:
+            return None
+        chunk = self.compiled.get(procedure)
+        return None if chunk is None or chunk.retired else chunk
+
+    def used(self, chunk: Chunk) -> None:
+        chunk.uses += 1
+
+    def retire(self, chunk: Chunk, why: str) -> None:
+        """Abandon a chunk and fall back to deliberating. The expanded form was never lost."""
+        chunk.retired, chunk.retired_because = True, why
+        chunk.fallbacks += 1
+        self.runs.setdefault(chunk.procedure, []).clear()
+        self.history.append(f"retired {chunk.procedure}/{chunk.shape}: {why}")
+
+    def stats(self) -> dict:
+        return {
+            "compiled": len([c for c in self.compiled.values() if not c.retired]),
+            "retired": len([c for c in self.compiled.values() if c.retired]),
+            "uses": sum(c.uses for c in self.compiled.values()),
+            "fallbacks": sum(c.fallbacks for c in self.compiled.values()),
+            "procedures": {p: c.describe() for p, c in self.compiled.items()},
+        }
+
+
+__all__ = ["Chunk", "Chunks", "Trace", "divergent", "mark", "TROUBLE"]

tensorcode/cognition.py

@@ -0,0 +1,311 @@
+"""A small cognitive substrate over the claim store.
+
+    perceive / parse   ->  Fragment          (claims extracted from one input, with locators)
+    integrate(mind, *fragments) -> Thought   (merge into working memory; report what changed)
+    think(mind, rules, since=thought) -> Thought   (bottom-up rules fire on what changed)
+    explain(mind, claim)                     (why a claim is believed: observations and rules)
+
+A ``Fragment`` can be a *snapshot* of a scope (for example, everything currently on
+screen). Claims in that scope that the new snapshot no longer contains are retracted,
+and derivations that depended only on them are withdrawn with them.
+
+Perception can be wrong for a frame. An optional ``Corroboration`` policy keeps chosen
+perceived claims *tentative* until they are seen the same way in ``k`` frames (or confirmed
+by an action's outcome); derivations inherit tentativeness from their premises, and a
+tentative claim a later frame contradicts is retracted. Rules and intentions that must not
+act on a single glance ask for the established view.
+
+Rules are ordinary Python: patterns select premises, a function derives claims. Every
+derived claim cites its premises, so every thought can be explained. Rules may call
+other operations (``tc.classify``, ``tc.parse``); those calls appear in the trace.
+Nothing here decides what to *do*; that is ``choose`` over intentions.
+"""
+
+from __future__ import annotations
+
+import time as _time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Callable, Iterable, Mapping, Sequence
+
+from .outcomes import Score
+from .records import Claim, ClaimRecord, Evidence, Patch, Put, Ref, Retract, Store, Tell
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+@dataclass(frozen=True)
+class Fragment:
+    """Claims extracted from one input."""
+
+    source: Ref  # the input these claims came from (a frame, a message, a document)
+    claims: tuple[tuple[Claim, str | None], ...]  # (claim, locator within the source)
+    entities: tuple[tuple[Ref, Any], ...] = ()  # typed payloads, e.g. geometry for a control
+    snapshot_of: Ref | None = None  # if set, this fragment is everything currently true in that scope
+    method: str = "parse"
+    observed_at: datetime = field(default_factory=_now)
+    confidence: Mapping[str, Score] = field(default_factory=dict)  # claim id -> extractor's score
+
+
+@dataclass(frozen=True)
+class Thought:
+    """What changed in the mind. Empty thoughts are normal."""
+
+    added: tuple[ClaimRecord, ...] = ()
+    retracted: tuple[ClaimRecord, ...] = ()
+    established: tuple[ClaimRecord, ...] = ()  # tentative claims that became established (see ``Corroboration``)
+
+    def __add__(self, other: Thought) -> Thought:
+        return Thought(self.added + other.added, self.retracted + other.retracted, self.established + other.established)
+
+    @property
+    def empty(self) -> bool:
+        return not self.added and not self.retracted and not self.established
+
+    def about(self, predicate: str) -> list[Claim]:
+        return [r.claim for r in self.added if r.claim.predicate == predicate]
+
+
+@dataclass
+class Corroboration:
+    """When a perceived claim may be acted on: after ``k`` consistent frames, or confirmation.
+
+    * ``predicates``: which perceived predicates this governs (None = all). Others are
+      established on sight, which keeps, say, button geometry instant while text readings
+      wait for a second look.
+    * A frame counts toward ``k`` only if the extractor's confidence (when it gave one) is at
+      least ``min_confidence``; below that a claim stays tentative until ``confirm``-ed.
+    * Derived claims are established only when every premise of some line of support is.
+    * A tentative claim that a later frame contradicts is retracted: in a snapshot scope by
+      no longer being perceived; elsewhere by a different object for a functional predicate.
+
+    State is per mind and per run: create a fresh policy for each episode.
+    """
+
+    k: int = 2
+    min_confidence: float = 0.0
+    predicates: frozenset[str] | None = None
+    frames: dict[str, int] = field(default_factory=dict)  # claim id -> frames that corroborated it
+    confirmed: set[str] = field(default_factory=set)
+
+    def governs(self, predicate: str) -> bool:
+        return self.predicates is None or predicate in self.predicates
+
+    def established(self, mind: Store, claim_id: str, _seen: frozenset[str] = frozenset()) -> bool:
+        rec = mind._claims.get(claim_id)
+        if rec is None or rec.retracted or claim_id in _seen:
+            return False
+        if claim_id in self.confirmed:
+            return True
+        for e in rec.evidence:
+            if e.derived_from:
+                if all(self.established(mind, p, _seen | {claim_id}) for p in e.derived_from):
+                    return True
+            elif not self.governs(rec.claim.predicate) or self.frames.get(claim_id, 0) >= self.k:
+                return True
+        return False
+
+    def tentative(self, mind: Store, claim_id: str) -> bool:
+        return claim_id in mind._claims and not self.established(mind, claim_id)
+
+    def confirm(self, mind: Store, claim_id: str) -> Thought:
+        """An action's outcome bore the claim out: establish it (and what now rests on it)."""
+        before = self.established(mind, claim_id)
+        self.confirmed.add(claim_id)
+        return Thought(established=tuple(mind.claim(i) for i in self._closure(mind, [claim_id]))) if not before else Thought()
+
+    def view(self, mind: Store) -> EstablishedView:
+        return EstablishedView(mind, self)
+
+    def _count(self, claim_id: str, score: Score | None) -> None:
+        if score is None or score.value >= self.min_confidence:
+            self.frames[claim_id] = self.frames.get(claim_id, 0) + 1
+
+    def _closure(self, mind: Store, ids: Iterable[str]) -> list[str]:
+        """Claims now established among ``ids`` and the derivations resting on them."""
+        out, stack, seen = [], [i for i in ids], set()
+        while stack:
+            cid = stack.pop()
+            if cid in seen:
+                continue
+            seen.add(cid)
+            if self.established(mind, cid):
+                out.append(cid)
+                stack.extend(sorted(mind._dependents.get(cid, ())))
+        return list(dict.fromkeys(out))
+
+
+class EstablishedView:
+    """A read-only look at a mind that hides tentative claims (everything else delegates)."""
+
+    def __init__(self, mind: Store, policy: Corroboration) -> None:
+        self._mind, self._policy = mind, policy
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self._mind, name)
+
+    def claims(self, *args: Any, **kwargs: Any) -> list[ClaimRecord]:
+        return [r for r in self._mind.claims(*args, **kwargs) if self._policy.established(self._mind, r.id)]
+
+    def match(self, *patterns: Any, with_support: bool = False) -> list:
+        found = [(b, s) for b, s in self._mind.match(*patterns, with_support=True) if all(self._policy.established(self._mind, c) for c in s)]
+        return found if with_support else [b for b, _ in found]
+
+
+def integrate(mind: Store, *fragments: Fragment, remember_retracted: bool = False, corroboration: Corroboration | None = None) -> Thought:
+    """Merge fragments into working memory.
+
+    Snapshot scopes are ephemeral: claims that fall out of a snapshot are retracted and,
+    unless ``remember_retracted``, forgotten along with derivations that depended on them.
+    With a ``Corroboration`` policy, each fragment is one frame of evidence for its claims
+    (see there); ``Thought.established`` lists claims that crossed the threshold.
+    """
+    policy = corroboration
+    was_tentative = {cid for f in fragments for c, _ in f.claims if (cid := c.id) in mind._claims and policy.tentative(mind, cid)} if policy else set()
+    edits: list[Any] = []
+    for f in fragments:
+        edits += [Put(ref, value) for ref, value in f.entities]
+        present = set()
+        for claim, locator in f.claims:
+            if f.snapshot_of is not None and claim.scope != f.snapshot_of:
+                raise ValueError(f"snapshot of {f.snapshot_of} contains a claim in scope {claim.scope}")
+            present.add(claim.id)
+            live = claim.id in mind._claims and not mind._claims[claim.id].retracted
+            if policy is not None:
+                policy._count(claim.id, f.confidence.get(claim.id))
+                if f.snapshot_of is None and claim.predicate in mind.functional:
+                    edits += [Retract(r.id, "contradicted by a later observation", (Evidence(f.source, f.observed_at, locator, f.method),))
+                              for r in mind.claims(claim.subject, claim.predicate, scope=claim.scope) if r.claim.object != claim.object and policy.tentative(mind, r.id)]
+            if f.snapshot_of is not None and live:
+                continue  # still perceived; no need to pile up evidence every frame
+            edits.append(Tell(claim, (Evidence(f.source, f.observed_at, locator, f.method, f.confidence.get(claim.id)),)))
+        if f.snapshot_of is not None:
+            edits += [Retract(rec.id, "no longer perceived", (Evidence(f.source, f.observed_at, method=f.method),)) for rec in mind.claims(scope=f.snapshot_of) if rec.id not in present]
+    if not edits:
+        return _newly_established(mind, policy, was_tentative, ())
+    commit = mind.apply(Patch(tuple(edits), mind.revision))
+    thought = Thought(tuple(mind.claim(i) for i in commit.added), tuple(mind.claim(i) for i in commit.retracted))
+    thought += _newly_established(mind, policy, was_tentative, commit.added)
+    if policy is not None:
+        for cid in commit.retracted:  # contradicted or no longer perceived: the count starts over
+            policy.frames.pop(cid, None)
+            policy.confirmed.discard(cid)
+    if not remember_retracted:
+        mind.forget(commit.retracted)
+    return thought
+
+
+def _newly_established(mind: Store, policy: Corroboration | None, was_tentative: set[str], added: Sequence[str]) -> Thought:
+    if policy is None or not was_tentative:
+        return Thought()
+    flipped = [cid for cid in was_tentative if cid not in added and policy.established(mind, cid)]
+    return Thought(established=tuple(mind.claim(i) for i in policy._closure(mind, flipped) if i not in added))
+
+
+@dataclass(frozen=True)
+class Rule:
+    """When the patterns match (and at least one matched claim is new), derive claims."""
+
+    name: str
+    when: tuple[tuple[Any, str, Any], ...]
+    then: Callable[[Mapping[str, Any], Store], Iterable[Claim | tuple[Claim, Score] | Fragment]]
+    version: str = "1"  # a Fragment output is an act of reading: integrated as knowledge, not as a derivation
+    reacts_to: str = "new_premises"  # or "any_change": also fire when something was retracted (e.g. a spinner vanished)
+    established_only: bool = False  # with a Corroboration policy: premises must be established, and ``then`` sees the established view
+
+
+@dataclass
+class ThinkStats:
+    rounds: int = 0
+    firings: int = 0
+    derived: int = 0
+    ms: float = 0.0
+
+
+def think(mind: Store, rules: Sequence[Rule], *, since: Thought, max_rounds: int = 8, stats: ThinkStats | None = None, forget_withdrawn: bool = True,
+          corroboration: Corroboration | None = None) -> Thought:
+    """Bottom-up inference: fire rules on matches that involve something new, until quiet.
+
+    Semi-naive evaluation: a match whose premises are all old already fired earlier. A claim
+    that just became established counts as new for ``established_only`` rules.
+    """
+    t0 = _time.perf_counter()
+    stats = stats if stats is not None else ThinkStats()
+    total = Thought()
+    frontier = {r.id for r in since.added} | {r.id for r in since.established}
+    changed = bool(since.added or since.retracted or since.established)
+    view = corroboration.view(mind) if corroboration is not None else None
+    for round_no in range(max_rounds):
+        if not frontier and not (round_no == 0 and changed):
+            break
+        stats.rounds += 1
+        edits: list[Tell] = []
+        readings: list[Fragment] = []
+        for rule in rules:
+            strict = rule.established_only and view is not None
+            for bindings, support in (view if strict else mind).match(*rule.when, with_support=True):
+                if frontier.isdisjoint(support) and not (rule.reacts_to == "any_change" and round_no == 0 and changed):
+                    continue
+                stats.firings += 1
+                for out in rule.then(bindings, view if strict else mind):
+                    if isinstance(out, Fragment):
+                        readings.append(out)
+                        continue
+                    claim, score = out if isinstance(out, tuple) else (out, None)
+                    edits.append(Tell(claim, (Evidence(Ref(f"rule:{rule.name}"), _now(), method=f"derive@{rule.version}", confidence=score, derived_from=tuple(sorted(set(support)))),)))
+        if not edits and not readings:
+            break
+        added_ids: list[str] = []
+        if edits:
+            added_ids += mind.apply(Patch(tuple(edits), mind.revision)).added
+        if readings:
+            added_ids += [r.id for r in integrate(mind, *readings).added]
+        added = tuple(mind.claim(i) for i in dict.fromkeys(added_ids))
+        stats.derived += len(added)
+        total += Thought(added)
+        frontier = {r.id for r in added}
+    stats.ms += (_time.perf_counter() - t0) * 1e3
+    return total
+
+
+def _shown(value: Any) -> Any:
+    """How a claim's object reads in an explanation.
+
+    ``Score`` and friends are worth unwrapping to their number, but a value that carries a
+    unit alongside it (a quantity) loses its point when reduced to a bare float: "60.0"
+    instead of "60 coin" is exactly the confusion the unit exists to prevent.
+    """
+    if hasattr(value, "value") and hasattr(value, "unit"):
+        return str(value)
+    return getattr(value, "value", value)
+
+
+def explain(mind: Store, claim_id: str, *, depth: int = 6) -> list[str]:
+    """Indented lines: the claim, then each line of support down to observations."""
+    lines: list[str] = []
+
+    def show(cid: str, level: int, seen: frozenset[str]) -> None:
+        rec = mind._claims.get(cid)
+        pad = "  " * level
+        if rec is None:
+            lines.append(f"{pad}(forgotten {cid})")
+            return
+        c = rec.claim
+        obj = _shown(c.object)
+        lines.append(f"{pad}{c.subject} {c.predicate} {obj!r}" + (" [retracted]" if rec.retracted else ""))
+        if level >= depth or cid in seen:
+            return
+        for e in rec.evidence:
+            conf = f" ({e.confidence.kind} {e.confidence.value:.2f})" if e.confidence else ""
+            if e.derived_from:
+                how = f" via {e.method}" if e.method else ""
+                lines.append(f"{pad}  ← {e.source}{conf}{how} from:")
+                for p in e.derived_from:
+                    show(p, level + 2, seen | {cid})
+            else:
+                lines.append(f"{pad}  ← observed in {e.source}" + (f" at {e.locator}" if e.locator else "") + f" via {e.method}{conf}")
+
+    show(claim_id, 0, frozenset())
+    return lines

tensorcode/context.py

@@ -0,0 +1,97 @@
+"""Deterministic context assembly: redundancy removal and budgeted packing.
+
+Packing is a lossy conversion, so the result says exactly what was dropped and why.
+Required evidence is never silently dropped: if it cannot fit, the answer is Unknown.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Callable, Generic, Hashable, Sequence, TypeVar
+
+from .outcomes import Score, Unknown
+
+T = TypeVar("T")
+
+_TOKEN = re.compile(r"\w+|[^\w\s]")
+
+
+def approx_tokens(text: str) -> int:
+    """Word-and-punctuation count; a stand-in for a real tokenizer.
+
+    Measured against the Qwen3 tokenizer on 12,187 HotpotQA sentences: 0.82x the true total
+    (per-sentence median 0.84, p5 0.64), so it *undercounts*. Pass a real tokenizer as ``cost``
+    when a budget is a hard model limit.
+    """
+    return len(_TOKEN.findall(text))
+
+
+def shingle_similarity(a: str, b: str, n: int = 3) -> Score:
+    """Jaccard overlap of word n-grams. A similarity, not a probability of duplication."""
+
+    def grams(s: str) -> set[tuple[str, ...]]:
+        words = s.lower().split()
+        return {tuple(words[i : i + n]) for i in range(max(1, len(words) - n + 1))}
+
+    ga, gb = grams(a), grams(b)
+    return Score(len(ga & gb) / len(ga | gb) if ga | gb else 0.0, "similarity")
+
+
+@dataclass(frozen=True)
+class Packed(Generic[T]):
+    items: tuple[T, ...]
+    used: int
+    budget: int
+    dropped: tuple[tuple[T, str], ...]  # (item, reason)
+
+
+def dedupe(
+    ranked: Sequence[tuple[T, Score]],
+    *,
+    similarity: Callable[[T, T], Score],
+    threshold: float,
+    keep: Callable[[T], bool] = lambda item: False,
+    key: Callable[[T], Hashable] = id,
+) -> tuple[list[tuple[T, Score]], list[tuple[T, str]]]:
+    """Walk in rank order; drop an item too similar to one already kept (unless ``keep`` says otherwise)."""
+    kept: list[tuple[T, Score]] = []
+    dropped: list[tuple[T, str]] = []
+    for item, score in ranked:
+        dup = next((k for k, _ in kept if similarity(item, k).value >= threshold), None)
+        if dup is not None and not keep(item):
+            dropped.append((item, f"near-duplicate of {key(dup)!r}"))
+        else:
+            kept.append((item, score))
+    return kept, dropped
+
+
+def pack(
+    ranked: Sequence[tuple[T, Score]],
+    *,
+    budget: int,
+    cost: Callable[[T], int],
+    required: Sequence[T] = (),
+    key: Callable[[T], Hashable] = id,
+    dropped: Sequence[tuple[T, str]] = (),
+) -> Packed[T] | Unknown:
+    """Required items first, then ranked items greedily while they fit.
+
+    ``dropped`` carries removals made earlier (e.g. by ``dedupe``) into the same report.
+    """
+    required_keys = {key(r) for r in required}
+    chosen: list[T] = list(required)
+    used = sum(cost(r) for r in required)
+    if used > budget:
+        return Unknown("required_evidence_exceeds_budget", f"required items cost {used} > budget {budget}")
+    dropped = list(dropped)
+    for item, _ in ranked:
+        if key(item) in required_keys:
+            continue
+        c = cost(item)
+        if used + c <= budget:
+            chosen.append(item)
+            used += c
+        else:
+            dropped.append((item, f"budget: needs {c}, {budget - used} left"))
+    return Packed(tuple(chosen), used, budget, tuple(dropped))