retainkit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
retainkit/__init__.py ADDED
@@ -0,0 +1,56 @@
1
+ """Context and memory policies for LLM agents, scored by what survives the budget."""
2
+
3
+ from .evaluate import DEFAULT_BUDGETS, Case, Report, run, score
4
+ from .io import dump_cases, load_cases
5
+ from .metrics import Outcome, PolicyResult, budget_for_recall
6
+ from .policies import (
7
+ DEFAULT_POLICIES,
8
+ FactMemory,
9
+ HeadTail,
10
+ KeepAll,
11
+ Oracle,
12
+ Policy,
13
+ Recency,
14
+ Retrieval,
15
+ SessionSummary,
16
+ )
17
+ from .report import age_table, frontier_table, markdown, sweep_table, to_json
18
+ from .tokens import char4, wordish
19
+ from .types import Fragment, Probe, Retained, Transcript, Turn
20
+
21
+ __version__ = "0.1.0"
22
+
23
+ __all__ = [
24
+ "DEFAULT_BUDGETS",
25
+ "DEFAULT_POLICIES",
26
+ "Case",
27
+ "FactMemory",
28
+ "Fragment",
29
+ "HeadTail",
30
+ "KeepAll",
31
+ "Oracle",
32
+ "Outcome",
33
+ "Policy",
34
+ "PolicyResult",
35
+ "Probe",
36
+ "Recency",
37
+ "Report",
38
+ "Retained",
39
+ "Retrieval",
40
+ "SessionSummary",
41
+ "Transcript",
42
+ "Turn",
43
+ "__version__",
44
+ "age_table",
45
+ "budget_for_recall",
46
+ "char4",
47
+ "dump_cases",
48
+ "frontier_table",
49
+ "load_cases",
50
+ "markdown",
51
+ "run",
52
+ "score",
53
+ "sweep_table",
54
+ "to_json",
55
+ "wordish",
56
+ ]
retainkit/bm25.py ADDED
@@ -0,0 +1,69 @@
1
+ """Okapi BM25 over short documents, standard library only.
2
+
3
+ Ties are broken by document index so that a ranking is reproducible on any
4
+ machine and any Python build.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import math
10
+ import re
11
+ from collections import Counter
12
+ from collections.abc import Sequence
13
+ from dataclasses import dataclass
14
+
15
+ _TOKEN = re.compile(r"[a-z0-9]+")
16
+
17
+ K1 = 1.5
18
+ B = 0.75
19
+
20
+
21
+ def tokenize(text: str) -> list[str]:
22
+ return _TOKEN.findall(text.lower())
23
+
24
+
25
+ @dataclass
26
+ class BM25:
27
+ """A small BM25 index built once and queried many times."""
28
+
29
+ documents: Sequence[str]
30
+ k1: float = K1
31
+ b: float = B
32
+
33
+ def __post_init__(self) -> None:
34
+ self._tokens = [tokenize(d) for d in self.documents]
35
+ self._lengths = [len(t) for t in self._tokens]
36
+ self._avg_length = (sum(self._lengths) / len(self._lengths)) if self._lengths else 0.0
37
+ self._frequencies = [Counter(t) for t in self._tokens]
38
+ document_frequency: Counter[str] = Counter()
39
+ for counts in self._frequencies:
40
+ document_frequency.update(counts.keys())
41
+ total = len(self.documents)
42
+ self._idf = {
43
+ term: math.log(1 + (total - freq + 0.5) / (freq + 0.5))
44
+ for term, freq in document_frequency.items()
45
+ }
46
+
47
+ def scores(self, query: str) -> list[float]:
48
+ terms = tokenize(query)
49
+ results = [0.0] * len(self.documents)
50
+ if not terms or self._avg_length == 0.0:
51
+ return results
52
+ for index, counts in enumerate(self._frequencies):
53
+ length = self._lengths[index]
54
+ total = 0.0
55
+ for term in terms:
56
+ frequency = counts.get(term, 0)
57
+ if not frequency:
58
+ continue
59
+ denominator = frequency + self.k1 * (
60
+ 1 - self.b + self.b * length / self._avg_length
61
+ )
62
+ total += self._idf[term] * frequency * (self.k1 + 1) / denominator
63
+ results[index] = total
64
+ return results
65
+
66
+ def ranked(self, query: str) -> list[int]:
67
+ """Document indices, best first, ties broken by original order."""
68
+ scored = self.scores(query)
69
+ return sorted(range(len(scored)), key=lambda i: (-scored[i], i))
retainkit/cli.py ADDED
@@ -0,0 +1,73 @@
1
+ """Command line: run the fixture, or run a sweep over your own conversations."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import sys
7
+ from pathlib import Path
8
+
9
+ from .demo import build_cases
10
+ from .evaluate import DEFAULT_BUDGETS, run
11
+ from .io import load_cases
12
+ from .policies import DEFAULT_POLICIES
13
+ from .report import markdown, sweep_table, to_json
14
+ from .tokens import char4, wordish
15
+
16
+ TOKENIZERS = {"char4": char4, "wordish": wordish}
17
+
18
+
19
+ def _write(report, out: Path, title: str, focus: int, stem: str) -> None:
20
+ out.mkdir(parents=True, exist_ok=True)
21
+ (out / "table.md").write_text(sweep_table(report) + "\n", encoding="utf-8")
22
+ (out / "report.md").write_text(markdown(report, title, focus) + "\n", encoding="utf-8")
23
+ (out / f"{stem}.json").write_text(to_json(report), encoding="utf-8")
24
+
25
+
26
+ def main(argv: list[str] | None = None) -> int:
27
+ parser = argparse.ArgumentParser(prog="retainkit", description=__doc__)
28
+ sub = parser.add_subparsers(dest="command", required=True)
29
+
30
+ demo = sub.add_parser("demo", help="run the built-in fixture and write results")
31
+ demo.add_argument("--out", default="results", type=Path)
32
+ demo.add_argument("--seed", default=7, type=int)
33
+ demo.add_argument("--tokenizer", default="char4", choices=sorted(TOKENIZERS))
34
+
35
+ evaluate = sub.add_parser("eval", help="sweep policies over conversations in a JSON file")
36
+ evaluate.add_argument("cases", type=Path)
37
+ evaluate.add_argument("--out", default=None, type=Path)
38
+ evaluate.add_argument("--title", default="retainkit sweep")
39
+ evaluate.add_argument("--tokenizer", default="char4", choices=sorted(TOKENIZERS))
40
+ evaluate.add_argument(
41
+ "--budget",
42
+ action="append",
43
+ type=int,
44
+ help="repeat to sweep several budgets; defaults to 256 … 8192",
45
+ )
46
+
47
+ args = parser.parse_args(argv)
48
+ tokenizer = TOKENIZERS[args.tokenizer]
49
+
50
+ if args.command == "demo":
51
+ cases = build_cases(seed=args.seed)
52
+ report = run(
53
+ cases,
54
+ DEFAULT_POLICIES,
55
+ DEFAULT_BUDGETS,
56
+ tokenizer=tokenizer,
57
+ notes={"source": "synthetic fixture, seed " + str(args.seed)},
58
+ )
59
+ _write(report, args.out, "retainkit fixture", 1024, "demo")
60
+ print(sweep_table(report))
61
+ return 0
62
+
63
+ cases = load_cases(args.cases)
64
+ budgets = tuple(args.budget) if args.budget else DEFAULT_BUDGETS
65
+ report = run(cases, DEFAULT_POLICIES, budgets, tokenizer=tokenizer)
66
+ if args.out:
67
+ _write(report, args.out, args.title, budgets[len(budgets) // 2], "sweep")
68
+ print(markdown(report, args.title, budgets[len(budgets) // 2]))
69
+ return 0
70
+
71
+
72
+ if __name__ == "__main__": # pragma: no cover
73
+ sys.exit(main())
retainkit/demo.py ADDED
@@ -0,0 +1,123 @@
1
+ """A synthetic fixture, built from a fixed seed, that CI diffs.
2
+
3
+ It is not a benchmark. It exists so that a change to a policy or a metric shows
4
+ up as a moved number in `results/`, and so the package can be tried without
5
+ downloading anything. The measured claims in the README come from LoCoMo, in
6
+ `examples/locomo/`.
7
+
8
+ The shape is the one a travel assistant meets: a standing fact about a
9
+ traveller stated once, small talk that mentions the same people, near misses
10
+ that use the same words as the question without answering it, and the question
11
+ arriving many sessions later.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import random
17
+
18
+ from .evaluate import Case
19
+ from .types import Probe, Transcript, Turn
20
+
21
+ FACTS: tuple[tuple[str, str, str, str], ...] = (
22
+ ("Nadia", "allergy", "shellfish", "What allergy does Nadia have?"),
23
+ ("Nadia", "seat", "aisle", "Which seat does Nadia book?"),
24
+ ("Tomas", "city", "Ljubljana", "Which city does Tomas return to?"),
25
+ ("Tomas", "loyalty", "Genius", "Which loyalty tier does Tomas hold?"),
26
+ ("Priya", "budget", "ninety euro", "What budget does Priya keep to?"),
27
+ ("Priya", "checkin", "late", "What check-in does Priya ask for?"),
28
+ ("Bram", "transport", "night train", "What transport does Bram prefer?"),
29
+ ("Bram", "room", "courtyard", "What room does Bram ask for?"),
30
+ ("Lena", "diet", "vegetarian", "What diet does Lena follow?"),
31
+ ("Lena", "trip", "walking", "What trip is Lena planning?"),
32
+ ("Sofia", "airport", "Schiphol", "Which airport does Sofia fly from?"),
33
+ ("Sofia", "luggage", "cabin only", "What luggage does Sofia travel with?"),
34
+ ("Ivan", "payment", "invoice", "What payment does Ivan use?"),
35
+ ("Ivan", "insurance", "annual", "What insurance does Ivan hold?"),
36
+ ("Mira", "language", "Portuguese", "What language does Mira speak?"),
37
+ ("Mira", "weekend", "shoulder season", "What weekend does Mira book?"),
38
+ )
39
+
40
+ CHAFF: tuple[str, ...] = (
41
+ "That sounds good, let me look at the dates again.",
42
+ "The weather looked fine for that week when I checked.",
43
+ "I will send the confirmation over once it comes through.",
44
+ "We can decide about the second night later.",
45
+ "It was busier than last time, but not by much.",
46
+ "I still have to sort out the trains on the way back.",
47
+ "No rush, whenever you get a moment.",
48
+ "Someone said the market is worth an early start.",
49
+ "I forgot to mention it earlier, sorry about that.",
50
+ "Fine by me either way, you choose.",
51
+ )
52
+
53
+ NAMED_CHAFF: tuple[str, ...] = (
54
+ "{person} said the same thing about the last trip.",
55
+ "I should check with {person} before anything is confirmed.",
56
+ "{person} was happy with how the previous booking went.",
57
+ "We spoke to {person} about the dates but nothing is fixed.",
58
+ )
59
+
60
+
61
+ def _statement(person: str, attribute: str, value: str) -> str:
62
+ return f"Note for the file: {person} has a {attribute} of {value}, so book accordingly."
63
+
64
+
65
+ def _near_miss(person: str, attribute: str) -> str:
66
+ """Same words as the question, no answer in it — what retrieval has to beat."""
67
+ return f"Note for the file: {person} raised the {attribute} again, nothing fixed yet."
68
+
69
+
70
+ def build_case(
71
+ case_id: str,
72
+ sessions: int,
73
+ turns_per_session: int,
74
+ rng: random.Random,
75
+ ) -> Case:
76
+ """One conversation with a fact planted in each session but the last."""
77
+ facts = list(FACTS)
78
+ rng.shuffle(facts)
79
+ planted = facts[: min(len(facts), sessions - 1)]
80
+
81
+ turns: list[Turn] = []
82
+ probes: list[Probe] = []
83
+ for session in range(1, sessions + 1):
84
+ fact_at = 1 + (session * 5) % turns_per_session
85
+ miss_at = {1 + (session * 7 + offset) % turns_per_session for offset in (3, 9, 15)} - {
86
+ fact_at
87
+ }
88
+ for position in range(1, turns_per_session + 1):
89
+ turn_id = f"D{session}:{position}"
90
+ speaker = "guest" if position % 2 else "agent"
91
+ if session <= len(planted) and position == fact_at:
92
+ person, attribute, value, question = planted[session - 1]
93
+ text = _statement(person, attribute, value)
94
+ probes.append(
95
+ Probe(
96
+ id=f"{case_id}-q{session}",
97
+ question=question,
98
+ evidence=(turn_id,),
99
+ answer=value,
100
+ tag=attribute,
101
+ )
102
+ )
103
+ elif session > 1 and position in miss_at:
104
+ person, attribute, _, _ = planted[rng.randrange(min(session - 1, len(planted)))]
105
+ text = _near_miss(person, attribute)
106
+ elif rng.random() < 0.25:
107
+ person = planted[rng.randrange(len(planted))][0]
108
+ text = rng.choice(NAMED_CHAFF).format(person=person)
109
+ else:
110
+ text = rng.choice(CHAFF)
111
+ turns.append(Turn(id=turn_id, session=session, speaker=speaker, text=text))
112
+ return Case(transcript=Transcript(id=case_id, turns=tuple(turns)), probes=tuple(probes))
113
+
114
+
115
+ def build_cases(
116
+ count: int = 8,
117
+ sessions: int = 16,
118
+ turns_per_session: int = 24,
119
+ seed: int = 7,
120
+ ) -> list[Case]:
121
+ """The fixture used by ``retainkit demo``."""
122
+ rng = random.Random(seed)
123
+ return [build_case(f"case-{i + 1}", sessions, turns_per_session, rng) for i in range(count)]
retainkit/evaluate.py ADDED
@@ -0,0 +1,134 @@
1
+ """Run policies over conversations and collect the sweep."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Sequence
6
+ from dataclasses import dataclass, field
7
+
8
+ from .metrics import Outcome, PolicyResult, summarise
9
+ from .policies import DEFAULT_POLICIES, Policy, turn_fragments
10
+ from .policies.base import spend
11
+ from .tokens import Tokenizer, char4
12
+ from .types import Probe, Retained, Transcript
13
+
14
+ #: Budgets swept unless the caller names their own, in tokens.
15
+ DEFAULT_BUDGETS: tuple[int, ...] = (256, 512, 1024, 2048, 4096, 8192)
16
+
17
+
18
+ @dataclass(frozen=True, slots=True)
19
+ class Case:
20
+ """One conversation and the probes asked against it."""
21
+
22
+ transcript: Transcript
23
+ probes: tuple[Probe, ...]
24
+
25
+
26
+ @dataclass(frozen=True, slots=True)
27
+ class Report:
28
+ """Every row of a sweep, plus what it was run over."""
29
+
30
+ rows: tuple[PolicyResult, ...]
31
+ cases: int
32
+ probes: int
33
+ full_tokens: float
34
+ budgets: tuple[int, ...]
35
+ tokenizer: str
36
+ notes: dict[str, str] = field(default_factory=dict)
37
+
38
+ def by_budget(self, budget: int) -> tuple[PolicyResult, ...]:
39
+ return tuple(r for r in self.rows if r.budget == budget)
40
+
41
+ def by_policy(self, policy: str) -> tuple[PolicyResult, ...]:
42
+ return tuple(r for r in self.rows if r.policy == policy)
43
+
44
+
45
+ def evidence_age(transcript: Transcript, probe: Probe) -> int:
46
+ """Sessions between the oldest evidence turn and the end of the conversation."""
47
+ sessions = [transcript.by_id(e).session for e in probe.evidence if _has(transcript, e)]
48
+ if not sessions:
49
+ return 0
50
+ return transcript.last_session - min(sessions)
51
+
52
+
53
+ def _has(transcript: Transcript, turn_id: str) -> bool:
54
+ try:
55
+ transcript.by_id(turn_id)
56
+ except KeyError:
57
+ return False
58
+ return True
59
+
60
+
61
+ def middle_share(retained: Retained, probe: Probe) -> float | None:
62
+ """How much of the surviving evidence sits in the middle half of the context.
63
+
64
+ Long-context models attend least well there, so a policy can retain the
65
+ evidence and still bury it. ``None`` when nothing survived.
66
+ """
67
+ positions = [retained.position_of(e) for e in probe.evidence]
68
+ found = [p for p in positions if p is not None]
69
+ if not found or len(retained.fragments) < 4:
70
+ return None
71
+ size = len(retained.fragments)
72
+ low, high = size * 0.25, size * 0.75
73
+ return sum(1 for p in found if low <= p <= high) / len(found)
74
+
75
+
76
+ def score(retained: Retained, transcript: Transcript, probe: Probe) -> Outcome:
77
+ """Turn one policy's selection into one measured outcome."""
78
+ evidence = set(probe.evidence)
79
+ kept = retained.source_ids & evidence
80
+ whole = retained.whole_source_ids & evidence
81
+ return Outcome(
82
+ probe_id=probe.id,
83
+ tag=probe.tag,
84
+ age=evidence_age(transcript, probe),
85
+ covered=kept == evidence,
86
+ covered_whole=whole == evidence,
87
+ partial=len(kept) / len(evidence),
88
+ tokens=retained.tokens,
89
+ middle=middle_share(retained, probe) if kept == evidence else None,
90
+ )
91
+
92
+
93
+ def run(
94
+ cases: Sequence[Case],
95
+ policies: Sequence[Policy] = DEFAULT_POLICIES,
96
+ budgets: Sequence[int] = DEFAULT_BUDGETS,
97
+ tokenizer: Tokenizer = char4,
98
+ seed: int = 0,
99
+ notes: dict[str, str] | None = None,
100
+ ) -> Report:
101
+ """Sweep every policy over every budget and summarise each combination."""
102
+ if not cases:
103
+ raise ValueError("nothing to evaluate")
104
+ full = [spend(turn_fragments(c.transcript), tokenizer) for c in cases]
105
+ full_tokens = sum(full) / len(full)
106
+
107
+ rows: list[PolicyResult] = []
108
+ probe_count = sum(len(c.probes) for c in cases)
109
+ for budget in budgets:
110
+ for policy in policies:
111
+ outcomes: list[Outcome] = []
112
+ for case in cases:
113
+ for probe in case.probes:
114
+ retained = policy.select(case.transcript, probe, budget, tokenizer)
115
+ outcomes.append(score(retained, case.transcript, probe))
116
+ rows.append(
117
+ summarise(
118
+ policy=policy.name,
119
+ budget=budget,
120
+ query_aware=policy.query_aware,
121
+ outcomes=outcomes,
122
+ full_tokens=full_tokens,
123
+ seed=seed,
124
+ )
125
+ )
126
+ return Report(
127
+ rows=tuple(rows),
128
+ cases=len(cases),
129
+ probes=probe_count,
130
+ full_tokens=full_tokens,
131
+ budgets=tuple(budgets),
132
+ tokenizer=getattr(tokenizer, "__name__", "custom"),
133
+ notes=notes or {},
134
+ )
retainkit/io.py ADDED
@@ -0,0 +1,88 @@
1
+ """Read conversations and probes from JSON, so the package works on your own data.
2
+
3
+ {"cases": [{"id": "trip-1",
4
+ "turns": [{"id": "D1:1", "session": 1, "speaker": "guest",
5
+ "text": "...", "timestamp": "2026-05-07"}],
6
+ "probes": [{"id": "q1", "question": "...", "evidence": ["D1:1"],
7
+ "answer": "...", "tag": "single-hop"}]}]}
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ from collections.abc import Iterable
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ from .evaluate import Case
18
+ from .types import Probe, Transcript, Turn
19
+
20
+
21
+ def case_from_dict(data: dict[str, Any]) -> Case:
22
+ turns = tuple(
23
+ Turn(
24
+ id=str(t["id"]),
25
+ session=int(t["session"]),
26
+ speaker=str(t.get("speaker", "")),
27
+ text=str(t["text"]),
28
+ timestamp=t.get("timestamp"),
29
+ )
30
+ for t in data["turns"]
31
+ )
32
+ known = {t.id for t in turns}
33
+ probes = []
34
+ for p in data.get("probes", []):
35
+ evidence = tuple(str(e) for e in p["evidence"] if str(e) in known)
36
+ if not evidence:
37
+ continue
38
+ probes.append(
39
+ Probe(
40
+ id=str(p["id"]),
41
+ question=str(p["question"]),
42
+ evidence=evidence,
43
+ answer=p.get("answer"),
44
+ tag=p.get("tag"),
45
+ )
46
+ )
47
+ return Case(
48
+ transcript=Transcript(id=str(data.get("id", "case")), turns=turns),
49
+ probes=tuple(probes),
50
+ )
51
+
52
+
53
+ def load_cases(path: str | Path) -> list[Case]:
54
+ payload = json.loads(Path(path).read_text(encoding="utf-8"))
55
+ cases = payload["cases"] if isinstance(payload, dict) else payload
56
+ return [case_from_dict(c) for c in cases]
57
+
58
+
59
+ def dump_cases(cases: Iterable[Case], path: str | Path) -> None:
60
+ payload = {
61
+ "cases": [
62
+ {
63
+ "id": case.transcript.id,
64
+ "turns": [
65
+ {
66
+ "id": t.id,
67
+ "session": t.session,
68
+ "speaker": t.speaker,
69
+ "text": t.text,
70
+ "timestamp": t.timestamp,
71
+ }
72
+ for t in case.transcript.turns
73
+ ],
74
+ "probes": [
75
+ {
76
+ "id": p.id,
77
+ "question": p.question,
78
+ "evidence": list(p.evidence),
79
+ "answer": p.answer,
80
+ "tag": p.tag,
81
+ }
82
+ for p in case.probes
83
+ ],
84
+ }
85
+ for case in cases
86
+ ]
87
+ }
88
+ Path(path).write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
retainkit/metrics.py ADDED
@@ -0,0 +1,140 @@
1
+ """What is measured, and how.
2
+
3
+ Every number here is about survival: whether the turns that support an answer
4
+ are still in the context a policy hands over. That is a necessary condition for
5
+ a correct answer, not a sufficient one — a model can be handed the evidence and
6
+ still get the answer wrong. Read these as an upper bound on what any model
7
+ behind the policy could do.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import random
13
+ from collections.abc import Sequence
14
+ from dataclasses import dataclass, field
15
+
16
+ #: Buckets for how many sessions back a probe's oldest evidence sits.
17
+ AGE_BUCKETS: tuple[tuple[str, int, int | None], ...] = (
18
+ ("same session", 0, 0),
19
+ ("1-2 back", 1, 2),
20
+ ("3-5 back", 3, 5),
21
+ ("6-10 back", 6, 10),
22
+ ("11+ back", 11, None),
23
+ )
24
+
25
+
26
+ def bucket_for(age: int) -> str:
27
+ for name, low, high in AGE_BUCKETS:
28
+ if age >= low and (high is None or age <= high):
29
+ return name
30
+ raise ValueError(f"no bucket for age {age}")
31
+
32
+
33
+ @dataclass(frozen=True, slots=True)
34
+ class Outcome:
35
+ """One probe, under one policy, at one budget."""
36
+
37
+ probe_id: str
38
+ tag: str | None
39
+ age: int
40
+ covered: bool
41
+ covered_whole: bool
42
+ partial: float
43
+ tokens: int
44
+ middle: float | None
45
+
46
+
47
+ @dataclass(frozen=True, slots=True)
48
+ class PolicyResult:
49
+ """One row of the sweep."""
50
+
51
+ policy: str
52
+ budget: int
53
+ query_aware: bool
54
+ probes: int
55
+ recall: float
56
+ recall_low: float
57
+ recall_high: float
58
+ whole_recall: float
59
+ partial: float
60
+ tokens: float
61
+ compression: float
62
+ middle_share: float
63
+ recall_by_age: dict[str, float] = field(default_factory=dict)
64
+ probes_by_age: dict[str, int] = field(default_factory=dict)
65
+ recall_by_tag: dict[str, float] = field(default_factory=dict)
66
+
67
+
68
+ def bootstrap_ci(
69
+ values: Sequence[float],
70
+ seed: int = 0,
71
+ resamples: int = 1000,
72
+ level: float = 0.95,
73
+ ) -> tuple[float, float]:
74
+ """Percentile interval on a mean, from a seeded resample."""
75
+ if not values:
76
+ return (0.0, 0.0)
77
+ rng = random.Random(seed)
78
+ size = len(values)
79
+ means = []
80
+ for _ in range(resamples):
81
+ total = 0.0
82
+ for _ in range(size):
83
+ total += values[rng.randrange(size)]
84
+ means.append(total / size)
85
+ means.sort()
86
+ tail = (1.0 - level) / 2.0
87
+ low = means[int(tail * (resamples - 1))]
88
+ high = means[int((1.0 - tail) * (resamples - 1))]
89
+ return (low, high)
90
+
91
+
92
+ def summarise(
93
+ policy: str,
94
+ budget: int,
95
+ query_aware: bool,
96
+ outcomes: Sequence[Outcome],
97
+ full_tokens: float,
98
+ seed: int = 0,
99
+ ) -> PolicyResult:
100
+ """Fold a policy's per-probe outcomes into one row."""
101
+ if not outcomes:
102
+ raise ValueError("no outcomes to summarise")
103
+ covered = [1.0 if o.covered else 0.0 for o in outcomes]
104
+ low, high = bootstrap_ci(covered, seed=seed)
105
+ tokens = sum(o.tokens for o in outcomes) / len(outcomes)
106
+ middles = [o.middle for o in outcomes if o.middle is not None]
107
+
108
+ by_age: dict[str, list[float]] = {}
109
+ for outcome in outcomes:
110
+ by_age.setdefault(bucket_for(outcome.age), []).append(1.0 if outcome.covered else 0.0)
111
+ by_tag: dict[str, list[float]] = {}
112
+ for outcome in outcomes:
113
+ if outcome.tag is not None:
114
+ by_tag.setdefault(outcome.tag, []).append(1.0 if outcome.covered else 0.0)
115
+
116
+ return PolicyResult(
117
+ policy=policy,
118
+ budget=budget,
119
+ query_aware=query_aware,
120
+ probes=len(outcomes),
121
+ recall=sum(covered) / len(covered),
122
+ recall_low=low,
123
+ recall_high=high,
124
+ whole_recall=sum(1 for o in outcomes if o.covered_whole) / len(outcomes),
125
+ partial=sum(o.partial for o in outcomes) / len(outcomes),
126
+ tokens=tokens,
127
+ compression=(tokens / full_tokens) if full_tokens else 0.0,
128
+ middle_share=(sum(middles) / len(middles)) if middles else 0.0,
129
+ recall_by_age={k: sum(v) / len(v) for k, v in by_age.items()},
130
+ probes_by_age={k: len(v) for k, v in by_age.items()},
131
+ recall_by_tag={k: sum(v) / len(v) for k, v in by_tag.items()},
132
+ )
133
+
134
+
135
+ def budget_for_recall(rows: Sequence[PolicyResult], policy: str, target: float) -> int | None:
136
+ """The smallest swept budget at which a policy reaches a recall target."""
137
+ reached = sorted(
138
+ (r.budget for r in rows if r.policy == policy and r.recall >= target),
139
+ )
140
+ return reached[0] if reached else None