retainkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- retainkit/__init__.py +56 -0
- retainkit/bm25.py +69 -0
- retainkit/cli.py +73 -0
- retainkit/demo.py +123 -0
- retainkit/evaluate.py +134 -0
- retainkit/io.py +88 -0
- retainkit/metrics.py +140 -0
- retainkit/policies/__init__.py +35 -0
- retainkit/policies/base.py +99 -0
- retainkit/policies/retrieval.py +125 -0
- retainkit/policies/simple.py +76 -0
- retainkit/policies/summary.py +72 -0
- retainkit/py.typed +0 -0
- retainkit/report.py +115 -0
- retainkit/tokens.py +38 -0
- retainkit/types.py +140 -0
- retainkit-0.1.0.dist-info/METADATA +215 -0
- retainkit-0.1.0.dist-info/RECORD +21 -0
- retainkit-0.1.0.dist-info/WHEEL +4 -0
- retainkit-0.1.0.dist-info/entry_points.txt +2 -0
- retainkit-0.1.0.dist-info/licenses/LICENSE +21 -0
retainkit/__init__.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Context and memory policies for LLM agents, scored by what survives the budget."""
|
|
2
|
+
|
|
3
|
+
from .evaluate import DEFAULT_BUDGETS, Case, Report, run, score
|
|
4
|
+
from .io import dump_cases, load_cases
|
|
5
|
+
from .metrics import Outcome, PolicyResult, budget_for_recall
|
|
6
|
+
from .policies import (
|
|
7
|
+
DEFAULT_POLICIES,
|
|
8
|
+
FactMemory,
|
|
9
|
+
HeadTail,
|
|
10
|
+
KeepAll,
|
|
11
|
+
Oracle,
|
|
12
|
+
Policy,
|
|
13
|
+
Recency,
|
|
14
|
+
Retrieval,
|
|
15
|
+
SessionSummary,
|
|
16
|
+
)
|
|
17
|
+
from .report import age_table, frontier_table, markdown, sweep_table, to_json
|
|
18
|
+
from .tokens import char4, wordish
|
|
19
|
+
from .types import Fragment, Probe, Retained, Transcript, Turn
|
|
20
|
+
|
|
21
|
+
__version__ = "0.1.0"
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"DEFAULT_BUDGETS",
|
|
25
|
+
"DEFAULT_POLICIES",
|
|
26
|
+
"Case",
|
|
27
|
+
"FactMemory",
|
|
28
|
+
"Fragment",
|
|
29
|
+
"HeadTail",
|
|
30
|
+
"KeepAll",
|
|
31
|
+
"Oracle",
|
|
32
|
+
"Outcome",
|
|
33
|
+
"Policy",
|
|
34
|
+
"PolicyResult",
|
|
35
|
+
"Probe",
|
|
36
|
+
"Recency",
|
|
37
|
+
"Report",
|
|
38
|
+
"Retained",
|
|
39
|
+
"Retrieval",
|
|
40
|
+
"SessionSummary",
|
|
41
|
+
"Transcript",
|
|
42
|
+
"Turn",
|
|
43
|
+
"__version__",
|
|
44
|
+
"age_table",
|
|
45
|
+
"budget_for_recall",
|
|
46
|
+
"char4",
|
|
47
|
+
"dump_cases",
|
|
48
|
+
"frontier_table",
|
|
49
|
+
"load_cases",
|
|
50
|
+
"markdown",
|
|
51
|
+
"run",
|
|
52
|
+
"score",
|
|
53
|
+
"sweep_table",
|
|
54
|
+
"to_json",
|
|
55
|
+
"wordish",
|
|
56
|
+
]
|
retainkit/bm25.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Okapi BM25 over short documents, standard library only.
|
|
2
|
+
|
|
3
|
+
Ties are broken by document index so that a ranking is reproducible on any
|
|
4
|
+
machine and any Python build.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
import re
|
|
11
|
+
from collections import Counter
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
|
|
15
|
+
_TOKEN = re.compile(r"[a-z0-9]+")
|
|
16
|
+
|
|
17
|
+
K1 = 1.5
|
|
18
|
+
B = 0.75
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def tokenize(text: str) -> list[str]:
|
|
22
|
+
return _TOKEN.findall(text.lower())
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class BM25:
|
|
27
|
+
"""A small BM25 index built once and queried many times."""
|
|
28
|
+
|
|
29
|
+
documents: Sequence[str]
|
|
30
|
+
k1: float = K1
|
|
31
|
+
b: float = B
|
|
32
|
+
|
|
33
|
+
def __post_init__(self) -> None:
|
|
34
|
+
self._tokens = [tokenize(d) for d in self.documents]
|
|
35
|
+
self._lengths = [len(t) for t in self._tokens]
|
|
36
|
+
self._avg_length = (sum(self._lengths) / len(self._lengths)) if self._lengths else 0.0
|
|
37
|
+
self._frequencies = [Counter(t) for t in self._tokens]
|
|
38
|
+
document_frequency: Counter[str] = Counter()
|
|
39
|
+
for counts in self._frequencies:
|
|
40
|
+
document_frequency.update(counts.keys())
|
|
41
|
+
total = len(self.documents)
|
|
42
|
+
self._idf = {
|
|
43
|
+
term: math.log(1 + (total - freq + 0.5) / (freq + 0.5))
|
|
44
|
+
for term, freq in document_frequency.items()
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
def scores(self, query: str) -> list[float]:
|
|
48
|
+
terms = tokenize(query)
|
|
49
|
+
results = [0.0] * len(self.documents)
|
|
50
|
+
if not terms or self._avg_length == 0.0:
|
|
51
|
+
return results
|
|
52
|
+
for index, counts in enumerate(self._frequencies):
|
|
53
|
+
length = self._lengths[index]
|
|
54
|
+
total = 0.0
|
|
55
|
+
for term in terms:
|
|
56
|
+
frequency = counts.get(term, 0)
|
|
57
|
+
if not frequency:
|
|
58
|
+
continue
|
|
59
|
+
denominator = frequency + self.k1 * (
|
|
60
|
+
1 - self.b + self.b * length / self._avg_length
|
|
61
|
+
)
|
|
62
|
+
total += self._idf[term] * frequency * (self.k1 + 1) / denominator
|
|
63
|
+
results[index] = total
|
|
64
|
+
return results
|
|
65
|
+
|
|
66
|
+
def ranked(self, query: str) -> list[int]:
|
|
67
|
+
"""Document indices, best first, ties broken by original order."""
|
|
68
|
+
scored = self.scores(query)
|
|
69
|
+
return sorted(range(len(scored)), key=lambda i: (-scored[i], i))
|
retainkit/cli.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Command line: run the fixture, or run a sweep over your own conversations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from .demo import build_cases
|
|
10
|
+
from .evaluate import DEFAULT_BUDGETS, run
|
|
11
|
+
from .io import load_cases
|
|
12
|
+
from .policies import DEFAULT_POLICIES
|
|
13
|
+
from .report import markdown, sweep_table, to_json
|
|
14
|
+
from .tokens import char4, wordish
|
|
15
|
+
|
|
16
|
+
TOKENIZERS = {"char4": char4, "wordish": wordish}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _write(report, out: Path, title: str, focus: int, stem: str) -> None:
|
|
20
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
21
|
+
(out / "table.md").write_text(sweep_table(report) + "\n", encoding="utf-8")
|
|
22
|
+
(out / "report.md").write_text(markdown(report, title, focus) + "\n", encoding="utf-8")
|
|
23
|
+
(out / f"{stem}.json").write_text(to_json(report), encoding="utf-8")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def main(argv: list[str] | None = None) -> int:
|
|
27
|
+
parser = argparse.ArgumentParser(prog="retainkit", description=__doc__)
|
|
28
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
29
|
+
|
|
30
|
+
demo = sub.add_parser("demo", help="run the built-in fixture and write results")
|
|
31
|
+
demo.add_argument("--out", default="results", type=Path)
|
|
32
|
+
demo.add_argument("--seed", default=7, type=int)
|
|
33
|
+
demo.add_argument("--tokenizer", default="char4", choices=sorted(TOKENIZERS))
|
|
34
|
+
|
|
35
|
+
evaluate = sub.add_parser("eval", help="sweep policies over conversations in a JSON file")
|
|
36
|
+
evaluate.add_argument("cases", type=Path)
|
|
37
|
+
evaluate.add_argument("--out", default=None, type=Path)
|
|
38
|
+
evaluate.add_argument("--title", default="retainkit sweep")
|
|
39
|
+
evaluate.add_argument("--tokenizer", default="char4", choices=sorted(TOKENIZERS))
|
|
40
|
+
evaluate.add_argument(
|
|
41
|
+
"--budget",
|
|
42
|
+
action="append",
|
|
43
|
+
type=int,
|
|
44
|
+
help="repeat to sweep several budgets; defaults to 256 … 8192",
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
args = parser.parse_args(argv)
|
|
48
|
+
tokenizer = TOKENIZERS[args.tokenizer]
|
|
49
|
+
|
|
50
|
+
if args.command == "demo":
|
|
51
|
+
cases = build_cases(seed=args.seed)
|
|
52
|
+
report = run(
|
|
53
|
+
cases,
|
|
54
|
+
DEFAULT_POLICIES,
|
|
55
|
+
DEFAULT_BUDGETS,
|
|
56
|
+
tokenizer=tokenizer,
|
|
57
|
+
notes={"source": "synthetic fixture, seed " + str(args.seed)},
|
|
58
|
+
)
|
|
59
|
+
_write(report, args.out, "retainkit fixture", 1024, "demo")
|
|
60
|
+
print(sweep_table(report))
|
|
61
|
+
return 0
|
|
62
|
+
|
|
63
|
+
cases = load_cases(args.cases)
|
|
64
|
+
budgets = tuple(args.budget) if args.budget else DEFAULT_BUDGETS
|
|
65
|
+
report = run(cases, DEFAULT_POLICIES, budgets, tokenizer=tokenizer)
|
|
66
|
+
if args.out:
|
|
67
|
+
_write(report, args.out, args.title, budgets[len(budgets) // 2], "sweep")
|
|
68
|
+
print(markdown(report, args.title, budgets[len(budgets) // 2]))
|
|
69
|
+
return 0
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
if __name__ == "__main__": # pragma: no cover
|
|
73
|
+
sys.exit(main())
|
retainkit/demo.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""A synthetic fixture, built from a fixed seed, that CI diffs.
|
|
2
|
+
|
|
3
|
+
It is not a benchmark. It exists so that a change to a policy or a metric shows
|
|
4
|
+
up as a moved number in `results/`, and so the package can be tried without
|
|
5
|
+
downloading anything. The measured claims in the README come from LoCoMo, in
|
|
6
|
+
`examples/locomo/`.
|
|
7
|
+
|
|
8
|
+
The shape is the one a travel assistant meets: a standing fact about a
|
|
9
|
+
traveller stated once, small talk that mentions the same people, near misses
|
|
10
|
+
that use the same words as the question without answering it, and the question
|
|
11
|
+
arriving many sessions later.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import random
|
|
17
|
+
|
|
18
|
+
from .evaluate import Case
|
|
19
|
+
from .types import Probe, Transcript, Turn
|
|
20
|
+
|
|
21
|
+
FACTS: tuple[tuple[str, str, str, str], ...] = (
|
|
22
|
+
("Nadia", "allergy", "shellfish", "What allergy does Nadia have?"),
|
|
23
|
+
("Nadia", "seat", "aisle", "Which seat does Nadia book?"),
|
|
24
|
+
("Tomas", "city", "Ljubljana", "Which city does Tomas return to?"),
|
|
25
|
+
("Tomas", "loyalty", "Genius", "Which loyalty tier does Tomas hold?"),
|
|
26
|
+
("Priya", "budget", "ninety euro", "What budget does Priya keep to?"),
|
|
27
|
+
("Priya", "checkin", "late", "What check-in does Priya ask for?"),
|
|
28
|
+
("Bram", "transport", "night train", "What transport does Bram prefer?"),
|
|
29
|
+
("Bram", "room", "courtyard", "What room does Bram ask for?"),
|
|
30
|
+
("Lena", "diet", "vegetarian", "What diet does Lena follow?"),
|
|
31
|
+
("Lena", "trip", "walking", "What trip is Lena planning?"),
|
|
32
|
+
("Sofia", "airport", "Schiphol", "Which airport does Sofia fly from?"),
|
|
33
|
+
("Sofia", "luggage", "cabin only", "What luggage does Sofia travel with?"),
|
|
34
|
+
("Ivan", "payment", "invoice", "What payment does Ivan use?"),
|
|
35
|
+
("Ivan", "insurance", "annual", "What insurance does Ivan hold?"),
|
|
36
|
+
("Mira", "language", "Portuguese", "What language does Mira speak?"),
|
|
37
|
+
("Mira", "weekend", "shoulder season", "What weekend does Mira book?"),
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
CHAFF: tuple[str, ...] = (
|
|
41
|
+
"That sounds good, let me look at the dates again.",
|
|
42
|
+
"The weather looked fine for that week when I checked.",
|
|
43
|
+
"I will send the confirmation over once it comes through.",
|
|
44
|
+
"We can decide about the second night later.",
|
|
45
|
+
"It was busier than last time, but not by much.",
|
|
46
|
+
"I still have to sort out the trains on the way back.",
|
|
47
|
+
"No rush, whenever you get a moment.",
|
|
48
|
+
"Someone said the market is worth an early start.",
|
|
49
|
+
"I forgot to mention it earlier, sorry about that.",
|
|
50
|
+
"Fine by me either way, you choose.",
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
NAMED_CHAFF: tuple[str, ...] = (
|
|
54
|
+
"{person} said the same thing about the last trip.",
|
|
55
|
+
"I should check with {person} before anything is confirmed.",
|
|
56
|
+
"{person} was happy with how the previous booking went.",
|
|
57
|
+
"We spoke to {person} about the dates but nothing is fixed.",
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _statement(person: str, attribute: str, value: str) -> str:
|
|
62
|
+
return f"Note for the file: {person} has a {attribute} of {value}, so book accordingly."
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _near_miss(person: str, attribute: str) -> str:
|
|
66
|
+
"""Same words as the question, no answer in it — what retrieval has to beat."""
|
|
67
|
+
return f"Note for the file: {person} raised the {attribute} again, nothing fixed yet."
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def build_case(
|
|
71
|
+
case_id: str,
|
|
72
|
+
sessions: int,
|
|
73
|
+
turns_per_session: int,
|
|
74
|
+
rng: random.Random,
|
|
75
|
+
) -> Case:
|
|
76
|
+
"""One conversation with a fact planted in each session but the last."""
|
|
77
|
+
facts = list(FACTS)
|
|
78
|
+
rng.shuffle(facts)
|
|
79
|
+
planted = facts[: min(len(facts), sessions - 1)]
|
|
80
|
+
|
|
81
|
+
turns: list[Turn] = []
|
|
82
|
+
probes: list[Probe] = []
|
|
83
|
+
for session in range(1, sessions + 1):
|
|
84
|
+
fact_at = 1 + (session * 5) % turns_per_session
|
|
85
|
+
miss_at = {1 + (session * 7 + offset) % turns_per_session for offset in (3, 9, 15)} - {
|
|
86
|
+
fact_at
|
|
87
|
+
}
|
|
88
|
+
for position in range(1, turns_per_session + 1):
|
|
89
|
+
turn_id = f"D{session}:{position}"
|
|
90
|
+
speaker = "guest" if position % 2 else "agent"
|
|
91
|
+
if session <= len(planted) and position == fact_at:
|
|
92
|
+
person, attribute, value, question = planted[session - 1]
|
|
93
|
+
text = _statement(person, attribute, value)
|
|
94
|
+
probes.append(
|
|
95
|
+
Probe(
|
|
96
|
+
id=f"{case_id}-q{session}",
|
|
97
|
+
question=question,
|
|
98
|
+
evidence=(turn_id,),
|
|
99
|
+
answer=value,
|
|
100
|
+
tag=attribute,
|
|
101
|
+
)
|
|
102
|
+
)
|
|
103
|
+
elif session > 1 and position in miss_at:
|
|
104
|
+
person, attribute, _, _ = planted[rng.randrange(min(session - 1, len(planted)))]
|
|
105
|
+
text = _near_miss(person, attribute)
|
|
106
|
+
elif rng.random() < 0.25:
|
|
107
|
+
person = planted[rng.randrange(len(planted))][0]
|
|
108
|
+
text = rng.choice(NAMED_CHAFF).format(person=person)
|
|
109
|
+
else:
|
|
110
|
+
text = rng.choice(CHAFF)
|
|
111
|
+
turns.append(Turn(id=turn_id, session=session, speaker=speaker, text=text))
|
|
112
|
+
return Case(transcript=Transcript(id=case_id, turns=tuple(turns)), probes=tuple(probes))
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def build_cases(
|
|
116
|
+
count: int = 8,
|
|
117
|
+
sessions: int = 16,
|
|
118
|
+
turns_per_session: int = 24,
|
|
119
|
+
seed: int = 7,
|
|
120
|
+
) -> list[Case]:
|
|
121
|
+
"""The fixture used by ``retainkit demo``."""
|
|
122
|
+
rng = random.Random(seed)
|
|
123
|
+
return [build_case(f"case-{i + 1}", sessions, turns_per_session, rng) for i in range(count)]
|
retainkit/evaluate.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Run policies over conversations and collect the sweep."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
|
|
8
|
+
from .metrics import Outcome, PolicyResult, summarise
|
|
9
|
+
from .policies import DEFAULT_POLICIES, Policy, turn_fragments
|
|
10
|
+
from .policies.base import spend
|
|
11
|
+
from .tokens import Tokenizer, char4
|
|
12
|
+
from .types import Probe, Retained, Transcript
|
|
13
|
+
|
|
14
|
+
#: Budgets swept unless the caller names their own, in tokens.
|
|
15
|
+
DEFAULT_BUDGETS: tuple[int, ...] = (256, 512, 1024, 2048, 4096, 8192)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True, slots=True)
|
|
19
|
+
class Case:
|
|
20
|
+
"""One conversation and the probes asked against it."""
|
|
21
|
+
|
|
22
|
+
transcript: Transcript
|
|
23
|
+
probes: tuple[Probe, ...]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True, slots=True)
|
|
27
|
+
class Report:
|
|
28
|
+
"""Every row of a sweep, plus what it was run over."""
|
|
29
|
+
|
|
30
|
+
rows: tuple[PolicyResult, ...]
|
|
31
|
+
cases: int
|
|
32
|
+
probes: int
|
|
33
|
+
full_tokens: float
|
|
34
|
+
budgets: tuple[int, ...]
|
|
35
|
+
tokenizer: str
|
|
36
|
+
notes: dict[str, str] = field(default_factory=dict)
|
|
37
|
+
|
|
38
|
+
def by_budget(self, budget: int) -> tuple[PolicyResult, ...]:
|
|
39
|
+
return tuple(r for r in self.rows if r.budget == budget)
|
|
40
|
+
|
|
41
|
+
def by_policy(self, policy: str) -> tuple[PolicyResult, ...]:
|
|
42
|
+
return tuple(r for r in self.rows if r.policy == policy)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def evidence_age(transcript: Transcript, probe: Probe) -> int:
|
|
46
|
+
"""Sessions between the oldest evidence turn and the end of the conversation."""
|
|
47
|
+
sessions = [transcript.by_id(e).session for e in probe.evidence if _has(transcript, e)]
|
|
48
|
+
if not sessions:
|
|
49
|
+
return 0
|
|
50
|
+
return transcript.last_session - min(sessions)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _has(transcript: Transcript, turn_id: str) -> bool:
|
|
54
|
+
try:
|
|
55
|
+
transcript.by_id(turn_id)
|
|
56
|
+
except KeyError:
|
|
57
|
+
return False
|
|
58
|
+
return True
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def middle_share(retained: Retained, probe: Probe) -> float | None:
|
|
62
|
+
"""How much of the surviving evidence sits in the middle half of the context.
|
|
63
|
+
|
|
64
|
+
Long-context models attend least well there, so a policy can retain the
|
|
65
|
+
evidence and still bury it. ``None`` when nothing survived.
|
|
66
|
+
"""
|
|
67
|
+
positions = [retained.position_of(e) for e in probe.evidence]
|
|
68
|
+
found = [p for p in positions if p is not None]
|
|
69
|
+
if not found or len(retained.fragments) < 4:
|
|
70
|
+
return None
|
|
71
|
+
size = len(retained.fragments)
|
|
72
|
+
low, high = size * 0.25, size * 0.75
|
|
73
|
+
return sum(1 for p in found if low <= p <= high) / len(found)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def score(retained: Retained, transcript: Transcript, probe: Probe) -> Outcome:
|
|
77
|
+
"""Turn one policy's selection into one measured outcome."""
|
|
78
|
+
evidence = set(probe.evidence)
|
|
79
|
+
kept = retained.source_ids & evidence
|
|
80
|
+
whole = retained.whole_source_ids & evidence
|
|
81
|
+
return Outcome(
|
|
82
|
+
probe_id=probe.id,
|
|
83
|
+
tag=probe.tag,
|
|
84
|
+
age=evidence_age(transcript, probe),
|
|
85
|
+
covered=kept == evidence,
|
|
86
|
+
covered_whole=whole == evidence,
|
|
87
|
+
partial=len(kept) / len(evidence),
|
|
88
|
+
tokens=retained.tokens,
|
|
89
|
+
middle=middle_share(retained, probe) if kept == evidence else None,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def run(
|
|
94
|
+
cases: Sequence[Case],
|
|
95
|
+
policies: Sequence[Policy] = DEFAULT_POLICIES,
|
|
96
|
+
budgets: Sequence[int] = DEFAULT_BUDGETS,
|
|
97
|
+
tokenizer: Tokenizer = char4,
|
|
98
|
+
seed: int = 0,
|
|
99
|
+
notes: dict[str, str] | None = None,
|
|
100
|
+
) -> Report:
|
|
101
|
+
"""Sweep every policy over every budget and summarise each combination."""
|
|
102
|
+
if not cases:
|
|
103
|
+
raise ValueError("nothing to evaluate")
|
|
104
|
+
full = [spend(turn_fragments(c.transcript), tokenizer) for c in cases]
|
|
105
|
+
full_tokens = sum(full) / len(full)
|
|
106
|
+
|
|
107
|
+
rows: list[PolicyResult] = []
|
|
108
|
+
probe_count = sum(len(c.probes) for c in cases)
|
|
109
|
+
for budget in budgets:
|
|
110
|
+
for policy in policies:
|
|
111
|
+
outcomes: list[Outcome] = []
|
|
112
|
+
for case in cases:
|
|
113
|
+
for probe in case.probes:
|
|
114
|
+
retained = policy.select(case.transcript, probe, budget, tokenizer)
|
|
115
|
+
outcomes.append(score(retained, case.transcript, probe))
|
|
116
|
+
rows.append(
|
|
117
|
+
summarise(
|
|
118
|
+
policy=policy.name,
|
|
119
|
+
budget=budget,
|
|
120
|
+
query_aware=policy.query_aware,
|
|
121
|
+
outcomes=outcomes,
|
|
122
|
+
full_tokens=full_tokens,
|
|
123
|
+
seed=seed,
|
|
124
|
+
)
|
|
125
|
+
)
|
|
126
|
+
return Report(
|
|
127
|
+
rows=tuple(rows),
|
|
128
|
+
cases=len(cases),
|
|
129
|
+
probes=probe_count,
|
|
130
|
+
full_tokens=full_tokens,
|
|
131
|
+
budgets=tuple(budgets),
|
|
132
|
+
tokenizer=getattr(tokenizer, "__name__", "custom"),
|
|
133
|
+
notes=notes or {},
|
|
134
|
+
)
|
retainkit/io.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Read conversations and probes from JSON, so the package works on your own data.
|
|
2
|
+
|
|
3
|
+
{"cases": [{"id": "trip-1",
|
|
4
|
+
"turns": [{"id": "D1:1", "session": 1, "speaker": "guest",
|
|
5
|
+
"text": "...", "timestamp": "2026-05-07"}],
|
|
6
|
+
"probes": [{"id": "q1", "question": "...", "evidence": ["D1:1"],
|
|
7
|
+
"answer": "...", "tag": "single-hop"}]}]}
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
from collections.abc import Iterable
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from .evaluate import Case
|
|
18
|
+
from .types import Probe, Transcript, Turn
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def case_from_dict(data: dict[str, Any]) -> Case:
|
|
22
|
+
turns = tuple(
|
|
23
|
+
Turn(
|
|
24
|
+
id=str(t["id"]),
|
|
25
|
+
session=int(t["session"]),
|
|
26
|
+
speaker=str(t.get("speaker", "")),
|
|
27
|
+
text=str(t["text"]),
|
|
28
|
+
timestamp=t.get("timestamp"),
|
|
29
|
+
)
|
|
30
|
+
for t in data["turns"]
|
|
31
|
+
)
|
|
32
|
+
known = {t.id for t in turns}
|
|
33
|
+
probes = []
|
|
34
|
+
for p in data.get("probes", []):
|
|
35
|
+
evidence = tuple(str(e) for e in p["evidence"] if str(e) in known)
|
|
36
|
+
if not evidence:
|
|
37
|
+
continue
|
|
38
|
+
probes.append(
|
|
39
|
+
Probe(
|
|
40
|
+
id=str(p["id"]),
|
|
41
|
+
question=str(p["question"]),
|
|
42
|
+
evidence=evidence,
|
|
43
|
+
answer=p.get("answer"),
|
|
44
|
+
tag=p.get("tag"),
|
|
45
|
+
)
|
|
46
|
+
)
|
|
47
|
+
return Case(
|
|
48
|
+
transcript=Transcript(id=str(data.get("id", "case")), turns=turns),
|
|
49
|
+
probes=tuple(probes),
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def load_cases(path: str | Path) -> list[Case]:
|
|
54
|
+
payload = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
55
|
+
cases = payload["cases"] if isinstance(payload, dict) else payload
|
|
56
|
+
return [case_from_dict(c) for c in cases]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def dump_cases(cases: Iterable[Case], path: str | Path) -> None:
|
|
60
|
+
payload = {
|
|
61
|
+
"cases": [
|
|
62
|
+
{
|
|
63
|
+
"id": case.transcript.id,
|
|
64
|
+
"turns": [
|
|
65
|
+
{
|
|
66
|
+
"id": t.id,
|
|
67
|
+
"session": t.session,
|
|
68
|
+
"speaker": t.speaker,
|
|
69
|
+
"text": t.text,
|
|
70
|
+
"timestamp": t.timestamp,
|
|
71
|
+
}
|
|
72
|
+
for t in case.transcript.turns
|
|
73
|
+
],
|
|
74
|
+
"probes": [
|
|
75
|
+
{
|
|
76
|
+
"id": p.id,
|
|
77
|
+
"question": p.question,
|
|
78
|
+
"evidence": list(p.evidence),
|
|
79
|
+
"answer": p.answer,
|
|
80
|
+
"tag": p.tag,
|
|
81
|
+
}
|
|
82
|
+
for p in case.probes
|
|
83
|
+
],
|
|
84
|
+
}
|
|
85
|
+
for case in cases
|
|
86
|
+
]
|
|
87
|
+
}
|
|
88
|
+
Path(path).write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|
retainkit/metrics.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""What is measured, and how.
|
|
2
|
+
|
|
3
|
+
Every number here is about survival: whether the turns that support an answer
|
|
4
|
+
are still in the context a policy hands over. That is a necessary condition for
|
|
5
|
+
a correct answer, not a sufficient one — a model can be handed the evidence and
|
|
6
|
+
still get the answer wrong. Read these as an upper bound on what any model
|
|
7
|
+
behind the policy could do.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import random
|
|
13
|
+
from collections.abc import Sequence
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
|
|
16
|
+
#: Buckets for how many sessions back a probe's oldest evidence sits.
|
|
17
|
+
AGE_BUCKETS: tuple[tuple[str, int, int | None], ...] = (
|
|
18
|
+
("same session", 0, 0),
|
|
19
|
+
("1-2 back", 1, 2),
|
|
20
|
+
("3-5 back", 3, 5),
|
|
21
|
+
("6-10 back", 6, 10),
|
|
22
|
+
("11+ back", 11, None),
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def bucket_for(age: int) -> str:
|
|
27
|
+
for name, low, high in AGE_BUCKETS:
|
|
28
|
+
if age >= low and (high is None or age <= high):
|
|
29
|
+
return name
|
|
30
|
+
raise ValueError(f"no bucket for age {age}")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True, slots=True)
|
|
34
|
+
class Outcome:
|
|
35
|
+
"""One probe, under one policy, at one budget."""
|
|
36
|
+
|
|
37
|
+
probe_id: str
|
|
38
|
+
tag: str | None
|
|
39
|
+
age: int
|
|
40
|
+
covered: bool
|
|
41
|
+
covered_whole: bool
|
|
42
|
+
partial: float
|
|
43
|
+
tokens: int
|
|
44
|
+
middle: float | None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True, slots=True)
|
|
48
|
+
class PolicyResult:
|
|
49
|
+
"""One row of the sweep."""
|
|
50
|
+
|
|
51
|
+
policy: str
|
|
52
|
+
budget: int
|
|
53
|
+
query_aware: bool
|
|
54
|
+
probes: int
|
|
55
|
+
recall: float
|
|
56
|
+
recall_low: float
|
|
57
|
+
recall_high: float
|
|
58
|
+
whole_recall: float
|
|
59
|
+
partial: float
|
|
60
|
+
tokens: float
|
|
61
|
+
compression: float
|
|
62
|
+
middle_share: float
|
|
63
|
+
recall_by_age: dict[str, float] = field(default_factory=dict)
|
|
64
|
+
probes_by_age: dict[str, int] = field(default_factory=dict)
|
|
65
|
+
recall_by_tag: dict[str, float] = field(default_factory=dict)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def bootstrap_ci(
|
|
69
|
+
values: Sequence[float],
|
|
70
|
+
seed: int = 0,
|
|
71
|
+
resamples: int = 1000,
|
|
72
|
+
level: float = 0.95,
|
|
73
|
+
) -> tuple[float, float]:
|
|
74
|
+
"""Percentile interval on a mean, from a seeded resample."""
|
|
75
|
+
if not values:
|
|
76
|
+
return (0.0, 0.0)
|
|
77
|
+
rng = random.Random(seed)
|
|
78
|
+
size = len(values)
|
|
79
|
+
means = []
|
|
80
|
+
for _ in range(resamples):
|
|
81
|
+
total = 0.0
|
|
82
|
+
for _ in range(size):
|
|
83
|
+
total += values[rng.randrange(size)]
|
|
84
|
+
means.append(total / size)
|
|
85
|
+
means.sort()
|
|
86
|
+
tail = (1.0 - level) / 2.0
|
|
87
|
+
low = means[int(tail * (resamples - 1))]
|
|
88
|
+
high = means[int((1.0 - tail) * (resamples - 1))]
|
|
89
|
+
return (low, high)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def summarise(
|
|
93
|
+
policy: str,
|
|
94
|
+
budget: int,
|
|
95
|
+
query_aware: bool,
|
|
96
|
+
outcomes: Sequence[Outcome],
|
|
97
|
+
full_tokens: float,
|
|
98
|
+
seed: int = 0,
|
|
99
|
+
) -> PolicyResult:
|
|
100
|
+
"""Fold a policy's per-probe outcomes into one row."""
|
|
101
|
+
if not outcomes:
|
|
102
|
+
raise ValueError("no outcomes to summarise")
|
|
103
|
+
covered = [1.0 if o.covered else 0.0 for o in outcomes]
|
|
104
|
+
low, high = bootstrap_ci(covered, seed=seed)
|
|
105
|
+
tokens = sum(o.tokens for o in outcomes) / len(outcomes)
|
|
106
|
+
middles = [o.middle for o in outcomes if o.middle is not None]
|
|
107
|
+
|
|
108
|
+
by_age: dict[str, list[float]] = {}
|
|
109
|
+
for outcome in outcomes:
|
|
110
|
+
by_age.setdefault(bucket_for(outcome.age), []).append(1.0 if outcome.covered else 0.0)
|
|
111
|
+
by_tag: dict[str, list[float]] = {}
|
|
112
|
+
for outcome in outcomes:
|
|
113
|
+
if outcome.tag is not None:
|
|
114
|
+
by_tag.setdefault(outcome.tag, []).append(1.0 if outcome.covered else 0.0)
|
|
115
|
+
|
|
116
|
+
return PolicyResult(
|
|
117
|
+
policy=policy,
|
|
118
|
+
budget=budget,
|
|
119
|
+
query_aware=query_aware,
|
|
120
|
+
probes=len(outcomes),
|
|
121
|
+
recall=sum(covered) / len(covered),
|
|
122
|
+
recall_low=low,
|
|
123
|
+
recall_high=high,
|
|
124
|
+
whole_recall=sum(1 for o in outcomes if o.covered_whole) / len(outcomes),
|
|
125
|
+
partial=sum(o.partial for o in outcomes) / len(outcomes),
|
|
126
|
+
tokens=tokens,
|
|
127
|
+
compression=(tokens / full_tokens) if full_tokens else 0.0,
|
|
128
|
+
middle_share=(sum(middles) / len(middles)) if middles else 0.0,
|
|
129
|
+
recall_by_age={k: sum(v) / len(v) for k, v in by_age.items()},
|
|
130
|
+
probes_by_age={k: len(v) for k, v in by_age.items()},
|
|
131
|
+
recall_by_tag={k: sum(v) / len(v) for k, v in by_tag.items()},
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def budget_for_recall(rows: Sequence[PolicyResult], policy: str, target: float) -> int | None:
|
|
136
|
+
"""The smallest swept budget at which a policy reaches a recall target."""
|
|
137
|
+
reached = sorted(
|
|
138
|
+
(r.budget for r in rows if r.policy == policy and r.recall >= target),
|
|
139
|
+
)
|
|
140
|
+
return reached[0] if reached else None
|