retainkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- retainkit/__init__.py +56 -0
- retainkit/bm25.py +69 -0
- retainkit/cli.py +73 -0
- retainkit/demo.py +123 -0
- retainkit/evaluate.py +134 -0
- retainkit/io.py +88 -0
- retainkit/metrics.py +140 -0
- retainkit/policies/__init__.py +35 -0
- retainkit/policies/base.py +99 -0
- retainkit/policies/retrieval.py +125 -0
- retainkit/policies/simple.py +76 -0
- retainkit/policies/summary.py +72 -0
- retainkit/py.typed +0 -0
- retainkit/report.py +115 -0
- retainkit/tokens.py +38 -0
- retainkit/types.py +140 -0
- retainkit-0.1.0.dist-info/METADATA +215 -0
- retainkit-0.1.0.dist-info/RECORD +21 -0
- retainkit-0.1.0.dist-info/WHEEL +4 -0
- retainkit-0.1.0.dist-info/entry_points.txt +2 -0
- retainkit-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""The context policies this package compares."""
|
|
2
|
+
|
|
3
|
+
from .base import ORDER_STRIDE, Policy, fill, package, sentence_fragments, spend, turn_fragments
|
|
4
|
+
from .retrieval import FactMemory, Oracle, Retrieval
|
|
5
|
+
from .simple import HeadTail, KeepAll, Recency
|
|
6
|
+
from .summary import SessionSummary, lead
|
|
7
|
+
|
|
8
|
+
#: The set reported by ``retainkit demo`` and by the LoCoMo example.
|
|
9
|
+
DEFAULT_POLICIES: tuple[Policy, ...] = (
|
|
10
|
+
Recency(),
|
|
11
|
+
HeadTail(),
|
|
12
|
+
SessionSummary(),
|
|
13
|
+
Retrieval(),
|
|
14
|
+
FactMemory(),
|
|
15
|
+
Oracle(),
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"DEFAULT_POLICIES",
|
|
20
|
+
"ORDER_STRIDE",
|
|
21
|
+
"FactMemory",
|
|
22
|
+
"HeadTail",
|
|
23
|
+
"KeepAll",
|
|
24
|
+
"Oracle",
|
|
25
|
+
"Policy",
|
|
26
|
+
"Recency",
|
|
27
|
+
"Retrieval",
|
|
28
|
+
"SessionSummary",
|
|
29
|
+
"fill",
|
|
30
|
+
"lead",
|
|
31
|
+
"package",
|
|
32
|
+
"sentence_fragments",
|
|
33
|
+
"spend",
|
|
34
|
+
"turn_fragments",
|
|
35
|
+
]
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""What every context policy has to provide, and the budget arithmetic they share."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterable, Sequence
|
|
6
|
+
from typing import Protocol, runtime_checkable
|
|
7
|
+
|
|
8
|
+
from ..tokens import Tokenizer, char4, cost
|
|
9
|
+
from ..types import Fragment, Probe, Retained, Transcript, chronological
|
|
10
|
+
|
|
11
|
+
#: Sentences of one turn are ordered inside the slot reserved for that turn.
|
|
12
|
+
ORDER_STRIDE = 1000
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def turn_fragments(transcript: Transcript) -> list[Fragment]:
|
|
16
|
+
"""One whole-turn fragment per turn, in conversation order."""
|
|
17
|
+
return [
|
|
18
|
+
Fragment(
|
|
19
|
+
source_id=turn.id,
|
|
20
|
+
session=turn.session,
|
|
21
|
+
order=index * ORDER_STRIDE,
|
|
22
|
+
text=turn.text,
|
|
23
|
+
whole=True,
|
|
24
|
+
)
|
|
25
|
+
for index, turn in enumerate(transcript.turns)
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def sentence_fragments(transcript: Transcript) -> list[Fragment]:
|
|
30
|
+
"""One fragment per sentence, carrying the turn it came from."""
|
|
31
|
+
fragments: list[Fragment] = []
|
|
32
|
+
for index, turn in enumerate(transcript.turns):
|
|
33
|
+
sentences = turn.sentences()
|
|
34
|
+
whole = len(sentences) == 1
|
|
35
|
+
for ordinal, sentence in enumerate(sentences):
|
|
36
|
+
fragments.append(
|
|
37
|
+
Fragment(
|
|
38
|
+
source_id=turn.id,
|
|
39
|
+
session=turn.session,
|
|
40
|
+
order=index * ORDER_STRIDE + min(ordinal, ORDER_STRIDE - 1),
|
|
41
|
+
text=sentence,
|
|
42
|
+
whole=whole,
|
|
43
|
+
)
|
|
44
|
+
)
|
|
45
|
+
return fragments
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def fill(
|
|
49
|
+
candidates: Iterable[Fragment],
|
|
50
|
+
budget: int,
|
|
51
|
+
tokenizer: Tokenizer = char4,
|
|
52
|
+
) -> tuple[list[Fragment], int]:
|
|
53
|
+
"""Take fragments in the order offered, skipping any that no longer fit.
|
|
54
|
+
|
|
55
|
+
Skipping rather than stopping means a long fragment does not shut the door
|
|
56
|
+
on the shorter ones behind it. It is greedy, and it is deterministic.
|
|
57
|
+
"""
|
|
58
|
+
kept: list[Fragment] = []
|
|
59
|
+
spent = 0
|
|
60
|
+
for fragment in candidates:
|
|
61
|
+
price = cost(fragment.text, tokenizer)
|
|
62
|
+
if spent + price > budget:
|
|
63
|
+
continue
|
|
64
|
+
kept.append(fragment)
|
|
65
|
+
spent += price
|
|
66
|
+
return kept, spent
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def spend(fragments: Sequence[Fragment], tokenizer: Tokenizer = char4) -> int:
|
|
70
|
+
return sum(cost(f.text, tokenizer) for f in fragments)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@runtime_checkable
|
|
74
|
+
class Policy(Protocol):
|
|
75
|
+
"""Decides what part of a conversation the model still gets to see."""
|
|
76
|
+
|
|
77
|
+
#: Name used in reports.
|
|
78
|
+
name: str
|
|
79
|
+
#: Whether the policy is allowed to look at the incoming question.
|
|
80
|
+
query_aware: bool
|
|
81
|
+
|
|
82
|
+
def select(
|
|
83
|
+
self,
|
|
84
|
+
transcript: Transcript,
|
|
85
|
+
probe: Probe,
|
|
86
|
+
budget: int,
|
|
87
|
+
tokenizer: Tokenizer = char4,
|
|
88
|
+
) -> Retained: ...
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def package(
|
|
92
|
+
name: str,
|
|
93
|
+
budget: int,
|
|
94
|
+
fragments: Sequence[Fragment],
|
|
95
|
+
tokenizer: Tokenizer = char4,
|
|
96
|
+
) -> Retained:
|
|
97
|
+
"""Put a selection back in conversation order and price it."""
|
|
98
|
+
ordered = chronological(fragments)
|
|
99
|
+
return Retained(policy=name, budget=budget, fragments=ordered, tokens=spend(ordered, tokenizer))
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Policies that read the incoming question before deciding what to keep."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
from ..bm25 import BM25
|
|
8
|
+
from ..tokens import Tokenizer, char4
|
|
9
|
+
from ..types import Fragment, Probe, Retained, Transcript
|
|
10
|
+
from .base import fill, package, sentence_fragments, turn_fragments
|
|
11
|
+
|
|
12
|
+
Index = tuple[list[Fragment], BM25]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _key(transcript: Transcript) -> tuple[str, int]:
|
|
16
|
+
"""Cache key. Transcripts are immutable, and an id names one conversation."""
|
|
17
|
+
return (transcript.id, len(transcript.turns))
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class Retrieval:
|
|
22
|
+
"""Rank whole turns against the question and keep the best ones that fit.
|
|
23
|
+
|
|
24
|
+
Selection is out of order; the kept turns are put back in conversation
|
|
25
|
+
order before the model sees them.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
name: str = "retrieval"
|
|
29
|
+
query_aware: bool = True
|
|
30
|
+
cache: dict[tuple[str, int], Index] = field(default_factory=dict, repr=False, compare=False)
|
|
31
|
+
|
|
32
|
+
def _index(self, transcript: Transcript) -> Index:
|
|
33
|
+
"""Build the index once per conversation, as a served system would."""
|
|
34
|
+
hit = self.cache.get(_key(transcript))
|
|
35
|
+
if hit is None:
|
|
36
|
+
fragments = turn_fragments(transcript)
|
|
37
|
+
hit = (fragments, BM25([f.text for f in fragments]))
|
|
38
|
+
self.cache[_key(transcript)] = hit
|
|
39
|
+
return hit
|
|
40
|
+
|
|
41
|
+
def select(
|
|
42
|
+
self,
|
|
43
|
+
transcript: Transcript,
|
|
44
|
+
probe: Probe,
|
|
45
|
+
budget: int,
|
|
46
|
+
tokenizer: Tokenizer = char4,
|
|
47
|
+
) -> Retained:
|
|
48
|
+
fragments, index = self._index(transcript)
|
|
49
|
+
ranked = [fragments[i] for i in index.ranked(probe.question)]
|
|
50
|
+
kept, _ = fill(ranked, budget, tokenizer)
|
|
51
|
+
return package(self.name, budget, kept, tokenizer)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class FactMemory:
|
|
56
|
+
"""A long-term store beside the live session.
|
|
57
|
+
|
|
58
|
+
Everything before the most recent session is broken into sentences and kept
|
|
59
|
+
in a store that is searched when a question arrives; the most recent session
|
|
60
|
+
stays in front of the model verbatim. ``recent_share`` is the part of the
|
|
61
|
+
budget the live session gets, and whatever it does not use goes to the store.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
recent_share: float = 0.4
|
|
65
|
+
name: str = "fact_memory"
|
|
66
|
+
query_aware: bool = True
|
|
67
|
+
cache: dict[tuple[str, int], Index] = field(default_factory=dict, repr=False, compare=False)
|
|
68
|
+
|
|
69
|
+
def _index(self, transcript: Transcript) -> Index:
|
|
70
|
+
"""The store is built once per conversation and searched per question."""
|
|
71
|
+
hit = self.cache.get(_key(transcript))
|
|
72
|
+
if hit is None:
|
|
73
|
+
current = transcript.last_session
|
|
74
|
+
stored = [f for f in sentence_fragments(transcript) if f.session != current]
|
|
75
|
+
hit = (stored, BM25([f.text for f in stored]))
|
|
76
|
+
self.cache[_key(transcript)] = hit
|
|
77
|
+
return hit
|
|
78
|
+
|
|
79
|
+
def __post_init__(self) -> None:
|
|
80
|
+
if not 0.0 <= self.recent_share <= 1.0:
|
|
81
|
+
raise ValueError("recent_share must be between 0 and 1")
|
|
82
|
+
|
|
83
|
+
def select(
|
|
84
|
+
self,
|
|
85
|
+
transcript: Transcript,
|
|
86
|
+
probe: Probe,
|
|
87
|
+
budget: int,
|
|
88
|
+
tokenizer: Tokenizer = char4,
|
|
89
|
+
) -> Retained:
|
|
90
|
+
if not transcript.turns:
|
|
91
|
+
return package(self.name, budget, [], tokenizer)
|
|
92
|
+
current = transcript.last_session
|
|
93
|
+
live = [f for f in turn_fragments(transcript) if f.session == current]
|
|
94
|
+
recent, recent_spend = fill(reversed(live), int(budget * self.recent_share), tokenizer)
|
|
95
|
+
|
|
96
|
+
stored, index = self._index(transcript)
|
|
97
|
+
if not stored:
|
|
98
|
+
return package(self.name, budget, recent, tokenizer)
|
|
99
|
+
ranked = [stored[i] for i in index.ranked(probe.question)]
|
|
100
|
+
recalled, _ = fill(ranked, budget - recent_spend, tokenizer)
|
|
101
|
+
return package(self.name, budget, recent + recalled, tokenizer)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@dataclass
|
|
105
|
+
class Oracle:
|
|
106
|
+
"""Keep the evidence and nothing else: the cheapest context that can work.
|
|
107
|
+
|
|
108
|
+
It cannot be run in production — it reads the answer key — but it is the
|
|
109
|
+
floor every other policy is priced against.
|
|
110
|
+
"""
|
|
111
|
+
|
|
112
|
+
name: str = "oracle"
|
|
113
|
+
query_aware: bool = True
|
|
114
|
+
|
|
115
|
+
def select(
|
|
116
|
+
self,
|
|
117
|
+
transcript: Transcript,
|
|
118
|
+
probe: Probe,
|
|
119
|
+
budget: int,
|
|
120
|
+
tokenizer: Tokenizer = char4,
|
|
121
|
+
) -> Retained:
|
|
122
|
+
wanted = set(probe.evidence)
|
|
123
|
+
fragments = [f for f in turn_fragments(transcript) if f.source_id in wanted]
|
|
124
|
+
kept, _ = fill(fragments, budget, tokenizer)
|
|
125
|
+
return package(self.name, budget, kept, tokenizer)
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Policies that do not look at the question: keep everything, keep the end, keep both ends."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
from ..tokens import Tokenizer, char4
|
|
8
|
+
from ..types import Probe, Retained, Transcript
|
|
9
|
+
from .base import fill, package, turn_fragments
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class KeepAll:
|
|
14
|
+
"""No management at all. Useful as the reference cost of a conversation."""
|
|
15
|
+
|
|
16
|
+
name: str = "keep_all"
|
|
17
|
+
query_aware: bool = False
|
|
18
|
+
|
|
19
|
+
def select(
|
|
20
|
+
self,
|
|
21
|
+
transcript: Transcript,
|
|
22
|
+
probe: Probe,
|
|
23
|
+
budget: int,
|
|
24
|
+
tokenizer: Tokenizer = char4,
|
|
25
|
+
) -> Retained:
|
|
26
|
+
return package(self.name, budget, turn_fragments(transcript), tokenizer)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class Recency:
|
|
31
|
+
"""The sliding window every agent framework starts with: keep the newest turns."""
|
|
32
|
+
|
|
33
|
+
name: str = "recency"
|
|
34
|
+
query_aware: bool = False
|
|
35
|
+
|
|
36
|
+
def select(
|
|
37
|
+
self,
|
|
38
|
+
transcript: Transcript,
|
|
39
|
+
probe: Probe,
|
|
40
|
+
budget: int,
|
|
41
|
+
tokenizer: Tokenizer = char4,
|
|
42
|
+
) -> Retained:
|
|
43
|
+
kept, _ = fill(reversed(turn_fragments(transcript)), budget, tokenizer)
|
|
44
|
+
return package(self.name, budget, kept, tokenizer)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass
|
|
48
|
+
class HeadTail:
|
|
49
|
+
"""Keep the opening of the conversation and its most recent turns.
|
|
50
|
+
|
|
51
|
+
The opening usually carries the standing facts about a user, which a plain
|
|
52
|
+
window drops first.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
head_share: float = 0.3
|
|
56
|
+
name: str = "head_tail"
|
|
57
|
+
query_aware: bool = False
|
|
58
|
+
|
|
59
|
+
def __post_init__(self) -> None:
|
|
60
|
+
if not 0.0 <= self.head_share <= 1.0:
|
|
61
|
+
raise ValueError("head_share must be between 0 and 1")
|
|
62
|
+
|
|
63
|
+
def select(
|
|
64
|
+
self,
|
|
65
|
+
transcript: Transcript,
|
|
66
|
+
probe: Probe,
|
|
67
|
+
budget: int,
|
|
68
|
+
tokenizer: Tokenizer = char4,
|
|
69
|
+
) -> Retained:
|
|
70
|
+
fragments = turn_fragments(transcript)
|
|
71
|
+
head_budget = int(budget * self.head_share)
|
|
72
|
+
head, head_spend = fill(fragments, head_budget, tokenizer)
|
|
73
|
+
taken = {f.order for f in head}
|
|
74
|
+
tail_candidates = [f for f in reversed(fragments) if f.order not in taken]
|
|
75
|
+
tail, _ = fill(tail_candidates, budget - head_spend, tokenizer)
|
|
76
|
+
return package(self.name, budget, head + tail, tokenizer)
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Session summaries, without a model in the loop.
|
|
2
|
+
|
|
3
|
+
An abstractive summariser rewrites what it keeps, and a rewritten sentence
|
|
4
|
+
cannot be traced back to the turn it came from, so this package cannot score
|
|
5
|
+
one: every metric here asks whether a named span survived. The default is
|
|
6
|
+
therefore extractive — the opening sentences of each closed session, the
|
|
7
|
+
oldest and most standard compression baseline there is. Pass a ``selector`` to
|
|
8
|
+
swap in your own extractive scheme.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from collections.abc import Callable, Sequence
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
|
|
16
|
+
from ..tokens import Tokenizer, char4
|
|
17
|
+
from ..types import Fragment, Probe, Retained, Transcript
|
|
18
|
+
from .base import fill, package, sentence_fragments, turn_fragments
|
|
19
|
+
|
|
20
|
+
Selector = Callable[[Sequence[Fragment], int], list[Fragment]]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def lead(fragments: Sequence[Fragment], per_session: int) -> list[Fragment]:
|
|
24
|
+
"""The first ``per_session`` sentences of a session, in order."""
|
|
25
|
+
return list(fragments[:per_session])
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class SessionSummary:
|
|
30
|
+
"""Summarise every closed session, keep the live one verbatim.
|
|
31
|
+
|
|
32
|
+
The budget is split: ``recent_share`` for the session in progress, the rest
|
|
33
|
+
spread evenly over the closed sessions so that an old session is not
|
|
34
|
+
crowded out by a recent one.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
per_session: int = 3
|
|
38
|
+
recent_share: float = 0.4
|
|
39
|
+
selector: Selector = field(default=lead)
|
|
40
|
+
name: str = "session_summary"
|
|
41
|
+
query_aware: bool = False
|
|
42
|
+
|
|
43
|
+
def select(
|
|
44
|
+
self,
|
|
45
|
+
transcript: Transcript,
|
|
46
|
+
probe: Probe,
|
|
47
|
+
budget: int,
|
|
48
|
+
tokenizer: Tokenizer = char4,
|
|
49
|
+
) -> Retained:
|
|
50
|
+
if not transcript.turns:
|
|
51
|
+
return package(self.name, budget, [], tokenizer)
|
|
52
|
+
current = transcript.last_session
|
|
53
|
+
live = [f for f in turn_fragments(transcript) if f.session == current]
|
|
54
|
+
recent, recent_spend = fill(reversed(live), int(budget * self.recent_share), tokenizer)
|
|
55
|
+
|
|
56
|
+
closed = [s for s in transcript.sessions if s != current]
|
|
57
|
+
remaining = budget - recent_spend
|
|
58
|
+
if not closed or remaining <= 0:
|
|
59
|
+
return package(self.name, budget, recent, tokenizer)
|
|
60
|
+
|
|
61
|
+
sentences = sentence_fragments(transcript)
|
|
62
|
+
share = remaining // len(closed)
|
|
63
|
+
kept: list[Fragment] = []
|
|
64
|
+
carried = remaining - share * len(closed)
|
|
65
|
+
for session in closed:
|
|
66
|
+
in_session = [f for f in sentences if f.session == session]
|
|
67
|
+
picked, spent = fill(
|
|
68
|
+
self.selector(in_session, self.per_session), share + carried, tokenizer
|
|
69
|
+
)
|
|
70
|
+
kept.extend(picked)
|
|
71
|
+
carried = share + carried - spent
|
|
72
|
+
return package(self.name, budget, recent + kept, tokenizer)
|
retainkit/py.typed
ADDED
|
File without changes
|
retainkit/report.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Turn a sweep into a table someone can read and a file a test can diff."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
from dataclasses import asdict
|
|
8
|
+
|
|
9
|
+
from .evaluate import Report
|
|
10
|
+
from .metrics import AGE_BUCKETS, PolicyResult, budget_for_recall
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _pct(value: float) -> str:
|
|
14
|
+
return f"{value * 100:.1f}%"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def sweep_table(report: Report) -> str:
|
|
18
|
+
"""Recall and cost for every policy at every budget."""
|
|
19
|
+
lines = [
|
|
20
|
+
"| Budget | Policy | Sees question | Recall | 95% CI | Whole turns | Partial | Tokens |",
|
|
21
|
+
"| ---: | --- | :---: | ---: | --- | ---: | ---: | ---: |",
|
|
22
|
+
]
|
|
23
|
+
for budget in report.budgets:
|
|
24
|
+
for row in sorted(report.by_budget(budget), key=lambda r: (-r.recall, r.policy)):
|
|
25
|
+
lines.append(
|
|
26
|
+
f"| {row.budget} | `{row.policy}` | {'yes' if row.query_aware else 'no'} "
|
|
27
|
+
f"| {_pct(row.recall)} | {_pct(row.recall_low)}–{_pct(row.recall_high)} "
|
|
28
|
+
f"| {_pct(row.whole_recall)} | {_pct(row.partial)} | {row.tokens:.0f} |"
|
|
29
|
+
)
|
|
30
|
+
return "\n".join(lines)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def age_table(report: Report, budget: int) -> str:
|
|
34
|
+
"""Recall split by how many sessions back the evidence sits."""
|
|
35
|
+
rows = sorted(report.by_budget(budget), key=lambda r: (-r.recall, r.policy))
|
|
36
|
+
names = [name for name, _, _ in AGE_BUCKETS if any(name in r.probes_by_age for r in rows)]
|
|
37
|
+
header = "| Policy | " + " | ".join(names) + " |"
|
|
38
|
+
rule = "| --- | " + " | ".join("---:" for _ in names) + " |"
|
|
39
|
+
lines = [header, rule]
|
|
40
|
+
for row in rows:
|
|
41
|
+
cells = [_pct(row.recall_by_age[n]) if n in row.recall_by_age else "—" for n in names]
|
|
42
|
+
lines.append(f"| `{row.policy}` | " + " | ".join(cells) + " |")
|
|
43
|
+
counts = [str(rows[0].probes_by_age.get(n, 0)) for n in names]
|
|
44
|
+
lines.append("| _probes_ | " + " | ".join(counts) + " |")
|
|
45
|
+
return "\n".join(lines)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def frontier_table(report: Report, targets: Sequence[float] = (0.5, 0.8, 0.9)) -> str:
|
|
49
|
+
"""The cheapest swept budget at which each policy reaches a recall target."""
|
|
50
|
+
policies = sorted({r.policy for r in report.rows})
|
|
51
|
+
header = "| Policy | " + " | ".join(f"{int(t * 100)}% recall" for t in targets) + " |"
|
|
52
|
+
rule = "| --- | " + " | ".join("---:" for _ in targets) + " |"
|
|
53
|
+
lines = [header, rule]
|
|
54
|
+
for policy in policies:
|
|
55
|
+
cells = []
|
|
56
|
+
for target in targets:
|
|
57
|
+
budget = budget_for_recall(report.rows, policy, target)
|
|
58
|
+
cells.append(f"{budget}" if budget is not None else "not reached")
|
|
59
|
+
lines.append(f"| `{policy}` | " + " | ".join(cells) + " |")
|
|
60
|
+
return "\n".join(lines)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def markdown(report: Report, title: str, budget_focus: int | None = None) -> str:
|
|
64
|
+
"""The full written report."""
|
|
65
|
+
focus = budget_focus if budget_focus is not None else report.budgets[len(report.budgets) // 2]
|
|
66
|
+
parts = [
|
|
67
|
+
f"# {title}",
|
|
68
|
+
"",
|
|
69
|
+
f"{report.cases} conversations, {report.probes} probes, "
|
|
70
|
+
f"{report.full_tokens:.0f} tokens of conversation on average "
|
|
71
|
+
f"(`{report.tokenizer}` estimator).",
|
|
72
|
+
"",
|
|
73
|
+
]
|
|
74
|
+
for key, value in sorted(report.notes.items()):
|
|
75
|
+
parts.append(f"- **{key}**: {value}")
|
|
76
|
+
if report.notes:
|
|
77
|
+
parts.append("")
|
|
78
|
+
parts += [
|
|
79
|
+
"## Recall against budget",
|
|
80
|
+
"",
|
|
81
|
+
sweep_table(report),
|
|
82
|
+
"",
|
|
83
|
+
f"## Recall by how old the evidence is, at {focus} tokens",
|
|
84
|
+
"",
|
|
85
|
+
age_table(report, focus),
|
|
86
|
+
"",
|
|
87
|
+
"## Cheapest budget that reaches a recall target",
|
|
88
|
+
"",
|
|
89
|
+
frontier_table(report),
|
|
90
|
+
"",
|
|
91
|
+
]
|
|
92
|
+
return "\n".join(parts)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def to_json(report: Report) -> str:
|
|
96
|
+
payload = {
|
|
97
|
+
"cases": report.cases,
|
|
98
|
+
"probes": report.probes,
|
|
99
|
+
"full_tokens": round(report.full_tokens, 3),
|
|
100
|
+
"budgets": list(report.budgets),
|
|
101
|
+
"tokenizer": report.tokenizer,
|
|
102
|
+
"notes": report.notes,
|
|
103
|
+
"rows": [_row(r) for r in report.rows],
|
|
104
|
+
}
|
|
105
|
+
return json.dumps(payload, indent=2, sort_keys=True) + "\n"
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _row(row: PolicyResult) -> dict[str, object]:
|
|
109
|
+
data = asdict(row)
|
|
110
|
+
for key, value in list(data.items()):
|
|
111
|
+
if isinstance(value, float):
|
|
112
|
+
data[key] = round(value, 4)
|
|
113
|
+
elif isinstance(value, dict):
|
|
114
|
+
data[key] = {k: (round(v, 4) if isinstance(v, float) else v) for k, v in value.items()}
|
|
115
|
+
return data
|
retainkit/tokens.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Token accounting.
|
|
2
|
+
|
|
3
|
+
The default estimator is the usual four-characters-per-token approximation. It
|
|
4
|
+
is an estimate, not a tokenizer: absolute numbers move if you swap it. On the
|
|
5
|
+
fixture, changing estimator moves one policy's recall by 22 points, so treat any
|
|
6
|
+
single figure here as approximate and pass your model's own tokenizer through
|
|
7
|
+
``Tokenizer`` when it has to be exact. What does not move is the comparison:
|
|
8
|
+
``tests/test_tokens.py`` holds the two estimators to the same verdict about
|
|
9
|
+
which policies beat which.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import math
|
|
15
|
+
import re
|
|
16
|
+
from collections.abc import Callable
|
|
17
|
+
|
|
18
|
+
Tokenizer = Callable[[str], int]
|
|
19
|
+
|
|
20
|
+
#: Tokens charged for the speaker label and separators around every fragment.
|
|
21
|
+
FRAGMENT_OVERHEAD = 4
|
|
22
|
+
|
|
23
|
+
_WORD = re.compile(r"\w+|[^\w\s]")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def char4(text: str) -> int:
|
|
27
|
+
"""Four characters per token, the common rule of thumb for English prose."""
|
|
28
|
+
return max(1, math.ceil(len(text) / 4))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def wordish(text: str) -> int:
|
|
32
|
+
"""Words and punctuation, inflated by the usual 4/3 sub-word factor."""
|
|
33
|
+
return max(1, math.ceil(len(_WORD.findall(text)) * 4 / 3))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def cost(text: str, tokenizer: Tokenizer = char4, overhead: int = FRAGMENT_OVERHEAD) -> int:
|
|
37
|
+
"""What one fragment costs once its speaker label and separators are counted."""
|
|
38
|
+
return tokenizer(text) + overhead
|