retainkit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,35 @@
1
+ """The context policies this package compares."""
2
+
3
+ from .base import ORDER_STRIDE, Policy, fill, package, sentence_fragments, spend, turn_fragments
4
+ from .retrieval import FactMemory, Oracle, Retrieval
5
+ from .simple import HeadTail, KeepAll, Recency
6
+ from .summary import SessionSummary, lead
7
+
8
+ #: The set reported by ``retainkit demo`` and by the LoCoMo example.
9
+ DEFAULT_POLICIES: tuple[Policy, ...] = (
10
+ Recency(),
11
+ HeadTail(),
12
+ SessionSummary(),
13
+ Retrieval(),
14
+ FactMemory(),
15
+ Oracle(),
16
+ )
17
+
18
+ __all__ = [
19
+ "DEFAULT_POLICIES",
20
+ "ORDER_STRIDE",
21
+ "FactMemory",
22
+ "HeadTail",
23
+ "KeepAll",
24
+ "Oracle",
25
+ "Policy",
26
+ "Recency",
27
+ "Retrieval",
28
+ "SessionSummary",
29
+ "fill",
30
+ "lead",
31
+ "package",
32
+ "sentence_fragments",
33
+ "spend",
34
+ "turn_fragments",
35
+ ]
@@ -0,0 +1,99 @@
1
+ """What every context policy has to provide, and the budget arithmetic they share."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Iterable, Sequence
6
+ from typing import Protocol, runtime_checkable
7
+
8
+ from ..tokens import Tokenizer, char4, cost
9
+ from ..types import Fragment, Probe, Retained, Transcript, chronological
10
+
11
+ #: Sentences of one turn are ordered inside the slot reserved for that turn.
12
+ ORDER_STRIDE = 1000
13
+
14
+
15
+ def turn_fragments(transcript: Transcript) -> list[Fragment]:
16
+ """One whole-turn fragment per turn, in conversation order."""
17
+ return [
18
+ Fragment(
19
+ source_id=turn.id,
20
+ session=turn.session,
21
+ order=index * ORDER_STRIDE,
22
+ text=turn.text,
23
+ whole=True,
24
+ )
25
+ for index, turn in enumerate(transcript.turns)
26
+ ]
27
+
28
+
29
+ def sentence_fragments(transcript: Transcript) -> list[Fragment]:
30
+ """One fragment per sentence, carrying the turn it came from."""
31
+ fragments: list[Fragment] = []
32
+ for index, turn in enumerate(transcript.turns):
33
+ sentences = turn.sentences()
34
+ whole = len(sentences) == 1
35
+ for ordinal, sentence in enumerate(sentences):
36
+ fragments.append(
37
+ Fragment(
38
+ source_id=turn.id,
39
+ session=turn.session,
40
+ order=index * ORDER_STRIDE + min(ordinal, ORDER_STRIDE - 1),
41
+ text=sentence,
42
+ whole=whole,
43
+ )
44
+ )
45
+ return fragments
46
+
47
+
48
+ def fill(
49
+ candidates: Iterable[Fragment],
50
+ budget: int,
51
+ tokenizer: Tokenizer = char4,
52
+ ) -> tuple[list[Fragment], int]:
53
+ """Take fragments in the order offered, skipping any that no longer fit.
54
+
55
+ Skipping rather than stopping means a long fragment does not shut the door
56
+ on the shorter ones behind it. It is greedy, and it is deterministic.
57
+ """
58
+ kept: list[Fragment] = []
59
+ spent = 0
60
+ for fragment in candidates:
61
+ price = cost(fragment.text, tokenizer)
62
+ if spent + price > budget:
63
+ continue
64
+ kept.append(fragment)
65
+ spent += price
66
+ return kept, spent
67
+
68
+
69
+ def spend(fragments: Sequence[Fragment], tokenizer: Tokenizer = char4) -> int:
70
+ return sum(cost(f.text, tokenizer) for f in fragments)
71
+
72
+
73
+ @runtime_checkable
74
+ class Policy(Protocol):
75
+ """Decides what part of a conversation the model still gets to see."""
76
+
77
+ #: Name used in reports.
78
+ name: str
79
+ #: Whether the policy is allowed to look at the incoming question.
80
+ query_aware: bool
81
+
82
+ def select(
83
+ self,
84
+ transcript: Transcript,
85
+ probe: Probe,
86
+ budget: int,
87
+ tokenizer: Tokenizer = char4,
88
+ ) -> Retained: ...
89
+
90
+
91
+ def package(
92
+ name: str,
93
+ budget: int,
94
+ fragments: Sequence[Fragment],
95
+ tokenizer: Tokenizer = char4,
96
+ ) -> Retained:
97
+ """Put a selection back in conversation order and price it."""
98
+ ordered = chronological(fragments)
99
+ return Retained(policy=name, budget=budget, fragments=ordered, tokens=spend(ordered, tokenizer))
@@ -0,0 +1,125 @@
1
+ """Policies that read the incoming question before deciding what to keep."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass, field
6
+
7
+ from ..bm25 import BM25
8
+ from ..tokens import Tokenizer, char4
9
+ from ..types import Fragment, Probe, Retained, Transcript
10
+ from .base import fill, package, sentence_fragments, turn_fragments
11
+
12
+ Index = tuple[list[Fragment], BM25]
13
+
14
+
15
+ def _key(transcript: Transcript) -> tuple[str, int]:
16
+ """Cache key. Transcripts are immutable, and an id names one conversation."""
17
+ return (transcript.id, len(transcript.turns))
18
+
19
+
20
+ @dataclass
21
+ class Retrieval:
22
+ """Rank whole turns against the question and keep the best ones that fit.
23
+
24
+ Selection is out of order; the kept turns are put back in conversation
25
+ order before the model sees them.
26
+ """
27
+
28
+ name: str = "retrieval"
29
+ query_aware: bool = True
30
+ cache: dict[tuple[str, int], Index] = field(default_factory=dict, repr=False, compare=False)
31
+
32
+ def _index(self, transcript: Transcript) -> Index:
33
+ """Build the index once per conversation, as a served system would."""
34
+ hit = self.cache.get(_key(transcript))
35
+ if hit is None:
36
+ fragments = turn_fragments(transcript)
37
+ hit = (fragments, BM25([f.text for f in fragments]))
38
+ self.cache[_key(transcript)] = hit
39
+ return hit
40
+
41
+ def select(
42
+ self,
43
+ transcript: Transcript,
44
+ probe: Probe,
45
+ budget: int,
46
+ tokenizer: Tokenizer = char4,
47
+ ) -> Retained:
48
+ fragments, index = self._index(transcript)
49
+ ranked = [fragments[i] for i in index.ranked(probe.question)]
50
+ kept, _ = fill(ranked, budget, tokenizer)
51
+ return package(self.name, budget, kept, tokenizer)
52
+
53
+
54
+ @dataclass
55
+ class FactMemory:
56
+ """A long-term store beside the live session.
57
+
58
+ Everything before the most recent session is broken into sentences and kept
59
+ in a store that is searched when a question arrives; the most recent session
60
+ stays in front of the model verbatim. ``recent_share`` is the part of the
61
+ budget the live session gets, and whatever it does not use goes to the store.
62
+ """
63
+
64
+ recent_share: float = 0.4
65
+ name: str = "fact_memory"
66
+ query_aware: bool = True
67
+ cache: dict[tuple[str, int], Index] = field(default_factory=dict, repr=False, compare=False)
68
+
69
+ def _index(self, transcript: Transcript) -> Index:
70
+ """The store is built once per conversation and searched per question."""
71
+ hit = self.cache.get(_key(transcript))
72
+ if hit is None:
73
+ current = transcript.last_session
74
+ stored = [f for f in sentence_fragments(transcript) if f.session != current]
75
+ hit = (stored, BM25([f.text for f in stored]))
76
+ self.cache[_key(transcript)] = hit
77
+ return hit
78
+
79
+ def __post_init__(self) -> None:
80
+ if not 0.0 <= self.recent_share <= 1.0:
81
+ raise ValueError("recent_share must be between 0 and 1")
82
+
83
+ def select(
84
+ self,
85
+ transcript: Transcript,
86
+ probe: Probe,
87
+ budget: int,
88
+ tokenizer: Tokenizer = char4,
89
+ ) -> Retained:
90
+ if not transcript.turns:
91
+ return package(self.name, budget, [], tokenizer)
92
+ current = transcript.last_session
93
+ live = [f for f in turn_fragments(transcript) if f.session == current]
94
+ recent, recent_spend = fill(reversed(live), int(budget * self.recent_share), tokenizer)
95
+
96
+ stored, index = self._index(transcript)
97
+ if not stored:
98
+ return package(self.name, budget, recent, tokenizer)
99
+ ranked = [stored[i] for i in index.ranked(probe.question)]
100
+ recalled, _ = fill(ranked, budget - recent_spend, tokenizer)
101
+ return package(self.name, budget, recent + recalled, tokenizer)
102
+
103
+
104
+ @dataclass
105
+ class Oracle:
106
+ """Keep the evidence and nothing else: the cheapest context that can work.
107
+
108
+ It cannot be run in production — it reads the answer key — but it is the
109
+ floor every other policy is priced against.
110
+ """
111
+
112
+ name: str = "oracle"
113
+ query_aware: bool = True
114
+
115
+ def select(
116
+ self,
117
+ transcript: Transcript,
118
+ probe: Probe,
119
+ budget: int,
120
+ tokenizer: Tokenizer = char4,
121
+ ) -> Retained:
122
+ wanted = set(probe.evidence)
123
+ fragments = [f for f in turn_fragments(transcript) if f.source_id in wanted]
124
+ kept, _ = fill(fragments, budget, tokenizer)
125
+ return package(self.name, budget, kept, tokenizer)
@@ -0,0 +1,76 @@
1
+ """Policies that do not look at the question: keep everything, keep the end, keep both ends."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+
7
+ from ..tokens import Tokenizer, char4
8
+ from ..types import Probe, Retained, Transcript
9
+ from .base import fill, package, turn_fragments
10
+
11
+
12
+ @dataclass
13
+ class KeepAll:
14
+ """No management at all. Useful as the reference cost of a conversation."""
15
+
16
+ name: str = "keep_all"
17
+ query_aware: bool = False
18
+
19
+ def select(
20
+ self,
21
+ transcript: Transcript,
22
+ probe: Probe,
23
+ budget: int,
24
+ tokenizer: Tokenizer = char4,
25
+ ) -> Retained:
26
+ return package(self.name, budget, turn_fragments(transcript), tokenizer)
27
+
28
+
29
+ @dataclass
30
+ class Recency:
31
+ """The sliding window every agent framework starts with: keep the newest turns."""
32
+
33
+ name: str = "recency"
34
+ query_aware: bool = False
35
+
36
+ def select(
37
+ self,
38
+ transcript: Transcript,
39
+ probe: Probe,
40
+ budget: int,
41
+ tokenizer: Tokenizer = char4,
42
+ ) -> Retained:
43
+ kept, _ = fill(reversed(turn_fragments(transcript)), budget, tokenizer)
44
+ return package(self.name, budget, kept, tokenizer)
45
+
46
+
47
+ @dataclass
48
+ class HeadTail:
49
+ """Keep the opening of the conversation and its most recent turns.
50
+
51
+ The opening usually carries the standing facts about a user, which a plain
52
+ window drops first.
53
+ """
54
+
55
+ head_share: float = 0.3
56
+ name: str = "head_tail"
57
+ query_aware: bool = False
58
+
59
+ def __post_init__(self) -> None:
60
+ if not 0.0 <= self.head_share <= 1.0:
61
+ raise ValueError("head_share must be between 0 and 1")
62
+
63
+ def select(
64
+ self,
65
+ transcript: Transcript,
66
+ probe: Probe,
67
+ budget: int,
68
+ tokenizer: Tokenizer = char4,
69
+ ) -> Retained:
70
+ fragments = turn_fragments(transcript)
71
+ head_budget = int(budget * self.head_share)
72
+ head, head_spend = fill(fragments, head_budget, tokenizer)
73
+ taken = {f.order for f in head}
74
+ tail_candidates = [f for f in reversed(fragments) if f.order not in taken]
75
+ tail, _ = fill(tail_candidates, budget - head_spend, tokenizer)
76
+ return package(self.name, budget, head + tail, tokenizer)
@@ -0,0 +1,72 @@
1
+ """Session summaries, without a model in the loop.
2
+
3
+ An abstractive summariser rewrites what it keeps, and a rewritten sentence
4
+ cannot be traced back to the turn it came from, so this package cannot score
5
+ one: every metric here asks whether a named span survived. The default is
6
+ therefore extractive — the opening sentences of each closed session, the
7
+ oldest and most standard compression baseline there is. Pass a ``selector`` to
8
+ swap in your own extractive scheme.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from collections.abc import Callable, Sequence
14
+ from dataclasses import dataclass, field
15
+
16
+ from ..tokens import Tokenizer, char4
17
+ from ..types import Fragment, Probe, Retained, Transcript
18
+ from .base import fill, package, sentence_fragments, turn_fragments
19
+
20
+ Selector = Callable[[Sequence[Fragment], int], list[Fragment]]
21
+
22
+
23
+ def lead(fragments: Sequence[Fragment], per_session: int) -> list[Fragment]:
24
+ """The first ``per_session`` sentences of a session, in order."""
25
+ return list(fragments[:per_session])
26
+
27
+
28
+ @dataclass
29
+ class SessionSummary:
30
+ """Summarise every closed session, keep the live one verbatim.
31
+
32
+ The budget is split: ``recent_share`` for the session in progress, the rest
33
+ spread evenly over the closed sessions so that an old session is not
34
+ crowded out by a recent one.
35
+ """
36
+
37
+ per_session: int = 3
38
+ recent_share: float = 0.4
39
+ selector: Selector = field(default=lead)
40
+ name: str = "session_summary"
41
+ query_aware: bool = False
42
+
43
+ def select(
44
+ self,
45
+ transcript: Transcript,
46
+ probe: Probe,
47
+ budget: int,
48
+ tokenizer: Tokenizer = char4,
49
+ ) -> Retained:
50
+ if not transcript.turns:
51
+ return package(self.name, budget, [], tokenizer)
52
+ current = transcript.last_session
53
+ live = [f for f in turn_fragments(transcript) if f.session == current]
54
+ recent, recent_spend = fill(reversed(live), int(budget * self.recent_share), tokenizer)
55
+
56
+ closed = [s for s in transcript.sessions if s != current]
57
+ remaining = budget - recent_spend
58
+ if not closed or remaining <= 0:
59
+ return package(self.name, budget, recent, tokenizer)
60
+
61
+ sentences = sentence_fragments(transcript)
62
+ share = remaining // len(closed)
63
+ kept: list[Fragment] = []
64
+ carried = remaining - share * len(closed)
65
+ for session in closed:
66
+ in_session = [f for f in sentences if f.session == session]
67
+ picked, spent = fill(
68
+ self.selector(in_session, self.per_session), share + carried, tokenizer
69
+ )
70
+ kept.extend(picked)
71
+ carried = share + carried - spent
72
+ return package(self.name, budget, recent + kept, tokenizer)
retainkit/py.typed ADDED
File without changes
retainkit/report.py ADDED
@@ -0,0 +1,115 @@
1
+ """Turn a sweep into a table someone can read and a file a test can diff."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from collections.abc import Sequence
7
+ from dataclasses import asdict
8
+
9
+ from .evaluate import Report
10
+ from .metrics import AGE_BUCKETS, PolicyResult, budget_for_recall
11
+
12
+
13
+ def _pct(value: float) -> str:
14
+ return f"{value * 100:.1f}%"
15
+
16
+
17
+ def sweep_table(report: Report) -> str:
18
+ """Recall and cost for every policy at every budget."""
19
+ lines = [
20
+ "| Budget | Policy | Sees question | Recall | 95% CI | Whole turns | Partial | Tokens |",
21
+ "| ---: | --- | :---: | ---: | --- | ---: | ---: | ---: |",
22
+ ]
23
+ for budget in report.budgets:
24
+ for row in sorted(report.by_budget(budget), key=lambda r: (-r.recall, r.policy)):
25
+ lines.append(
26
+ f"| {row.budget} | `{row.policy}` | {'yes' if row.query_aware else 'no'} "
27
+ f"| {_pct(row.recall)} | {_pct(row.recall_low)}–{_pct(row.recall_high)} "
28
+ f"| {_pct(row.whole_recall)} | {_pct(row.partial)} | {row.tokens:.0f} |"
29
+ )
30
+ return "\n".join(lines)
31
+
32
+
33
+ def age_table(report: Report, budget: int) -> str:
34
+ """Recall split by how many sessions back the evidence sits."""
35
+ rows = sorted(report.by_budget(budget), key=lambda r: (-r.recall, r.policy))
36
+ names = [name for name, _, _ in AGE_BUCKETS if any(name in r.probes_by_age for r in rows)]
37
+ header = "| Policy | " + " | ".join(names) + " |"
38
+ rule = "| --- | " + " | ".join("---:" for _ in names) + " |"
39
+ lines = [header, rule]
40
+ for row in rows:
41
+ cells = [_pct(row.recall_by_age[n]) if n in row.recall_by_age else "—" for n in names]
42
+ lines.append(f"| `{row.policy}` | " + " | ".join(cells) + " |")
43
+ counts = [str(rows[0].probes_by_age.get(n, 0)) for n in names]
44
+ lines.append("| _probes_ | " + " | ".join(counts) + " |")
45
+ return "\n".join(lines)
46
+
47
+
48
+ def frontier_table(report: Report, targets: Sequence[float] = (0.5, 0.8, 0.9)) -> str:
49
+ """The cheapest swept budget at which each policy reaches a recall target."""
50
+ policies = sorted({r.policy for r in report.rows})
51
+ header = "| Policy | " + " | ".join(f"{int(t * 100)}% recall" for t in targets) + " |"
52
+ rule = "| --- | " + " | ".join("---:" for _ in targets) + " |"
53
+ lines = [header, rule]
54
+ for policy in policies:
55
+ cells = []
56
+ for target in targets:
57
+ budget = budget_for_recall(report.rows, policy, target)
58
+ cells.append(f"{budget}" if budget is not None else "not reached")
59
+ lines.append(f"| `{policy}` | " + " | ".join(cells) + " |")
60
+ return "\n".join(lines)
61
+
62
+
63
+ def markdown(report: Report, title: str, budget_focus: int | None = None) -> str:
64
+ """The full written report."""
65
+ focus = budget_focus if budget_focus is not None else report.budgets[len(report.budgets) // 2]
66
+ parts = [
67
+ f"# {title}",
68
+ "",
69
+ f"{report.cases} conversations, {report.probes} probes, "
70
+ f"{report.full_tokens:.0f} tokens of conversation on average "
71
+ f"(`{report.tokenizer}` estimator).",
72
+ "",
73
+ ]
74
+ for key, value in sorted(report.notes.items()):
75
+ parts.append(f"- **{key}**: {value}")
76
+ if report.notes:
77
+ parts.append("")
78
+ parts += [
79
+ "## Recall against budget",
80
+ "",
81
+ sweep_table(report),
82
+ "",
83
+ f"## Recall by how old the evidence is, at {focus} tokens",
84
+ "",
85
+ age_table(report, focus),
86
+ "",
87
+ "## Cheapest budget that reaches a recall target",
88
+ "",
89
+ frontier_table(report),
90
+ "",
91
+ ]
92
+ return "\n".join(parts)
93
+
94
+
95
+ def to_json(report: Report) -> str:
96
+ payload = {
97
+ "cases": report.cases,
98
+ "probes": report.probes,
99
+ "full_tokens": round(report.full_tokens, 3),
100
+ "budgets": list(report.budgets),
101
+ "tokenizer": report.tokenizer,
102
+ "notes": report.notes,
103
+ "rows": [_row(r) for r in report.rows],
104
+ }
105
+ return json.dumps(payload, indent=2, sort_keys=True) + "\n"
106
+
107
+
108
+ def _row(row: PolicyResult) -> dict[str, object]:
109
+ data = asdict(row)
110
+ for key, value in list(data.items()):
111
+ if isinstance(value, float):
112
+ data[key] = round(value, 4)
113
+ elif isinstance(value, dict):
114
+ data[key] = {k: (round(v, 4) if isinstance(v, float) else v) for k, v in value.items()}
115
+ return data
retainkit/tokens.py ADDED
@@ -0,0 +1,38 @@
1
+ """Token accounting.
2
+
3
+ The default estimator is the usual four-characters-per-token approximation. It
4
+ is an estimate, not a tokenizer: absolute numbers move if you swap it. On the
5
+ fixture, changing estimator moves one policy's recall by 22 points, so treat any
6
+ single figure here as approximate and pass your model's own tokenizer through
7
+ ``Tokenizer`` when it has to be exact. What does not move is the comparison:
8
+ ``tests/test_tokens.py`` holds the two estimators to the same verdict about
9
+ which policies beat which.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import math
15
+ import re
16
+ from collections.abc import Callable
17
+
18
+ Tokenizer = Callable[[str], int]
19
+
20
+ #: Tokens charged for the speaker label and separators around every fragment.
21
+ FRAGMENT_OVERHEAD = 4
22
+
23
+ _WORD = re.compile(r"\w+|[^\w\s]")
24
+
25
+
26
+ def char4(text: str) -> int:
27
+ """Four characters per token, the common rule of thumb for English prose."""
28
+ return max(1, math.ceil(len(text) / 4))
29
+
30
+
31
+ def wordish(text: str) -> int:
32
+ """Words and punctuation, inflated by the usual 4/3 sub-word factor."""
33
+ return max(1, math.ceil(len(_WORD.findall(text)) * 4 / 3))
34
+
35
+
36
+ def cost(text: str, tokenizer: Tokenizer = char4, overhead: int = FRAGMENT_OVERHEAD) -> int:
37
+ """What one fragment costs once its speaker label and separators are counted."""
38
+ return tokenizer(text) + overhead