retainkit 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. retainkit-0.1.0/.github/FUNDING.yml +1 -0
  2. retainkit-0.1.0/.github/workflows/ci.yml +41 -0
  3. retainkit-0.1.0/.github/workflows/citation.yml +19 -0
  4. retainkit-0.1.0/.github/workflows/release.yml +53 -0
  5. retainkit-0.1.0/.zenodo.json +23 -0
  6. retainkit-0.1.0/CITATION.cff +13 -0
  7. retainkit-0.1.0/LICENSE +21 -0
  8. retainkit-0.1.0/Makefile +37 -0
  9. retainkit-0.1.0/PKG-INFO +215 -0
  10. retainkit-0.1.0/README.md +189 -0
  11. retainkit-0.1.0/examples/locomo/fetch_data.py +33 -0
  12. retainkit-0.1.0/examples/locomo/plot_frontier.py +96 -0
  13. retainkit-0.1.0/examples/locomo/results/figures/locomo.png +0 -0
  14. retainkit-0.1.0/examples/locomo/results/locomo.json +1450 -0
  15. retainkit-0.1.0/examples/locomo/results/locomo.md +88 -0
  16. retainkit-0.1.0/examples/locomo/run_locomo.py +156 -0
  17. retainkit-0.1.0/pyproject.toml +63 -0
  18. retainkit-0.1.0/results/demo.json +1603 -0
  19. retainkit-0.1.0/results/report.md +70 -0
  20. retainkit-0.1.0/results/table.md +38 -0
  21. retainkit-0.1.0/src/retainkit/__init__.py +56 -0
  22. retainkit-0.1.0/src/retainkit/bm25.py +69 -0
  23. retainkit-0.1.0/src/retainkit/cli.py +73 -0
  24. retainkit-0.1.0/src/retainkit/demo.py +123 -0
  25. retainkit-0.1.0/src/retainkit/evaluate.py +134 -0
  26. retainkit-0.1.0/src/retainkit/io.py +88 -0
  27. retainkit-0.1.0/src/retainkit/metrics.py +140 -0
  28. retainkit-0.1.0/src/retainkit/policies/__init__.py +35 -0
  29. retainkit-0.1.0/src/retainkit/policies/base.py +99 -0
  30. retainkit-0.1.0/src/retainkit/policies/retrieval.py +125 -0
  31. retainkit-0.1.0/src/retainkit/policies/simple.py +76 -0
  32. retainkit-0.1.0/src/retainkit/policies/summary.py +72 -0
  33. retainkit-0.1.0/src/retainkit/py.typed +0 -0
  34. retainkit-0.1.0/src/retainkit/report.py +115 -0
  35. retainkit-0.1.0/src/retainkit/tokens.py +38 -0
  36. retainkit-0.1.0/src/retainkit/types.py +140 -0
  37. retainkit-0.1.0/tests/test_bm25.py +31 -0
  38. retainkit-0.1.0/tests/test_demo.py +43 -0
  39. retainkit-0.1.0/tests/test_evaluate.py +78 -0
  40. retainkit-0.1.0/tests/test_io.py +40 -0
  41. retainkit-0.1.0/tests/test_metrics.py +73 -0
  42. retainkit-0.1.0/tests/test_policies.py +111 -0
  43. retainkit-0.1.0/tests/test_report.py +50 -0
  44. retainkit-0.1.0/tests/test_tokens.py +32 -0
  45. retainkit-0.1.0/tests/test_types.py +43 -0
@@ -0,0 +1 @@
1
+ github: mohammadi-hadi
@@ -0,0 +1,41 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ test:
10
+ runs-on: ubuntu-latest
11
+ strategy:
12
+ fail-fast: false
13
+ matrix:
14
+ python-version: ["3.11", "3.12", "3.13"]
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+ - uses: actions/setup-python@v5
18
+ with:
19
+ python-version: ${{ matrix.python-version }}
20
+ - run: python -m pip install --upgrade pip
21
+ - run: pip install -e ".[dev]"
22
+ - run: ruff check src tests examples
23
+ - run: ruff format --check src tests examples
24
+ - run: mypy src
25
+ - run: pytest -q
26
+
27
+ results:
28
+ # The committed fixture results are regenerated from the code and diffed, so
29
+ # a change that moves a number fails here until the results are regenerated
30
+ # with it. The README can then never quote a figure the code no longer produces.
31
+ runs-on: ubuntu-latest
32
+ steps:
33
+ - uses: actions/checkout@v4
34
+ - uses: actions/setup-python@v5
35
+ with:
36
+ python-version: "3.12"
37
+ - run: python -m pip install --upgrade pip
38
+ - run: pip install -e .
39
+ - run: python -m retainkit.cli demo --out results
40
+ - name: Fail if the committed results moved
41
+ run: git diff --exit-code -- results/table.md results/report.md results/demo.json
@@ -0,0 +1,19 @@
1
+ name: Citation
2
+
3
+ on:
4
+ push:
5
+ paths: ["CITATION.cff"]
6
+ pull_request:
7
+ paths: ["CITATION.cff"]
8
+ workflow_dispatch:
9
+
10
+ permissions:
11
+ contents: read
12
+
13
+ jobs:
14
+ validate:
15
+ runs-on: ubuntu-latest
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+ - name: Validate CITATION.cff
19
+ run: pipx install cffconvert && cffconvert --validate
@@ -0,0 +1,53 @@
1
+ name: Release
2
+
3
+ on:
4
+ push:
5
+ tags: ["v*"]
6
+
7
+ jobs:
8
+ build:
9
+ runs-on: ubuntu-latest
10
+ steps:
11
+ - uses: actions/checkout@v4
12
+ - uses: actions/setup-python@v5
13
+ with:
14
+ python-version: "3.12"
15
+ - run: python -m pip install --upgrade pip build
16
+ - run: python -m build
17
+ - uses: actions/upload-artifact@v4
18
+ with:
19
+ name: dist
20
+ path: dist/
21
+
22
+ publish:
23
+ needs: build
24
+ runs-on: ubuntu-latest
25
+ environment: pypi
26
+ permissions:
27
+ id-token: write
28
+ steps:
29
+ - uses: actions/download-artifact@v4
30
+ with:
31
+ name: dist
32
+ path: dist/
33
+ - uses: pypa/gh-action-pypi-publish@release/v1
34
+ with:
35
+ # PyPI is immutable, so a re-run of a version that is
36
+ # already published carries no new information.
37
+ skip-existing: true
38
+
39
+ github-release:
40
+ needs: publish
41
+ runs-on: ubuntu-latest
42
+ permissions:
43
+ contents: write
44
+ steps:
45
+ - uses: actions/checkout@v4
46
+ - uses: actions/download-artifact@v4
47
+ with:
48
+ name: dist
49
+ path: dist/
50
+ - uses: softprops/action-gh-release@v2
51
+ with:
52
+ files: dist/*
53
+ generate_release_notes: true
@@ -0,0 +1,23 @@
1
+ {
2
+ "title": "retainkit: context and memory policies for LLM agents, scored by the evidence that survives the token budget",
3
+ "description": "Context and memory policies for LLM agents, scored by the evidence that survives the token budget — across 1,527 LoCoMo questions a sliding window holds 7.7% of the supporting turns at 2,048 tokens, where retrieval holds 59.5% and a window needs 16 times the budget to match it.",
4
+ "creators": [
5
+ {
6
+ "name": "Mohammadi, Hadi",
7
+ "orcid": "0000-0003-0860-9200",
8
+ "affiliation": "Utrecht University"
9
+ }
10
+ ],
11
+ "license": "mit",
12
+ "upload_type": "software",
13
+ "keywords": [
14
+ "agents",
15
+ "context management",
16
+ "conversational memory",
17
+ "llm evaluation",
18
+ "locomo",
19
+ "long-term memory",
20
+ "python",
21
+ "retrieval"
22
+ ]
23
+ }
@@ -0,0 +1,13 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use retainkit, please cite it as below."
3
+ title: "retainkit: context and memory policies for LLM agents, scored by the evidence that survives the token budget"
4
+ type: software
5
+ authors:
6
+ - family-names: Mohammadi
7
+ given-names: Hadi
8
+ orcid: "https://orcid.org/0000-0003-0860-9200"
9
+ affiliation: "Utrecht University"
10
+ repository-code: "https://github.com/mohammadi-hadi/retainkit"
11
+ license: MIT
12
+ version: 0.1.0
13
+ date-released: 2026-09-05
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Hadi Mohammadi
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,37 @@
1
+ .PHONY: install lint test demo results locomo build
2
+
3
+ install:
4
+ python3 -m venv .venv && .venv/bin/pip install -e ".[dev]"
5
+
6
+ lint:
7
+ .venv/bin/ruff check src tests examples
8
+ .venv/bin/ruff format --check src tests examples
9
+ .venv/bin/mypy src
10
+
11
+ test:
12
+ .venv/bin/python -m pytest -q
13
+
14
+ demo:
15
+ .venv/bin/retainkit demo --out results
16
+
17
+ results: demo
18
+
19
+ locomo:
20
+ .venv/bin/python examples/locomo/fetch_data.py
21
+ .venv/bin/python examples/locomo/run_locomo.py
22
+
23
+ # Build sdist + wheel from a clean `git archive` export in a temp directory
24
+ # OUTSIDE the working tree: hatchling walks upward looking for ignore files and
25
+ # would otherwise pick up local ones. Only committed files can reach a
26
+ # distribution, and the artifacts are then checked for stray hidden files.
27
+ build:
28
+ rm -rf dist
29
+ @export_dir=$$(mktemp -d) && \
30
+ git archive HEAD | tar -x -C "$$export_dir" && \
31
+ .venv/bin/python -m build --outdir dist "$$export_dir" && \
32
+ rm -rf "$$export_dir"
33
+ @if tar -tzf dist/*.tar.gz | grep -E '/\.' | grep -vE '/\.(github/|zenodo\.json$$)' ; then \
34
+ echo "unexpected hidden files reached the sdist"; exit 1; fi
35
+ @if unzip -Z1 dist/*.whl | grep -E '(^|/)\.' ; then \
36
+ echo "unexpected hidden files reached the wheel"; exit 1; fi
37
+ @echo "artifacts clean:" && ls dist
@@ -0,0 +1,215 @@
1
+ Metadata-Version: 2.5
2
+ Name: retainkit
3
+ Version: 0.1.0
4
+ Summary: Context and memory policies for LLM agents, scored by the evidence that survives the token budget
5
+ Project-URL: Repository, https://github.com/mohammadi-hadi/retainkit
6
+ Project-URL: Issues, https://github.com/mohammadi-hadi/retainkit/issues
7
+ Author: Hadi Mohammadi
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: agents,context-management,llm-evaluation,long-term-memory,retrieval
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
17
+ Requires-Python: >=3.11
18
+ Provides-Extra: dev
19
+ Requires-Dist: build>=1.2; extra == 'dev'
20
+ Requires-Dist: mypy>=1.11; extra == 'dev'
21
+ Requires-Dist: pytest>=8; extra == 'dev'
22
+ Requires-Dist: ruff>=0.6; extra == 'dev'
23
+ Provides-Extra: viz
24
+ Requires-Dist: matplotlib>=3.8; extra == 'viz'
25
+ Description-Content-Type: text/markdown
26
+
27
+ <div align="center">
28
+
29
+ # retainkit
30
+
31
+ [![CI](https://github.com/mohammadi-hadi/retainkit/actions/workflows/ci.yml/badge.svg)](https://github.com/mohammadi-hadi/retainkit/actions/workflows/ci.yml)
32
+ [![PyPI](https://img.shields.io/pypi/v/retainkit)](https://pypi.org/project/retainkit/)
33
+ [![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/)
34
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
35
+
36
+ *Context and memory policies for LLM agents, scored by the evidence that survives the token budget.*
37
+
38
+ </div>
39
+
40
+ An agent that has been talking to someone for thirty sessions cannot put all of
41
+ it in the prompt. Something is dropped on every turn, and what is dropped
42
+ decides which questions the agent can still answer. That decision is usually
43
+ made by a default in a framework: keep the last N messages, summarise the rest.
44
+
45
+ retainkit measures the decision. Give it a conversation, a question, the turns
46
+ that support the answer, and a budget; it reports whether those turns are still
47
+ in front of the model, how much of the budget went on them, and how the answer
48
+ gets worse the further back the evidence sits.
49
+
50
+ ## 1,527 questions over ten long conversations
51
+
52
+ [LoCoMo](https://github.com/snap-research/locomo) records ten conversations
53
+ running 19 to 32 sessions each. Every question names the turns that answer it.
54
+ An average conversation is 23,056 tokens; the evidence for the average question
55
+ is **84 of them**, so 0.4% of the transcript carries the answer and the other
56
+ 99.6% is the search problem.
57
+
58
+ At 2,048 tokens, 8.9% of the conversation:
59
+
60
+ | Policy | Sees the question | Recall | 95% CI | Whole turns |
61
+ | --- | :---: | ---: | --- | ---: |
62
+ | `oracle` (keeps only the evidence) | yes | 100.0% | 100.0–100.0% | 100.0% |
63
+ | `fact_memory` | yes | 62.7% | 60.2–65.1% | 3.9% |
64
+ | `retrieval` | yes | 59.5% | 56.8–62.0% | 59.5% |
65
+ | `session_summary` | no | 13.4% | 11.7–15.2% | 2.0% |
66
+ | `head_tail` | no | 8.2% | 6.9–9.6% | 8.2% |
67
+ | `recency` | no | 7.7% | 6.4–9.0% | 7.7% |
68
+
69
+ A sliding window at this budget answers 7.7% of what it could. Reading the
70
+ question first takes that to 59.5% for the same money. Put the other way: a
71
+ window needs 16,384 tokens to reach the 50% that retrieval reaches at 1,024.
72
+
73
+ ![LoCoMo results](examples/locomo/results/figures/locomo.png)
74
+
75
+ The right panel is where the money is. Recency holds every question whose
76
+ evidence is in the session in progress and 62.3% of the ones a session or two
77
+ back. Three sessions back it holds **none of them**, while retrieval is still at
78
+ 72.1%. Long-term memory is a different problem from the window problem, and the
79
+ window scores zero on it.
80
+
81
+ Full numbers, including every budget from 512 to 32,768 and a split by question
82
+ type: [`examples/locomo/results/locomo.md`](examples/locomo/results/locomo.md).
83
+
84
+ ## Two results that do not flatter the tooling
85
+
86
+ **Multi-hop questions are not solved by retrieval.** At 2,048 tokens the best
87
+ policy holds the evidence for 76.8% of single-hop questions and 16.2% of
88
+ multi-hop ones. A multi-hop question needs several turns from several sessions
89
+ at once, and one query against a store tends to bring back one of them. Anyone
90
+ reporting a single memory-recall number is averaging over this gap.
91
+
92
+ **`fact_memory` wins on a technicality worth knowing about.** It stores
93
+ sentences rather than turns, so it fits more of the conversation into the same
94
+ budget and scores 62.7% against retrieval's 59.5%. But the whole evidence turn
95
+ survives only 3.9% of the time; the rest of its wins hand the model one sentence
96
+ out of several. Whichever column you take as the real one, take the same column
97
+ for every policy.
98
+
99
+ ## What is being measured
100
+
101
+ Whether a named span survived, and nothing else. A model handed the evidence can
102
+ still answer wrongly, so every number here is a ceiling on what any model behind
103
+ the policy could do, not a score for the model. The point of a ceiling is that it
104
+ is cheap: no API calls, no judge, no variance between runs, and a policy that
105
+ loses the evidence is not worth benchmarking end to end.
106
+
107
+ The same convention rules out one thing: an abstractive summariser rewrites what
108
+ it keeps, and a rewritten sentence cannot be traced to a turn. The summary policy
109
+ here is extractive so that survival stays answerable. Token counts come from a
110
+ four-characters-per-token estimate, which moves absolute numbers around by up to
111
+ 22 points for a window policy, so pass your own tokenizer if the budget has to be
112
+ exact. It does not move which policies win, and there is a test holding it to that.
113
+
114
+ ## Install
115
+
116
+ ```bash
117
+ pip install retainkit
118
+ ```
119
+
120
+ No runtime dependencies. `pip install "retainkit[viz]"` adds matplotlib for the
121
+ figures.
122
+
123
+ ## Use
124
+
125
+ ```python
126
+ from retainkit import Case, Probe, Recency, Retrieval, Transcript, Turn, run, sweep_table
127
+
128
+ case = Case(
129
+ transcript=Transcript(
130
+ id="guest-42",
131
+ turns=(
132
+ Turn(id="s1:1", session=1, speaker="guest", text="I never eat shellfish."),
133
+ Turn(id="s1:2", session=1, speaker="agent", text="Noted, thank you."),
134
+ Turn(id="s9:7", session=9, speaker="guest", text="Book me somewhere near the water."),
135
+ ),
136
+ ),
137
+ probes=(Probe(id="q1", question="What can this guest not eat?", evidence=("s1:1",)),),
138
+ )
139
+
140
+ print(sweep_table(run([case], (Recency(), Retrieval()), budgets=(64, 512))))
141
+ ```
142
+
143
+ Your own conversations go in as JSON and out as a report:
144
+
145
+ ```bash
146
+ retainkit eval conversations.json --budget 2048 --budget 8192 --out results/
147
+ ```
148
+
149
+ The file format is in [`src/retainkit/io.py`](src/retainkit/io.py). `retainkit
150
+ demo` runs a seeded fixture and needs no data at all.
151
+
152
+ ## The policies
153
+
154
+ | Policy | Keeps | Reads the question |
155
+ | --- | --- | :---: |
156
+ | `recency` | The newest turns that fit. | no |
157
+ | `head_tail` | The opening of the conversation and the newest turns. | no |
158
+ | `session_summary` | The live session, plus the lead sentences of each closed one. | no |
159
+ | `retrieval` | The turns that rank highest against the question, put back in order. | yes |
160
+ | `fact_memory` | The live session, plus sentences pulled from a long-term store. | yes |
161
+ | `oracle` | The evidence and nothing else. Not runnable in production. | yes |
162
+
163
+ A policy is anything with a `name`, a `query_aware` flag and a `select` method,
164
+ so your own goes straight into the same sweep:
165
+
166
+ ```python
167
+ class KeepEverySecondTurn:
168
+ name = "every_second"
169
+ query_aware = False
170
+
171
+ def select(self, transcript, probe, budget, tokenizer):
172
+ from retainkit.policies import fill, package
173
+ from retainkit.policies.base import turn_fragments
174
+
175
+ kept, _ = fill(turn_fragments(transcript)[::2], budget, tokenizer)
176
+ return package(self.name, budget, kept, tokenizer)
177
+ ```
178
+
179
+ ## Reproducing
180
+
181
+ ```bash
182
+ make install
183
+ make test
184
+ make demo # the seeded fixture, written to results/
185
+ python examples/locomo/fetch_data.py
186
+ python examples/locomo/run_locomo.py
187
+ ```
188
+
189
+ LoCoMo is fetched at run time and is not redistributed here; it carries the terms
190
+ of its own release. The fixture results in `results/` are rebuilt by CI on every
191
+ push and the build fails if a committed number no longer matches the code, so
192
+ nothing quoted above can drift away from what the package produces.
193
+
194
+ ## Citing
195
+
196
+ ```bibtex
197
+ @software{mohammadi_retainkit,
198
+ author = {Mohammadi, Hadi},
199
+ title = {retainkit: context and memory policies for LLM agents,
200
+ scored by the evidence that survives the token budget},
201
+ year = {2026},
202
+ url = {https://github.com/mohammadi-hadi/retainkit},
203
+ license = {MIT}
204
+ }
205
+ ```
206
+
207
+ LoCoMo is by Maharana et al., *Evaluating Very Long-Term Conversational Memory
208
+ of LLM Agents*, ACL 2024. Cite it if you use these results.
209
+
210
+ ## Related
211
+
212
+ [judgekit](https://github.com/mohammadi-hadi/judgekit) audits an LLM judge,
213
+ [arenakit](https://github.com/mohammadi-hadi/arenakit) audits a pairwise
214
+ leaderboard, [trajectory-judge](https://github.com/mohammadi-hadi/trajectory-judge)
215
+ scores agent trajectories. More at [mohammadi.cv](https://mohammadi.cv).
@@ -0,0 +1,189 @@
1
+ <div align="center">
2
+
3
+ # retainkit
4
+
5
+ [![CI](https://github.com/mohammadi-hadi/retainkit/actions/workflows/ci.yml/badge.svg)](https://github.com/mohammadi-hadi/retainkit/actions/workflows/ci.yml)
6
+ [![PyPI](https://img.shields.io/pypi/v/retainkit)](https://pypi.org/project/retainkit/)
7
+ [![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/)
8
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
9
+
10
+ *Context and memory policies for LLM agents, scored by the evidence that survives the token budget.*
11
+
12
+ </div>
13
+
14
+ An agent that has been talking to someone for thirty sessions cannot put all of
15
+ it in the prompt. Something is dropped on every turn, and what is dropped
16
+ decides which questions the agent can still answer. That decision is usually
17
+ made by a default in a framework: keep the last N messages, summarise the rest.
18
+
19
+ retainkit measures the decision. Give it a conversation, a question, the turns
20
+ that support the answer, and a budget; it reports whether those turns are still
21
+ in front of the model, how much of the budget went on them, and how the answer
22
+ gets worse the further back the evidence sits.
23
+
24
+ ## 1,527 questions over ten long conversations
25
+
26
+ [LoCoMo](https://github.com/snap-research/locomo) records ten conversations
27
+ running 19 to 32 sessions each. Every question names the turns that answer it.
28
+ An average conversation is 23,056 tokens; the evidence for the average question
29
+ is **84 of them**, so 0.4% of the transcript carries the answer and the other
30
+ 99.6% is the search problem.
31
+
32
+ At 2,048 tokens, 8.9% of the conversation:
33
+
34
+ | Policy | Sees the question | Recall | 95% CI | Whole turns |
35
+ | --- | :---: | ---: | --- | ---: |
36
+ | `oracle` (keeps only the evidence) | yes | 100.0% | 100.0–100.0% | 100.0% |
37
+ | `fact_memory` | yes | 62.7% | 60.2–65.1% | 3.9% |
38
+ | `retrieval` | yes | 59.5% | 56.8–62.0% | 59.5% |
39
+ | `session_summary` | no | 13.4% | 11.7–15.2% | 2.0% |
40
+ | `head_tail` | no | 8.2% | 6.9–9.6% | 8.2% |
41
+ | `recency` | no | 7.7% | 6.4–9.0% | 7.7% |
42
+
43
+ A sliding window at this budget answers 7.7% of what it could. Reading the
44
+ question first takes that to 59.5% for the same money. Put the other way: a
45
+ window needs 16,384 tokens to reach the 50% that retrieval reaches at 1,024.
46
+
47
+ ![LoCoMo results](examples/locomo/results/figures/locomo.png)
48
+
49
+ The right panel is where the money is. Recency holds every question whose
50
+ evidence is in the session in progress and 62.3% of the ones a session or two
51
+ back. Three sessions back it holds **none of them**, while retrieval is still at
52
+ 72.1%. Long-term memory is a different problem from the window problem, and the
53
+ window scores zero on it.
54
+
55
+ Full numbers, including every budget from 512 to 32,768 and a split by question
56
+ type: [`examples/locomo/results/locomo.md`](examples/locomo/results/locomo.md).
57
+
58
+ ## Two results that do not flatter the tooling
59
+
60
+ **Multi-hop questions are not solved by retrieval.** At 2,048 tokens the best
61
+ policy holds the evidence for 76.8% of single-hop questions and 16.2% of
62
+ multi-hop ones. A multi-hop question needs several turns from several sessions
63
+ at once, and one query against a store tends to bring back one of them. Anyone
64
+ reporting a single memory-recall number is averaging over this gap.
65
+
66
+ **`fact_memory` wins on a technicality worth knowing about.** It stores
67
+ sentences rather than turns, so it fits more of the conversation into the same
68
+ budget and scores 62.7% against retrieval's 59.5%. But the whole evidence turn
69
+ survives only 3.9% of the time; the rest of its wins hand the model one sentence
70
+ out of several. Whichever column you take as the real one, take the same column
71
+ for every policy.
72
+
73
+ ## What is being measured
74
+
75
+ Whether a named span survived, and nothing else. A model handed the evidence can
76
+ still answer wrongly, so every number here is a ceiling on what any model behind
77
+ the policy could do, not a score for the model. The point of a ceiling is that it
78
+ is cheap: no API calls, no judge, no variance between runs, and a policy that
79
+ loses the evidence is not worth benchmarking end to end.
80
+
81
+ The same convention rules out one thing: an abstractive summariser rewrites what
82
+ it keeps, and a rewritten sentence cannot be traced to a turn. The summary policy
83
+ here is extractive so that survival stays answerable. Token counts come from a
84
+ four-characters-per-token estimate, which moves absolute numbers around by up to
85
+ 22 points for a window policy, so pass your own tokenizer if the budget has to be
86
+ exact. It does not move which policies win, and there is a test holding it to that.
87
+
88
+ ## Install
89
+
90
+ ```bash
91
+ pip install retainkit
92
+ ```
93
+
94
+ No runtime dependencies. `pip install "retainkit[viz]"` adds matplotlib for the
95
+ figures.
96
+
97
+ ## Use
98
+
99
+ ```python
100
+ from retainkit import Case, Probe, Recency, Retrieval, Transcript, Turn, run, sweep_table
101
+
102
+ case = Case(
103
+ transcript=Transcript(
104
+ id="guest-42",
105
+ turns=(
106
+ Turn(id="s1:1", session=1, speaker="guest", text="I never eat shellfish."),
107
+ Turn(id="s1:2", session=1, speaker="agent", text="Noted, thank you."),
108
+ Turn(id="s9:7", session=9, speaker="guest", text="Book me somewhere near the water."),
109
+ ),
110
+ ),
111
+ probes=(Probe(id="q1", question="What can this guest not eat?", evidence=("s1:1",)),),
112
+ )
113
+
114
+ print(sweep_table(run([case], (Recency(), Retrieval()), budgets=(64, 512))))
115
+ ```
116
+
117
+ Your own conversations go in as JSON and out as a report:
118
+
119
+ ```bash
120
+ retainkit eval conversations.json --budget 2048 --budget 8192 --out results/
121
+ ```
122
+
123
+ The file format is in [`src/retainkit/io.py`](src/retainkit/io.py). `retainkit
124
+ demo` runs a seeded fixture and needs no data at all.
125
+
126
+ ## The policies
127
+
128
+ | Policy | Keeps | Reads the question |
129
+ | --- | --- | :---: |
130
+ | `recency` | The newest turns that fit. | no |
131
+ | `head_tail` | The opening of the conversation and the newest turns. | no |
132
+ | `session_summary` | The live session, plus the lead sentences of each closed one. | no |
133
+ | `retrieval` | The turns that rank highest against the question, put back in order. | yes |
134
+ | `fact_memory` | The live session, plus sentences pulled from a long-term store. | yes |
135
+ | `oracle` | The evidence and nothing else. Not runnable in production. | yes |
136
+
137
+ A policy is anything with a `name`, a `query_aware` flag and a `select` method,
138
+ so your own goes straight into the same sweep:
139
+
140
+ ```python
141
+ class KeepEverySecondTurn:
142
+ name = "every_second"
143
+ query_aware = False
144
+
145
+ def select(self, transcript, probe, budget, tokenizer):
146
+ from retainkit.policies import fill, package
147
+ from retainkit.policies.base import turn_fragments
148
+
149
+ kept, _ = fill(turn_fragments(transcript)[::2], budget, tokenizer)
150
+ return package(self.name, budget, kept, tokenizer)
151
+ ```
152
+
153
+ ## Reproducing
154
+
155
+ ```bash
156
+ make install
157
+ make test
158
+ make demo # the seeded fixture, written to results/
159
+ python examples/locomo/fetch_data.py
160
+ python examples/locomo/run_locomo.py
161
+ ```
162
+
163
+ LoCoMo is fetched at run time and is not redistributed here; it carries the terms
164
+ of its own release. The fixture results in `results/` are rebuilt by CI on every
165
+ push and the build fails if a committed number no longer matches the code, so
166
+ nothing quoted above can drift away from what the package produces.
167
+
168
+ ## Citing
169
+
170
+ ```bibtex
171
+ @software{mohammadi_retainkit,
172
+ author = {Mohammadi, Hadi},
173
+ title = {retainkit: context and memory policies for LLM agents,
174
+ scored by the evidence that survives the token budget},
175
+ year = {2026},
176
+ url = {https://github.com/mohammadi-hadi/retainkit},
177
+ license = {MIT}
178
+ }
179
+ ```
180
+
181
+ LoCoMo is by Maharana et al., *Evaluating Very Long-Term Conversational Memory
182
+ of LLM Agents*, ACL 2024. Cite it if you use these results.
183
+
184
+ ## Related
185
+
186
+ [judgekit](https://github.com/mohammadi-hadi/judgekit) audits an LLM judge,
187
+ [arenakit](https://github.com/mohammadi-hadi/arenakit) audits a pairwise
188
+ leaderboard, [trajectory-judge](https://github.com/mohammadi-hadi/trajectory-judge)
189
+ scores agent trajectories. More at [mohammadi.cv](https://mohammadi.cv).
@@ -0,0 +1,33 @@
1
+ """Download the LoCoMo conversations.
2
+
3
+ LoCoMo is released by Snap Research with the paper *Evaluating Very Long-Term
4
+ Conversational Memory of LLM Agents* (Maharana et al., ACL 2024). The file is
5
+ fetched into `examples/locomo/data/`, which is not tracked here: the dataset
6
+ carries its own terms and is not redistributed with this package.
7
+
8
+ python examples/locomo/fetch_data.py
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import sys
14
+ import urllib.request
15
+ from pathlib import Path
16
+
17
+ URL = "https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json"
18
+ DATA = Path(__file__).parent / "data" / "locomo10.json"
19
+
20
+
21
+ def main() -> int:
22
+ DATA.parent.mkdir(parents=True, exist_ok=True)
23
+ if DATA.exists():
24
+ print(f"already here: {DATA} ({DATA.stat().st_size / 1e6:.1f} MB)")
25
+ return 0
26
+ print(f"fetching {URL}")
27
+ urllib.request.urlretrieve(URL, DATA) # noqa: S310
28
+ print(f"wrote {DATA} ({DATA.stat().st_size / 1e6:.1f} MB)")
29
+ return 0
30
+
31
+
32
+ if __name__ == "__main__":
33
+ sys.exit(main())