retainkit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- retainkit-0.1.0/.github/FUNDING.yml +1 -0
- retainkit-0.1.0/.github/workflows/ci.yml +41 -0
- retainkit-0.1.0/.github/workflows/citation.yml +19 -0
- retainkit-0.1.0/.github/workflows/release.yml +53 -0
- retainkit-0.1.0/.zenodo.json +23 -0
- retainkit-0.1.0/CITATION.cff +13 -0
- retainkit-0.1.0/LICENSE +21 -0
- retainkit-0.1.0/Makefile +37 -0
- retainkit-0.1.0/PKG-INFO +215 -0
- retainkit-0.1.0/README.md +189 -0
- retainkit-0.1.0/examples/locomo/fetch_data.py +33 -0
- retainkit-0.1.0/examples/locomo/plot_frontier.py +96 -0
- retainkit-0.1.0/examples/locomo/results/figures/locomo.png +0 -0
- retainkit-0.1.0/examples/locomo/results/locomo.json +1450 -0
- retainkit-0.1.0/examples/locomo/results/locomo.md +88 -0
- retainkit-0.1.0/examples/locomo/run_locomo.py +156 -0
- retainkit-0.1.0/pyproject.toml +63 -0
- retainkit-0.1.0/results/demo.json +1603 -0
- retainkit-0.1.0/results/report.md +70 -0
- retainkit-0.1.0/results/table.md +38 -0
- retainkit-0.1.0/src/retainkit/__init__.py +56 -0
- retainkit-0.1.0/src/retainkit/bm25.py +69 -0
- retainkit-0.1.0/src/retainkit/cli.py +73 -0
- retainkit-0.1.0/src/retainkit/demo.py +123 -0
- retainkit-0.1.0/src/retainkit/evaluate.py +134 -0
- retainkit-0.1.0/src/retainkit/io.py +88 -0
- retainkit-0.1.0/src/retainkit/metrics.py +140 -0
- retainkit-0.1.0/src/retainkit/policies/__init__.py +35 -0
- retainkit-0.1.0/src/retainkit/policies/base.py +99 -0
- retainkit-0.1.0/src/retainkit/policies/retrieval.py +125 -0
- retainkit-0.1.0/src/retainkit/policies/simple.py +76 -0
- retainkit-0.1.0/src/retainkit/policies/summary.py +72 -0
- retainkit-0.1.0/src/retainkit/py.typed +0 -0
- retainkit-0.1.0/src/retainkit/report.py +115 -0
- retainkit-0.1.0/src/retainkit/tokens.py +38 -0
- retainkit-0.1.0/src/retainkit/types.py +140 -0
- retainkit-0.1.0/tests/test_bm25.py +31 -0
- retainkit-0.1.0/tests/test_demo.py +43 -0
- retainkit-0.1.0/tests/test_evaluate.py +78 -0
- retainkit-0.1.0/tests/test_io.py +40 -0
- retainkit-0.1.0/tests/test_metrics.py +73 -0
- retainkit-0.1.0/tests/test_policies.py +111 -0
- retainkit-0.1.0/tests/test_report.py +50 -0
- retainkit-0.1.0/tests/test_tokens.py +32 -0
- retainkit-0.1.0/tests/test_types.py +43 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
github: mohammadi-hadi
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
fail-fast: false
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ["3.11", "3.12", "3.13"]
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
- uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: ${{ matrix.python-version }}
|
|
20
|
+
- run: python -m pip install --upgrade pip
|
|
21
|
+
- run: pip install -e ".[dev]"
|
|
22
|
+
- run: ruff check src tests examples
|
|
23
|
+
- run: ruff format --check src tests examples
|
|
24
|
+
- run: mypy src
|
|
25
|
+
- run: pytest -q
|
|
26
|
+
|
|
27
|
+
results:
|
|
28
|
+
# The committed fixture results are regenerated from the code and diffed, so
|
|
29
|
+
# a change that moves a number fails here until the results are regenerated
|
|
30
|
+
# with it. The README can then never quote a figure the code no longer produces.
|
|
31
|
+
runs-on: ubuntu-latest
|
|
32
|
+
steps:
|
|
33
|
+
- uses: actions/checkout@v4
|
|
34
|
+
- uses: actions/setup-python@v5
|
|
35
|
+
with:
|
|
36
|
+
python-version: "3.12"
|
|
37
|
+
- run: python -m pip install --upgrade pip
|
|
38
|
+
- run: pip install -e .
|
|
39
|
+
- run: python -m retainkit.cli demo --out results
|
|
40
|
+
- name: Fail if the committed results moved
|
|
41
|
+
run: git diff --exit-code -- results/table.md results/report.md results/demo.json
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
name: Citation
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
paths: ["CITATION.cff"]
|
|
6
|
+
pull_request:
|
|
7
|
+
paths: ["CITATION.cff"]
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
validate:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
- name: Validate CITATION.cff
|
|
19
|
+
run: pipx install cffconvert && cffconvert --validate
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
build:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
steps:
|
|
11
|
+
- uses: actions/checkout@v4
|
|
12
|
+
- uses: actions/setup-python@v5
|
|
13
|
+
with:
|
|
14
|
+
python-version: "3.12"
|
|
15
|
+
- run: python -m pip install --upgrade pip build
|
|
16
|
+
- run: python -m build
|
|
17
|
+
- uses: actions/upload-artifact@v4
|
|
18
|
+
with:
|
|
19
|
+
name: dist
|
|
20
|
+
path: dist/
|
|
21
|
+
|
|
22
|
+
publish:
|
|
23
|
+
needs: build
|
|
24
|
+
runs-on: ubuntu-latest
|
|
25
|
+
environment: pypi
|
|
26
|
+
permissions:
|
|
27
|
+
id-token: write
|
|
28
|
+
steps:
|
|
29
|
+
- uses: actions/download-artifact@v4
|
|
30
|
+
with:
|
|
31
|
+
name: dist
|
|
32
|
+
path: dist/
|
|
33
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
34
|
+
with:
|
|
35
|
+
# PyPI is immutable, so a re-run of a version that is
|
|
36
|
+
# already published carries no new information.
|
|
37
|
+
skip-existing: true
|
|
38
|
+
|
|
39
|
+
github-release:
|
|
40
|
+
needs: publish
|
|
41
|
+
runs-on: ubuntu-latest
|
|
42
|
+
permissions:
|
|
43
|
+
contents: write
|
|
44
|
+
steps:
|
|
45
|
+
- uses: actions/checkout@v4
|
|
46
|
+
- uses: actions/download-artifact@v4
|
|
47
|
+
with:
|
|
48
|
+
name: dist
|
|
49
|
+
path: dist/
|
|
50
|
+
- uses: softprops/action-gh-release@v2
|
|
51
|
+
with:
|
|
52
|
+
files: dist/*
|
|
53
|
+
generate_release_notes: true
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"title": "retainkit: context and memory policies for LLM agents, scored by the evidence that survives the token budget",
|
|
3
|
+
"description": "Context and memory policies for LLM agents, scored by the evidence that survives the token budget — across 1,527 LoCoMo questions a sliding window holds 7.7% of the supporting turns at 2,048 tokens, where retrieval holds 59.5% and a window needs 16 times the budget to match it.",
|
|
4
|
+
"creators": [
|
|
5
|
+
{
|
|
6
|
+
"name": "Mohammadi, Hadi",
|
|
7
|
+
"orcid": "0000-0003-0860-9200",
|
|
8
|
+
"affiliation": "Utrecht University"
|
|
9
|
+
}
|
|
10
|
+
],
|
|
11
|
+
"license": "mit",
|
|
12
|
+
"upload_type": "software",
|
|
13
|
+
"keywords": [
|
|
14
|
+
"agents",
|
|
15
|
+
"context management",
|
|
16
|
+
"conversational memory",
|
|
17
|
+
"llm evaluation",
|
|
18
|
+
"locomo",
|
|
19
|
+
"long-term memory",
|
|
20
|
+
"python",
|
|
21
|
+
"retrieval"
|
|
22
|
+
]
|
|
23
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use retainkit, please cite it as below."
|
|
3
|
+
title: "retainkit: context and memory policies for LLM agents, scored by the evidence that survives the token budget"
|
|
4
|
+
type: software
|
|
5
|
+
authors:
|
|
6
|
+
- family-names: Mohammadi
|
|
7
|
+
given-names: Hadi
|
|
8
|
+
orcid: "https://orcid.org/0000-0003-0860-9200"
|
|
9
|
+
affiliation: "Utrecht University"
|
|
10
|
+
repository-code: "https://github.com/mohammadi-hadi/retainkit"
|
|
11
|
+
license: MIT
|
|
12
|
+
version: 0.1.0
|
|
13
|
+
date-released: 2026-09-05
|
retainkit-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Hadi Mohammadi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
retainkit-0.1.0/Makefile
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
.PHONY: install lint test demo results locomo build
|
|
2
|
+
|
|
3
|
+
install:
|
|
4
|
+
python3 -m venv .venv && .venv/bin/pip install -e ".[dev]"
|
|
5
|
+
|
|
6
|
+
lint:
|
|
7
|
+
.venv/bin/ruff check src tests examples
|
|
8
|
+
.venv/bin/ruff format --check src tests examples
|
|
9
|
+
.venv/bin/mypy src
|
|
10
|
+
|
|
11
|
+
test:
|
|
12
|
+
.venv/bin/python -m pytest -q
|
|
13
|
+
|
|
14
|
+
demo:
|
|
15
|
+
.venv/bin/retainkit demo --out results
|
|
16
|
+
|
|
17
|
+
results: demo
|
|
18
|
+
|
|
19
|
+
locomo:
|
|
20
|
+
.venv/bin/python examples/locomo/fetch_data.py
|
|
21
|
+
.venv/bin/python examples/locomo/run_locomo.py
|
|
22
|
+
|
|
23
|
+
# Build sdist + wheel from a clean `git archive` export in a temp directory
|
|
24
|
+
# OUTSIDE the working tree: hatchling walks upward looking for ignore files and
|
|
25
|
+
# would otherwise pick up local ones. Only committed files can reach a
|
|
26
|
+
# distribution, and the artifacts are then checked for stray hidden files.
|
|
27
|
+
build:
|
|
28
|
+
rm -rf dist
|
|
29
|
+
@export_dir=$$(mktemp -d) && \
|
|
30
|
+
git archive HEAD | tar -x -C "$$export_dir" && \
|
|
31
|
+
.venv/bin/python -m build --outdir dist "$$export_dir" && \
|
|
32
|
+
rm -rf "$$export_dir"
|
|
33
|
+
@if tar -tzf dist/*.tar.gz | grep -E '/\.' | grep -vE '/\.(github/|zenodo\.json$$)' ; then \
|
|
34
|
+
echo "unexpected hidden files reached the sdist"; exit 1; fi
|
|
35
|
+
@if unzip -Z1 dist/*.whl | grep -E '(^|/)\.' ; then \
|
|
36
|
+
echo "unexpected hidden files reached the wheel"; exit 1; fi
|
|
37
|
+
@echo "artifacts clean:" && ls dist
|
retainkit-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: retainkit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Context and memory policies for LLM agents, scored by the evidence that survives the token budget
|
|
5
|
+
Project-URL: Repository, https://github.com/mohammadi-hadi/retainkit
|
|
6
|
+
Project-URL: Issues, https://github.com/mohammadi-hadi/retainkit/issues
|
|
7
|
+
Author: Hadi Mohammadi
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: agents,context-management,llm-evaluation,long-term-memory,retrieval
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
20
|
+
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
21
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
22
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
23
|
+
Provides-Extra: viz
|
|
24
|
+
Requires-Dist: matplotlib>=3.8; extra == 'viz'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
<div align="center">
|
|
28
|
+
|
|
29
|
+
# retainkit
|
|
30
|
+
|
|
31
|
+
[](https://github.com/mohammadi-hadi/retainkit/actions/workflows/ci.yml)
|
|
32
|
+
[](https://pypi.org/project/retainkit/)
|
|
33
|
+
[](https://www.python.org/downloads/)
|
|
34
|
+
[](LICENSE)
|
|
35
|
+
|
|
36
|
+
*Context and memory policies for LLM agents, scored by the evidence that survives the token budget.*
|
|
37
|
+
|
|
38
|
+
</div>
|
|
39
|
+
|
|
40
|
+
An agent that has been talking to someone for thirty sessions cannot put all of
|
|
41
|
+
it in the prompt. Something is dropped on every turn, and what is dropped
|
|
42
|
+
decides which questions the agent can still answer. That decision is usually
|
|
43
|
+
made by a default in a framework: keep the last N messages, summarise the rest.
|
|
44
|
+
|
|
45
|
+
retainkit measures the decision. Give it a conversation, a question, the turns
|
|
46
|
+
that support the answer, and a budget; it reports whether those turns are still
|
|
47
|
+
in front of the model, how much of the budget went on them, and how the answer
|
|
48
|
+
gets worse the further back the evidence sits.
|
|
49
|
+
|
|
50
|
+
## 1,527 questions over ten long conversations
|
|
51
|
+
|
|
52
|
+
[LoCoMo](https://github.com/snap-research/locomo) records ten conversations
|
|
53
|
+
running 19 to 32 sessions each. Every question names the turns that answer it.
|
|
54
|
+
An average conversation is 23,056 tokens; the evidence for the average question
|
|
55
|
+
is **84 of them**, so 0.4% of the transcript carries the answer and the other
|
|
56
|
+
99.6% is the search problem.
|
|
57
|
+
|
|
58
|
+
At 2,048 tokens, 8.9% of the conversation:
|
|
59
|
+
|
|
60
|
+
| Policy | Sees the question | Recall | 95% CI | Whole turns |
|
|
61
|
+
| --- | :---: | ---: | --- | ---: |
|
|
62
|
+
| `oracle` (keeps only the evidence) | yes | 100.0% | 100.0–100.0% | 100.0% |
|
|
63
|
+
| `fact_memory` | yes | 62.7% | 60.2–65.1% | 3.9% |
|
|
64
|
+
| `retrieval` | yes | 59.5% | 56.8–62.0% | 59.5% |
|
|
65
|
+
| `session_summary` | no | 13.4% | 11.7–15.2% | 2.0% |
|
|
66
|
+
| `head_tail` | no | 8.2% | 6.9–9.6% | 8.2% |
|
|
67
|
+
| `recency` | no | 7.7% | 6.4–9.0% | 7.7% |
|
|
68
|
+
|
|
69
|
+
A sliding window at this budget answers 7.7% of what it could. Reading the
|
|
70
|
+
question first takes that to 59.5% for the same money. Put the other way: a
|
|
71
|
+
window needs 16,384 tokens to reach the 50% that retrieval reaches at 1,024.
|
|
72
|
+
|
|
73
|
+

|
|
74
|
+
|
|
75
|
+
The right panel is where the money is. Recency holds every question whose
|
|
76
|
+
evidence is in the session in progress and 62.3% of the ones a session or two
|
|
77
|
+
back. Three sessions back it holds **none of them**, while retrieval is still at
|
|
78
|
+
72.1%. Long-term memory is a different problem from the window problem, and the
|
|
79
|
+
window scores zero on it.
|
|
80
|
+
|
|
81
|
+
Full numbers, including every budget from 512 to 32,768 and a split by question
|
|
82
|
+
type: [`examples/locomo/results/locomo.md`](examples/locomo/results/locomo.md).
|
|
83
|
+
|
|
84
|
+
## Two results that do not flatter the tooling
|
|
85
|
+
|
|
86
|
+
**Multi-hop questions are not solved by retrieval.** At 2,048 tokens the best
|
|
87
|
+
policy holds the evidence for 76.8% of single-hop questions and 16.2% of
|
|
88
|
+
multi-hop ones. A multi-hop question needs several turns from several sessions
|
|
89
|
+
at once, and one query against a store tends to bring back one of them. Anyone
|
|
90
|
+
reporting a single memory-recall number is averaging over this gap.
|
|
91
|
+
|
|
92
|
+
**`fact_memory` wins on a technicality worth knowing about.** It stores
|
|
93
|
+
sentences rather than turns, so it fits more of the conversation into the same
|
|
94
|
+
budget and scores 62.7% against retrieval's 59.5%. But the whole evidence turn
|
|
95
|
+
survives only 3.9% of the time; the rest of its wins hand the model one sentence
|
|
96
|
+
out of several. Whichever column you take as the real one, take the same column
|
|
97
|
+
for every policy.
|
|
98
|
+
|
|
99
|
+
## What is being measured
|
|
100
|
+
|
|
101
|
+
Whether a named span survived, and nothing else. A model handed the evidence can
|
|
102
|
+
still answer wrongly, so every number here is a ceiling on what any model behind
|
|
103
|
+
the policy could do, not a score for the model. The point of a ceiling is that it
|
|
104
|
+
is cheap: no API calls, no judge, no variance between runs, and a policy that
|
|
105
|
+
loses the evidence is not worth benchmarking end to end.
|
|
106
|
+
|
|
107
|
+
The same convention rules out one thing: an abstractive summariser rewrites what
|
|
108
|
+
it keeps, and a rewritten sentence cannot be traced to a turn. The summary policy
|
|
109
|
+
here is extractive so that survival stays answerable. Token counts come from a
|
|
110
|
+
four-characters-per-token estimate, which moves absolute numbers around by up to
|
|
111
|
+
22 points for a window policy, so pass your own tokenizer if the budget has to be
|
|
112
|
+
exact. It does not move which policies win, and there is a test holding it to that.
|
|
113
|
+
|
|
114
|
+
## Install
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
pip install retainkit
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
No runtime dependencies. `pip install "retainkit[viz]"` adds matplotlib for the
|
|
121
|
+
figures.
|
|
122
|
+
|
|
123
|
+
## Use
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
from retainkit import Case, Probe, Recency, Retrieval, Transcript, Turn, run, sweep_table
|
|
127
|
+
|
|
128
|
+
case = Case(
|
|
129
|
+
transcript=Transcript(
|
|
130
|
+
id="guest-42",
|
|
131
|
+
turns=(
|
|
132
|
+
Turn(id="s1:1", session=1, speaker="guest", text="I never eat shellfish."),
|
|
133
|
+
Turn(id="s1:2", session=1, speaker="agent", text="Noted, thank you."),
|
|
134
|
+
Turn(id="s9:7", session=9, speaker="guest", text="Book me somewhere near the water."),
|
|
135
|
+
),
|
|
136
|
+
),
|
|
137
|
+
probes=(Probe(id="q1", question="What can this guest not eat?", evidence=("s1:1",)),),
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
print(sweep_table(run([case], (Recency(), Retrieval()), budgets=(64, 512))))
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Your own conversations go in as JSON and out as a report:
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
retainkit eval conversations.json --budget 2048 --budget 8192 --out results/
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
The file format is in [`src/retainkit/io.py`](src/retainkit/io.py). `retainkit
|
|
150
|
+
demo` runs a seeded fixture and needs no data at all.
|
|
151
|
+
|
|
152
|
+
## The policies
|
|
153
|
+
|
|
154
|
+
| Policy | Keeps | Reads the question |
|
|
155
|
+
| --- | --- | :---: |
|
|
156
|
+
| `recency` | The newest turns that fit. | no |
|
|
157
|
+
| `head_tail` | The opening of the conversation and the newest turns. | no |
|
|
158
|
+
| `session_summary` | The live session, plus the lead sentences of each closed one. | no |
|
|
159
|
+
| `retrieval` | The turns that rank highest against the question, put back in order. | yes |
|
|
160
|
+
| `fact_memory` | The live session, plus sentences pulled from a long-term store. | yes |
|
|
161
|
+
| `oracle` | The evidence and nothing else. Not runnable in production. | yes |
|
|
162
|
+
|
|
163
|
+
A policy is anything with a `name`, a `query_aware` flag and a `select` method,
|
|
164
|
+
so your own goes straight into the same sweep:
|
|
165
|
+
|
|
166
|
+
```python
|
|
167
|
+
class KeepEverySecondTurn:
|
|
168
|
+
name = "every_second"
|
|
169
|
+
query_aware = False
|
|
170
|
+
|
|
171
|
+
def select(self, transcript, probe, budget, tokenizer):
|
|
172
|
+
from retainkit.policies import fill, package
|
|
173
|
+
from retainkit.policies.base import turn_fragments
|
|
174
|
+
|
|
175
|
+
kept, _ = fill(turn_fragments(transcript)[::2], budget, tokenizer)
|
|
176
|
+
return package(self.name, budget, kept, tokenizer)
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
## Reproducing
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
make install
|
|
183
|
+
make test
|
|
184
|
+
make demo # the seeded fixture, written to results/
|
|
185
|
+
python examples/locomo/fetch_data.py
|
|
186
|
+
python examples/locomo/run_locomo.py
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
LoCoMo is fetched at run time and is not redistributed here; it carries the terms
|
|
190
|
+
of its own release. The fixture results in `results/` are rebuilt by CI on every
|
|
191
|
+
push and the build fails if a committed number no longer matches the code, so
|
|
192
|
+
nothing quoted above can drift away from what the package produces.
|
|
193
|
+
|
|
194
|
+
## Citing
|
|
195
|
+
|
|
196
|
+
```bibtex
|
|
197
|
+
@software{mohammadi_retainkit,
|
|
198
|
+
author = {Mohammadi, Hadi},
|
|
199
|
+
title = {retainkit: context and memory policies for LLM agents,
|
|
200
|
+
scored by the evidence that survives the token budget},
|
|
201
|
+
year = {2026},
|
|
202
|
+
url = {https://github.com/mohammadi-hadi/retainkit},
|
|
203
|
+
license = {MIT}
|
|
204
|
+
}
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
LoCoMo is by Maharana et al., *Evaluating Very Long-Term Conversational Memory
|
|
208
|
+
of LLM Agents*, ACL 2024. Cite it if you use these results.
|
|
209
|
+
|
|
210
|
+
## Related
|
|
211
|
+
|
|
212
|
+
[judgekit](https://github.com/mohammadi-hadi/judgekit) audits an LLM judge,
|
|
213
|
+
[arenakit](https://github.com/mohammadi-hadi/arenakit) audits a pairwise
|
|
214
|
+
leaderboard, [trajectory-judge](https://github.com/mohammadi-hadi/trajectory-judge)
|
|
215
|
+
scores agent trajectories. More at [mohammadi.cv](https://mohammadi.cv).
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
# retainkit
|
|
4
|
+
|
|
5
|
+
[](https://github.com/mohammadi-hadi/retainkit/actions/workflows/ci.yml)
|
|
6
|
+
[](https://pypi.org/project/retainkit/)
|
|
7
|
+
[](https://www.python.org/downloads/)
|
|
8
|
+
[](LICENSE)
|
|
9
|
+
|
|
10
|
+
*Context and memory policies for LLM agents, scored by the evidence that survives the token budget.*
|
|
11
|
+
|
|
12
|
+
</div>
|
|
13
|
+
|
|
14
|
+
An agent that has been talking to someone for thirty sessions cannot put all of
|
|
15
|
+
it in the prompt. Something is dropped on every turn, and what is dropped
|
|
16
|
+
decides which questions the agent can still answer. That decision is usually
|
|
17
|
+
made by a default in a framework: keep the last N messages, summarise the rest.
|
|
18
|
+
|
|
19
|
+
retainkit measures the decision. Give it a conversation, a question, the turns
|
|
20
|
+
that support the answer, and a budget; it reports whether those turns are still
|
|
21
|
+
in front of the model, how much of the budget went on them, and how the answer
|
|
22
|
+
gets worse the further back the evidence sits.
|
|
23
|
+
|
|
24
|
+
## 1,527 questions over ten long conversations
|
|
25
|
+
|
|
26
|
+
[LoCoMo](https://github.com/snap-research/locomo) records ten conversations
|
|
27
|
+
running 19 to 32 sessions each. Every question names the turns that answer it.
|
|
28
|
+
An average conversation is 23,056 tokens; the evidence for the average question
|
|
29
|
+
is **84 of them**, so 0.4% of the transcript carries the answer and the other
|
|
30
|
+
99.6% is the search problem.
|
|
31
|
+
|
|
32
|
+
At 2,048 tokens, 8.9% of the conversation:
|
|
33
|
+
|
|
34
|
+
| Policy | Sees the question | Recall | 95% CI | Whole turns |
|
|
35
|
+
| --- | :---: | ---: | --- | ---: |
|
|
36
|
+
| `oracle` (keeps only the evidence) | yes | 100.0% | 100.0–100.0% | 100.0% |
|
|
37
|
+
| `fact_memory` | yes | 62.7% | 60.2–65.1% | 3.9% |
|
|
38
|
+
| `retrieval` | yes | 59.5% | 56.8–62.0% | 59.5% |
|
|
39
|
+
| `session_summary` | no | 13.4% | 11.7–15.2% | 2.0% |
|
|
40
|
+
| `head_tail` | no | 8.2% | 6.9–9.6% | 8.2% |
|
|
41
|
+
| `recency` | no | 7.7% | 6.4–9.0% | 7.7% |
|
|
42
|
+
|
|
43
|
+
A sliding window at this budget answers 7.7% of what it could. Reading the
|
|
44
|
+
question first takes that to 59.5% for the same money. Put the other way: a
|
|
45
|
+
window needs 16,384 tokens to reach the 50% that retrieval reaches at 1,024.
|
|
46
|
+
|
|
47
|
+

|
|
48
|
+
|
|
49
|
+
The right panel is where the money is. Recency holds every question whose
|
|
50
|
+
evidence is in the session in progress and 62.3% of the ones a session or two
|
|
51
|
+
back. Three sessions back it holds **none of them**, while retrieval is still at
|
|
52
|
+
72.1%. Long-term memory is a different problem from the window problem, and the
|
|
53
|
+
window scores zero on it.
|
|
54
|
+
|
|
55
|
+
Full numbers, including every budget from 512 to 32,768 and a split by question
|
|
56
|
+
type: [`examples/locomo/results/locomo.md`](examples/locomo/results/locomo.md).
|
|
57
|
+
|
|
58
|
+
## Two results that do not flatter the tooling
|
|
59
|
+
|
|
60
|
+
**Multi-hop questions are not solved by retrieval.** At 2,048 tokens the best
|
|
61
|
+
policy holds the evidence for 76.8% of single-hop questions and 16.2% of
|
|
62
|
+
multi-hop ones. A multi-hop question needs several turns from several sessions
|
|
63
|
+
at once, and one query against a store tends to bring back one of them. Anyone
|
|
64
|
+
reporting a single memory-recall number is averaging over this gap.
|
|
65
|
+
|
|
66
|
+
**`fact_memory` wins on a technicality worth knowing about.** It stores
|
|
67
|
+
sentences rather than turns, so it fits more of the conversation into the same
|
|
68
|
+
budget and scores 62.7% against retrieval's 59.5%. But the whole evidence turn
|
|
69
|
+
survives only 3.9% of the time; the rest of its wins hand the model one sentence
|
|
70
|
+
out of several. Whichever column you take as the real one, take the same column
|
|
71
|
+
for every policy.
|
|
72
|
+
|
|
73
|
+
## What is being measured
|
|
74
|
+
|
|
75
|
+
Whether a named span survived, and nothing else. A model handed the evidence can
|
|
76
|
+
still answer wrongly, so every number here is a ceiling on what any model behind
|
|
77
|
+
the policy could do, not a score for the model. The point of a ceiling is that it
|
|
78
|
+
is cheap: no API calls, no judge, no variance between runs, and a policy that
|
|
79
|
+
loses the evidence is not worth benchmarking end to end.
|
|
80
|
+
|
|
81
|
+
The same convention rules out one thing: an abstractive summariser rewrites what
|
|
82
|
+
it keeps, and a rewritten sentence cannot be traced to a turn. The summary policy
|
|
83
|
+
here is extractive so that survival stays answerable. Token counts come from a
|
|
84
|
+
four-characters-per-token estimate, which moves absolute numbers around by up to
|
|
85
|
+
22 points for a window policy, so pass your own tokenizer if the budget has to be
|
|
86
|
+
exact. It does not move which policies win, and there is a test holding it to that.
|
|
87
|
+
|
|
88
|
+
## Install
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
pip install retainkit
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
No runtime dependencies. `pip install "retainkit[viz]"` adds matplotlib for the
|
|
95
|
+
figures.
|
|
96
|
+
|
|
97
|
+
## Use
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from retainkit import Case, Probe, Recency, Retrieval, Transcript, Turn, run, sweep_table
|
|
101
|
+
|
|
102
|
+
case = Case(
|
|
103
|
+
transcript=Transcript(
|
|
104
|
+
id="guest-42",
|
|
105
|
+
turns=(
|
|
106
|
+
Turn(id="s1:1", session=1, speaker="guest", text="I never eat shellfish."),
|
|
107
|
+
Turn(id="s1:2", session=1, speaker="agent", text="Noted, thank you."),
|
|
108
|
+
Turn(id="s9:7", session=9, speaker="guest", text="Book me somewhere near the water."),
|
|
109
|
+
),
|
|
110
|
+
),
|
|
111
|
+
probes=(Probe(id="q1", question="What can this guest not eat?", evidence=("s1:1",)),),
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
print(sweep_table(run([case], (Recency(), Retrieval()), budgets=(64, 512))))
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Your own conversations go in as JSON and out as a report:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
retainkit eval conversations.json --budget 2048 --budget 8192 --out results/
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
The file format is in [`src/retainkit/io.py`](src/retainkit/io.py). `retainkit
|
|
124
|
+
demo` runs a seeded fixture and needs no data at all.
|
|
125
|
+
|
|
126
|
+
## The policies
|
|
127
|
+
|
|
128
|
+
| Policy | Keeps | Reads the question |
|
|
129
|
+
| --- | --- | :---: |
|
|
130
|
+
| `recency` | The newest turns that fit. | no |
|
|
131
|
+
| `head_tail` | The opening of the conversation and the newest turns. | no |
|
|
132
|
+
| `session_summary` | The live session, plus the lead sentences of each closed one. | no |
|
|
133
|
+
| `retrieval` | The turns that rank highest against the question, put back in order. | yes |
|
|
134
|
+
| `fact_memory` | The live session, plus sentences pulled from a long-term store. | yes |
|
|
135
|
+
| `oracle` | The evidence and nothing else. Not runnable in production. | yes |
|
|
136
|
+
|
|
137
|
+
A policy is anything with a `name`, a `query_aware` flag and a `select` method,
|
|
138
|
+
so your own goes straight into the same sweep:
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
class KeepEverySecondTurn:
|
|
142
|
+
name = "every_second"
|
|
143
|
+
query_aware = False
|
|
144
|
+
|
|
145
|
+
def select(self, transcript, probe, budget, tokenizer):
|
|
146
|
+
from retainkit.policies import fill, package
|
|
147
|
+
from retainkit.policies.base import turn_fragments
|
|
148
|
+
|
|
149
|
+
kept, _ = fill(turn_fragments(transcript)[::2], budget, tokenizer)
|
|
150
|
+
return package(self.name, budget, kept, tokenizer)
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
## Reproducing
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
make install
|
|
157
|
+
make test
|
|
158
|
+
make demo # the seeded fixture, written to results/
|
|
159
|
+
python examples/locomo/fetch_data.py
|
|
160
|
+
python examples/locomo/run_locomo.py
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
LoCoMo is fetched at run time and is not redistributed here; it carries the terms
|
|
164
|
+
of its own release. The fixture results in `results/` are rebuilt by CI on every
|
|
165
|
+
push and the build fails if a committed number no longer matches the code, so
|
|
166
|
+
nothing quoted above can drift away from what the package produces.
|
|
167
|
+
|
|
168
|
+
## Citing
|
|
169
|
+
|
|
170
|
+
```bibtex
|
|
171
|
+
@software{mohammadi_retainkit,
|
|
172
|
+
author = {Mohammadi, Hadi},
|
|
173
|
+
title = {retainkit: context and memory policies for LLM agents,
|
|
174
|
+
scored by the evidence that survives the token budget},
|
|
175
|
+
year = {2026},
|
|
176
|
+
url = {https://github.com/mohammadi-hadi/retainkit},
|
|
177
|
+
license = {MIT}
|
|
178
|
+
}
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
LoCoMo is by Maharana et al., *Evaluating Very Long-Term Conversational Memory
|
|
182
|
+
of LLM Agents*, ACL 2024. Cite it if you use these results.
|
|
183
|
+
|
|
184
|
+
## Related
|
|
185
|
+
|
|
186
|
+
[judgekit](https://github.com/mohammadi-hadi/judgekit) audits an LLM judge,
|
|
187
|
+
[arenakit](https://github.com/mohammadi-hadi/arenakit) audits a pairwise
|
|
188
|
+
leaderboard, [trajectory-judge](https://github.com/mohammadi-hadi/trajectory-judge)
|
|
189
|
+
scores agent trajectories. More at [mohammadi.cv](https://mohammadi.cv).
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""Download the LoCoMo conversations.
|
|
2
|
+
|
|
3
|
+
LoCoMo is released by Snap Research with the paper *Evaluating Very Long-Term
|
|
4
|
+
Conversational Memory of LLM Agents* (Maharana et al., ACL 2024). The file is
|
|
5
|
+
fetched into `examples/locomo/data/`, which is not tracked here: the dataset
|
|
6
|
+
carries its own terms and is not redistributed with this package.
|
|
7
|
+
|
|
8
|
+
python examples/locomo/fetch_data.py
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import sys
|
|
14
|
+
import urllib.request
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
URL = "https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json"
|
|
18
|
+
DATA = Path(__file__).parent / "data" / "locomo10.json"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def main() -> int:
|
|
22
|
+
DATA.parent.mkdir(parents=True, exist_ok=True)
|
|
23
|
+
if DATA.exists():
|
|
24
|
+
print(f"already here: {DATA} ({DATA.stat().st_size / 1e6:.1f} MB)")
|
|
25
|
+
return 0
|
|
26
|
+
print(f"fetching {URL}")
|
|
27
|
+
urllib.request.urlretrieve(URL, DATA) # noqa: S310
|
|
28
|
+
print(f"wrote {DATA} ({DATA.stat().st_size / 1e6:.1f} MB)")
|
|
29
|
+
return 0
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
if __name__ == "__main__":
|
|
33
|
+
sys.exit(main())
|