ctx-slim 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ctx_slim-0.1.0/.github/workflows/ci.yml +48 -0
- ctx_slim-0.1.0/.github/workflows/publish-test.yml +20 -0
- ctx_slim-0.1.0/.github/workflows/publish.yml +19 -0
- ctx_slim-0.1.0/.gitignore +14 -0
- ctx_slim-0.1.0/LICENSE +21 -0
- ctx_slim-0.1.0/METHODS.md +70 -0
- ctx_slim-0.1.0/PKG-INFO +223 -0
- ctx_slim-0.1.0/README.md +190 -0
- ctx_slim-0.1.0/bench/__init__.py +0 -0
- ctx_slim-0.1.0/bench/bench_dedupe.py +78 -0
- ctx_slim-0.1.0/bench/figures/fig1_divergence.pdf +0 -0
- ctx_slim-0.1.0/bench/figures/fig1_divergence.png +0 -0
- ctx_slim-0.1.0/bench/figures/fig2_cache_decay.pdf +0 -0
- ctx_slim-0.1.0/bench/figures/fig2_cache_decay.png +0 -0
- ctx_slim-0.1.0/bench/figures/fig3_price_per_token.pdf +0 -0
- ctx_slim-0.1.0/bench/figures/fig3_price_per_token.png +0 -0
- ctx_slim-0.1.0/bench/figures.py +187 -0
- ctx_slim-0.1.0/bench/killgate.py +536 -0
- ctx_slim-0.1.0/bench/results/.killgate_progress.json +3002 -0
- ctx_slim-0.1.0/bench/results/.spend_ledger.json +84 -0
- ctx_slim-0.1.0/bench/results/killgate-run1-contaminated.json +1644 -0
- ctx_slim-0.1.0/bench/results/killgate-run2-confounded.json +1644 -0
- ctx_slim-0.1.0/bench/results/killgate.json +3025 -0
- ctx_slim-0.1.0/bench/results/validation.json +194 -0
- ctx_slim-0.1.0/bench/run_until_done.sh +39 -0
- ctx_slim-0.1.0/bench/validate.py +182 -0
- ctx_slim-0.1.0/benchmarks/run_benchmark.py +149 -0
- ctx_slim-0.1.0/benchmarks/traces/sample_agent_trace.json +213 -0
- ctx_slim-0.1.0/pyproject.toml +78 -0
- ctx_slim-0.1.0/src/context_slim/__init__.py +69 -0
- ctx_slim-0.1.0/src/context_slim/audit.py +91 -0
- ctx_slim-0.1.0/src/context_slim/cache/__init__.py +5 -0
- ctx_slim-0.1.0/src/context_slim/cache/model.py +127 -0
- ctx_slim-0.1.0/src/context_slim/cache/prefix.py +225 -0
- ctx_slim-0.1.0/src/context_slim/cache/rates.py +181 -0
- ctx_slim-0.1.0/src/context_slim/core.py +220 -0
- ctx_slim-0.1.0/src/context_slim/expiry.py +222 -0
- ctx_slim-0.1.0/src/context_slim/json_ast.py +238 -0
- ctx_slim-0.1.0/src/context_slim/ledger.py +163 -0
- ctx_slim-0.1.0/src/context_slim/policy.py +142 -0
- ctx_slim-0.1.0/src/context_slim/presets.py +87 -0
- ctx_slim-0.1.0/src/context_slim/providers/__init__.py +13 -0
- ctx_slim-0.1.0/src/context_slim/providers/_base.py +142 -0
- ctx_slim-0.1.0/src/context_slim/pruner.py +113 -0
- ctx_slim-0.1.0/src/context_slim/py.typed +0 -0
- ctx_slim-0.1.0/src/context_slim/schemas.py +168 -0
- ctx_slim-0.1.0/tests/conftest.py +58 -0
- ctx_slim-0.1.0/tests/test_api.py +140 -0
- ctx_slim-0.1.0/tests/test_audit.py +94 -0
- ctx_slim-0.1.0/tests/test_cost_model.py +148 -0
- ctx_slim-0.1.0/tests/test_json_ast.py +137 -0
- ctx_slim-0.1.0/tests/test_killgate_fixture.py +52 -0
- ctx_slim-0.1.0/tests/test_ledger.py +73 -0
- ctx_slim-0.1.0/tests/test_performance.py +143 -0
- ctx_slim-0.1.0/tests/test_policy.py +96 -0
- ctx_slim-0.1.0/tests/test_prefix.py +65 -0
- ctx_slim-0.1.0/tests/test_presets.py +63 -0
- ctx_slim-0.1.0/tests/test_providers.py +65 -0
- ctx_slim-0.1.0/tests/test_pruner.py +59 -0
- ctx_slim-0.1.0/tests/test_tool_integrity.py +118 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push: { branches: [master] }
|
|
5
|
+
pull_request:
|
|
6
|
+
schedule:
|
|
7
|
+
# Rates go stale at 90 days; catch it on a Monday, not in a launch thread.
|
|
8
|
+
- cron: "0 6 * * 1"
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
test:
|
|
12
|
+
runs-on: ${{ matrix.os }}
|
|
13
|
+
strategy:
|
|
14
|
+
fail-fast: false
|
|
15
|
+
matrix:
|
|
16
|
+
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
17
|
+
python: ["3.9", "3.11", "3.13"]
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
- uses: actions/setup-python@v5
|
|
21
|
+
with: { python-version: "${{ matrix.python }}" }
|
|
22
|
+
- run: pip install -e ".[dev]"
|
|
23
|
+
- run: pytest -q --cov=context_slim --cov-report=term-missing
|
|
24
|
+
- name: performance (uninstrumented)
|
|
25
|
+
# The timing assertions skip themselves under a coverage tracer, which
|
|
26
|
+
# would otherwise measure the tracer rather than the code. Run them
|
|
27
|
+
# once more with coverage off so the sub-5ms budget is still enforced.
|
|
28
|
+
# Linux only: hosted macOS/Windows runners are too noisy for a
|
|
29
|
+
# wall-clock budget, and a flaky perf gate gets ignored, not fixed.
|
|
30
|
+
run: pytest -q --no-cov tests/test_performance.py
|
|
31
|
+
if: matrix.os == 'ubuntu-latest'
|
|
32
|
+
- name: no runtime dependencies
|
|
33
|
+
run: |
|
|
34
|
+
pip install pipdeptree
|
|
35
|
+
pipdeptree -p context-slim --warn silence | tee /tmp/deps.txt
|
|
36
|
+
test "$(grep -c '^ ' /tmp/deps.txt || true)" -eq 0
|
|
37
|
+
shell: bash
|
|
38
|
+
if: matrix.os == 'ubuntu-latest'
|
|
39
|
+
|
|
40
|
+
lint:
|
|
41
|
+
runs-on: ubuntu-latest
|
|
42
|
+
steps:
|
|
43
|
+
- uses: actions/checkout@v4
|
|
44
|
+
- uses: actions/setup-python@v5
|
|
45
|
+
with: { python-version: "3.11" }
|
|
46
|
+
- run: pip install -e ".[dev]"
|
|
47
|
+
- run: ruff check .
|
|
48
|
+
- run: mypy --strict src/context_slim
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
name: publish-test
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch: # manual only - a test upload should never fire by accident
|
|
5
|
+
|
|
6
|
+
jobs:
|
|
7
|
+
testpypi:
|
|
8
|
+
runs-on: ubuntu-latest
|
|
9
|
+
environment: testpypi
|
|
10
|
+
permissions:
|
|
11
|
+
id-token: write # OIDC token for trusted publishing - no API token or secret needed
|
|
12
|
+
steps:
|
|
13
|
+
- uses: actions/checkout@v4
|
|
14
|
+
- uses: actions/setup-python@v5
|
|
15
|
+
with: { python-version: "3.11" }
|
|
16
|
+
- run: pip install build
|
|
17
|
+
- run: python -m build
|
|
18
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
19
|
+
with:
|
|
20
|
+
repository-url: https://test.pypi.org/legacy/
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
name: publish
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
pypi:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
environment: pypi
|
|
11
|
+
permissions:
|
|
12
|
+
id-token: write # OIDC token for trusted publishing - no API token or secret needed
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
- uses: actions/setup-python@v5
|
|
16
|
+
with: { python-version: "3.11" }
|
|
17
|
+
- run: pip install build
|
|
18
|
+
- run: python -m build
|
|
19
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
ctx_slim-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ramtin
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# Methods and threats to validity
|
|
2
|
+
|
|
3
|
+
Four runs. Three produced confident headline numbers that were artifacts, each
|
|
4
|
+
pointing the opposite way to the one before it. Run 1 said pruning saves 29.6%;
|
|
5
|
+
run 3 said it costs 41.6% more. Same code, same provider, same week.
|
|
6
|
+
|
|
7
|
+
That instability is itself the most transferable finding here, so it is
|
|
8
|
+
documented rather than quietly dropped.
|
|
9
|
+
|
|
10
|
+
## Confound 1 — cross-arm cache sharing (runs 1-2, invalidated)
|
|
11
|
+
|
|
12
|
+
Provider prompt caches are content-addressed and scoped to the **account**, not
|
|
13
|
+
to the process or the experiment. All arms were built from one identical prefix
|
|
14
|
+
and executed in blocks, so whichever arm ran first paid the cache write and
|
|
15
|
+
every later arm free-rode on it. Execution order was confounded with treatment.
|
|
16
|
+
|
|
17
|
+
Caught by a variance check, not by inspection: sigma was roughly half the mean;
|
|
18
|
+
every condition declined monotonically across repeats; and different conditions
|
|
19
|
+
returned bit-identical costs ($0.003214 in three arms), which only happens when
|
|
20
|
+
arms are reading the same cache entries.
|
|
21
|
+
|
|
22
|
+
**Fix.** A unique high-entropy salt at the head of each arm's system prompt, so
|
|
23
|
+
each of the 15 arms occupies its own cache namespace and pays its own way. Arms
|
|
24
|
+
shuffled under a fixed seed so position cannot correlate with treatment.
|
|
25
|
+
|
|
26
|
+
## Confound 2 — non-monotonic pruning policy (run 3, half invalidated)
|
|
27
|
+
|
|
28
|
+
`tail_first` came out worst, contradicting prefix-cache theory. Replaying the
|
|
29
|
+
stub sets showed the policy recomputed "newest half of candidates" from scratch
|
|
30
|
+
each turn, so previously-stubbed messages fell out of that half and reverted to
|
|
31
|
+
full content - a deep prefix change on every turn.
|
|
32
|
+
|
|
33
|
+
Real pruners never un-clear. `clear_tool_uses_20250919` and LangChain's
|
|
34
|
+
truncation are both monotonic.
|
|
35
|
+
|
|
36
|
+
**Fix.** A persistent cleared-set that only grows.
|
|
37
|
+
|
|
38
|
+
## Confound 3 — fraction-based clearing erases the variable (found pre-run-4)
|
|
39
|
+
|
|
40
|
+
Accumulating a fixed fraction converges on clearing every candidate, at which
|
|
41
|
+
point both orderings produce the same set and the independent variable
|
|
42
|
+
disappears (8585 vs 8566 tokens - indistinguishable).
|
|
43
|
+
|
|
44
|
+
**Fix.** Budget-triggered clearing: clear the minimum needed to get back under a
|
|
45
|
+
token threshold, which is what `clear_tool_uses_20250919` does.
|
|
46
|
+
|
|
47
|
+
## Run 4 design (the valid one)
|
|
48
|
+
|
|
49
|
+
Salted namespaces, shuffled arms, monotonic budget-triggered policies, n=5,
|
|
50
|
+
bootstrap 95% CIs on arm-level means. Predictions were pre-registered before the
|
|
51
|
+
data landed: tail-first's hit rate would rise from 69.1%, tail-first would be
|
|
52
|
+
cheaper than oldest-first, and no-prune would stay cheapest. All three held.
|
|
53
|
+
|
|
54
|
+
## Known limitations
|
|
55
|
+
|
|
56
|
+
- 8k prefixes, 20 turns, synthetic loops. Larger regimes untested.
|
|
57
|
+
- One model, one provider, one account, one region.
|
|
58
|
+
- `no_prune` sends more tokens by construction; that it still wins on cost is
|
|
59
|
+
the point, but it is not a like-for-like token comparison.
|
|
60
|
+
- A cache-hit dip at turn 14 appears in all three conditions including
|
|
61
|
+
`no_prune`, so it is not caused by pruning. Unexplained.
|
|
62
|
+
- Provider cache internals are opaque. We observe billing, not the cache.
|
|
63
|
+
- n=5 arms: adequate for the observed effect sizes, thin for subtle ones.
|
|
64
|
+
|
|
65
|
+
## A note for anyone benchmarking prompt caches
|
|
66
|
+
|
|
67
|
+
Prompt-cache experiments are **stateful across arms** in a way ordinary A/B
|
|
68
|
+
benchmarks are not. The cache is a hidden channel between conditions. Any
|
|
69
|
+
comparison that does not explicitly namespace its cache keys is measuring
|
|
70
|
+
execution order as much as treatment.
|
ctx_slim-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ctx-slim
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Prune your LLM agent's context without destroying your prompt cache.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Ramtin2000/context-slim
|
|
6
|
+
Project-URL: Repository, https://github.com/Ramtin2000/context-slim
|
|
7
|
+
Project-URL: Issues, https://github.com/Ramtin2000/context-slim/issues
|
|
8
|
+
Author: Ramtin
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: agents,context-engineering,finops,llm,prompt-caching
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Provides-Extra: bench
|
|
25
|
+
Requires-Dist: openai>=1.40; extra == 'bench'
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: hypothesis; extra == 'dev'
|
|
28
|
+
Requires-Dist: mypy>=1.8; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest-cov; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
31
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# context-slim
|
|
35
|
+
|
|
36
|
+
**Pruning your LLM agent's context can cost more than leaving it alone.**
|
|
37
|
+
|
|
38
|
+
Measured against a live API, n=5 arms per condition, bootstrap 95% CIs:
|
|
39
|
+
|
|
40
|
+
| strategy | tokens sent | cache hit | cost/arm | vs no pruning |
|
|
41
|
+
|---|---|---|---|---|
|
|
42
|
+
| don't prune | 928,400 | **92.2%** | **$0.007053** | — |
|
|
43
|
+
| prune oldest-first | 735,790 | 75.5% | $0.011239 | **+59.4%** |
|
|
44
|
+
| prune newest-first | 735,770 | 81.0% | $0.009376 | **+32.9%** |
|
|
45
|
+
|
|
46
|
+
**20.7% fewer tokens. 33–59% more money.** All three differences significant.
|
|
47
|
+
|
|
48
|
+
Prompt caches are *prefix* caches, so an un-pruned loop is append-only — every
|
|
49
|
+
turn extends the last and the whole prompt is reusable. Pruning breaks that, and
|
|
50
|
+
the re-write costs more than the tokens saved.
|
|
51
|
+
|
|
52
|
+
Where you cut still matters: newest-first is **16.6% cheaper** than oldest-first
|
|
53
|
+
at identical token counts (a 20-token difference). Anthropic's context-editing
|
|
54
|
+
API clears oldest-first by default.
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
cache hit rate by turn
|
|
58
|
+
t2 t5 t8 t11 t14 t17 t20
|
|
59
|
+
don't prune 95% 96% 96% 96% 77% 97% 97%
|
|
60
|
+
oldest-first 95% 96% 63% 77% 76% 77% 75%
|
|
61
|
+
newest-first 95% 96% 93% 91% 76% 77% 60%
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
> **Caveat this properly.** 8k prefixes, 20 turns, synthetic loops, one model
|
|
65
|
+
> (`gpt-5.6-luna`), one account. Larger contexts over longer horizons are
|
|
66
|
+
> untested and may behave differently. Three earlier revisions of this
|
|
67
|
+
> experiment produced confident numbers that were artifacts — see
|
|
68
|
+
> [`METHODS.md`](METHODS.md) for what went wrong and how it was caught.
|
|
69
|
+
|
|
70
|
+
Reproduce: `python -m bench.killgate --repeats 5` (~$0.14). Raw usage blocks in
|
|
71
|
+
[`bench/results/`](bench/results/).
|
|
72
|
+
|
|
73
|
+
## Why the dedupe is built the way it is
|
|
74
|
+
|
|
75
|
+
The textbook approach to block deduplication is a byte-level rolling hash with
|
|
76
|
+
content-defined boundaries. In pure Python that is one interpreter iteration per
|
|
77
|
+
byte, and it does not fit a sub-5ms budget. `str.split` plus `hashlib.blake2b`
|
|
78
|
+
does the same job at block granularity with both halves running in C.
|
|
79
|
+
|
|
80
|
+
Measured on ~93 KB (`python -m bench.bench_dedupe`):
|
|
81
|
+
|
|
82
|
+
| | time |
|
|
83
|
+
|---|---|
|
|
84
|
+
| rejected: 64-byte rolling hash (per-byte Python loop) | 25.26 ms |
|
|
85
|
+
| shipped: `str.split` + `blake2b` (per-block loop) | **0.24 ms** |
|
|
86
|
+
| shipped: full `dedupe_blocks` pass | 0.54 ms |
|
|
87
|
+
| shipped: `collapse_whitespace` | 1.06 ms |
|
|
88
|
+
|
|
89
|
+
**104× on the hashing step.** The rejected implementation is kept in
|
|
90
|
+
`bench/bench_dedupe.py` so the comparison is measured rather than asserted.
|
|
91
|
+
|
|
92
|
+
## The cost model is checkable
|
|
93
|
+
|
|
94
|
+
Most token accounting asks you to trust it. This one predicts how much of a
|
|
95
|
+
request the API will report as cached, *before* the call, then diffs against
|
|
96
|
+
`usage.prompt_tokens_details`.
|
|
97
|
+
|
|
98
|
+
Measured over 24 live requests (`python -m bench.validate`, ~$0.01):
|
|
99
|
+
|
|
100
|
+
| | raw | calibrated |
|
|
101
|
+
|---|---|---|
|
|
102
|
+
| prompt-token error (median) | 26.11% | **0.64%** |
|
|
103
|
+
| cached-token error (median) | 25.80% | **0.84%** |
|
|
104
|
+
|
|
105
|
+
The raw 26% was a single wrong constant, not a broken model — the
|
|
106
|
+
predicted/actual ratio had a spread of 0.9%, so dividing it out left a max
|
|
107
|
+
residual of 1.78%. The estimator is calibrated by that constant in
|
|
108
|
+
`cache/prefix.py`, with both caveats stated there: it is fit to one tokenizer
|
|
109
|
+
family, and it sits inside the pruning policy's own budget, so it cannot be
|
|
110
|
+
recalibrated and replayed against an old run.
|
|
111
|
+
|
|
112
|
+
No dollar figure in this repo comes from the estimator. Those all read the
|
|
113
|
+
provider's usage counters.
|
|
114
|
+
|
|
115
|
+
## Install
|
|
116
|
+
|
|
117
|
+
> **Not on PyPI yet.** Install from source until v0.1.0 ships:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
pip install git+https://github.com/Ramtin2000/context-slim
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Zero runtime dependencies. No model, no GPU, no network. Python 3.9+.
|
|
124
|
+
|
|
125
|
+
## Use
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
from context_slim import doctor, plan, apply
|
|
129
|
+
|
|
130
|
+
# 1. Find cache pathologies that cost money silently.
|
|
131
|
+
for d in doctor(messages, model="openai/gpt-5.6-luna"):
|
|
132
|
+
print(d.code, d.message)
|
|
133
|
+
|
|
134
|
+
# 2. Decide what is worth pruning. Pure — no I/O, no mutation.
|
|
135
|
+
p = plan(messages, model="openai/gpt-5.6-luna", horizon=30)
|
|
136
|
+
for v in p.verdicts:
|
|
137
|
+
print(v.decision.value, v.reason)
|
|
138
|
+
|
|
139
|
+
# 3. Execute only the approved edits.
|
|
140
|
+
messages, report = apply(messages, p)
|
|
141
|
+
print(report)
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
`plan()` and `apply()` are separate so that **"don't prune" is an ordinary
|
|
145
|
+
outcome you can inspect**, not an exception or a silent no-op:
|
|
146
|
+
|
|
147
|
+
```
|
|
148
|
+
REFUSE msg 2 structurally unprofitable: W/S = 41.2 means 461.3 turns to pay
|
|
149
|
+
back $0.000412, against a horizon of 20. Prune closer to the tail.
|
|
150
|
+
PLAN msg 14 pays back after 4.1 turns (horizon 20); costs $0.000082 now,
|
|
151
|
+
saves $0.000020/turn, net $0.000318 at horizon
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
## The `doctor` check
|
|
155
|
+
|
|
156
|
+
Two pathologies cost money with no pruning involved at all:
|
|
157
|
+
|
|
158
|
+
- **`lookback-overrun`** — Anthropic checks at most 20 positions behind a cache
|
|
159
|
+
breakpoint. Grow past that and the hit is missed silently. No error. Just a bill.
|
|
160
|
+
- **`no-breakpoint`** — Anthropic caching is opt-in. Without a breakpoint,
|
|
161
|
+
nothing is cached and every turn pays full price.
|
|
162
|
+
|
|
163
|
+
`context-slim doctor conversation.json` exits non-zero on an error-severity
|
|
164
|
+
finding, so it can sit in CI.
|
|
165
|
+
|
|
166
|
+
## What this is NOT
|
|
167
|
+
|
|
168
|
+
- ❌ Not a summarizer, embedder, or tokenizer. No model is ever loaded.
|
|
169
|
+
- ❌ Not a competitor to Anthropic's context editing or LangChain's compaction —
|
|
170
|
+
a **cost-aware controller** that decides whether and where to invoke them.
|
|
171
|
+
- ❌ Not "fewer tokens at any cost." Sometimes the answer is *don't*, and this
|
|
172
|
+
is the only tool that will tell you so.
|
|
173
|
+
|
|
174
|
+
## When NOT to use it
|
|
175
|
+
|
|
176
|
+
Short loops, uncached workloads, and any prefix below the model's cache minimum
|
|
177
|
+
(512 tokens on Claude Opus 5, 1024 on GPT-5.6). `doctor` reports all three.
|
|
178
|
+
|
|
179
|
+
## Repository layout
|
|
180
|
+
|
|
181
|
+
```
|
|
182
|
+
src/context_slim/
|
|
183
|
+
core.py CacheAlignedContext — pins the Anchor Zone, orchestrates plan/apply
|
|
184
|
+
expiry.py candidate generation, tail-first ordering, ATOMIC_PURGE / TOMBSTONE
|
|
185
|
+
pruner.py block dedupe and whitespace collapse (Layer 4 text ops)
|
|
186
|
+
json_ast.py schema-preserving trim, and dependency-free JSON minification
|
|
187
|
+
schemas.py frozen dataclasses — Money is exact-integer, never float
|
|
188
|
+
audit.py CLI: `context-slim doctor|plan|apply|simulate`
|
|
189
|
+
policy.py the break-even decision engine (Law 1 lives here)
|
|
190
|
+
ledger.py defers unprofitable prunes until the cache breaks anyway
|
|
191
|
+
cache/ the cost model, rate tables, and prefix diagnostics
|
|
192
|
+
providers/ OpenAI / Anthropic wire-shape adapters
|
|
193
|
+
bench/ the real benchmark: live API calls, salted cache namespaces,
|
|
194
|
+
bootstrap CIs. Costs real money to run — see METHODS.md for
|
|
195
|
+
what it took to make it trustworthy.
|
|
196
|
+
benchmarks/ an offline, zero-cost illustration of the same idea, using
|
|
197
|
+
the same cost primitives against a synthetic trace. Useful to
|
|
198
|
+
see the shape of the argument before spending anything
|
|
199
|
+
confirming it — not a substitute for bench/.
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
`CacheAlignedContext` (`core.py`) is an optional stateful wrapper around the
|
|
203
|
+
same four functions exported at the package root — it computes the Anchor
|
|
204
|
+
Zone boundary once and threads it through every call, so a candidate can
|
|
205
|
+
never be proposed inside the immutable prefix regardless of preset:
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
from context_slim import CacheAlignedContext
|
|
209
|
+
|
|
210
|
+
ctx = CacheAlignedContext(messages, model="openai/gpt-5.6-luna", preset="balanced")
|
|
211
|
+
ctx.anchor # the immutable prefix — system prompt, tools, Turn 1
|
|
212
|
+
ctx.compaction_zone # everything eligible for pruning
|
|
213
|
+
messages, report = ctx.apply(ctx.plan(horizon=30))
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
## Status
|
|
217
|
+
|
|
218
|
+
Pre-release, built in public over 14 days. The cost model is validated against
|
|
219
|
+
providers' own `cached_tokens` counters — see `bench/killgate.py`.
|
|
220
|
+
|
|
221
|
+
## License
|
|
222
|
+
|
|
223
|
+
MIT
|
ctx_slim-0.1.0/README.md
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
# context-slim
|
|
2
|
+
|
|
3
|
+
**Pruning your LLM agent's context can cost more than leaving it alone.**
|
|
4
|
+
|
|
5
|
+
Measured against a live API, n=5 arms per condition, bootstrap 95% CIs:
|
|
6
|
+
|
|
7
|
+
| strategy | tokens sent | cache hit | cost/arm | vs no pruning |
|
|
8
|
+
|---|---|---|---|---|
|
|
9
|
+
| don't prune | 928,400 | **92.2%** | **$0.007053** | — |
|
|
10
|
+
| prune oldest-first | 735,790 | 75.5% | $0.011239 | **+59.4%** |
|
|
11
|
+
| prune newest-first | 735,770 | 81.0% | $0.009376 | **+32.9%** |
|
|
12
|
+
|
|
13
|
+
**20.7% fewer tokens. 33–59% more money.** All three differences significant.
|
|
14
|
+
|
|
15
|
+
Prompt caches are *prefix* caches, so an un-pruned loop is append-only — every
|
|
16
|
+
turn extends the last and the whole prompt is reusable. Pruning breaks that, and
|
|
17
|
+
the re-write costs more than the tokens saved.
|
|
18
|
+
|
|
19
|
+
Where you cut still matters: newest-first is **16.6% cheaper** than oldest-first
|
|
20
|
+
at identical token counts (a 20-token difference). Anthropic's context-editing
|
|
21
|
+
API clears oldest-first by default.
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
cache hit rate by turn
|
|
25
|
+
t2 t5 t8 t11 t14 t17 t20
|
|
26
|
+
don't prune 95% 96% 96% 96% 77% 97% 97%
|
|
27
|
+
oldest-first 95% 96% 63% 77% 76% 77% 75%
|
|
28
|
+
newest-first 95% 96% 93% 91% 76% 77% 60%
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
> **Caveat this properly.** 8k prefixes, 20 turns, synthetic loops, one model
|
|
32
|
+
> (`gpt-5.6-luna`), one account. Larger contexts over longer horizons are
|
|
33
|
+
> untested and may behave differently. Three earlier revisions of this
|
|
34
|
+
> experiment produced confident numbers that were artifacts — see
|
|
35
|
+
> [`METHODS.md`](METHODS.md) for what went wrong and how it was caught.
|
|
36
|
+
|
|
37
|
+
Reproduce: `python -m bench.killgate --repeats 5` (~$0.14). Raw usage blocks in
|
|
38
|
+
[`bench/results/`](bench/results/).
|
|
39
|
+
|
|
40
|
+
## Why the dedupe is built the way it is
|
|
41
|
+
|
|
42
|
+
The textbook approach to block deduplication is a byte-level rolling hash with
|
|
43
|
+
content-defined boundaries. In pure Python that is one interpreter iteration per
|
|
44
|
+
byte, and it does not fit a sub-5ms budget. `str.split` plus `hashlib.blake2b`
|
|
45
|
+
does the same job at block granularity with both halves running in C.
|
|
46
|
+
|
|
47
|
+
Measured on ~93 KB (`python -m bench.bench_dedupe`):
|
|
48
|
+
|
|
49
|
+
| | time |
|
|
50
|
+
|---|---|
|
|
51
|
+
| rejected: 64-byte rolling hash (per-byte Python loop) | 25.26 ms |
|
|
52
|
+
| shipped: `str.split` + `blake2b` (per-block loop) | **0.24 ms** |
|
|
53
|
+
| shipped: full `dedupe_blocks` pass | 0.54 ms |
|
|
54
|
+
| shipped: `collapse_whitespace` | 1.06 ms |
|
|
55
|
+
|
|
56
|
+
**104× on the hashing step.** The rejected implementation is kept in
|
|
57
|
+
`bench/bench_dedupe.py` so the comparison is measured rather than asserted.
|
|
58
|
+
|
|
59
|
+
## The cost model is checkable
|
|
60
|
+
|
|
61
|
+
Most token accounting asks you to trust it. This one predicts how much of a
|
|
62
|
+
request the API will report as cached, *before* the call, then diffs against
|
|
63
|
+
`usage.prompt_tokens_details`.
|
|
64
|
+
|
|
65
|
+
Measured over 24 live requests (`python -m bench.validate`, ~$0.01):
|
|
66
|
+
|
|
67
|
+
| | raw | calibrated |
|
|
68
|
+
|---|---|---|
|
|
69
|
+
| prompt-token error (median) | 26.11% | **0.64%** |
|
|
70
|
+
| cached-token error (median) | 25.80% | **0.84%** |
|
|
71
|
+
|
|
72
|
+
The raw 26% was a single wrong constant, not a broken model — the
|
|
73
|
+
predicted/actual ratio had a spread of 0.9%, so dividing it out left a max
|
|
74
|
+
residual of 1.78%. The estimator is calibrated by that constant in
|
|
75
|
+
`cache/prefix.py`, with both caveats stated there: it is fit to one tokenizer
|
|
76
|
+
family, and it sits inside the pruning policy's own budget, so it cannot be
|
|
77
|
+
recalibrated and replayed against an old run.
|
|
78
|
+
|
|
79
|
+
No dollar figure in this repo comes from the estimator. Those all read the
|
|
80
|
+
provider's usage counters.
|
|
81
|
+
|
|
82
|
+
## Install
|
|
83
|
+
|
|
84
|
+
> **Not on PyPI yet.** Install from source until v0.1.0 ships:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
pip install git+https://github.com/Ramtin2000/context-slim
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Zero runtime dependencies. No model, no GPU, no network. Python 3.9+.
|
|
91
|
+
|
|
92
|
+
## Use
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
from context_slim import doctor, plan, apply
|
|
96
|
+
|
|
97
|
+
# 1. Find cache pathologies that cost money silently.
|
|
98
|
+
for d in doctor(messages, model="openai/gpt-5.6-luna"):
|
|
99
|
+
print(d.code, d.message)
|
|
100
|
+
|
|
101
|
+
# 2. Decide what is worth pruning. Pure — no I/O, no mutation.
|
|
102
|
+
p = plan(messages, model="openai/gpt-5.6-luna", horizon=30)
|
|
103
|
+
for v in p.verdicts:
|
|
104
|
+
print(v.decision.value, v.reason)
|
|
105
|
+
|
|
106
|
+
# 3. Execute only the approved edits.
|
|
107
|
+
messages, report = apply(messages, p)
|
|
108
|
+
print(report)
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
`plan()` and `apply()` are separate so that **"don't prune" is an ordinary
|
|
112
|
+
outcome you can inspect**, not an exception or a silent no-op:
|
|
113
|
+
|
|
114
|
+
```
|
|
115
|
+
REFUSE msg 2 structurally unprofitable: W/S = 41.2 means 461.3 turns to pay
|
|
116
|
+
back $0.000412, against a horizon of 20. Prune closer to the tail.
|
|
117
|
+
PLAN msg 14 pays back after 4.1 turns (horizon 20); costs $0.000082 now,
|
|
118
|
+
saves $0.000020/turn, net $0.000318 at horizon
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
## The `doctor` check
|
|
122
|
+
|
|
123
|
+
Two pathologies cost money with no pruning involved at all:
|
|
124
|
+
|
|
125
|
+
- **`lookback-overrun`** — Anthropic checks at most 20 positions behind a cache
|
|
126
|
+
breakpoint. Grow past that and the hit is missed silently. No error. Just a bill.
|
|
127
|
+
- **`no-breakpoint`** — Anthropic caching is opt-in. Without a breakpoint,
|
|
128
|
+
nothing is cached and every turn pays full price.
|
|
129
|
+
|
|
130
|
+
`context-slim doctor conversation.json` exits non-zero on an error-severity
|
|
131
|
+
finding, so it can sit in CI.
|
|
132
|
+
|
|
133
|
+
## What this is NOT
|
|
134
|
+
|
|
135
|
+
- ❌ Not a summarizer, embedder, or tokenizer. No model is ever loaded.
|
|
136
|
+
- ❌ Not a competitor to Anthropic's context editing or LangChain's compaction —
|
|
137
|
+
a **cost-aware controller** that decides whether and where to invoke them.
|
|
138
|
+
- ❌ Not "fewer tokens at any cost." Sometimes the answer is *don't*, and this
|
|
139
|
+
is the only tool that will tell you so.
|
|
140
|
+
|
|
141
|
+
## When NOT to use it
|
|
142
|
+
|
|
143
|
+
Short loops, uncached workloads, and any prefix below the model's cache minimum
|
|
144
|
+
(512 tokens on Claude Opus 5, 1024 on GPT-5.6). `doctor` reports all three.
|
|
145
|
+
|
|
146
|
+
## Repository layout
|
|
147
|
+
|
|
148
|
+
```
|
|
149
|
+
src/context_slim/
|
|
150
|
+
core.py CacheAlignedContext — pins the Anchor Zone, orchestrates plan/apply
|
|
151
|
+
expiry.py candidate generation, tail-first ordering, ATOMIC_PURGE / TOMBSTONE
|
|
152
|
+
pruner.py block dedupe and whitespace collapse (Layer 4 text ops)
|
|
153
|
+
json_ast.py schema-preserving trim, and dependency-free JSON minification
|
|
154
|
+
schemas.py frozen dataclasses — Money is exact-integer, never float
|
|
155
|
+
audit.py CLI: `context-slim doctor|plan|apply|simulate`
|
|
156
|
+
policy.py the break-even decision engine (Law 1 lives here)
|
|
157
|
+
ledger.py defers unprofitable prunes until the cache breaks anyway
|
|
158
|
+
cache/ the cost model, rate tables, and prefix diagnostics
|
|
159
|
+
providers/ OpenAI / Anthropic wire-shape adapters
|
|
160
|
+
bench/ the real benchmark: live API calls, salted cache namespaces,
|
|
161
|
+
bootstrap CIs. Costs real money to run — see METHODS.md for
|
|
162
|
+
what it took to make it trustworthy.
|
|
163
|
+
benchmarks/ an offline, zero-cost illustration of the same idea, using
|
|
164
|
+
the same cost primitives against a synthetic trace. Useful to
|
|
165
|
+
see the shape of the argument before spending anything
|
|
166
|
+
confirming it — not a substitute for bench/.
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
`CacheAlignedContext` (`core.py`) is an optional stateful wrapper around the
|
|
170
|
+
same four functions exported at the package root — it computes the Anchor
|
|
171
|
+
Zone boundary once and threads it through every call, so a candidate can
|
|
172
|
+
never be proposed inside the immutable prefix regardless of preset:
|
|
173
|
+
|
|
174
|
+
```python
|
|
175
|
+
from context_slim import CacheAlignedContext
|
|
176
|
+
|
|
177
|
+
ctx = CacheAlignedContext(messages, model="openai/gpt-5.6-luna", preset="balanced")
|
|
178
|
+
ctx.anchor # the immutable prefix — system prompt, tools, Turn 1
|
|
179
|
+
ctx.compaction_zone # everything eligible for pruning
|
|
180
|
+
messages, report = ctx.apply(ctx.plan(horizon=30))
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
## Status
|
|
184
|
+
|
|
185
|
+
Pre-release, built in public over 14 days. The cost model is validated against
|
|
186
|
+
providers' own `cached_tokens` counters — see `bench/killgate.py`.
|
|
187
|
+
|
|
188
|
+
## License
|
|
189
|
+
|
|
190
|
+
MIT
|
|
File without changes
|