ctxprune 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ctxprune-0.3.0/.gitignore +17 -0
- ctxprune-0.3.0/CHANGELOG.md +144 -0
- ctxprune-0.3.0/LICENSE +12 -0
- ctxprune-0.3.0/PACKAGE.md +143 -0
- ctxprune-0.3.0/PKG-INFO +514 -0
- ctxprune-0.3.0/README.md +470 -0
- ctxprune-0.3.0/THIRD_PARTY_NOTICES +28 -0
- ctxprune-0.3.0/docs/BENCHMARKS.md +461 -0
- ctxprune-0.3.0/docs/SAFETY.md +280 -0
- ctxprune-0.3.0/docs/bench/bench_shipped.py +20 -0
- ctxprune-0.3.0/docs/bench/latency_scripts_106bcea.json +26 -0
- ctxprune-0.3.0/docs/bench/latency_shipped_106bcea.json +9 -0
- ctxprune-0.3.0/docs/bench/measure_variant_dedup.diff +11 -0
- ctxprune-0.3.0/docs/bench/measure_variant_powershell.diff +29 -0
- ctxprune-0.3.0/docs/bench/real_cc_split.py +35 -0
- ctxprune-0.3.0/docs/bench/real_cc_split.txt +6 -0
- ctxprune-0.3.0/docs/bench/spill_gain_12c7b79.json +534 -0
- ctxprune-0.3.0/docs/bench/typical_dedup_results.json +4080 -0
- ctxprune-0.3.0/docs/bench/typical_dedup_summary.json +354 -0
- ctxprune-0.3.0/docs/bench/typical_safe_results.json +3702 -0
- ctxprune-0.3.0/docs/bench/typical_safe_summary.json +354 -0
- ctxprune-0.3.0/docs/ideas/README.md +51 -0
- ctxprune-0.3.0/docs/ideas/build.py +89 -0
- ctxprune-0.3.0/docs/ideas/dropped.md +355 -0
- ctxprune-0.3.0/docs/ideas/gate/README.md +155 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_00_examples.py +277 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_alias_model_cache.py +2401 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_dedup.py +3167 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_hook_infra.py +2192 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_hook_notes.py +1613 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_logs.py +2136 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_new_cloud_k8s_iac.py +1482 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_new_data_formats_text.py +1900 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_new_pkgmgr_langtools.py +2138 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_new_protocol_envelope.py +1754 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_new_system_ops_terminal.py +1713 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_new_vcs_ci_review.py +2521 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_noise_pretool.py +2062 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_rules_ci_infra_cloud.py +5613 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_rules_compilers_build_native.py +8111 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_rules_logs_ops_ml.py +4548 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_rules_media_docs_rag_misc.py +6663 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_rules_stateful_agents_code.py +4030 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_rules_structured_json.py +5656 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_struct_a.py +2730 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_struct_b.py +2680 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_struct_c.py +2915 -0
- ctxprune-0.3.0/docs/ideas/gate/cases_terminal.py +2253 -0
- ctxprune-0.3.0/docs/ideas/gate/harness.py +604 -0
- ctxprune-0.3.0/docs/ideas/gate/run.py +133 -0
- ctxprune-0.3.0/docs/ideas/ideas.jsonl +1463 -0
- ctxprune-0.3.0/docs/ideas/ideas.md +2569 -0
- ctxprune-0.3.0/docs/ideas/shortlist.md +469 -0
- ctxprune-0.3.0/docs/later-integrations.md +59 -0
- ctxprune-0.3.0/pyproject.toml +128 -0
- ctxprune-0.3.0/scripts/ctx_audit.py +1464 -0
- ctxprune-0.3.0/scripts/ctxprune_compress.py +344 -0
- ctxprune-0.3.0/scripts/ctxprune_core.py +594 -0
- ctxprune-0.3.0/scripts/ctxprune_note.py +53 -0
- ctxprune-0.3.0/scripts/ctxprune_proxy.py +1341 -0
- ctxprune-0.3.0/scripts/dedup.py +1535 -0
- ctxprune-0.3.0/scripts/emitted.py +76 -0
- ctxprune-0.3.0/scripts/export_v2_onnx.py +538 -0
- ctxprune-0.3.0/scripts/hook_budget.py +91 -0
- ctxprune-0.3.0/scripts/integrations/dsh/ctxprune.mjs +132 -0
- ctxprune-0.3.0/scripts/integrations/hermes/ctxprune/__init__.py +516 -0
- ctxprune-0.3.0/scripts/integrations/hermes/ctxprune/plugin.yaml +8 -0
- ctxprune-0.3.0/scripts/integrations/opencode/ctxprune.js +336 -0
- ctxprune-0.3.0/scripts/integrations/pi/ctxprune.ts +358 -0
- ctxprune-0.3.0/scripts/line_encode.py +53 -0
- ctxprune-0.3.0/scripts/lossless.py +1243 -0
- ctxprune-0.3.0/scripts/lossless2.py +1644 -0
- ctxprune-0.3.0/scripts/lossless_hook.py +346 -0
- ctxprune-0.3.0/scripts/methods.py +376 -0
- ctxprune-0.3.0/scripts/read_trim.py +160 -0
- ctxprune-0.3.0/scripts/recover.py +286 -0
- ctxprune-0.3.0/scripts/render_store.py +470 -0
- ctxprune-0.3.0/scripts/spill.py +300 -0
- ctxprune-0.3.0/scripts/v2_hook.py +144 -0
- ctxprune-0.3.0/scripts/v2_server.py +262 -0
- ctxprune-0.3.0/scripts/view_safe.py +155 -0
- ctxprune-0.3.0/src/ctxprune/__init__.py +31 -0
- ctxprune-0.3.0/src/ctxprune/__main__.py +5 -0
- ctxprune-0.3.0/src/ctxprune/bundle.py +310 -0
- ctxprune-0.3.0/src/ctxprune/cli.py +382 -0
- ctxprune-0.3.0/src/ctxprune/doctor.py +442 -0
- ctxprune-0.3.0/src/ctxprune/glue.py +304 -0
- ctxprune-0.3.0/src/ctxprune/harness_audit.py +358 -0
- ctxprune-0.3.0/src/ctxprune/installers.py +781 -0
- ctxprune-0.3.0/src/ctxprune/legacy_cli.py +79 -0
- ctxprune-0.3.0/src/ctxprune/model.py +84 -0
- ctxprune-0.3.0/src/ctxprune/plugin_cli.py +51 -0
- {ctxprune-0.2.0 → ctxprune-0.3.0/src}/ctxprune/serve.py +3 -3
- ctxprune-0.3.0/src/ctxprune/stats.py +273 -0
- ctxprune-0.3.0/src/ctxprune/usage.py +327 -0
- ctxprune-0.3.0/tests/packaging/cc_live.py +222 -0
- ctxprune-0.3.0/tests/packaging/codepage_hook.py +81 -0
- ctxprune-0.3.0/tests/packaging/determinism.py +167 -0
- ctxprune-0.3.0/tests/packaging/e2e.py +468 -0
- ctxprune-0.3.0/tests/packaging/test_harness_usage.py +242 -0
- ctxprune-0.3.0/tests/packaging/test_units.py +212 -0
- ctxprune-0.2.0/.gitignore +0 -6
- ctxprune-0.2.0/LICENSE +0 -202
- ctxprune-0.2.0/PKG-INFO +0 -178
- ctxprune-0.2.0/README.md +0 -130
- ctxprune-0.2.0/ctxprune/__init__.py +0 -22
- ctxprune-0.2.0/ctxprune/__main__.py +0 -3
- ctxprune-0.2.0/ctxprune/align.py +0 -93
- ctxprune-0.2.0/ctxprune/cli.py +0 -110
- ctxprune-0.2.0/ctxprune/hook.py +0 -191
- ctxprune-0.2.0/ctxprune/sources.py +0 -256
- ctxprune-0.2.0/ctxprune/teacher.py +0 -241
- ctxprune-0.2.0/pyproject.toml +0 -65
- ctxprune-0.2.0/space/static/README.md +0 -17
- ctxprune-0.2.0/tests/test_core.py +0 -143
- ctxprune-0.2.0/tests/test_hook.py +0 -67
- ctxprune-0.2.0/tests/test_integrations.py +0 -31
- {ctxprune-0.2.0 → ctxprune-0.3.0/src}/ctxprune/atoms.py +0 -0
- {ctxprune-0.2.0 → ctxprune-0.3.0/src}/ctxprune/chunking.py +0 -0
- {ctxprune-0.2.0 → ctxprune-0.3.0/src}/ctxprune/compress.py +0 -0
- {ctxprune-0.2.0 → ctxprune-0.3.0/src}/ctxprune/idcheck.py +0 -0
- {ctxprune-0.2.0 → ctxprune-0.3.0/src}/ctxprune/integrations.py +0 -0
- {ctxprune-0.2.0 → ctxprune-0.3.0/src}/ctxprune/tokenlabels.py +0 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
private/
|
|
2
|
+
data/
|
|
3
|
+
venv-enc/
|
|
4
|
+
research/enc-bench/keep/
|
|
5
|
+
research/enc-bench/**/*.onnx
|
|
6
|
+
results/t2/*.json
|
|
7
|
+
results/t3/*.json
|
|
8
|
+
*.outs.jsonl
|
|
9
|
+
logs/
|
|
10
|
+
__pycache__/
|
|
11
|
+
runbook/.*.json
|
|
12
|
+
runbook/.last_rental
|
|
13
|
+
runbook/reap.log
|
|
14
|
+
bench/longhorizon/_selftest/
|
|
15
|
+
runbook/.known_hosts_*
|
|
16
|
+
results/rented/
|
|
17
|
+
out/
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.3.0 (2026-11-18)
|
|
4
|
+
|
|
5
|
+
0.3.0 is a rewrite of the agent integrations. 0.2.1's Claude Code hook deleted words with a token model. 0.3.0's hooks
|
|
6
|
+
by default rewrite tool outputs into a shorter form that keeps the content, and they run inside more agents. The
|
|
7
|
+
model-based line selection for agents is an opt-in preset that's still experimental. The 0.2 token-pruning API is
|
|
8
|
+
unchanged (see "Kept from 0.2.1").
|
|
9
|
+
|
|
10
|
+
### New
|
|
11
|
+
- **The `safe` preset is the default.**
|
|
12
|
+
- Aligned tables are single-spaced, and redundant `---`/`+++` diff headers are dropped.
|
|
13
|
+
- JSON record lists with nested values (typical MCP results) become rows. `\uXXXX`-escaped JSON is shown with its
|
|
14
|
+
characters (`json_ascii`). Playwright MCP snapshots lose the YAML dashes.
|
|
15
|
+
- BOM and CRLF text is handled exactly. Windows `\` path lists stay raw.
|
|
16
|
+
- Runs of identical lines are counted. Colour codes and progress redraws are removed.
|
|
17
|
+
- Standard library only, with no model and no network.
|
|
18
|
+
- [README](README.md#what-is-exact-and-what-isnt) says which of these are byte-exact and which aren't.
|
|
19
|
+
- **Integrations:**
|
|
20
|
+
- Claude Code: a hook for Bash, PowerShell, Grep and MCP tools.
|
|
21
|
+
- Hermes, Pi, OpenCode and dsh: plugins.
|
|
22
|
+
- Codex: a local proxy.
|
|
23
|
+
- `ctxprune install --all`, `uninstall`, `upgrade`, `doctor`, `stats`.
|
|
24
|
+
- **`ctxprune audit`:** for Claude Code, where the bill goes (system prompt, MCP listings, skills, CLAUDE.md, hooks,
|
|
25
|
+
thinking, tool calls, tool results) and which static context you could drop; for Codex, Pi, OpenCode, Hermes and
|
|
26
|
+
dsh, the static-overhead checks that apply (session prefix, instruction files, MCP servers and their use, skills,
|
|
27
|
+
plugins, prompt-cache share, provider caching rules), each marked verified or unverified. Local and advice-only:
|
|
28
|
+
numbers and names, never content, no config changes.
|
|
29
|
+
- **`ctxprune stats`** shows one row per agent plus a total, with the share of input ctxprune removed where the
|
|
30
|
+
agent keeps usage records locally, the subscription reading of it (usage-limit headroom), and the dollar figure
|
|
31
|
+
at API prices. `ctxprune doctor` ends with a per-agent status summary.
|
|
32
|
+
`ctxprune install` names `stats` and `audit` when it's done. `--weekly-summary` (opt-in) adds a once-a-week line at
|
|
33
|
+
session start that only you see.
|
|
34
|
+
- **`ctxprune-safe` Claude Code plugin:** self-contained, with no pip needed. There's also a self-contained Hermes
|
|
35
|
+
plugin build (`ctxprune bundle`).
|
|
36
|
+
- **Spill files (Claude Code):** when Claude Code saves a big output to a file and shows only a preview, the file is
|
|
37
|
+
rewritten losslessly. `CTXPRUNE_SPILL=0` turns it off.
|
|
38
|
+
- **Long-context probe.** Every default rule was re-checked with the output inside a 16k–56k-token agent transcript
|
|
39
|
+
(Qwen3.6, the shipped bytes, tool schemas sent). Four rules that passed the short probes did worse there and are
|
|
40
|
+
opt-in now. On public Linux Claude Code sessions they were most of what the default saved without dedup (0.98% of
|
|
41
|
+
tool tokens before, 0.29% now); on MCP-heavy mixes they were under 1 point:
|
|
42
|
+
- grep grouping (`CTXPRUNE_HITS=1`): without thinking, hits filed under the wrong file, −3.7 [−9.7, +3.2] points
|
|
43
|
+
(with thinking it helped, +8.3; the bar was both modes);
|
|
44
|
+
- `paths` directory grouping, with Hermes `sffiles` (`CTXPRUNE_PATHS=1`, or `CTXPRUNE_PATHS_MODELS=<hosted model
|
|
45
|
+
ids>`): files under the wrong directory, −8.3 [−20.9, −0.5] points, −11 at 56k;
|
|
46
|
+
- flat-record `jsonrows`, e.g. search hits or a git log as JSON (`CTXPRUNE_JSONROWS_FLAT=1`): the reader re-ran the
|
|
47
|
+
search instead of answering, +7.4 [+3.7, +12.0] points of re-fetches (+12.0 with thinking);
|
|
48
|
+
- `numbered`, the unpadded `cat -n` gutter (`CTXPRUNE_NUMBERED=1`): +5.1 [+0.9, +10.2] points of re-fetches.
|
|
49
|
+
- **Local / self-hosted endpoints:** no `jsonrows` (thinking models spent +139 to +407 tokens per lookup on nested
|
|
50
|
+
records, against ~176 prompt tokens saved).
|
|
51
|
+
- **Cross-turn dedup by model.** It is on for `claude-opus-*` served by Anthropic and off for everything else, decided
|
|
52
|
+
once per session. `CTXPRUNE_DEDUP=0|1` overrides it.
|
|
53
|
+
- Claude Opus 5.5 passed the dedup comprehension probe (−0.2 [−1.8, +1.3] points).
|
|
54
|
+
- The open models failed it (Qwen3.8: −7 to −21 points).
|
|
55
|
+
- **First-render pins.** The proxy and the plugins' request-time dedup keep serving the bytes they first sent for each
|
|
56
|
+
result, so an upgrade or a settings change never re-renders history the provider has cached. The pins live in a
|
|
57
|
+
bounded sqlite store.
|
|
58
|
+
- **Same bytes on every OS and Python.** CI checked Windows, macOS and Ubuntu × Python 3.9–3.13.
|
|
59
|
+
- **Opt-in flags:** `CTXPRUNE_BLOCKREF`, `CTXPRUNE_PREFIX`, `CTXPRUNE_JSONROWS_EXT`, `CTXPRUNE_READ_TRIM`, and the four
|
|
60
|
+
above. Each is off
|
|
61
|
+
because a comprehension probe on open models found a cost; see the README.
|
|
62
|
+
- **Lossy presets:** `balanced` and `aggressive` (experimental; the model weights are not published yet).
|
|
63
|
+
- They keep a recovery file for every dropped range and leave file views whole.
|
|
64
|
+
- A refetch guard in the Claude Code hook and the proxy falls back to `safe` for a while when the agent re-fetches
|
|
65
|
+
what was dropped.
|
|
66
|
+
- Codex, dsh, OpenCode and Pi run `balanced` as `safe` unless `CTXPRUNE_FORCE_LOSSY=1`.
|
|
67
|
+
- **Security fixes** from a red team and a runtime audit, listed in [docs/SAFETY.md](docs/SAFETY.md):
|
|
68
|
+
- forged-marker escaping;
|
|
69
|
+
- private state files;
|
|
70
|
+
- bounded parsing;
|
|
71
|
+
- a kill switch in every integration, including the proxy.
|
|
72
|
+
|
|
73
|
+
### Changed
|
|
74
|
+
- **License: 0BSD** (0.2.1 was Apache-2.0).
|
|
75
|
+
- The optional line-scoring model weights are not 0BSD and are not part of the package. They are a fine-tune of
|
|
76
|
+
mmBERT-small (MIT) trained on data that includes CC-BY-4.0 sets (SWE-rebench-openhands, Nemotron-SWE).
|
|
77
|
+
- The weights are not published yet; their license and attributions will be on the Hugging Face model card
|
|
78
|
+
when they are. See
|
|
79
|
+
[THIRD_PARTY_NOTICES](THIRD_PARTY_NOTICES).
|
|
80
|
+
- **The Claude Code hook** now also matches `PowerShell` and `Grep`. Its timeout went from 60 s to 30 s, because the
|
|
81
|
+
default no longer loads a model.
|
|
82
|
+
- **Cross-turn dedup** was on for every model in pre-release builds of 0.3.0. It is now on for Claude Opus only
|
|
83
|
+
(see above).
|
|
84
|
+
- **Hermes plugin:** each call has a time budget (`CTXPRUNE_BUDGET`, default 2 s). After that, Hermes keeps the
|
|
85
|
+
original output.
|
|
86
|
+
- **Release gate:** the PyPI publish job (trusted publishing, self-hosted Linux runner) runs only after the tests, the
|
|
87
|
+
install e2e and identical determinism digests on Python 3.9–3.13 pass on the tagged commit. macOS and Windows were
|
|
88
|
+
last checked in CI at runtime a4bd87e (all 15 cells identical); they are not part of this release's gate.
|
|
89
|
+
|
|
90
|
+
### Kept from 0.2.1 (same behaviour, same outputs)
|
|
91
|
+
- **The Python API:** `ctxprune.prune()`, `Compressor`, `DEFAULT_MODEL`, loaded lazily so the hooks never import them.
|
|
92
|
+
- **The LangChain and LlamaIndex integrations:** `ctxprune.integrations.CtxpruneDocumentCompressor` and
|
|
93
|
+
`CtxpruneNodePostprocessor`, and the `langchain`, `llamaindex`, `torch` and `onnx` extras.
|
|
94
|
+
- **`ctxprune FILE` / `some_command | ctxprune`** with `--keep` / `--protect` / `--keep-structure` / `--stats`.
|
|
95
|
+
- **The 0.2.1 `ctxprune serve`** (HTTP `POST 127.0.0.1:8719/compress`). With `--model-dir DIR`, `ctxprune serve` is
|
|
96
|
+
the 0.3 line-model server for the lossy presets instead.
|
|
97
|
+
- **`ctxprune install|uninstall claude-code --scope user|project|local`.** project/local write the project's
|
|
98
|
+
`.claude/settings.json` / `settings.local.json`, as in 0.2.1, and stay out of the user-level install manifest.
|
|
99
|
+
- **The dependencies** (onnxruntime, tokenizers, numpy, huggingface_hub), so a fresh `pip install ctxprune` still
|
|
100
|
+
gives a working `prune()`. The hooks and plugins don't import them.
|
|
101
|
+
- Checked against 0.2.1 on the same inputs: `prune()` (16 cases), `Compressor`, both integrations, the four CLI forms,
|
|
102
|
+
`--stats` and `serve` give byte-identical output.
|
|
103
|
+
|
|
104
|
+
### Removed
|
|
105
|
+
- **`CTXPRUNE_RATE`** in the Claude Code hook. The hook is lossless by default now; `CTXPRUNE_KEEP` sets the keep
|
|
106
|
+
ratio of the lossy presets. `CTXPRUNE_MIN_CHARS` now applies only to them.
|
|
107
|
+
|
|
108
|
+
## Migrating from 0.2.1
|
|
109
|
+
|
|
110
|
+
**If you used `ctxprune install claude-code`:**
|
|
111
|
+
- Run `uv tool upgrade ctxprune` (or `pipx upgrade ctxprune` / `pip install -U ctxprune`), then
|
|
112
|
+
`ctxprune install claude-code`.
|
|
113
|
+
- The 0.2.1 hook calls `ctxprune hook claude-code`, which keeps working after the upgrade, but its matcher stays
|
|
114
|
+
`Bash|mcp__.*` until you re-run `install`. Re-running replaces the old ctxprune hook entry with the new one.
|
|
115
|
+
- After that, `ctxprune upgrade` handles future upgrades, including rewriting the integrations.
|
|
116
|
+
|
|
117
|
+
**If you used the 0.2.1 Claude Code plugin** (`riceharvest/ctxprune`, `claude-code-plugin/`):
|
|
118
|
+
- Its hook script runs `ctxprune hook claude-code` if `ctxprune` is on your PATH, else
|
|
119
|
+
`uvx --from "ctxprune>=0.2.1" ctxprune hook claude-code`.
|
|
120
|
+
- Once 0.3.0 is on PyPI, that `uvx` range resolves to 0.3.0. **So plugin users without a local install move from the
|
|
121
|
+
0.2.1 lossy model to the 0.3.0 `safe` hook automatically**, with no action on their side.
|
|
122
|
+
- What they get is lossless, with no model download, but the old plugin's matcher doesn't include `Grep` or
|
|
123
|
+
`PowerShell`, so those stay unhooked until you update the plugin.
|
|
124
|
+
- If you have 0.2.1 installed on your PATH, the plugin keeps running 0.2.1 until you upgrade it.
|
|
125
|
+
- This release updates `claude-code-plugin/` itself (matcher `Bash|PowerShell|Grep|mcp__.*`, `uvx` pinned to
|
|
126
|
+
`ctxprune>=0.3,<0.4`). Run `/plugin marketplace update ctxprune` to get it. The no-pip alternative is
|
|
127
|
+
`/plugin install ctxprune-safe@ctxprune`.
|
|
128
|
+
|
|
129
|
+
**If you relied on the 0.2.1 lossy compression:**
|
|
130
|
+
- 0.3.0's default saves fewer tokens than 0.2.1 at `--keep 0.5`, but it doesn't drop content.
|
|
131
|
+
- The lossy replacement is `--preset balanced` with `ctxprune[model]` and `ctxprune serve --model-dir DIR`. Its
|
|
132
|
+
weights are not published yet.
|
|
133
|
+
- In my agent-session measurements, `balanced` did not lower cost reliably on most harnesses
|
|
134
|
+
([docs/BENCHMARKS.md](docs/BENCHMARKS.md#6-presets-the-lossy-balanced-preset)).
|
|
135
|
+
|
|
136
|
+
**If you used the Python API, the file/pipe CLI, `ctxprune serve` or the LangChain/LlamaIndex classes:** nothing to
|
|
137
|
+
do. They work as in 0.2.1, with the same model and the same outputs.
|
|
138
|
+
|
|
139
|
+
**Kill switch:** `CTXPRUNE_OFF=1` works as before, in the environment or inside one command.
|
|
140
|
+
|
|
141
|
+
## 0.2.1
|
|
142
|
+
|
|
143
|
+
Token-level pruning model (mmBERT-small fine-tune) for files, pipes, Python, LangChain/LlamaIndex and a Claude Code
|
|
144
|
+
hook/plugin. Apache-2.0.
|
ctxprune-0.3.0/LICENSE
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
Copyright (C) 2026 by riceharvest
|
|
2
|
+
|
|
3
|
+
Permission to use, copy, modify, and/or distribute this software for any
|
|
4
|
+
purpose with or without fee is hereby granted.
|
|
5
|
+
|
|
6
|
+
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
7
|
+
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
8
|
+
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
|
9
|
+
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
10
|
+
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
|
11
|
+
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
|
12
|
+
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# ctxprune
|
|
2
|
+
|
|
3
|
+
Install and integration reference. For what ctxprune does, what it saves and what is exact, start with
|
|
4
|
+
[README.md](README.md).
|
|
5
|
+
|
|
6
|
+
The default `safe` preset drops no content: it counts repeated lines, writes JSON record lists as rows, shows
|
|
7
|
+
escaped JSON as text and strips ANSI colour codes. No model, no server, standard library only. (The pip package
|
|
8
|
+
also installs onnxruntime, tokenizers, numpy and huggingface_hub for the 0.2 token-pruning API; the hooks never
|
|
9
|
+
import them.)
|
|
10
|
+
|
|
11
|
+
Works with Claude Code (hook), Hermes, Pi, OpenCode, dsh (plugins) and Codex (local proxy).
|
|
12
|
+
|
|
13
|
+
## Install
|
|
14
|
+
|
|
15
|
+
```sh
|
|
16
|
+
uv tool install ctxprune # or: pipx install ctxprune
|
|
17
|
+
ctxprune install --all # every supported agent found on this machine
|
|
18
|
+
ctxprune doctor # checks versions and runs a test call through each integration
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
One agent only: `ctxprune install claude-code` (also `hermes`, `pi`, `opencode`, `dsh`, `codex`).
|
|
22
|
+
Don't run `install` through `uvx`: it writes the interpreter path into your agent configs, and uvx deletes that
|
|
23
|
+
environment later. `ctxprune install` refuses to run from one.
|
|
24
|
+
|
|
25
|
+
## Use
|
|
26
|
+
|
|
27
|
+
- `ctxprune stats`: tokens saved so far, one row per agent plus a total, the share of input where the agent keeps
|
|
28
|
+
usage records locally, and a rough dollar estimate at API prices (`--price` = your model's USD per million input
|
|
29
|
+
tokens). The log keeps counts only, never tool output or commands.
|
|
30
|
+
- `ctxprune audit [--harness NAME] [--days 30]`: static overhead per agent, and for Claude Code where the bill goes
|
|
31
|
+
(local, advice only; numbers and names, never content).
|
|
32
|
+
- `ctxprune install claude-code --weekly-summary`: a once-a-week savings line at session start, shown to you only
|
|
33
|
+
(off by default).
|
|
34
|
+
- `CTXPRUNE_OFF=1`: turns ctxprune off, either in the agent's environment or inside one command
|
|
35
|
+
(`CTXPRUNE_OFF=1 cat big.log`).
|
|
36
|
+
- `ctxprune upgrade`: upgrades the package (uv tool, pipx or pip, whichever installed it) and rewrites
|
|
37
|
+
every installed integration for the new version.
|
|
38
|
+
- `ctxprune uninstall --all`: removes everything it added. Your original config files stay in the backups
|
|
39
|
+
folder (see below).
|
|
40
|
+
|
|
41
|
+
## What `install` changes
|
|
42
|
+
|
|
43
|
+
Each install is idempotent, and the first time it changes a file it copies the original to
|
|
44
|
+
`<config>/backups/<agent>/`.
|
|
45
|
+
|
|
46
|
+
| Agent | File | Change |
|
|
47
|
+
|---|---|---|
|
|
48
|
+
| Claude Code (2.1.121 or newer) | `~/.claude/settings.json` (`$CLAUDE_CONFIG_DIR`) | one `PostToolUse` hook, matcher `Bash\|PowerShell\|Grep\|mcp__.*`; older ctxprune hooks are replaced |
|
|
49
|
+
| Hermes (0.21+) | `~/.hermes/plugins/ctxprune/`, `config.yaml` (`$HERMES_HOME`) | the self-contained plugin (runtime copied in, since Hermes runs in its own venv), added to `plugins.enabled` |
|
|
50
|
+
| Pi | `~/.pi/agent/extensions/ctxprune.ts` (`$PI_CODING_AGENT_DIR`) | the extension |
|
|
51
|
+
| OpenCode | `~/.config/opencode/plugins/ctxprune.js` (`$XDG_CONFIG_HOME`) | the plugin |
|
|
52
|
+
| dsh | `~/.dsh/ctxprune-plugin/ctxprune.mjs`, `~/.dsh/cordis.patch.yml` (`$DSH_HOME`) | the plugin, one `insert` patch item |
|
|
53
|
+
| Codex | `~/.codex/config.toml` (`$CODEX_HOME`), a user service | provider + profile `ctxprune`; the proxy on 127.0.0.1:8787 |
|
|
54
|
+
|
|
55
|
+
Codex: run `codex --profile ctxprune`. `install codex` writes the service file and prints the command that
|
|
56
|
+
starts it (`--start` runs that command for you). What has been checked so far is the proxy itself: it rewrites a
|
|
57
|
+
Responses-API tool output on its way to an upstream. Codex has not yet been run through it, so the provider
|
|
58
|
+
block (`env_key = "OPENAI_API_KEY"`, `wire_api = "responses"`) still needs a live check. ChatGPT-login accounts
|
|
59
|
+
are not expected to work through it.
|
|
60
|
+
|
|
61
|
+
## Presets
|
|
62
|
+
|
|
63
|
+
The preset table lives in one place: `ctxprune_core.PRESETS` / `PRESET_HARNESS` in the runtime. Packaging reads
|
|
64
|
+
it and adds only ports and timeouts (`ctxprune/glue.py` `DEFAULTS`).
|
|
65
|
+
|
|
66
|
+
- `safe` (default): the lossless rules. Cross-turn dedup goes by model in every preset (the table says `"model"`):
|
|
67
|
+
on for `claude-opus-*` served by Anthropic (`ctxprune_core.DEDUP_MODELS`), off otherwise; `CTXPRUNE_DEDUP=0|1`
|
|
68
|
+
overrides. It runs in the Claude Code hook (decided once per session), the proxy (per conversation) and per request
|
|
69
|
+
in the Hermes, Pi and OpenCode plugins; dsh has no request-level hook, so no dedup there. Packaging passes
|
|
70
|
+
`--dedup` to the hook / proxy only when `CTXPRUNE_DEDUP` forces it (`glue.dedup_flag`).
|
|
71
|
+
- `balanced` / `aggressive` (`ctxprune install --all --preset balanced`): also drop low-value lines with the v2
|
|
72
|
+
line model. They need `pip install "ctxprune[model]"` (onnxruntime, tokenizers, numpy) and a running
|
|
73
|
+
`ctxprune serve --model-dir DIR`. On Codex, dsh, OpenCode and Pi, `balanced` stays `safe` unless you set
|
|
74
|
+
`CTXPRUNE_FORCE_LOSSY=1`. Experimental: the weights are not published yet.
|
|
75
|
+
|
|
76
|
+
## No-pip plugins (catalog / marketplace builds)
|
|
77
|
+
|
|
78
|
+
`ctxprune bundle hermes DIR` and `ctxprune bundle claude-code DIR` write self-contained plugin folders. They
|
|
79
|
+
contain a byte copy of the stdlib runtime, so the target machine needs only Python 3.9+:
|
|
80
|
+
|
|
81
|
+
- Hermes (validated on 0.21.6 and 0.22.2): a plugin directory for `hermes plugins install`. It passes Hermes' plugin
|
|
82
|
+
validator and declares `transform_terminal_output` and `transform_tool_result`. The safe build ships no model
|
|
83
|
+
code and writes no files (`--stats` turns on the counts-only log).
|
|
84
|
+
- Claude Code: `.claude-plugin/marketplace.json` plus `claude-code-plugin-safe/`, which installs with
|
|
85
|
+
`/plugin marketplace add riceharvest/ctxprune` and then `/plugin install ctxprune-safe@ctxprune`. Its hook
|
|
86
|
+
runs `scripts/hook.sh`, which runs the bundled `lossless_hook.py` with the first `python3`/`python` it finds.
|
|
87
|
+
Use either the plugin or `ctxprune install claude-code`, not both: `ctxprune doctor` warns if both are on.
|
|
88
|
+
|
|
89
|
+
## Paths
|
|
90
|
+
|
|
91
|
+
| | Linux / macOS | Windows |
|
|
92
|
+
|---|---|---|
|
|
93
|
+
| settings, install manifest, backups | `$XDG_CONFIG_HOME/ctxprune` (`~/.config/ctxprune`) | `%APPDATA%\ctxprune` |
|
|
94
|
+
| saved outputs, dedup state (0700) | `$XDG_CACHE_HOME/ctxprune` (`~/.cache/ctxprune`) | `%LOCALAPPDATA%\ctxprune\cache` |
|
|
95
|
+
| stats log (counts only, 0600) | `$XDG_STATE_HOME/ctxprune/stats.jsonl` | `%LOCALAPPDATA%\ctxprune\stats.jsonl` |
|
|
96
|
+
|
|
97
|
+
`CTXPRUNE_CONFIG_DIR`, `CTXPRUNE_STATE` and `CTXPRUNE_STATS` override these (`CTXPRUNE_STATS=off` turns the log off).
|
|
98
|
+
|
|
99
|
+
### Windows notes
|
|
100
|
+
|
|
101
|
+
- `ctxprune install` writes the full path of the Python that runs ctxprune (`sys.executable`, e.g.
|
|
102
|
+
`C:\Users\you\AppData\Roaming\uv\tools\ctxprune\Scripts\python.exe`) into every config. The installed
|
|
103
|
+
plugins never call a bare `python3`, which on Windows can be the Microsoft Store stub that does nothing.
|
|
104
|
+
- The `ctxprune-safe` marketplace plugin has no install step, so its `hook.sh` looks for an interpreter itself:
|
|
105
|
+
`py -3`, then `python`, then `python3` on Windows. If none works, the hook prints nothing and Claude Code
|
|
106
|
+
keeps the original output.
|
|
107
|
+
- On Windows, Claude Code runs hook commands through Git Bash (not tested here; check it on a Windows machine).
|
|
108
|
+
The command is `"C:\...\python.exe" "C:\...\ctxprune\plugin_cli.py" hook claude-code`. Backslashes inside double quotes pass
|
|
109
|
+
through bash unchanged.
|
|
110
|
+
- OpenCode reads `%USERPROFILE%\.config\opencode` on Windows as well (or `XDG_CONFIG_HOME` when it is set).
|
|
111
|
+
- Codex: Windows has no user-service manager that ctxprune writes to. `install codex` prints a `schtasks`
|
|
112
|
+
command that starts the proxy at logon.
|
|
113
|
+
- Tool outputs with `\r\n` line endings are handled exactly by the safe preset. The opt-in grep grouping
|
|
114
|
+
(`CTXPRUNE_HITS=1`) is tuned for `/` paths, so it saves less on `C:\` paths.
|
|
115
|
+
- WSL: install ctxprune on the same side as the agent. A Windows-side agent can't run a WSL Python path,
|
|
116
|
+
and the reverse is true too.
|
|
117
|
+
|
|
118
|
+
## The Codex proxy
|
|
119
|
+
|
|
120
|
+
The proxy sits in the request path, so it is a hard dependency: if it dies, the harness behind it fails (from the
|
|
121
|
+
live canary on 2026-10-17: Claude Code exits 1 after ~180 s of retries, Hermes hangs, Pi/OpenCode exit 0 with the
|
|
122
|
+
task unfinished). So:
|
|
123
|
+
|
|
124
|
+
- `install --all` uses native plugins for Claude Code, Hermes, Pi, OpenCode and dsh, and the proxy only for Codex.
|
|
125
|
+
- The systemd unit has `Restart=always` + `RestartSec=1` (back within ~1 s after `kill -9` or OOM); launchd uses
|
|
126
|
+
`KeepAlive`. The Windows `schtasks` task does **not** restart the proxy: if it dies, Codex fails until you start it.
|
|
127
|
+
- Kill switch: the proxy passes requests through untouched while `CTXPRUNE_OFF=1` is in its env or the file
|
|
128
|
+
`<state>/OFF` exists (checked per request, no restart). The native hooks read only `CTXPRUNE_OFF`, not the file.
|
|
129
|
+
- Hooks/plugins record what they emit in `<state>/emitted/`, and the proxy forwards that text untouched. It only
|
|
130
|
+
works with one shared state dir: the unit uses the same default as the hooks (`CTXPRUNE_STATE`, else
|
|
131
|
+
`~/.cache/ctxprune`); don't give the proxy a different `--state`.
|
|
132
|
+
|
|
133
|
+
## Upgrading from 0.2.1
|
|
134
|
+
|
|
135
|
+
The 0.2.1 Claude Code plugin (`riceharvest/ctxprune`, `claude-code-plugin/hooks/hooks.json`) matches
|
|
136
|
+
`Bash|mcp__.*` and runs `uvx --from "ctxprune>=0.2.1" ctxprune hook claude-code` when ctxprune isn't installed.
|
|
137
|
+
Once 0.3.0 is on PyPI, that range resolves to 0.3.0, so existing plugin users silently switch from the 0.2.1 lossy
|
|
138
|
+
token model to the 0.3.0 `safe` hook (lossless rules; dedup by model), and Grep/PowerShell stay unhooked until the
|
|
139
|
+
plugin's matcher is updated. See [CHANGELOG.md](CHANGELOG.md#migrating-from-021). The 0.2 token-pruning API
|
|
140
|
+
(`ctxprune.prune()`, `Compressor`, `ctxprune FILE --keep`, `ctxprune serve` without `--model-dir`, the LangChain /
|
|
141
|
+
LlamaIndex classes, `install claude-code --scope`) works as in 0.2.1, with byte-identical outputs.
|
|
142
|
+
0.3.0 updates that plugin in place (matcher `Bash|PowerShell|Grep|mcp__.*`, pinned to `ctxprune>=0.3,<0.4`):
|
|
143
|
+
`/plugin marketplace update ctxprune` picks it up.
|