ctxprune 0.3.1__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ctxprune-0.3.1 → ctxprune-0.3.2}/CHANGELOG.md +11 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/PKG-INFO +26 -2
- {ctxprune-0.3.1 → ctxprune-0.3.2}/README.md +25 -1
- ctxprune-0.3.2/docs/bench/llmlingua_facts.md +39 -0
- ctxprune-0.3.2/docs/bench/llmlingua_facts.py +83 -0
- ctxprune-0.3.2/docs/media/banner.png +0 -0
- ctxprune-0.3.2/docs/media/icon.png +0 -0
- ctxprune-0.3.2/docs/media/mcp-before-after.png +0 -0
- ctxprune-0.3.2/docs/media/replay-vs-compactors.png +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/pyproject.toml +1 -1
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/__init__.py +1 -1
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/cli.py +4 -0
- ctxprune-0.3.2/src/ctxprune/mcp_server.py +153 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/.gitignore +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/LICENSE +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/PACKAGE.md +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/THIRD_PARTY_NOTICES +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/BENCHMARKS.md +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/SAFETY.md +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/bench_shipped.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/latency_scripts_106bcea.json +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/latency_shipped_106bcea.json +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/measure_variant_dedup.diff +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/measure_variant_powershell.diff +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/real_cc_split.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/real_cc_split.txt +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/spill_gain_12c7b79.json +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/typical_dedup_results.json +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/typical_dedup_summary.json +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/typical_safe_results.json +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/typical_safe_summary.json +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/README.md +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/build.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/dropped.md +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/README.md +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_00_examples.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_alias_model_cache.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_dedup.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_hook_infra.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_hook_notes.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_logs.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_new_cloud_k8s_iac.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_new_data_formats_text.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_new_pkgmgr_langtools.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_new_protocol_envelope.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_new_system_ops_terminal.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_new_vcs_ci_review.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_noise_pretool.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_rules_ci_infra_cloud.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_rules_compilers_build_native.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_rules_logs_ops_ml.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_rules_media_docs_rag_misc.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_rules_stateful_agents_code.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_rules_structured_json.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_struct_a.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_struct_b.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_struct_c.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_terminal.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/harness.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/run.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/ideas.jsonl +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/ideas.md +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/shortlist.md +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/later-integrations.md +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/ctx_audit.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/ctxprune_compress.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/ctxprune_core.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/ctxprune_note.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/ctxprune_proxy.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/dedup.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/emitted.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/export_v2_onnx.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/hook_budget.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/integrations/dsh/ctxprune.mjs +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/integrations/hermes/ctxprune/__init__.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/integrations/hermes/ctxprune/plugin.yaml +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/integrations/opencode/ctxprune.js +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/integrations/pi/ctxprune.ts +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/line_encode.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/lossless.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/lossless2.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/lossless_hook.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/methods.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/read_trim.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/recover.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/render_store.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/spill.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/v2_hook.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/v2_server.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/view_safe.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/__main__.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/atoms.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/bundle.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/chunking.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/compress.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/doctor.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/glue.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/harness_audit.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/idcheck.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/installers.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/integrations.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/legacy_cli.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/model.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/plugin_cli.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/serve.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/stats.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/tokenlabels.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/usage.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/tests/packaging/cc_live.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/tests/packaging/codepage_hook.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/tests/packaging/determinism.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/tests/packaging/e2e.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/tests/packaging/test_harness_usage.py +0 -0
- {ctxprune-0.3.1 → ctxprune-0.3.2}/tests/packaging/test_units.py +0 -0
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.2 (2026-11-18)
|
|
4
|
+
|
|
5
|
+
- **`ctxprune mcp`**: a stdio MCP server for clients without tool-output hooks (Claude Desktop, Cursor, Cline, ...).
|
|
6
|
+
`read_compact` reads a file through the lossless `safe` pipeline; `read_pruned` token-prunes it with the 0.2 model,
|
|
7
|
+
identifiers kept by default. Stdlib JSON-RPC, no MCP SDK. Listed as `io.github.riceharvest/ctxprune` in the MCP
|
|
8
|
+
Registry (`server.json`).
|
|
9
|
+
- README / model card: use `protect=True` for token pruning of tool output and logs. Measured on 120 held-out chunks
|
|
10
|
+
against LLMLingua-2 in [docs/bench/llmlingua_facts.md](docs/bench/llmlingua_facts.md).
|
|
11
|
+
- Note: `ctxprune mcp` is now a subcommand, so a 0.2-style `ctxprune mcp` on a file literally named `mcp` needs
|
|
12
|
+
`ctxprune ./mcp`.
|
|
13
|
+
|
|
3
14
|
## 0.3.1 (2026-11-18)
|
|
4
15
|
|
|
5
16
|
Docs only; the code and every output are the same as 0.3.0.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ctxprune
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Shrink what your LLM reads: lossless tool-output compression for coding agents (Claude Code, Hermes, Pi, OpenCode, dsh, Codex) and token pruning for prompts and RAG (LangChain, LlamaIndex)
|
|
5
5
|
Project-URL: Homepage, https://github.com/riceharvest/ctxprune
|
|
6
6
|
Project-URL: Model, https://huggingface.co/darioooooo0o/ctxprune-small
|
|
@@ -137,6 +137,24 @@ Python 3.9+ on the machine:
|
|
|
137
137
|
|
|
138
138
|
Use either the plugin or `ctxprune install claude-code`, not both. `ctxprune doctor` warns if both are on.
|
|
139
139
|
|
|
140
|
+
## MCP server (Claude Desktop, Cursor, Cline and other MCP clients)
|
|
141
|
+
|
|
142
|
+
For clients without tool-output hooks, `ctxprune mcp` is a local stdio MCP server with two read tools:
|
|
143
|
+
|
|
144
|
+
- `read_compact(path, start_line?, end_line?)`: the lossless `safe` pipeline, for big logs, JSON/MCP dumps, CSV and
|
|
145
|
+
test output. Nothing is dropped.
|
|
146
|
+
- `read_pruned(path, keep=0.5, protect=true)`: token pruning with the model below (lossy). Identifiers, numbers and
|
|
147
|
+
paths are kept unless `protect=false`.
|
|
148
|
+
|
|
149
|
+
```json
|
|
150
|
+
{ "mcpServers": { "ctxprune": { "command": "uvx", "args": ["ctxprune", "mcp"] } } }
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
It reads only the files the client asks for, makes no network calls (`read_pruned` downloads the model once on first
|
|
154
|
+
use), and needs no MCP SDK.
|
|
155
|
+
|
|
156
|
+
<!-- mcp-name: io.github.riceharvest/ctxprune -->
|
|
157
|
+
|
|
140
158
|
## Token pruning: the 0.2 API (Python, LangChain, LlamaIndex)
|
|
141
159
|
|
|
142
160
|
This is lossy compression for prompts, retrieved documents and logs: a 140M-parameter token classifier
|
|
@@ -180,7 +198,13 @@ With LLMLingua: `PromptCompressor(model_name="darioooooo0o/ctxprune-small", use_
|
|
|
180
198
|
How good it is: at 50% of tokens kept, a reader LLM (DeepSeek V4 Flash) answered 68.8% of 378 held-out questions on
|
|
181
199
|
agent text exactly, vs 53.4% for LLMLingua-2-large and 41.0% for LLMLingua-2-base (uncompressed: 91.8%). It is weaker
|
|
182
200
|
than LLMLingua-2-large on non-English web prose, and the question set comes from our own training sources. Per-source
|
|
183
|
-
tables and caveats are on the [model card](https://huggingface.co/darioooooo0o/ctxprune-small).
|
|
201
|
+
tables and caveats are on the [model card](https://huggingface.co/darioooooo0o/ctxprune-small).
|
|
202
|
+
|
|
203
|
+
**Use `protect=True` on tool output, logs and anything with identifiers.** ctxprune never splits or re-spaces an
|
|
204
|
+
identifier, but without `protect` it may delete one. On 30 held-out agent tool outputs at 50% kept, the share of IDs,
|
|
205
|
+
paths, versions, URLs and numbers that survived verbatim was 99.6% with `protect=True` (55.9% of tokens kept),
|
|
206
|
+
45.8% without it, 72.9% for LLMLingua-2-base and 94.6% for LLMLingua-2-large. Every type and rate:
|
|
207
|
+
[docs/bench/llmlingua_facts.md](https://github.com/riceharvest/ctxprune/blob/main/docs/bench/llmlingua_facts.md). The training and
|
|
184
208
|
evaluation code for this model is at tag [`v0.2.1`](https://github.com/riceharvest/ctxprune/tree/v0.2.1)
|
|
185
209
|
(`scripts/`, `ctxprune/`).
|
|
186
210
|
|
|
@@ -93,6 +93,24 @@ Python 3.9+ on the machine:
|
|
|
93
93
|
|
|
94
94
|
Use either the plugin or `ctxprune install claude-code`, not both. `ctxprune doctor` warns if both are on.
|
|
95
95
|
|
|
96
|
+
## MCP server (Claude Desktop, Cursor, Cline and other MCP clients)
|
|
97
|
+
|
|
98
|
+
For clients without tool-output hooks, `ctxprune mcp` is a local stdio MCP server with two read tools:
|
|
99
|
+
|
|
100
|
+
- `read_compact(path, start_line?, end_line?)`: the lossless `safe` pipeline, for big logs, JSON/MCP dumps, CSV and
|
|
101
|
+
test output. Nothing is dropped.
|
|
102
|
+
- `read_pruned(path, keep=0.5, protect=true)`: token pruning with the model below (lossy). Identifiers, numbers and
|
|
103
|
+
paths are kept unless `protect=false`.
|
|
104
|
+
|
|
105
|
+
```json
|
|
106
|
+
{ "mcpServers": { "ctxprune": { "command": "uvx", "args": ["ctxprune", "mcp"] } } }
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
It reads only the files the client asks for, makes no network calls (`read_pruned` downloads the model once on first
|
|
110
|
+
use), and needs no MCP SDK.
|
|
111
|
+
|
|
112
|
+
<!-- mcp-name: io.github.riceharvest/ctxprune -->
|
|
113
|
+
|
|
96
114
|
## Token pruning: the 0.2 API (Python, LangChain, LlamaIndex)
|
|
97
115
|
|
|
98
116
|
This is lossy compression for prompts, retrieved documents and logs: a 140M-parameter token classifier
|
|
@@ -136,7 +154,13 @@ With LLMLingua: `PromptCompressor(model_name="darioooooo0o/ctxprune-small", use_
|
|
|
136
154
|
How good it is: at 50% of tokens kept, a reader LLM (DeepSeek V4 Flash) answered 68.8% of 378 held-out questions on
|
|
137
155
|
agent text exactly, vs 53.4% for LLMLingua-2-large and 41.0% for LLMLingua-2-base (uncompressed: 91.8%). It is weaker
|
|
138
156
|
than LLMLingua-2-large on non-English web prose, and the question set comes from our own training sources. Per-source
|
|
139
|
-
tables and caveats are on the [model card](https://huggingface.co/darioooooo0o/ctxprune-small).
|
|
157
|
+
tables and caveats are on the [model card](https://huggingface.co/darioooooo0o/ctxprune-small).
|
|
158
|
+
|
|
159
|
+
**Use `protect=True` on tool output, logs and anything with identifiers.** ctxprune never splits or re-spaces an
|
|
160
|
+
identifier, but without `protect` it may delete one. On 30 held-out agent tool outputs at 50% kept, the share of IDs,
|
|
161
|
+
paths, versions, URLs and numbers that survived verbatim was 99.6% with `protect=True` (55.9% of tokens kept),
|
|
162
|
+
45.8% without it, 72.9% for LLMLingua-2-base and 94.6% for LLMLingua-2-large. Every type and rate:
|
|
163
|
+
[docs/bench/llmlingua_facts.md](https://github.com/riceharvest/ctxprune/blob/main/docs/bench/llmlingua_facts.md). The training and
|
|
140
164
|
evaluation code for this model is at tag [`v0.2.1`](https://github.com/riceharvest/ctxprune/tree/v0.2.1)
|
|
141
165
|
(`scripts/`, `ctxprune/`).
|
|
142
166
|
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Token pruning: which facts survive? (ctxprune-small vs LLMLingua-2)
|
|
2
|
+
|
|
3
|
+
**Question.** After token pruning, how many of the things a reader needs verbatim are still there? That means IDs and
|
|
4
|
+
hashes, paths, versions, URLs, dates, emails and numbers of two or more digits.
|
|
5
|
+
|
|
6
|
+
**Data.** 120 held-out chunks (test split, 400–6,000 chars), 30 per type, from the v2 corpus:
|
|
7
|
+
- agent tool output: SWE-agent bash output (Kwai SWE-smith) and Terminal-Bench observations;
|
|
8
|
+
- MCP results: Toucan MCP tool calls;
|
|
9
|
+
- chat: oasst2;
|
|
10
|
+
- multilingual web prose: FineWeb-2.
|
|
11
|
+
|
|
12
|
+
**Method.** [`llmlingua_facts.py`](llmlingua_facts.py), measured 2026-11-18.
|
|
13
|
+
- LLMLingua-2 runs through stock `llmlingua` 0.2.2 (`compress_prompt([text], rate=r, force_tokens=["\n"],
|
|
14
|
+
drop_consecutive=True)`) with Microsoft's base (mBERT) and large (XLM-R) checkpoints.
|
|
15
|
+
- ctxprune is `ctxprune.prune(text, keep=r)` with and without `protect=True`.
|
|
16
|
+
- A fact counts as kept if it appears verbatim in the output. Tokens are o200k.
|
|
17
|
+
- `items` picks the chunks with a fixed seed; `score` prints the table.
|
|
18
|
+
|
|
19
|
+
**Results** (facts kept / tokens kept):
|
|
20
|
+
|
|
21
|
+
| type | rate | LLMLingua-2 base | LLMLingua-2 large | ctxprune | ctxprune `protect=True` |
|
|
22
|
+
|---|--:|---|---|---|---|
|
|
23
|
+
| agent tool output (240 facts) | 0.5 | 72.9% / 58.9% | 94.6% / 52.9% | 45.8% / 52.1% | **99.6%** / 55.9% |
|
|
24
|
+
| MCP results (245) | 0.5 | 71.8% / 61.4% | 75.9% / 49.3% | 66.1% / 52.4% | **95.1%** / 54.9% |
|
|
25
|
+
| chat (27) | 0.5 | 92.6% / 48.5% | 96.3% / 49.7% | 77.8% / 50.4% | **100%** / 50.5% |
|
|
26
|
+
| multilingual web prose (93) | 0.5 | 81.7% / 55.3% | 91.4% / 51.2% | 89.2% / 53.2% | **98.9%** / 53.4% |
|
|
27
|
+
| agent tool output | 0.8 | 79.2% / 94.7% | 100% / 79.8% | – | 100% / 82.2% |
|
|
28
|
+
| MCP results | 0.8 | 72.2% / 97.9% | 91.8% / 78.1% | – | 98.4% / 81.1% |
|
|
29
|
+
|
|
30
|
+
**How to read it.**
|
|
31
|
+
- **Use `protect=True` on tool output, logs and anything with identifiers.** Without it, ctxprune drops more facts
|
|
32
|
+
than LLMLingua-2 on agent output (45.8% kept). It never splits or re-spaces them, but it does delete them.
|
|
33
|
+
- With `protect=True`, ctxprune keeps more facts than LLMLingua-2-base everywhere, often with fewer tokens. Against
|
|
34
|
+
LLMLingua-2-large the gap on agent tool output is small (99.6% vs 94.6%) and ctxprune keeps about 3 points more
|
|
35
|
+
tokens.
|
|
36
|
+
- `rate` is not a hard budget for every tool: LLMLingua-2-base at 0.8 kept 95–98% of tokens on tool output.
|
|
37
|
+
- Facts kept is not reading accuracy. On QA from the compressed text, LLMLingua-2-large is better than ctxprune on
|
|
38
|
+
non-English web prose (see the [model card](https://huggingface.co/darioooooo0o/ctxprune-small)). The chat row has
|
|
39
|
+
only 27 facts.
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""LLMLingua-2 (stock llmlingua 0.2.2, microsoft mBERT/XLM-R models) vs ctxprune-small on held-out chunks.
|
|
2
|
+
|
|
3
|
+
python compare.py items # -> items.jsonl (held-out test chunks per domain)
|
|
4
|
+
<stock venv>/bin/python compare.py lingua MODEL RATE OUT
|
|
5
|
+
<ctxprune venv>/bin/python compare.py ctxprune RATE OUT
|
|
6
|
+
python compare.py score items.jsonl OUT... # facts kept verbatim + o200k tokens kept, per domain
|
|
7
|
+
"""
|
|
8
|
+
import json, os, random, re, sys
|
|
9
|
+
|
|
10
|
+
D = os.environ.get("CTXPRUNE_CHUNKS", "data/v2/chunks") # the v2 corpus chunks (research repo)
|
|
11
|
+
DOMAINS = {"agent tool output": ["kwai_swe_tool", "terminalbench_obs"], "MCP result": ["toucan_mcp"],
|
|
12
|
+
"chat": ["oasst2"], "multilingual web prose": ["fineweb2"]}
|
|
13
|
+
N = 30
|
|
14
|
+
|
|
15
|
+
# "facts": the things a reader needs verbatim: IDs/hashes, paths, versions, numbers >= 2 digits, URLs, dates, emails
|
|
16
|
+
# paths need a word on both sides of a slash (so `</returncode>` closing tags are not counted as paths)
|
|
17
|
+
FACT = re.compile(r"https?://\S+|[\w.+-]+@[\w-]+\.[\w.]+|\b\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2})?)?"
|
|
18
|
+
r"|(?<![<\w])[\w.-]*\w/(?:[\w.-]+/)*[\w.-]*\w|\b[vV]?\d+(?:\.\d+){1,3}\b"
|
|
19
|
+
r"|\b(?=\w*\d)(?=\w*[A-Za-z])[\w-]{6,}\b|\b\d{2,}\b")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def items():
|
|
23
|
+
rnd = random.Random(7)
|
|
24
|
+
out = open("items.jsonl", "w")
|
|
25
|
+
for dom, srcs in DOMAINS.items():
|
|
26
|
+
pool = []
|
|
27
|
+
for s in srcs:
|
|
28
|
+
for line in open(f"{D}/{s}.jsonl"):
|
|
29
|
+
r = json.loads(line)
|
|
30
|
+
if r["split"] == "test" and 400 <= len(r["text"]) <= 6000:
|
|
31
|
+
pool.append(r)
|
|
32
|
+
rnd.shuffle(pool)
|
|
33
|
+
for r in pool[:N]:
|
|
34
|
+
out.write(json.dumps({"id": r["id"], "domain": dom, "text": r["text"]}) + "\n")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def run_lingua(model, rate, outp):
|
|
38
|
+
from llmlingua import PromptCompressor
|
|
39
|
+
pc = PromptCompressor(model_name=model, use_llmlingua2=True, device_map="cpu")
|
|
40
|
+
with open(outp, "w") as o:
|
|
41
|
+
for line in open("items.jsonl"):
|
|
42
|
+
it = json.loads(line)
|
|
43
|
+
try:
|
|
44
|
+
txt = pc.compress_prompt([it["text"]], rate=rate, force_tokens=["\n"], drop_consecutive=True)["compressed_prompt"]
|
|
45
|
+
except Exception as e:
|
|
46
|
+
txt = f"<ERROR {type(e).__name__}: {e}>"
|
|
47
|
+
o.write(json.dumps({"id": it["id"], "out": txt}) + "\n")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def run_ctxprune(rate, outp):
|
|
51
|
+
import ctxprune
|
|
52
|
+
with open(outp, "w") as o:
|
|
53
|
+
for line in open("items.jsonl"):
|
|
54
|
+
it = json.loads(line)
|
|
55
|
+
o.write(json.dumps({"id": it["id"], "out": ctxprune.prune(it["text"], keep=rate)}) + "\n")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def score(itemsp, outs):
|
|
59
|
+
import tiktoken
|
|
60
|
+
enc = tiktoken.get_encoding("o200k_base")
|
|
61
|
+
its = {json.loads(l)["id"]: json.loads(l) for l in open(itemsp)}
|
|
62
|
+
res = {}
|
|
63
|
+
for op in outs:
|
|
64
|
+
rows = {json.loads(l)["id"]: json.loads(l)["out"] for l in open(op)}
|
|
65
|
+
for iid, it in its.items():
|
|
66
|
+
facts = set(FACT.findall(it["text"]))
|
|
67
|
+
o = rows.get(iid, "")
|
|
68
|
+
k = (op, it["domain"])
|
|
69
|
+
a = res.setdefault(k, [0, 0, 0, 0, 0])
|
|
70
|
+
a[0] += len(facts); a[1] += sum(1 for f in facts if f in o)
|
|
71
|
+
a[2] += len(enc.encode(it["text"])); a[3] += len(enc.encode(o)); a[4] += o.startswith("<ERROR")
|
|
72
|
+
print(f"{'output':34} {'domain':24} {'facts kept':>12} {'tokens kept':>12} errors")
|
|
73
|
+
for (op, dom), (nf, kf, t0, t1, er) in sorted(res.items(), key=lambda kv: (kv[0][1], kv[0][0])):
|
|
74
|
+
print(f"{op:34} {dom:24} {kf / max(nf, 1):11.1%} {t1 / max(t0, 1):11.1%} {er:5d} (n_facts={nf})")
|
|
75
|
+
json.dump({f"{op}|{dom}": v for (op, dom), v in res.items()}, open("score.json", "w"), indent=1)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
if __name__ == "__main__":
|
|
79
|
+
cmd = sys.argv[1]
|
|
80
|
+
if cmd == "items": items()
|
|
81
|
+
elif cmd == "lingua": run_lingua(sys.argv[2], float(sys.argv[3]), sys.argv[4])
|
|
82
|
+
elif cmd == "ctxprune": run_ctxprune(float(sys.argv[2]), sys.argv[3])
|
|
83
|
+
elif cmd == "score": score(sys.argv[2], sys.argv[3:])
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ctxprune"
|
|
3
|
-
version = "0.3.
|
|
3
|
+
version = "0.3.2"
|
|
4
4
|
description = "Shrink what your LLM reads: lossless tool-output compression for coding agents (Claude Code, Hermes, Pi, OpenCode, dsh, Codex) and token pruning for prompts and RAG (LangChain, LlamaIndex)"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
# 0BSD from 0.3.0 (0.2.1 was Apache-2.0). The optional model weights are not part of the package; see THIRD_PARTY_NOTICES
|
|
@@ -8,7 +8,7 @@ API (`ctxprune.prune`, `Compressor`, `DEFAULT_MODEL`) is still here and loads it
|
|
|
8
8
|
ctxprune.prune(text, keep=0.33, protect=True) # never drop identifiers, numbers or paths
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
|
-
__version__ = "0.3.
|
|
11
|
+
__version__ = "0.3.2"
|
|
12
12
|
__all__ = ["Compressor", "prune", "DEFAULT_MODEL"]
|
|
13
13
|
|
|
14
14
|
_default = None
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
ctxprune serve --model-dir DIR line-model server for lossy presets (ctxprune[model])
|
|
13
13
|
ctxprune bundle hermes|claude-code DEST self-contained plugin dirs (catalog / marketplace builds)
|
|
14
14
|
ctxprune hook claude-code (called by Claude Code: hook JSON on stdin)
|
|
15
|
+
ctxprune mcp stdio MCP server: read_compact (lossless), read_pruned (lossy)
|
|
15
16
|
0.2 token pruning (lossy, still supported; model darioooooo0o/ctxprune-small):
|
|
16
17
|
ctxprune FILE [--keep 0.5] [--protect] [--stats] token-prune a file (or stdin) and print it
|
|
17
18
|
ctxprune serve [--port 8719] 0.2 HTTP API: POST /compress {"text": ..., "rate": 0.5}
|
|
@@ -360,6 +361,9 @@ def main(argv: list | None = None) -> int:
|
|
|
360
361
|
return _compress(rest)
|
|
361
362
|
if cmd == "proxy":
|
|
362
363
|
return _proxy(rest)
|
|
364
|
+
if cmd == "mcp":
|
|
365
|
+
from .mcp_server import main as mcp_main
|
|
366
|
+
return mcp_main(rest)
|
|
363
367
|
if cmd == "bundle":
|
|
364
368
|
from .bundle import main as bundle_main
|
|
365
369
|
return bundle_main(rest)
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""`ctxprune mcp`: a stdio MCP server for clients without tool-output hooks (Claude Desktop, Cursor, Cline, ...).
|
|
2
|
+
|
|
3
|
+
Two read tools, so a big log, JSON dump or report enters the context already shortened:
|
|
4
|
+
read_compact the lossless `safe` pipeline (the same bytes the agent hooks produce); nothing is dropped
|
|
5
|
+
read_pruned opt-in token pruning with the 0.2 model (lossy); identifiers, numbers and paths kept by default
|
|
6
|
+
|
|
7
|
+
Stdlib JSON-RPC 2.0 over stdin/stdout (one message per line), protocol 2025-06-18; no MCP SDK dependency.
|
|
8
|
+
Local only: it reads the files the client asks for and makes no network calls (read_pruned downloads the model from
|
|
9
|
+
Hugging Face once, on first use).
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import sys
|
|
16
|
+
|
|
17
|
+
from . import __version__
|
|
18
|
+
|
|
19
|
+
PROTOCOL = "2025-06-18"
|
|
20
|
+
MAX_BYTES = int(os.environ.get("CTXPRUNE_MCP_MAX_BYTES", str(20 * 1024 * 1024)))
|
|
21
|
+
|
|
22
|
+
TOOLS = [
|
|
23
|
+
{
|
|
24
|
+
"name": "read_compact",
|
|
25
|
+
"title": "Read a file, losslessly compacted",
|
|
26
|
+
"description": ("Read a text file (log, JSON/MCP dump, CSV, test output, report) and return it rewritten into a "
|
|
27
|
+
"shorter form that keeps every value: runs of identical lines counted, JSON record lists as "
|
|
28
|
+
"rows, escaped JSON shown as text, terminal colour codes and progress redraws removed. Use it "
|
|
29
|
+
"instead of a plain read for big machine-generated files. Not for source files you will edit."),
|
|
30
|
+
"inputSchema": {
|
|
31
|
+
"type": "object",
|
|
32
|
+
"properties": {
|
|
33
|
+
"path": {"type": "string", "description": "Path of the file to read."},
|
|
34
|
+
"start_line": {"type": "integer", "minimum": 1, "description": "First line to read (1-based)."},
|
|
35
|
+
"end_line": {"type": "integer", "minimum": 1, "description": "Last line to read (inclusive)."},
|
|
36
|
+
},
|
|
37
|
+
"required": ["path"],
|
|
38
|
+
},
|
|
39
|
+
"annotations": {"readOnlyHint": True, "openWorldHint": False},
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
"name": "read_pruned",
|
|
43
|
+
"title": "Read a file, token-pruned (lossy)",
|
|
44
|
+
"description": ("Read a text file and delete its least useful words with a small local model, keeping about "
|
|
45
|
+
"`keep` of its tokens. Lossy: use it for long prose, retrieved documents or logs when the "
|
|
46
|
+
"gist matters more than every word. Identifiers, numbers and paths are kept unless "
|
|
47
|
+
"protect=false."),
|
|
48
|
+
"inputSchema": {
|
|
49
|
+
"type": "object",
|
|
50
|
+
"properties": {
|
|
51
|
+
"path": {"type": "string", "description": "Path of the file to read."},
|
|
52
|
+
"keep": {"type": "number", "exclusiveMinimum": 0, "maximum": 1, "default": 0.5,
|
|
53
|
+
"description": "Share of tokens to keep (0-1]."},
|
|
54
|
+
"protect": {"type": "boolean", "default": True,
|
|
55
|
+
"description": "Never drop identifiers, numbers or paths."},
|
|
56
|
+
},
|
|
57
|
+
"required": ["path"],
|
|
58
|
+
},
|
|
59
|
+
"annotations": {"readOnlyHint": True, "openWorldHint": False},
|
|
60
|
+
},
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _read(args: dict) -> str:
|
|
65
|
+
path = os.path.expanduser(str(args.get("path", "")))
|
|
66
|
+
if not path:
|
|
67
|
+
raise ValueError("path is required")
|
|
68
|
+
if not os.path.isfile(path):
|
|
69
|
+
raise ValueError(f"not a file: {path}")
|
|
70
|
+
if os.path.getsize(path) > MAX_BYTES:
|
|
71
|
+
raise ValueError(f"file is over {MAX_BYTES} bytes (CTXPRUNE_MCP_MAX_BYTES)")
|
|
72
|
+
with open(path, "rb") as f:
|
|
73
|
+
text = f.read().decode("utf-8", "surrogateescape")
|
|
74
|
+
a, b = args.get("start_line"), args.get("end_line")
|
|
75
|
+
if a or b:
|
|
76
|
+
lines = text.splitlines(keepends=True)
|
|
77
|
+
text = "".join(lines[(int(a) - 1 if a else 0):(int(b) if b else len(lines))])
|
|
78
|
+
return text
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _compact(text: str) -> tuple[str, str]:
|
|
82
|
+
from . import glue
|
|
83
|
+
glue.boot()
|
|
84
|
+
import ctxprune_compress
|
|
85
|
+
out, rec = ctxprune_compress.run(text, command="", harness="mcp")
|
|
86
|
+
return out, f"[ctxprune: {len(text):,} -> {len(out):,} chars, lossless]"
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _pruned(text: str, keep: float, protect: bool) -> tuple[str, str]:
|
|
90
|
+
from . import prune
|
|
91
|
+
out = prune(text, keep=keep, protect=protect)
|
|
92
|
+
return out, (f"[ctxprune: {len(text):,} -> {len(out):,} chars, token-pruned, target {keep:.0%} of tokens"
|
|
93
|
+
f"{', identifiers kept' if protect else ''} (lossy)]")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def call_tool(name: str, args: dict) -> dict:
|
|
97
|
+
try:
|
|
98
|
+
text = _read(args)
|
|
99
|
+
if name == "read_compact":
|
|
100
|
+
out, note = _compact(text)
|
|
101
|
+
elif name == "read_pruned":
|
|
102
|
+
keep = float(args.get("keep", 0.5))
|
|
103
|
+
if not 0 < keep <= 1:
|
|
104
|
+
raise ValueError("keep must be in (0, 1]")
|
|
105
|
+
out, note = _pruned(text, keep, bool(args.get("protect", True)))
|
|
106
|
+
else:
|
|
107
|
+
return {"content": [{"type": "text", "text": f"unknown tool {name}"}], "isError": True}
|
|
108
|
+
return {"content": [{"type": "text", "text": out}, {"type": "text", "text": note}], "isError": False}
|
|
109
|
+
except Exception as e: # tool errors go back to the model, not as protocol errors
|
|
110
|
+
return {"content": [{"type": "text", "text": f"ctxprune: {e}"}], "isError": True}
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def handle(msg: dict) -> dict | None:
|
|
114
|
+
mid, method = msg.get("id"), msg.get("method")
|
|
115
|
+
if mid is None: # notification (initialized, cancelled, ...)
|
|
116
|
+
return None
|
|
117
|
+
if method == "initialize":
|
|
118
|
+
req = (msg.get("params") or {}).get("protocolVersion") or PROTOCOL
|
|
119
|
+
result = {"protocolVersion": req if isinstance(req, str) else PROTOCOL,
|
|
120
|
+
"capabilities": {"tools": {"listChanged": False}},
|
|
121
|
+
"serverInfo": {"name": "ctxprune", "title": "ctxprune", "version": __version__},
|
|
122
|
+
"instructions": "Use read_compact for big logs, JSON/MCP dumps and test output: it is lossless."}
|
|
123
|
+
elif method == "ping":
|
|
124
|
+
result = {}
|
|
125
|
+
elif method == "tools/list":
|
|
126
|
+
result = {"tools": TOOLS}
|
|
127
|
+
elif method == "tools/call":
|
|
128
|
+
p = msg.get("params") or {}
|
|
129
|
+
result = call_tool(str(p.get("name")), p.get("arguments") or {})
|
|
130
|
+
else:
|
|
131
|
+
return {"jsonrpc": "2.0", "id": mid, "error": {"code": -32601, "message": f"method not found: {method}"}}
|
|
132
|
+
return {"jsonrpc": "2.0", "id": mid, "result": result}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def main(argv: list | None = None) -> int:
|
|
136
|
+
out = sys.stdout
|
|
137
|
+
for line in sys.stdin:
|
|
138
|
+
line = line.strip()
|
|
139
|
+
if not line:
|
|
140
|
+
continue
|
|
141
|
+
try:
|
|
142
|
+
msg = json.loads(line)
|
|
143
|
+
except ValueError:
|
|
144
|
+
out.write(json.dumps({"jsonrpc": "2.0", "id": None,
|
|
145
|
+
"error": {"code": -32700, "message": "parse error"}}) + "\n")
|
|
146
|
+
out.flush()
|
|
147
|
+
continue
|
|
148
|
+
msgs = msg if isinstance(msg, list) else [msg]
|
|
149
|
+
replies = [r for r in (handle(m) for m in msgs if isinstance(m, dict)) if r is not None]
|
|
150
|
+
for r in replies:
|
|
151
|
+
out.write(json.dumps(r) + "\n")
|
|
152
|
+
out.flush()
|
|
153
|
+
return 0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|