ctxprune 0.3.1__tar.gz → 0.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. {ctxprune-0.3.1 → ctxprune-0.3.2}/CHANGELOG.md +11 -0
  2. {ctxprune-0.3.1 → ctxprune-0.3.2}/PKG-INFO +26 -2
  3. {ctxprune-0.3.1 → ctxprune-0.3.2}/README.md +25 -1
  4. ctxprune-0.3.2/docs/bench/llmlingua_facts.md +39 -0
  5. ctxprune-0.3.2/docs/bench/llmlingua_facts.py +83 -0
  6. ctxprune-0.3.2/docs/media/banner.png +0 -0
  7. ctxprune-0.3.2/docs/media/icon.png +0 -0
  8. ctxprune-0.3.2/docs/media/mcp-before-after.png +0 -0
  9. ctxprune-0.3.2/docs/media/replay-vs-compactors.png +0 -0
  10. {ctxprune-0.3.1 → ctxprune-0.3.2}/pyproject.toml +1 -1
  11. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/__init__.py +1 -1
  12. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/cli.py +4 -0
  13. ctxprune-0.3.2/src/ctxprune/mcp_server.py +153 -0
  14. {ctxprune-0.3.1 → ctxprune-0.3.2}/.gitignore +0 -0
  15. {ctxprune-0.3.1 → ctxprune-0.3.2}/LICENSE +0 -0
  16. {ctxprune-0.3.1 → ctxprune-0.3.2}/PACKAGE.md +0 -0
  17. {ctxprune-0.3.1 → ctxprune-0.3.2}/THIRD_PARTY_NOTICES +0 -0
  18. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/BENCHMARKS.md +0 -0
  19. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/SAFETY.md +0 -0
  20. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/bench_shipped.py +0 -0
  21. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/latency_scripts_106bcea.json +0 -0
  22. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/latency_shipped_106bcea.json +0 -0
  23. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/measure_variant_dedup.diff +0 -0
  24. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/measure_variant_powershell.diff +0 -0
  25. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/real_cc_split.py +0 -0
  26. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/real_cc_split.txt +0 -0
  27. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/spill_gain_12c7b79.json +0 -0
  28. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/typical_dedup_results.json +0 -0
  29. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/typical_dedup_summary.json +0 -0
  30. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/typical_safe_results.json +0 -0
  31. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/bench/typical_safe_summary.json +0 -0
  32. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/README.md +0 -0
  33. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/build.py +0 -0
  34. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/dropped.md +0 -0
  35. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/README.md +0 -0
  36. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_00_examples.py +0 -0
  37. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_alias_model_cache.py +0 -0
  38. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_dedup.py +0 -0
  39. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_hook_infra.py +0 -0
  40. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_hook_notes.py +0 -0
  41. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_logs.py +0 -0
  42. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_new_cloud_k8s_iac.py +0 -0
  43. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_new_data_formats_text.py +0 -0
  44. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_new_pkgmgr_langtools.py +0 -0
  45. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_new_protocol_envelope.py +0 -0
  46. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_new_system_ops_terminal.py +0 -0
  47. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_new_vcs_ci_review.py +0 -0
  48. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_noise_pretool.py +0 -0
  49. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_rules_ci_infra_cloud.py +0 -0
  50. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_rules_compilers_build_native.py +0 -0
  51. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_rules_logs_ops_ml.py +0 -0
  52. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_rules_media_docs_rag_misc.py +0 -0
  53. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_rules_stateful_agents_code.py +0 -0
  54. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_rules_structured_json.py +0 -0
  55. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_struct_a.py +0 -0
  56. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_struct_b.py +0 -0
  57. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_struct_c.py +0 -0
  58. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/cases_terminal.py +0 -0
  59. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/harness.py +0 -0
  60. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/gate/run.py +0 -0
  61. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/ideas.jsonl +0 -0
  62. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/ideas.md +0 -0
  63. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/ideas/shortlist.md +0 -0
  64. {ctxprune-0.3.1 → ctxprune-0.3.2}/docs/later-integrations.md +0 -0
  65. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/ctx_audit.py +0 -0
  66. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/ctxprune_compress.py +0 -0
  67. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/ctxprune_core.py +0 -0
  68. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/ctxprune_note.py +0 -0
  69. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/ctxprune_proxy.py +0 -0
  70. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/dedup.py +0 -0
  71. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/emitted.py +0 -0
  72. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/export_v2_onnx.py +0 -0
  73. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/hook_budget.py +0 -0
  74. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/integrations/dsh/ctxprune.mjs +0 -0
  75. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/integrations/hermes/ctxprune/__init__.py +0 -0
  76. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/integrations/hermes/ctxprune/plugin.yaml +0 -0
  77. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/integrations/opencode/ctxprune.js +0 -0
  78. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/integrations/pi/ctxprune.ts +0 -0
  79. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/line_encode.py +0 -0
  80. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/lossless.py +0 -0
  81. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/lossless2.py +0 -0
  82. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/lossless_hook.py +0 -0
  83. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/methods.py +0 -0
  84. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/read_trim.py +0 -0
  85. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/recover.py +0 -0
  86. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/render_store.py +0 -0
  87. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/spill.py +0 -0
  88. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/v2_hook.py +0 -0
  89. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/v2_server.py +0 -0
  90. {ctxprune-0.3.1 → ctxprune-0.3.2}/scripts/view_safe.py +0 -0
  91. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/__main__.py +0 -0
  92. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/atoms.py +0 -0
  93. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/bundle.py +0 -0
  94. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/chunking.py +0 -0
  95. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/compress.py +0 -0
  96. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/doctor.py +0 -0
  97. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/glue.py +0 -0
  98. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/harness_audit.py +0 -0
  99. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/idcheck.py +0 -0
  100. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/installers.py +0 -0
  101. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/integrations.py +0 -0
  102. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/legacy_cli.py +0 -0
  103. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/model.py +0 -0
  104. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/plugin_cli.py +0 -0
  105. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/serve.py +0 -0
  106. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/stats.py +0 -0
  107. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/tokenlabels.py +0 -0
  108. {ctxprune-0.3.1 → ctxprune-0.3.2}/src/ctxprune/usage.py +0 -0
  109. {ctxprune-0.3.1 → ctxprune-0.3.2}/tests/packaging/cc_live.py +0 -0
  110. {ctxprune-0.3.1 → ctxprune-0.3.2}/tests/packaging/codepage_hook.py +0 -0
  111. {ctxprune-0.3.1 → ctxprune-0.3.2}/tests/packaging/determinism.py +0 -0
  112. {ctxprune-0.3.1 → ctxprune-0.3.2}/tests/packaging/e2e.py +0 -0
  113. {ctxprune-0.3.1 → ctxprune-0.3.2}/tests/packaging/test_harness_usage.py +0 -0
  114. {ctxprune-0.3.1 → ctxprune-0.3.2}/tests/packaging/test_units.py +0 -0
@@ -1,5 +1,16 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.3.2 (2026-11-18)
4
+
5
+ - **`ctxprune mcp`**: a stdio MCP server for clients without tool-output hooks (Claude Desktop, Cursor, Cline, ...).
6
+ `read_compact` reads a file through the lossless `safe` pipeline; `read_pruned` token-prunes it with the 0.2 model,
7
+ identifiers kept by default. Stdlib JSON-RPC, no MCP SDK. Listed as `io.github.riceharvest/ctxprune` in the MCP
8
+ Registry (`server.json`).
9
+ - README / model card: use `protect=True` for token pruning of tool output and logs. Measured on 120 held-out chunks
10
+ against LLMLingua-2 in [docs/bench/llmlingua_facts.md](docs/bench/llmlingua_facts.md).
11
+ - Note: `ctxprune mcp` is now a subcommand, so a 0.2-style `ctxprune mcp` on a file literally named `mcp` needs
12
+ `ctxprune ./mcp`.
13
+
3
14
  ## 0.3.1 (2026-11-18)
4
15
 
5
16
  Docs only; the code and every output are the same as 0.3.0.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: ctxprune
3
- Version: 0.3.1
3
+ Version: 0.3.2
4
4
  Summary: Shrink what your LLM reads: lossless tool-output compression for coding agents (Claude Code, Hermes, Pi, OpenCode, dsh, Codex) and token pruning for prompts and RAG (LangChain, LlamaIndex)
5
5
  Project-URL: Homepage, https://github.com/riceharvest/ctxprune
6
6
  Project-URL: Model, https://huggingface.co/darioooooo0o/ctxprune-small
@@ -137,6 +137,24 @@ Python 3.9+ on the machine:
137
137
 
138
138
  Use either the plugin or `ctxprune install claude-code`, not both. `ctxprune doctor` warns if both are on.
139
139
 
140
+ ## MCP server (Claude Desktop, Cursor, Cline and other MCP clients)
141
+
142
+ For clients without tool-output hooks, `ctxprune mcp` is a local stdio MCP server with two read tools:
143
+
144
+ - `read_compact(path, start_line?, end_line?)`: the lossless `safe` pipeline, for big logs, JSON/MCP dumps, CSV and
145
+ test output. Nothing is dropped.
146
+ - `read_pruned(path, keep=0.5, protect=true)`: token pruning with the model below (lossy). Identifiers, numbers and
147
+ paths are kept unless `protect=false`.
148
+
149
+ ```json
150
+ { "mcpServers": { "ctxprune": { "command": "uvx", "args": ["ctxprune", "mcp"] } } }
151
+ ```
152
+
153
+ It reads only the files the client asks for, makes no network calls (`read_pruned` downloads the model once on first
154
+ use), and needs no MCP SDK.
155
+
156
+ <!-- mcp-name: io.github.riceharvest/ctxprune -->
157
+
140
158
  ## Token pruning: the 0.2 API (Python, LangChain, LlamaIndex)
141
159
 
142
160
  This is lossy compression for prompts, retrieved documents and logs: a 140M-parameter token classifier
@@ -180,7 +198,13 @@ With LLMLingua: `PromptCompressor(model_name="darioooooo0o/ctxprune-small", use_
180
198
  How good it is: at 50% of tokens kept, a reader LLM (DeepSeek V4 Flash) answered 68.8% of 378 held-out questions on
181
199
  agent text exactly, vs 53.4% for LLMLingua-2-large and 41.0% for LLMLingua-2-base (uncompressed: 91.8%). It is weaker
182
200
  than LLMLingua-2-large on non-English web prose, and the question set comes from our own training sources. Per-source
183
- tables and caveats are on the [model card](https://huggingface.co/darioooooo0o/ctxprune-small). The training and
201
+ tables and caveats are on the [model card](https://huggingface.co/darioooooo0o/ctxprune-small).
202
+
203
+ **Use `protect=True` on tool output, logs and anything with identifiers.** ctxprune never splits or re-spaces an
204
+ identifier, but without `protect` it may delete one. On 30 held-out agent tool outputs at 50% kept, the share of IDs,
205
+ paths, versions, URLs and numbers that survived verbatim was 99.6% with `protect=True` (55.9% of tokens kept),
206
+ 45.8% without it, 72.9% for LLMLingua-2-base and 94.6% for LLMLingua-2-large. Every type and rate:
207
+ [docs/bench/llmlingua_facts.md](https://github.com/riceharvest/ctxprune/blob/main/docs/bench/llmlingua_facts.md). The training and
184
208
  evaluation code for this model is at tag [`v0.2.1`](https://github.com/riceharvest/ctxprune/tree/v0.2.1)
185
209
  (`scripts/`, `ctxprune/`).
186
210
 
@@ -93,6 +93,24 @@ Python 3.9+ on the machine:
93
93
 
94
94
  Use either the plugin or `ctxprune install claude-code`, not both. `ctxprune doctor` warns if both are on.
95
95
 
96
+ ## MCP server (Claude Desktop, Cursor, Cline and other MCP clients)
97
+
98
+ For clients without tool-output hooks, `ctxprune mcp` is a local stdio MCP server with two read tools:
99
+
100
+ - `read_compact(path, start_line?, end_line?)`: the lossless `safe` pipeline, for big logs, JSON/MCP dumps, CSV and
101
+ test output. Nothing is dropped.
102
+ - `read_pruned(path, keep=0.5, protect=true)`: token pruning with the model below (lossy). Identifiers, numbers and
103
+ paths are kept unless `protect=false`.
104
+
105
+ ```json
106
+ { "mcpServers": { "ctxprune": { "command": "uvx", "args": ["ctxprune", "mcp"] } } }
107
+ ```
108
+
109
+ It reads only the files the client asks for, makes no network calls (`read_pruned` downloads the model once on first
110
+ use), and needs no MCP SDK.
111
+
112
+ <!-- mcp-name: io.github.riceharvest/ctxprune -->
113
+
96
114
  ## Token pruning: the 0.2 API (Python, LangChain, LlamaIndex)
97
115
 
98
116
  This is lossy compression for prompts, retrieved documents and logs: a 140M-parameter token classifier
@@ -136,7 +154,13 @@ With LLMLingua: `PromptCompressor(model_name="darioooooo0o/ctxprune-small", use_
136
154
  How good it is: at 50% of tokens kept, a reader LLM (DeepSeek V4 Flash) answered 68.8% of 378 held-out questions on
137
155
  agent text exactly, vs 53.4% for LLMLingua-2-large and 41.0% for LLMLingua-2-base (uncompressed: 91.8%). It is weaker
138
156
  than LLMLingua-2-large on non-English web prose, and the question set comes from our own training sources. Per-source
139
- tables and caveats are on the [model card](https://huggingface.co/darioooooo0o/ctxprune-small). The training and
157
+ tables and caveats are on the [model card](https://huggingface.co/darioooooo0o/ctxprune-small).
158
+
159
+ **Use `protect=True` on tool output, logs and anything with identifiers.** ctxprune never splits or re-spaces an
160
+ identifier, but without `protect` it may delete one. On 30 held-out agent tool outputs at 50% kept, the share of IDs,
161
+ paths, versions, URLs and numbers that survived verbatim was 99.6% with `protect=True` (55.9% of tokens kept),
162
+ 45.8% without it, 72.9% for LLMLingua-2-base and 94.6% for LLMLingua-2-large. Every type and rate:
163
+ [docs/bench/llmlingua_facts.md](https://github.com/riceharvest/ctxprune/blob/main/docs/bench/llmlingua_facts.md). The training and
140
164
  evaluation code for this model is at tag [`v0.2.1`](https://github.com/riceharvest/ctxprune/tree/v0.2.1)
141
165
  (`scripts/`, `ctxprune/`).
142
166
 
@@ -0,0 +1,39 @@
1
+ # Token pruning: which facts survive? (ctxprune-small vs LLMLingua-2)
2
+
3
+ **Question.** After token pruning, how many of the things a reader needs verbatim are still there? That means IDs and
4
+ hashes, paths, versions, URLs, dates, emails and numbers of two or more digits.
5
+
6
+ **Data.** 120 held-out chunks (test split, 400–6,000 chars), 30 per type, from the v2 corpus:
7
+ - agent tool output: SWE-agent bash output (Kwai SWE-smith) and Terminal-Bench observations;
8
+ - MCP results: Toucan MCP tool calls;
9
+ - chat: oasst2;
10
+ - multilingual web prose: FineWeb-2.
11
+
12
+ **Method.** [`llmlingua_facts.py`](llmlingua_facts.py), measured 2026-11-18.
13
+ - LLMLingua-2 runs through stock `llmlingua` 0.2.2 (`compress_prompt([text], rate=r, force_tokens=["\n"],
14
+ drop_consecutive=True)`) with Microsoft's base (mBERT) and large (XLM-R) checkpoints.
15
+ - ctxprune is `ctxprune.prune(text, keep=r)` with and without `protect=True`.
16
+ - A fact counts as kept if it appears verbatim in the output. Tokens are o200k.
17
+ - `items` picks the chunks with a fixed seed; `score` prints the table.
18
+
19
+ **Results** (facts kept / tokens kept):
20
+
21
+ | type | rate | LLMLingua-2 base | LLMLingua-2 large | ctxprune | ctxprune `protect=True` |
22
+ |---|--:|---|---|---|---|
23
+ | agent tool output (240 facts) | 0.5 | 72.9% / 58.9% | 94.6% / 52.9% | 45.8% / 52.1% | **99.6%** / 55.9% |
24
+ | MCP results (245) | 0.5 | 71.8% / 61.4% | 75.9% / 49.3% | 66.1% / 52.4% | **95.1%** / 54.9% |
25
+ | chat (27) | 0.5 | 92.6% / 48.5% | 96.3% / 49.7% | 77.8% / 50.4% | **100%** / 50.5% |
26
+ | multilingual web prose (93) | 0.5 | 81.7% / 55.3% | 91.4% / 51.2% | 89.2% / 53.2% | **98.9%** / 53.4% |
27
+ | agent tool output | 0.8 | 79.2% / 94.7% | 100% / 79.8% | – | 100% / 82.2% |
28
+ | MCP results | 0.8 | 72.2% / 97.9% | 91.8% / 78.1% | – | 98.4% / 81.1% |
29
+
30
+ **How to read it.**
31
+ - **Use `protect=True` on tool output, logs and anything with identifiers.** Without it, ctxprune drops more facts
32
+ than LLMLingua-2 on agent output (45.8% kept). It never splits or re-spaces them, but it does delete them.
33
+ - With `protect=True`, ctxprune keeps more facts than LLMLingua-2-base everywhere, often with fewer tokens. Against
34
+ LLMLingua-2-large the gap on agent tool output is small (99.6% vs 94.6%) and ctxprune keeps about 3 points more
35
+ tokens.
36
+ - `rate` is not a hard budget for every tool: LLMLingua-2-base at 0.8 kept 95–98% of tokens on tool output.
37
+ - Facts kept is not reading accuracy. On QA from the compressed text, LLMLingua-2-large is better than ctxprune on
38
+ non-English web prose (see the [model card](https://huggingface.co/darioooooo0o/ctxprune-small)). The chat row has
39
+ only 27 facts.
@@ -0,0 +1,83 @@
1
+ """LLMLingua-2 (stock llmlingua 0.2.2, microsoft mBERT/XLM-R models) vs ctxprune-small on held-out chunks.
2
+
3
+ python compare.py items # -> items.jsonl (held-out test chunks per domain)
4
+ <stock venv>/bin/python compare.py lingua MODEL RATE OUT
5
+ <ctxprune venv>/bin/python compare.py ctxprune RATE OUT
6
+ python compare.py score items.jsonl OUT... # facts kept verbatim + o200k tokens kept, per domain
7
+ """
8
+ import json, os, random, re, sys
9
+
10
+ D = os.environ.get("CTXPRUNE_CHUNKS", "data/v2/chunks") # the v2 corpus chunks (research repo)
11
+ DOMAINS = {"agent tool output": ["kwai_swe_tool", "terminalbench_obs"], "MCP result": ["toucan_mcp"],
12
+ "chat": ["oasst2"], "multilingual web prose": ["fineweb2"]}
13
+ N = 30
14
+
15
+ # "facts": the things a reader needs verbatim: IDs/hashes, paths, versions, numbers >= 2 digits, URLs, dates, emails
16
+ # paths need a word on both sides of a slash (so `</returncode>` closing tags are not counted as paths)
17
+ FACT = re.compile(r"https?://\S+|[\w.+-]+@[\w-]+\.[\w.]+|\b\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2})?)?"
18
+ r"|(?<![<\w])[\w.-]*\w/(?:[\w.-]+/)*[\w.-]*\w|\b[vV]?\d+(?:\.\d+){1,3}\b"
19
+ r"|\b(?=\w*\d)(?=\w*[A-Za-z])[\w-]{6,}\b|\b\d{2,}\b")
20
+
21
+
22
+ def items():
23
+ rnd = random.Random(7)
24
+ out = open("items.jsonl", "w")
25
+ for dom, srcs in DOMAINS.items():
26
+ pool = []
27
+ for s in srcs:
28
+ for line in open(f"{D}/{s}.jsonl"):
29
+ r = json.loads(line)
30
+ if r["split"] == "test" and 400 <= len(r["text"]) <= 6000:
31
+ pool.append(r)
32
+ rnd.shuffle(pool)
33
+ for r in pool[:N]:
34
+ out.write(json.dumps({"id": r["id"], "domain": dom, "text": r["text"]}) + "\n")
35
+
36
+
37
+ def run_lingua(model, rate, outp):
38
+ from llmlingua import PromptCompressor
39
+ pc = PromptCompressor(model_name=model, use_llmlingua2=True, device_map="cpu")
40
+ with open(outp, "w") as o:
41
+ for line in open("items.jsonl"):
42
+ it = json.loads(line)
43
+ try:
44
+ txt = pc.compress_prompt([it["text"]], rate=rate, force_tokens=["\n"], drop_consecutive=True)["compressed_prompt"]
45
+ except Exception as e:
46
+ txt = f"<ERROR {type(e).__name__}: {e}>"
47
+ o.write(json.dumps({"id": it["id"], "out": txt}) + "\n")
48
+
49
+
50
+ def run_ctxprune(rate, outp):
51
+ import ctxprune
52
+ with open(outp, "w") as o:
53
+ for line in open("items.jsonl"):
54
+ it = json.loads(line)
55
+ o.write(json.dumps({"id": it["id"], "out": ctxprune.prune(it["text"], keep=rate)}) + "\n")
56
+
57
+
58
+ def score(itemsp, outs):
59
+ import tiktoken
60
+ enc = tiktoken.get_encoding("o200k_base")
61
+ its = {json.loads(l)["id"]: json.loads(l) for l in open(itemsp)}
62
+ res = {}
63
+ for op in outs:
64
+ rows = {json.loads(l)["id"]: json.loads(l)["out"] for l in open(op)}
65
+ for iid, it in its.items():
66
+ facts = set(FACT.findall(it["text"]))
67
+ o = rows.get(iid, "")
68
+ k = (op, it["domain"])
69
+ a = res.setdefault(k, [0, 0, 0, 0, 0])
70
+ a[0] += len(facts); a[1] += sum(1 for f in facts if f in o)
71
+ a[2] += len(enc.encode(it["text"])); a[3] += len(enc.encode(o)); a[4] += o.startswith("<ERROR")
72
+ print(f"{'output':34} {'domain':24} {'facts kept':>12} {'tokens kept':>12} errors")
73
+ for (op, dom), (nf, kf, t0, t1, er) in sorted(res.items(), key=lambda kv: (kv[0][1], kv[0][0])):
74
+ print(f"{op:34} {dom:24} {kf / max(nf, 1):11.1%} {t1 / max(t0, 1):11.1%} {er:5d} (n_facts={nf})")
75
+ json.dump({f"{op}|{dom}": v for (op, dom), v in res.items()}, open("score.json", "w"), indent=1)
76
+
77
+
78
+ if __name__ == "__main__":
79
+ cmd = sys.argv[1]
80
+ if cmd == "items": items()
81
+ elif cmd == "lingua": run_lingua(sys.argv[2], float(sys.argv[3]), sys.argv[4])
82
+ elif cmd == "ctxprune": run_ctxprune(float(sys.argv[2]), sys.argv[3])
83
+ elif cmd == "score": score(sys.argv[2], sys.argv[3:])
Binary file
Binary file
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "ctxprune"
3
- version = "0.3.1"
3
+ version = "0.3.2"
4
4
  description = "Shrink what your LLM reads: lossless tool-output compression for coding agents (Claude Code, Hermes, Pi, OpenCode, dsh, Codex) and token pruning for prompts and RAG (LangChain, LlamaIndex)"
5
5
  readme = "README.md"
6
6
  # 0BSD from 0.3.0 (0.2.1 was Apache-2.0). The optional model weights are not part of the package; see THIRD_PARTY_NOTICES
@@ -8,7 +8,7 @@ API (`ctxprune.prune`, `Compressor`, `DEFAULT_MODEL`) is still here and loads it
8
8
  ctxprune.prune(text, keep=0.33, protect=True) # never drop identifiers, numbers or paths
9
9
  """
10
10
 
11
- __version__ = "0.3.1"
11
+ __version__ = "0.3.2"
12
12
  __all__ = ["Compressor", "prune", "DEFAULT_MODEL"]
13
13
 
14
14
  _default = None
@@ -12,6 +12,7 @@
12
12
  ctxprune serve --model-dir DIR line-model server for lossy presets (ctxprune[model])
13
13
  ctxprune bundle hermes|claude-code DEST self-contained plugin dirs (catalog / marketplace builds)
14
14
  ctxprune hook claude-code (called by Claude Code: hook JSON on stdin)
15
+ ctxprune mcp stdio MCP server: read_compact (lossless), read_pruned (lossy)
15
16
  0.2 token pruning (lossy, still supported; model darioooooo0o/ctxprune-small):
16
17
  ctxprune FILE [--keep 0.5] [--protect] [--stats] token-prune a file (or stdin) and print it
17
18
  ctxprune serve [--port 8719] 0.2 HTTP API: POST /compress {"text": ..., "rate": 0.5}
@@ -360,6 +361,9 @@ def main(argv: list | None = None) -> int:
360
361
  return _compress(rest)
361
362
  if cmd == "proxy":
362
363
  return _proxy(rest)
364
+ if cmd == "mcp":
365
+ from .mcp_server import main as mcp_main
366
+ return mcp_main(rest)
363
367
  if cmd == "bundle":
364
368
  from .bundle import main as bundle_main
365
369
  return bundle_main(rest)
@@ -0,0 +1,153 @@
1
+ """`ctxprune mcp`: a stdio MCP server for clients without tool-output hooks (Claude Desktop, Cursor, Cline, ...).
2
+
3
+ Two read tools, so a big log, JSON dump or report enters the context already shortened:
4
+ read_compact the lossless `safe` pipeline (the same bytes the agent hooks produce); nothing is dropped
5
+ read_pruned opt-in token pruning with the 0.2 model (lossy); identifiers, numbers and paths kept by default
6
+
7
+ Stdlib JSON-RPC 2.0 over stdin/stdout (one message per line), protocol 2025-06-18; no MCP SDK dependency.
8
+ Local only: it reads the files the client asks for and makes no network calls (read_pruned downloads the model from
9
+ Hugging Face once, on first use).
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import json
14
+ import os
15
+ import sys
16
+
17
+ from . import __version__
18
+
19
+ PROTOCOL = "2025-06-18"
20
+ MAX_BYTES = int(os.environ.get("CTXPRUNE_MCP_MAX_BYTES", str(20 * 1024 * 1024)))
21
+
22
+ TOOLS = [
23
+ {
24
+ "name": "read_compact",
25
+ "title": "Read a file, losslessly compacted",
26
+ "description": ("Read a text file (log, JSON/MCP dump, CSV, test output, report) and return it rewritten into a "
27
+ "shorter form that keeps every value: runs of identical lines counted, JSON record lists as "
28
+ "rows, escaped JSON shown as text, terminal colour codes and progress redraws removed. Use it "
29
+ "instead of a plain read for big machine-generated files. Not for source files you will edit."),
30
+ "inputSchema": {
31
+ "type": "object",
32
+ "properties": {
33
+ "path": {"type": "string", "description": "Path of the file to read."},
34
+ "start_line": {"type": "integer", "minimum": 1, "description": "First line to read (1-based)."},
35
+ "end_line": {"type": "integer", "minimum": 1, "description": "Last line to read (inclusive)."},
36
+ },
37
+ "required": ["path"],
38
+ },
39
+ "annotations": {"readOnlyHint": True, "openWorldHint": False},
40
+ },
41
+ {
42
+ "name": "read_pruned",
43
+ "title": "Read a file, token-pruned (lossy)",
44
+ "description": ("Read a text file and delete its least useful words with a small local model, keeping about "
45
+ "`keep` of its tokens. Lossy: use it for long prose, retrieved documents or logs when the "
46
+ "gist matters more than every word. Identifiers, numbers and paths are kept unless "
47
+ "protect=false."),
48
+ "inputSchema": {
49
+ "type": "object",
50
+ "properties": {
51
+ "path": {"type": "string", "description": "Path of the file to read."},
52
+ "keep": {"type": "number", "exclusiveMinimum": 0, "maximum": 1, "default": 0.5,
53
+ "description": "Share of tokens to keep (0-1]."},
54
+ "protect": {"type": "boolean", "default": True,
55
+ "description": "Never drop identifiers, numbers or paths."},
56
+ },
57
+ "required": ["path"],
58
+ },
59
+ "annotations": {"readOnlyHint": True, "openWorldHint": False},
60
+ },
61
+ ]
62
+
63
+
64
+ def _read(args: dict) -> str:
65
+ path = os.path.expanduser(str(args.get("path", "")))
66
+ if not path:
67
+ raise ValueError("path is required")
68
+ if not os.path.isfile(path):
69
+ raise ValueError(f"not a file: {path}")
70
+ if os.path.getsize(path) > MAX_BYTES:
71
+ raise ValueError(f"file is over {MAX_BYTES} bytes (CTXPRUNE_MCP_MAX_BYTES)")
72
+ with open(path, "rb") as f:
73
+ text = f.read().decode("utf-8", "surrogateescape")
74
+ a, b = args.get("start_line"), args.get("end_line")
75
+ if a or b:
76
+ lines = text.splitlines(keepends=True)
77
+ text = "".join(lines[(int(a) - 1 if a else 0):(int(b) if b else len(lines))])
78
+ return text
79
+
80
+
81
+ def _compact(text: str) -> tuple[str, str]:
82
+ from . import glue
83
+ glue.boot()
84
+ import ctxprune_compress
85
+ out, rec = ctxprune_compress.run(text, command="", harness="mcp")
86
+ return out, f"[ctxprune: {len(text):,} -> {len(out):,} chars, lossless]"
87
+
88
+
89
+ def _pruned(text: str, keep: float, protect: bool) -> tuple[str, str]:
90
+ from . import prune
91
+ out = prune(text, keep=keep, protect=protect)
92
+ return out, (f"[ctxprune: {len(text):,} -> {len(out):,} chars, token-pruned, target {keep:.0%} of tokens"
93
+ f"{', identifiers kept' if protect else ''} (lossy)]")
94
+
95
+
96
+ def call_tool(name: str, args: dict) -> dict:
97
+ try:
98
+ text = _read(args)
99
+ if name == "read_compact":
100
+ out, note = _compact(text)
101
+ elif name == "read_pruned":
102
+ keep = float(args.get("keep", 0.5))
103
+ if not 0 < keep <= 1:
104
+ raise ValueError("keep must be in (0, 1]")
105
+ out, note = _pruned(text, keep, bool(args.get("protect", True)))
106
+ else:
107
+ return {"content": [{"type": "text", "text": f"unknown tool {name}"}], "isError": True}
108
+ return {"content": [{"type": "text", "text": out}, {"type": "text", "text": note}], "isError": False}
109
+ except Exception as e: # tool errors go back to the model, not as protocol errors
110
+ return {"content": [{"type": "text", "text": f"ctxprune: {e}"}], "isError": True}
111
+
112
+
113
+ def handle(msg: dict) -> dict | None:
114
+ mid, method = msg.get("id"), msg.get("method")
115
+ if mid is None: # notification (initialized, cancelled, ...)
116
+ return None
117
+ if method == "initialize":
118
+ req = (msg.get("params") or {}).get("protocolVersion") or PROTOCOL
119
+ result = {"protocolVersion": req if isinstance(req, str) else PROTOCOL,
120
+ "capabilities": {"tools": {"listChanged": False}},
121
+ "serverInfo": {"name": "ctxprune", "title": "ctxprune", "version": __version__},
122
+ "instructions": "Use read_compact for big logs, JSON/MCP dumps and test output: it is lossless."}
123
+ elif method == "ping":
124
+ result = {}
125
+ elif method == "tools/list":
126
+ result = {"tools": TOOLS}
127
+ elif method == "tools/call":
128
+ p = msg.get("params") or {}
129
+ result = call_tool(str(p.get("name")), p.get("arguments") or {})
130
+ else:
131
+ return {"jsonrpc": "2.0", "id": mid, "error": {"code": -32601, "message": f"method not found: {method}"}}
132
+ return {"jsonrpc": "2.0", "id": mid, "result": result}
133
+
134
+
135
+ def main(argv: list | None = None) -> int:
136
+ out = sys.stdout
137
+ for line in sys.stdin:
138
+ line = line.strip()
139
+ if not line:
140
+ continue
141
+ try:
142
+ msg = json.loads(line)
143
+ except ValueError:
144
+ out.write(json.dumps({"jsonrpc": "2.0", "id": None,
145
+ "error": {"code": -32700, "message": "parse error"}}) + "\n")
146
+ out.flush()
147
+ continue
148
+ msgs = msg if isinstance(msg, list) else [msg]
149
+ replies = [r for r in (handle(m) for m in msgs if isinstance(m, dict)) if r is not None]
150
+ for r in replies:
151
+ out.write(json.dumps(r) + "\n")
152
+ out.flush()
153
+ return 0
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes