ctxprune 0.1.0__tar.gz

ctxprune-0.1.0/.gitignore

@@ -0,0 +1,6 @@
+.venv/
+__pycache__/
+*.pyc
+.pytest_cache/
+uv.lock.bak
+dist/

ctxprune-0.1.0/LICENSE

@@ -0,0 +1,202 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

ctxprune-0.1.0/PKG-INFO

@@ -0,0 +1,143 @@
+Metadata-Version: 2.5
+Name: ctxprune
+Version: 0.1.0
+Summary: Context compression for AI agents: an LLMLingua-2 successor that keeps identifiers intact
+Project-URL: Homepage, https://github.com/riceharvest/ctxprune
+Project-URL: Model, https://huggingface.co/darioooooo0o/ctxprune-small
+Project-URL: Demo, https://huggingface.co/spaces/darioooooo0o/ctxprune
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: agents,context-compression,llm,llmlingua,prompt-compression,rag,token-classification
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing
+Requires-Python: >=3.11
+Requires-Dist: huggingface-hub>=0.25
+Requires-Dist: numpy
+Requires-Dist: onnxruntime>=1.18
+Requires-Dist: transformers>=4.45
+Provides-Extra: baseline
+Requires-Dist: llmlingua; extra == 'baseline'
+Requires-Dist: torch; extra == 'baseline'
+Provides-Extra: data
+Requires-Dist: httpx; extra == 'data'
+Requires-Dist: pyarrow; extra == 'data'
+Requires-Dist: tokenizers; extra == 'data'
+Provides-Extra: dev
+Requires-Dist: httpx; extra == 'dev'
+Requires-Dist: pyarrow; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Provides-Extra: export
+Requires-Dist: onnx; extra == 'export'
+Requires-Dist: onnxruntime>=1.18; extra == 'export'
+Requires-Dist: torch; extra == 'export'
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.3; extra == 'langchain'
+Provides-Extra: llamaindex
+Requires-Dist: llama-index-core>=0.12; extra == 'llamaindex'
+Provides-Extra: onnx
+Provides-Extra: torch
+Requires-Dist: torch; extra == 'torch'
+Description-Content-Type: text/markdown
+
+# ctxprune
+
+Context compression for AI agents. A small token classifier deletes the least useful words from
+tool outputs, code, documents and chat before they reach an LLM. It is a successor to Microsoft's
+LLMLingua-2, built for agent text, and it never splits or re-spaces identifiers.
+
+Model: [darioooooo0o/ctxprune-small](https://huggingface.co/darioooooo0o/ctxprune-small) (Apache-2.0).
+Results and limitations are on the model card. **Try it next to LLMLingua-2 on your own text:
+[live demo](https://huggingface.co/spaces/darioooooo0o/ctxprune).**
+
+```bash
+pip install "ctxprune @ git+https://github.com/riceharvest/ctxprune"   # CPU, ONNX; no torch needed
+```
+
+```python
+from ctxprune import Compressor
+
+c = Compressor("darioooooo0o/ctxprune-small", backend="onnx")
+c.compress(tool_output, rate=0.5)["text"]                         # keep ~50%
+c.compress(tool_output, rate=0.33, force_protected=True)["text"]  # never drop IDs/numbers/paths
+```
+
+```bash
+ctxprune order.json --rate 0.5          # CLI, same thing
+
+```
+
+LangChain and LlamaIndex:
+
+```python
+from ctxprune.integrations import CtxpruneDocumentCompressor    # LangChain
+retriever = ContextualCompressionRetriever(base_compressor=CtxpruneDocumentCompressor(rate=0.5),
+                                           base_retriever=base_retriever)
+
+from ctxprune.integrations import CtxpruneNodePostprocessor     # LlamaIndex
+engine = index.as_query_engine(node_postprocessors=[CtxpruneNodePostprocessor(rate=0.5)])
+```
+
+Works inside LLMLingua too, once its tokenizer-detection fix is merged:
+[microsoft/LLMLingua#261](https://github.com/microsoft/LLMLingua/pull/261) (`upstream/llmlingua-subword-style.patch`).
+
+At 50% of tokens kept, an independent LLM answers 68.8% of held-out questions correctly from ctxprune's
+output vs 53.4% from LLMLingua-2-large's (91.8% uncompressed); at 33%, 53.4% vs 32.5%. Details, per-source
+results and limitations: [model card](https://huggingface.co/darioooooo0o/ctxprune-small).
+
+By [@imdariotoo](https://x.com/imdariotoo).
+
+---
+
+## Reproducing the model
+
+Why: LLMLingua-2 (Microsoft, Mar 2024) was trained only on meeting transcripts
+under CC BY-NC-SA labels. On agent tool outputs it loses or mangles about half
+of all identifiers (UUIDs, IPs, paths, versions) and breaks JSON.
+
+## Pipeline
+
+| step | script | where | notes |
+|---|---|---|---|
+| corpus | `scripts/build_corpus.py` | CPU, ~2 min | ranged parquet reads from 8 permissive sources; split per repo/server/site |
+| labels | `scripts/label.py` | teacher LLM (`scripts/teacher_up.sh`) | deletion-only compression, LCS-aligned to atoms; cached |
+| train | `scripts/train.py` | any torch device (we used an Intel Arc Pro B70) or CPU | token classifier, label 1 = keep (LLMLingua-2 convention) |
+| quick eval | `scripts/eval_quick.py` | CPU | identifiers lost, JSON validity, char ratio vs LLMLingua-2 |
+| QA eval | `scripts/eval_qa.py questions/answer` | teacher LLM | can a reader still answer from the compressed text |
+
+`scripts/run_v1.sh` runs corpus and labels; `scripts/after_label.sh` runs everything after them.
+
+## Data sources (all permissive)
+
+SWE-smith trajectories (MIT), SWE-rebench OpenHands trajectories (CC BY 4.0),
+Toucan-1.5M MCP tool calls (Apache-2.0), github-code-clean filtered to
+MIT/Apache/BSD/ISC files, FineWeb-Edu and FineWeb-2 (ODC-BY), oasst2
+(Apache-2.0). Every chunk records its dataset, license and source reference.
+Teacher: Qwen3.8-27B GPTQ (Apache-2.0 outputs).
+
+## Training more after an eval
+
+Nothing has to be redone:
+
+1. **More data, same mix.** Rerun `build_corpus.py` into the same directory
+   with a larger `--total`. Sampling is deterministic, and dataset revisions
+   are pinned in `chunks/revisions.json`, so every source file becomes a
+   superset of the old one. To weight a domain up, add `--mix toucan_mcp=0.3`.
+2. **Label only the new chunks.** Rerun `label.py`. The teacher cache is keyed
+   by (prompt version, model, text), so old chunks come back from disk.
+3. **Continue training** from the previous model:
+   `train.py --data <dir> --init-from runs/v1-mmbert-small/final --out runs/v2-...`
+   (or `--resume` to continue an interrupted run from its checkpoint).
+   `--data` accepts several directories, to mix label sets.
+4. **Re-run evals.** Question and answer caches mean only new contexts cost LLM calls.
+
+Changing the teacher prompt bumps `PROMPT_VERSION` in `ctxprune/teacher.py`, which
+invalidates the cache on purpose.
+
+## Tests
+
+`uv run pytest`: atom round-trips, identifier atoms, CJK handling, LCS
+alignment, chunk budgets, split stability and cache resume.

ctxprune-0.1.0/README.md

@@ -0,0 +1,98 @@
+# ctxprune
+
+Context compression for AI agents. A small token classifier deletes the least useful words from
+tool outputs, code, documents and chat before they reach an LLM. It is a successor to Microsoft's
+LLMLingua-2, built for agent text, and it never splits or re-spaces identifiers.
+
+Model: [darioooooo0o/ctxprune-small](https://huggingface.co/darioooooo0o/ctxprune-small) (Apache-2.0).
+Results and limitations are on the model card. **Try it next to LLMLingua-2 on your own text:
+[live demo](https://huggingface.co/spaces/darioooooo0o/ctxprune).**
+
+```bash
+pip install "ctxprune @ git+https://github.com/riceharvest/ctxprune"   # CPU, ONNX; no torch needed
+```
+
+```python
+from ctxprune import Compressor
+
+c = Compressor("darioooooo0o/ctxprune-small", backend="onnx")
+c.compress(tool_output, rate=0.5)["text"]                         # keep ~50%
+c.compress(tool_output, rate=0.33, force_protected=True)["text"]  # never drop IDs/numbers/paths
+```
+
+```bash
+ctxprune order.json --rate 0.5          # CLI, same thing
+
+```
+
+LangChain and LlamaIndex:
+
+```python
+from ctxprune.integrations import CtxpruneDocumentCompressor    # LangChain
+retriever = ContextualCompressionRetriever(base_compressor=CtxpruneDocumentCompressor(rate=0.5),
+                                           base_retriever=base_retriever)
+
+from ctxprune.integrations import CtxpruneNodePostprocessor     # LlamaIndex
+engine = index.as_query_engine(node_postprocessors=[CtxpruneNodePostprocessor(rate=0.5)])
+```
+
+Works inside LLMLingua too, once its tokenizer-detection fix is merged:
+[microsoft/LLMLingua#261](https://github.com/microsoft/LLMLingua/pull/261) (`upstream/llmlingua-subword-style.patch`).
+
+At 50% of tokens kept, an independent LLM answers 68.8% of held-out questions correctly from ctxprune's
+output vs 53.4% from LLMLingua-2-large's (91.8% uncompressed); at 33%, 53.4% vs 32.5%. Details, per-source
+results and limitations: [model card](https://huggingface.co/darioooooo0o/ctxprune-small).
+
+By [@imdariotoo](https://x.com/imdariotoo).
+
+---
+
+## Reproducing the model
+
+Why: LLMLingua-2 (Microsoft, Mar 2024) was trained only on meeting transcripts
+under CC BY-NC-SA labels. On agent tool outputs it loses or mangles about half
+of all identifiers (UUIDs, IPs, paths, versions) and breaks JSON.
+
+## Pipeline
+
+| step | script | where | notes |
+|---|---|---|---|
+| corpus | `scripts/build_corpus.py` | CPU, ~2 min | ranged parquet reads from 8 permissive sources; split per repo/server/site |
+| labels | `scripts/label.py` | teacher LLM (`scripts/teacher_up.sh`) | deletion-only compression, LCS-aligned to atoms; cached |
+| train | `scripts/train.py` | any torch device (we used an Intel Arc Pro B70) or CPU | token classifier, label 1 = keep (LLMLingua-2 convention) |
+| quick eval | `scripts/eval_quick.py` | CPU | identifiers lost, JSON validity, char ratio vs LLMLingua-2 |
+| QA eval | `scripts/eval_qa.py questions/answer` | teacher LLM | can a reader still answer from the compressed text |
+
+`scripts/run_v1.sh` runs corpus and labels; `scripts/after_label.sh` runs everything after them.
+
+## Data sources (all permissive)
+
+SWE-smith trajectories (MIT), SWE-rebench OpenHands trajectories (CC BY 4.0),
+Toucan-1.5M MCP tool calls (Apache-2.0), github-code-clean filtered to
+MIT/Apache/BSD/ISC files, FineWeb-Edu and FineWeb-2 (ODC-BY), oasst2
+(Apache-2.0). Every chunk records its dataset, license and source reference.
+Teacher: Qwen3.8-27B GPTQ (Apache-2.0 outputs).
+
+## Training more after an eval
+
+Nothing has to be redone:
+
+1. **More data, same mix.** Rerun `build_corpus.py` into the same directory
+   with a larger `--total`. Sampling is deterministic, and dataset revisions
+   are pinned in `chunks/revisions.json`, so every source file becomes a
+   superset of the old one. To weight a domain up, add `--mix toucan_mcp=0.3`.
+2. **Label only the new chunks.** Rerun `label.py`. The teacher cache is keyed
+   by (prompt version, model, text), so old chunks come back from disk.
+3. **Continue training** from the previous model:
+   `train.py --data <dir> --init-from runs/v1-mmbert-small/final --out runs/v2-...`
+   (or `--resume` to continue an interrupted run from its checkpoint).
+   `--data` accepts several directories, to mix label sets.
+4. **Re-run evals.** Question and answer caches mean only new contexts cost LLM calls.
+
+Changing the teacher prompt bumps `PROMPT_VERSION` in `ctxprune/teacher.py`, which
+invalidates the cache on purpose.
+
+## Tests
+
+`uv run pytest`: atom round-trips, identifier atoms, CJK handling, LCS
+alignment, chunk budgets, split stability and cache resume.

ctxprune-0.1.0/ctxprune/__init__.py

@@ -0,0 +1,6 @@
+"""ctxprune: context compression for AI agents."""
+
+from .compress import Compressor
+
+__all__ = ["Compressor"]
+__version__ = "0.1.0"

ctxprune-0.1.0/ctxprune/align.py

@@ -0,0 +1,93 @@
+"""Turn a teacher's deletion-only compression into per-atom keep labels.
+
+The teacher is asked to only delete atoms, so its output should be an
+ordered subsequence of the source atoms. We find the longest common
+subsequence (exact match first, casefold second) and label matched source
+atoms as kept. Teacher atoms that match nothing are "variations" (the
+teacher rewrote or invented text); a high variation rate means the sample
+is untrustworthy and gets filtered.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .atoms import Atom, atomize, is_protected, norm
+
+
+@dataclass
+class Alignment:
+    keep: list[bool]
+    n_src: int
+    n_comp: int
+    n_matched: int
+    variation_rate: float  # teacher atoms with no source match
+    comp_rate: float  # kept / source, in atoms
+    protected_total: int
+    protected_kept: int
+
+
+def _lcs_pairs(a: list[str], b: list[str]) -> list[tuple[int, int]]:
+    n, m = len(a), len(b)
+    if n == 0 or m == 0:
+        return []
+    # dp[i][j] = LCS length of a[i:], b[j:]; rows built bottom-up.
+    dp = [[0] * (m + 1) for _ in range(n + 1)]
+    for i in range(n - 1, -1, -1):
+        ai, row, nxt = a[i], dp[i], dp[i + 1]
+        for j in range(m - 1, -1, -1):
+            if ai == b[j]:
+                row[j] = nxt[j + 1] + 1
+            else:
+                x, y = nxt[j], row[j + 1]
+                row[j] = x if x >= y else y
+    pairs, i, j = [], 0, 0
+    while i < n and j < m:
+        if a[i] == b[j]:
+            pairs.append((i, j))
+            i += 1
+            j += 1
+        elif dp[i + 1][j] >= dp[i][j + 1]:
+            i += 1
+        else:
+            j += 1
+    return pairs
+
+
+def align(src_atoms: list[Atom], comp_text: str) -> Alignment:
+    src = [a.text for a in src_atoms]
+    comp = [a.text for a in atomize(comp_text) if a.text]
+    keep = [False] * len(src)
+    matched_comp = [False] * len(comp)
+
+    # Pass 1: exact. Pass 2: casefold over what is left, in the gaps between
+    # pass-1 anchors so order is preserved.
+    pairs = _lcs_pairs(src, comp)
+    for i, j in pairs:
+        keep[i] = True
+        matched_comp[j] = True
+    anchors = [(-1, -1)] + pairs + [(len(src), len(comp))]
+    for (i0, j0), (i1, j1) in zip(anchors, anchors[1:]):
+        si = [i for i in range(i0 + 1, i1) if not keep[i]]
+        cj = [j for j in range(j0 + 1, j1) if not matched_comp[j]]
+        if not si or not cj:
+            continue
+        sub = _lcs_pairs([norm(src[i]) for i in si], [norm(comp[j]) for j in cj])
+        for a, b in sub:
+            keep[si[a]] = True
+            matched_comp[cj[b]] = True
+
+    real = [i for i, t in enumerate(src) if t]
+    n_src = len(real)
+    n_matched = sum(matched_comp)
+    prot = [i for i in real if is_protected(src[i])]
+    return Alignment(
+        keep=keep,
+        n_src=n_src,
+        n_comp=len(comp),
+        n_matched=n_matched,
+        variation_rate=(1 - n_matched / len(comp)) if comp else 0.0,
+        comp_rate=(sum(keep[i] for i in real) / n_src) if n_src else 0.0,
+        protected_total=len(prot),
+        protected_kept=sum(keep[i] for i in prot),
+    )

ctxprune-0.1.0/ctxprune/atoms.py

@@ -0,0 +1,106 @@
+"""Atom segmentation: the unit the compressor keeps or drops.
+
+An atom is either a "word" that glues identifier-ish characters together
+(paths, URLs, versions, IPs, snake_case, UUIDs) or a single punctuation
+character. Each atom remembers the whitespace that preceded it, so
+``detok(atoms, keep)`` rebuilds kept text exactly as it appeared in the
+source, without the "0. 21" / "ord _ 8f3a" artifacts LLMLingua-2 produces.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+# Scripts written without spaces (Han, kana) get one atom per character, so
+# a deletion inside a sentence is still a deletion and not a "rewrite".
+_CJK = "\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff"
+_W = rf"[^\W{_CJK}]"  # word char that is not CJK
+# A word may start with one path/flag sigil, then word chars, with
+# identifier punctuation allowed only *between* word chars.
+_WORD = rf"(?:--?|[/~.$@#])?{_W}(?:(?:{_W}|[.:/@%+#~=?&'\-])*{_W})?"
+_ATOM_RE = re.compile(rf"(\s*)({_WORD}|[{_CJK}]|[^\s])", re.UNICODE)
+
+# All-caps English that is prose, not an identifier (license headers, shouting).
+_CAPS_WORDS = set(
+    "THE AND FOR ANY NOT BUT ARE WAS YOU ALL WITH FROM THIS THAT WITHOUT WARRANTIES WARRANTY SOFTWARE KIND "
+    "IMPLIED INCLUDING LIMITED COPYRIGHT OTHER LIABILITY USE BASIS CONDITIONS SUCH EVENT SHALL HOLDERS "
+    "CONTRIBUTORS DAMAGES PURPOSE PARTICULAR FITNESS MERCHANTABILITY EXPRESS OUT CONNECTION ARISING "
+    "WHETHER CONTRACT TORT OTHERWISE PROVIDED LICENSE NOTICE NOTE IMPORTANT".split()
+)
+_HEX_RE = re.compile(r"^[0-9a-fA-F]{7,}$")
+_CAMEL_RE = re.compile(r"[a-z][A-Z]|[A-Z]{2}[a-z]")
+
+
+@dataclass(frozen=True)
+class Atom:
+    ws: str  # whitespace preceding the atom in the source
+    text: str
+
+
+def atomize(text: str) -> list[Atom]:
+    atoms: list[Atom] = []
+    pos = 0
+    for m in _ATOM_RE.finditer(text):
+        if m.start() != pos:  # unreachable by construction; guard anyway
+            raise ValueError(f"atomizer skipped {text[pos:m.start()]!r}")
+        atoms.append(Atom(m.group(1), m.group(2)))
+        pos = m.end()
+    tail = text[pos:]
+    if tail.strip():
+        raise ValueError(f"atomizer left non-space tail {tail!r}")
+    if tail:  # trailing whitespace rides on a sentinel-free empty atom
+        atoms.append(Atom(tail, ""))
+    return atoms
+
+
+def detok(atoms: list[Atom], keep: list[bool], collapse_ws: bool = True) -> str:
+    """Rebuild text from kept atoms.
+
+    Kept atoms keep their own leading whitespace. When an atom's predecessor
+    was dropped, the dropped run's whitespace is collapsed to the "largest"
+    whitespace seen in that run (a newline wins over a space), so line
+    structure survives deletions.
+    """
+    out: list[str] = []
+    pending = ""
+    for a, k in zip(atoms, keep):
+        if k:
+            ws = a.ws
+            if pending and collapse_ws:
+                ws = _merge_ws(pending, a.ws)
+            if not out:
+                ws = ws if not collapse_ws else ""
+            out.append(ws + a.text)
+            pending = ""
+        else:
+            pending = _merge_ws(pending, a.ws) if pending else (a.ws or " ")
+    return "".join(out)
+
+
+def _merge_ws(a: str, b: str) -> str:
+    if "\n" in a or "\n" in b:
+        return "\n"
+    return " " if (a or b) else ""
+
+
+def is_protected(atom: str) -> bool:
+    """Atoms an agent is likely to need verbatim later (IDs, numbers, paths)."""
+    if not atom or not any(c.isalnum() for c in atom):
+        return False
+    if any(c.isdigit() for c in atom):
+        return True
+    core = atom.lstrip("-/~.$@#")
+    if any(c in core for c in "_/.:@="):
+        return True
+    if _HEX_RE.match(core):
+        return True
+    if len(core) >= 3 and core.isupper() and core not in _CAPS_WORDS:
+        return True
+    if _CAMEL_RE.search(core):
+        return True
+    return atom != core  # flags like --format, paths like /tmp
+
+
+def norm(atom: str) -> str:
+    return atom.casefold()