chunkguard 0.1.0__tar.gz

chunkguard-0.1.0/.env.example

@@ -0,0 +1,40 @@
+# Used by chunker_from_env(), which needs: pip install "chunkguard[litellm]"
+# Models are LiteLLM strings "<provider>/<model>", see https://docs.litellm.ai/docs/providers
+# LLM and embedding model are chosen independently, from any provider.
+
+CHUNKGUARD_LLM_MODEL=openai/gpt-4.1-mini
+CHUNKGUARD_EMBEDDING_MODEL=openai/text-embedding-3-small
+
+# Optional. Unset values fall back to the provider's own variables
+# (OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY, AWS_*, ...) in the process environment.
+CHUNKGUARD_LLM_API_KEY=
+CHUNKGUARD_LLM_API_BASE=
+CHUNKGUARD_LLM_API_VERSION=
+CHUNKGUARD_EMBEDDING_API_KEY=
+CHUNKGUARD_EMBEDDING_API_BASE=
+CHUNKGUARD_EMBEDDING_API_VERSION=
+
+# Other setups, for example:
+# CHUNKGUARD_LLM_MODEL=anthropic/claude-sonnet-5-5
+# CHUNKGUARD_LLM_MODEL=gemini/gemini-2.5-flash
+# CHUNKGUARD_LLM_MODEL=azure/<deployment>            + API_BASE and API_VERSION
+# CHUNKGUARD_LLM_MODEL=ollama/llama3.1                 (local, API_BASE=http://localhost:11434)
+# CHUNKGUARD_EMBEDDING_MODEL=cohere/embed-multilingual-v3.0
+# CHUNKGUARD_EMBEDDING_MODEL=mistral/mistral-embed
+# CHUNKGUARD_EMBEDDING_MODEL=openai/<model>            + API_BASE for any OpenAI-compatible server
+
+# Texts per embedding request; lower it if your provider rejects large batches.
+CHUNKGUARD_EMBEDDING_BATCH_SIZE=64
+
+# Per request, for LLM and embedding calls alike.
+CHUNKGUARD_REQUEST_TIMEOUT=120
+CHUNKGUARD_MAX_RETRIES=2
+
+# Chunk size limits in characters; 0 switches a limit off (the paper uses none).
+# Sentences are never cut, so a single sentence longer than the maximum stays one chunk.
+CHUNKGUARD_MAX_CHUNK_CHARS=2000
+CHUNKGUARD_MIN_CHUNK_CHARS=200
+
+# Longer documents reach the LLM as evenly spread excerpts of this total size;
+# 0 sends the whole document.
+CHUNKGUARD_SUMMARY_MAX_CHARS=100000

chunkguard-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.11", "3.12", "3.13"]
+        resolution: [highest]
+        include:
+          # Proves the lower bounds in pyproject.toml still hold.
+          - python: "3.11"
+            resolution: lowest-direct
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - run: uv run --python ${{ matrix.python }} --resolution ${{ matrix.resolution }} pytest -q
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .

chunkguard-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,21 @@
+name: Release
+
+# Push a tag like v0.1.0 to publish that version to PyPI.
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # PyPI trusted publishing, no token needed
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - name: Check tag matches package version
+        run: test "${GITHUB_REF_NAME#v}" = "$(uv version --short)"
+      - run: uv run pytest -q
+      - run: uv build
+      - uses: pypa/gh-action-pypi-publish@release/v1

chunkguard-0.1.0/.gitignore

@@ -0,0 +1,7 @@
+.env*
+!.env.example
+.venv/
+__pycache__/
+.pytest_cache/
+.ruff_cache/
+dist/

chunkguard-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Zakariae Fakhri
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

chunkguard-0.1.0/PKG-INFO

@@ -0,0 +1,254 @@
+Metadata-Version: 2.5
+Name: chunkguard
+Version: 0.1.0
+Summary: PIC document chunking for RAG with any LLM and embedding model
+Project-URL: Homepage, https://github.com/zakariaefakhri/chunkguard
+Project-URL: Issues, https://github.com/zakariaefakhri/chunkguard/issues
+Project-URL: Paper, https://aclanthology.org/2025.findings-acl.422/
+Author: Zakariae Fakhri
+License-Expression: MIT
+License-File: LICENSE
+Keywords: chunking,embeddings,llm,rag,retrieval,text-splitter
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: pydantic>=2.0
+Requires-Dist: sentencex>=1.0.30
+Provides-Extra: litellm
+Requires-Dist: litellm>=1.83.0; extra == 'litellm'
+Requires-Dist: pydantic-settings>=2.2; extra == 'litellm'
+Description-Content-Type: text/markdown
+
+# chunkguard
+
+[![CI](https://github.com/zakariaefakhri/chunkguard/actions/workflows/ci.yml/badge.svg)](https://github.com/zakariaefakhri/chunkguard/actions/workflows/ci.yml)
+
+Document chunking for RAG with **PIC** (Pseudo-Instruction for document Chunking), with any
+LLM and any embedding model. From
+[*Document Segmentation Matters for Retrieval-Augmented Generation*](https://aclanthology.org/2025.findings-acl.422/)
+(Wang et al., Findings of ACL 2025). This is an independent implementation, not the authors' code.
+
+When you build a knowledge base, you don't know the questions users will ask. PIC stands in
+for them with an LLM summary of the document. Every sentence is embedded and compared with
+the summary; the threshold is the mean similarity. Neighbouring sentences at or above it
+form a chunk of core content, neighbouring sentences below it a chunk of side details. So
+chunk boundaries sit exactly where the text turns towards or away from its topic, and every
+sentence is judged against the whole document rather than just its neighbour, as in
+semantic chunking.
+
+Order is kept, no sentence is cut, nothing is dropped. Cost per document: one LLM call and
+one embedding pass over its sentences. No training.
+
+## Install
+
+```bash
+pip install chunkguard              # core: bring your own model functions (Pydantic + sentencex)
+pip install "chunkguard[litellm]"   # plus .env configuration for 100+ providers via LiteLLM
+```
+
+## Choose your models
+
+chunkguard needs two things, and you pick both: an LLM that writes one summary per document,
+and an embedding model. They can come from different providers.
+
+**Option 1: plain functions.** `PICChunker` takes any `complete(prompt) -> str` and any
+`embed(texts) -> list[vector]`. No provider SDK is involved:
+
+```python
+from chunkguard import PICChunker
+from sentence_transformers import SentenceTransformer  # local embeddings, as an example
+import anthropic  # any LLM, as an example
+
+claude = anthropic.Anthropic()
+encoder = SentenceTransformer("BAAI/bge-m3")
+
+
+def complete(prompt: str) -> str:
+    message = claude.messages.create(
+        model="claude-sonnet-5-5", max_tokens=512, messages=[{"role": "user", "content": prompt}]
+    )
+    return message.content[0].text
+
+
+chunker = PICChunker(complete=complete, embed=lambda texts: encoder.encode(texts).tolist())
+chunks = chunker.split(doc_id="report-7", text=cleaned_text, language="en")
+```
+
+**Option 2: `.env` with LiteLLM.** Install the extra, copy [`.env.example`](.env.example) to
+`.env` and set the two models as [LiteLLM strings](https://docs.litellm.ai/docs/providers):
+
+```bash
+CHUNKGUARD_LLM_MODEL=anthropic/claude-sonnet-5-5
+CHUNKGUARD_EMBEDDING_MODEL=cohere/embed-multilingual-v3.0
+```
+
+Put keys in `.env` as `CHUNKGUARD_LLM_API_KEY` / `CHUNKGUARD_EMBEDDING_API_KEY`. If those are
+unset, LiteLLM uses the provider's usual variables (`ANTHROPIC_API_KEY`, `COHERE_API_KEY`,
+`AWS_*`, ...) from the process environment. Azure, Bedrock, Vertex, Ollama, vLLM and any
+OpenAI-compatible server work the same way. Then:
+
+```python
+from chunkguard import chunker_from_env
+
+chunker = chunker_from_env()
+chunks = chunker.split(doc_id="report-7", text=cleaned_text, language="en")
+
+for chunk in chunks:
+    print(chunk.index, chunk.relevant, round(chunk.similarity, 3), chunk.text[:60])
+```
+
+`split` takes plain text that extraction has already cleaned (line breaks inside sentences
+joined). Each `Chunk` carries:
+
+| Field | Meaning |
+|---|---|
+| `doc_id`, `index` | Document and position of the chunk in it |
+| `text` | Exact slice `input[start:end]` |
+| `start`, `end` | Character offsets into the input (end exclusive) |
+| `sentence_start`, `sentence_end` | Sentence range (end exclusive) |
+| `relevant` | Core content (`True`) or side details (`False`) |
+| `similarity` | Mean cosine similarity of the chunk's sentences to the summary |
+| `summary` | The document summary, for use as metadata in the index |
+
+`Chunk` is a Pydantic model; `chunk.model_dump()` gives a dict for your vector store.
+
+### Async
+
+```python
+from chunkguard import async_chunker_from_env
+
+chunker = async_chunker_from_env()
+chunks = await chunker.split("report-7", cleaned_text, "en")
+```
+
+`AsyncPICChunker` takes async `complete` and `embed` functions in the same way.
+
+### Your own summary prompt
+
+The default is the paper's prompt (`chunkguard.SUMMARY_PROMPT`) plus an instruction to answer
+in the document's language. Pass `summary_prompt` with `{text}` and `{language}` placeholders
+to change it.
+
+### Your own sentence splitter
+
+Sentences come from [sentencex](https://github.com/wikimedia/sentencex) (see [Limits](#limits)).
+To use another splitter, pass a function
+that returns `(start, end)` character spans. For example, NLTK's Punkt, which the paper uses
+for sentence splitting:
+
+```python
+from nltk.tokenize.punkt import PunktSentenceTokenizer
+
+punkt = PunktSentenceTokenizer()
+
+
+def nltk_sentences(text, language):
+    return list(punkt.span_tokenize(text))
+
+
+chunker = PICChunker(complete, embed, sentence_splitter=nltk_sentences)
+```
+
+## How it works
+
+1. **Sentences.** Split into sentences with sentencex, keeping character offsets. Every line
+   break also ends a sentence, so headings and list items stand alone.
+2. **Summary.** One LLM call with the summarizer prompt from the paper (Appendix C), plus an
+   instruction to answer in the document's language.
+3. **Similarity.** Embed the summary and every sentence; cosine similarity of each sentence
+   to the summary.
+4. **Threshold.** The mean of those similarities. A sentence is relevant at or above it.
+5. **Grouping.** Each run of neighbouring sentences with the same flag becomes one chunk.
+
+Steps 1 to 5 are the method as published. chunkguard adds three things for production use;
+set `max_chars=None, min_chars=0, summary_max_chars=None` to switch them off:
+
+6. **Long chunks** (over `max_chars`) are split at sentence boundaries into near-equal pieces.
+7. **Short chunks** (under `min_chars`) merge into the neighbour with the closest mean
+   similarity, unless that would exceed `max_chars`. A merged chunk is `relevant` when its
+   relevant sentences make up at least half of its text.
+8. **Long documents** reach the summarizer as eight evenly spread excerpts totalling
+   `summary_max_chars`, so the call fits the context window and every part of the document
+   still shapes the summary.
+
+## Configuration
+
+With plain functions, pass the options to `PICChunker` (`max_chars`, `min_chars`,
+`summary_max_chars`, `summary_prompt`, `sentence_splitter`). With the LiteLLM extra, they come
+from environment variables or `.env`, prefixed `CHUNKGUARD_`:
+
+| Variable | Default | Meaning |
+|---|---|---|
+| `LLM_MODEL` | required | LiteLLM model string, e.g. `gemini/gemini-2.5-flash` |
+| `LLM_API_KEY` | provider variable | API key |
+| `LLM_API_BASE` | – | Endpoint, e.g. Azure resource, Ollama or any OpenAI-compatible server |
+| `LLM_API_VERSION` | – | Azure only |
+| `EMBEDDING_MODEL`, `EMBEDDING_API_*` | | Same settings for the embedding model |
+| `EMBEDDING_BATCH_SIZE` | `64` | Texts per embedding request |
+| `REQUEST_TIMEOUT` | `120` | Seconds per request |
+| `MAX_RETRIES` | `2` | Retries per request on provider errors |
+| `MAX_CHUNK_CHARS` | `2000` | Split longer chunks; `0` = no limit |
+| `MIN_CHUNK_CHARS` | `200` | Merge shorter chunks; `0` = no merging |
+| `SUMMARY_MAX_CHARS` | `100000` | Excerpt longer documents for the summary; `0` = send all |
+
+Embedding requests stay below 400,000 characters each. A sentence longer than 8,000
+characters is truncated for embedding only; its chunk keeps the full text.
+
+LiteLLM versions 1.82.7 and 1.82.8 were compromised on PyPI in March 2026; the extra requires
+1.83.0 or later.
+
+## Limits
+
+- **Sentence splitting is rule-based.** sentencex carries punctuation and abbreviation rules
+  for many languages and falls back to related ones. chunkguard adds two language-neutral
+  rules: no cut before a lowercase letter, and no cut after a number of up to three digits
+  ("im 18. Jahrhundert"), which merges the rare English "We sold 3. Then ..." into one
+  sentence. Some boundaries are still missed (a sentence ending in "etc."), and languages
+  without sentence punctuation (Thai) are not split. Pass your own splitter if that matters.
+- **Sizes are in characters, not tokens.** Pick `MAX_CHUNK_CHARS` with your embedding model's
+  token limit in mind.
+- **The relevance flag is relative to the document.** Nearly every document has relevant and
+  irrelevant chunks, however on-topic it is overall. Use it to weight or filter retrieval,
+  not as an absolute quality score.
+- **Retrieval gains are the paper's results** on English Wikipedia and open-domain QA. This
+  repository does not reproduce them.
+
+## Development
+
+```bash
+uv sync
+uv run pytest
+uv run ruff check . && uv run ruff format --check .
+```
+
+Releases: bump `version` in `pyproject.toml`, then push a matching tag (`v0.1.0`). The
+release workflow publishes to PyPI via trusted publishing.
+
+## Citation
+
+If you use PIC in research, cite the paper:
+
+```bibtex
+@inproceedings{wang-etal-2025-document,
+  title     = {Document Segmentation Matters for Retrieval-Augmented Generation},
+  author    = {Wang, Zhitong and Gao, Cheng and Xiao, Chaojun and Huang, Yufei and Si, Shuzheng and
+               Luo, Kangyang and Bai, Yuzhuo and Li, Wenhao and Duan, Tangjian and Lv, Chuancheng and
+               Lu, Guoshan and Chen, Gang and Qi, Fanchao and Sun, Maosong},
+  booktitle = {Findings of the Association for Computational Linguistics: ACL 2025},
+  pages     = {8063--8075},
+  year      = {2025},
+  url       = {https://aclanthology.org/2025.findings-acl.422/}
+}
+```
+
+## License
+
+[MIT](LICENSE)

chunkguard-0.1.0/README.md

@@ -0,0 +1,225 @@
+# chunkguard
+
+[![CI](https://github.com/zakariaefakhri/chunkguard/actions/workflows/ci.yml/badge.svg)](https://github.com/zakariaefakhri/chunkguard/actions/workflows/ci.yml)
+
+Document chunking for RAG with **PIC** (Pseudo-Instruction for document Chunking), with any
+LLM and any embedding model. From
+[*Document Segmentation Matters for Retrieval-Augmented Generation*](https://aclanthology.org/2025.findings-acl.422/)
+(Wang et al., Findings of ACL 2025). This is an independent implementation, not the authors' code.
+
+When you build a knowledge base, you don't know the questions users will ask. PIC stands in
+for them with an LLM summary of the document. Every sentence is embedded and compared with
+the summary; the threshold is the mean similarity. Neighbouring sentences at or above it
+form a chunk of core content, neighbouring sentences below it a chunk of side details. So
+chunk boundaries sit exactly where the text turns towards or away from its topic, and every
+sentence is judged against the whole document rather than just its neighbour, as in
+semantic chunking.
+
+Order is kept, no sentence is cut, nothing is dropped. Cost per document: one LLM call and
+one embedding pass over its sentences. No training.
+
+## Install
+
+```bash
+pip install chunkguard              # core: bring your own model functions (Pydantic + sentencex)
+pip install "chunkguard[litellm]"   # plus .env configuration for 100+ providers via LiteLLM
+```
+
+## Choose your models
+
+chunkguard needs two things, and you pick both: an LLM that writes one summary per document,
+and an embedding model. They can come from different providers.
+
+**Option 1: plain functions.** `PICChunker` takes any `complete(prompt) -> str` and any
+`embed(texts) -> list[vector]`. No provider SDK is involved:
+
+```python
+from chunkguard import PICChunker
+from sentence_transformers import SentenceTransformer  # local embeddings, as an example
+import anthropic  # any LLM, as an example
+
+claude = anthropic.Anthropic()
+encoder = SentenceTransformer("BAAI/bge-m3")
+
+
+def complete(prompt: str) -> str:
+    message = claude.messages.create(
+        model="claude-sonnet-5-5", max_tokens=512, messages=[{"role": "user", "content": prompt}]
+    )
+    return message.content[0].text
+
+
+chunker = PICChunker(complete=complete, embed=lambda texts: encoder.encode(texts).tolist())
+chunks = chunker.split(doc_id="report-7", text=cleaned_text, language="en")
+```
+
+**Option 2: `.env` with LiteLLM.** Install the extra, copy [`.env.example`](.env.example) to
+`.env` and set the two models as [LiteLLM strings](https://docs.litellm.ai/docs/providers):
+
+```bash
+CHUNKGUARD_LLM_MODEL=anthropic/claude-sonnet-5-5
+CHUNKGUARD_EMBEDDING_MODEL=cohere/embed-multilingual-v3.0
+```
+
+Put keys in `.env` as `CHUNKGUARD_LLM_API_KEY` / `CHUNKGUARD_EMBEDDING_API_KEY`. If those are
+unset, LiteLLM uses the provider's usual variables (`ANTHROPIC_API_KEY`, `COHERE_API_KEY`,
+`AWS_*`, ...) from the process environment. Azure, Bedrock, Vertex, Ollama, vLLM and any
+OpenAI-compatible server work the same way. Then:
+
+```python
+from chunkguard import chunker_from_env
+
+chunker = chunker_from_env()
+chunks = chunker.split(doc_id="report-7", text=cleaned_text, language="en")
+
+for chunk in chunks:
+    print(chunk.index, chunk.relevant, round(chunk.similarity, 3), chunk.text[:60])
+```
+
+`split` takes plain text that extraction has already cleaned (line breaks inside sentences
+joined). Each `Chunk` carries:
+
+| Field | Meaning |
+|---|---|
+| `doc_id`, `index` | Document and position of the chunk in it |
+| `text` | Exact slice `input[start:end]` |
+| `start`, `end` | Character offsets into the input (end exclusive) |
+| `sentence_start`, `sentence_end` | Sentence range (end exclusive) |
+| `relevant` | Core content (`True`) or side details (`False`) |
+| `similarity` | Mean cosine similarity of the chunk's sentences to the summary |
+| `summary` | The document summary, for use as metadata in the index |
+
+`Chunk` is a Pydantic model; `chunk.model_dump()` gives a dict for your vector store.
+
+### Async
+
+```python
+from chunkguard import async_chunker_from_env
+
+chunker = async_chunker_from_env()
+chunks = await chunker.split("report-7", cleaned_text, "en")
+```
+
+`AsyncPICChunker` takes async `complete` and `embed` functions in the same way.
+
+### Your own summary prompt
+
+The default is the paper's prompt (`chunkguard.SUMMARY_PROMPT`) plus an instruction to answer
+in the document's language. Pass `summary_prompt` with `{text}` and `{language}` placeholders
+to change it.
+
+### Your own sentence splitter
+
+Sentences come from [sentencex](https://github.com/wikimedia/sentencex) (see [Limits](#limits)).
+To use another splitter, pass a function
+that returns `(start, end)` character spans. For example, NLTK's Punkt, which the paper uses
+for sentence splitting:
+
+```python
+from nltk.tokenize.punkt import PunktSentenceTokenizer
+
+punkt = PunktSentenceTokenizer()
+
+
+def nltk_sentences(text, language):
+    return list(punkt.span_tokenize(text))
+
+
+chunker = PICChunker(complete, embed, sentence_splitter=nltk_sentences)
+```
+
+## How it works
+
+1. **Sentences.** Split into sentences with sentencex, keeping character offsets. Every line
+   break also ends a sentence, so headings and list items stand alone.
+2. **Summary.** One LLM call with the summarizer prompt from the paper (Appendix C), plus an
+   instruction to answer in the document's language.
+3. **Similarity.** Embed the summary and every sentence; cosine similarity of each sentence
+   to the summary.
+4. **Threshold.** The mean of those similarities. A sentence is relevant at or above it.
+5. **Grouping.** Each run of neighbouring sentences with the same flag becomes one chunk.
+
+Steps 1 to 5 are the method as published. chunkguard adds three things for production use;
+set `max_chars=None, min_chars=0, summary_max_chars=None` to switch them off:
+
+6. **Long chunks** (over `max_chars`) are split at sentence boundaries into near-equal pieces.
+7. **Short chunks** (under `min_chars`) merge into the neighbour with the closest mean
+   similarity, unless that would exceed `max_chars`. A merged chunk is `relevant` when its
+   relevant sentences make up at least half of its text.
+8. **Long documents** reach the summarizer as eight evenly spread excerpts totalling
+   `summary_max_chars`, so the call fits the context window and every part of the document
+   still shapes the summary.
+
+## Configuration
+
+With plain functions, pass the options to `PICChunker` (`max_chars`, `min_chars`,
+`summary_max_chars`, `summary_prompt`, `sentence_splitter`). With the LiteLLM extra, they come
+from environment variables or `.env`, prefixed `CHUNKGUARD_`:
+
+| Variable | Default | Meaning |
+|---|---|---|
+| `LLM_MODEL` | required | LiteLLM model string, e.g. `gemini/gemini-2.5-flash` |
+| `LLM_API_KEY` | provider variable | API key |
+| `LLM_API_BASE` | – | Endpoint, e.g. Azure resource, Ollama or any OpenAI-compatible server |
+| `LLM_API_VERSION` | – | Azure only |
+| `EMBEDDING_MODEL`, `EMBEDDING_API_*` | | Same settings for the embedding model |
+| `EMBEDDING_BATCH_SIZE` | `64` | Texts per embedding request |
+| `REQUEST_TIMEOUT` | `120` | Seconds per request |
+| `MAX_RETRIES` | `2` | Retries per request on provider errors |
+| `MAX_CHUNK_CHARS` | `2000` | Split longer chunks; `0` = no limit |
+| `MIN_CHUNK_CHARS` | `200` | Merge shorter chunks; `0` = no merging |
+| `SUMMARY_MAX_CHARS` | `100000` | Excerpt longer documents for the summary; `0` = send all |
+
+Embedding requests stay below 400,000 characters each. A sentence longer than 8,000
+characters is truncated for embedding only; its chunk keeps the full text.
+
+LiteLLM versions 1.82.7 and 1.82.8 were compromised on PyPI in March 2026; the extra requires
+1.83.0 or later.
+
+## Limits
+
+- **Sentence splitting is rule-based.** sentencex carries punctuation and abbreviation rules
+  for many languages and falls back to related ones. chunkguard adds two language-neutral
+  rules: no cut before a lowercase letter, and no cut after a number of up to three digits
+  ("im 18. Jahrhundert"), which merges the rare English "We sold 3. Then ..." into one
+  sentence. Some boundaries are still missed (a sentence ending in "etc."), and languages
+  without sentence punctuation (Thai) are not split. Pass your own splitter if that matters.
+- **Sizes are in characters, not tokens.** Pick `MAX_CHUNK_CHARS` with your embedding model's
+  token limit in mind.
+- **The relevance flag is relative to the document.** Nearly every document has relevant and
+  irrelevant chunks, however on-topic it is overall. Use it to weight or filter retrieval,
+  not as an absolute quality score.
+- **Retrieval gains are the paper's results** on English Wikipedia and open-domain QA. This
+  repository does not reproduce them.
+
+## Development
+
+```bash
+uv sync
+uv run pytest
+uv run ruff check . && uv run ruff format --check .
+```
+
+Releases: bump `version` in `pyproject.toml`, then push a matching tag (`v0.1.0`). The
+release workflow publishes to PyPI via trusted publishing.
+
+## Citation
+
+If you use PIC in research, cite the paper:
+
+```bibtex
+@inproceedings{wang-etal-2025-document,
+  title     = {Document Segmentation Matters for Retrieval-Augmented Generation},
+  author    = {Wang, Zhitong and Gao, Cheng and Xiao, Chaojun and Huang, Yufei and Si, Shuzheng and
+               Luo, Kangyang and Bai, Yuzhuo and Li, Wenhao and Duan, Tangjian and Lv, Chuancheng and
+               Lu, Guoshan and Chen, Gang and Qi, Fanchao and Sun, Maosong},
+  booktitle = {Findings of the Association for Computational Linguistics: ACL 2025},
+  pages     = {8063--8075},
+  year      = {2025},
+  url       = {https://aclanthology.org/2025.findings-acl.422/}
+}
+```
+
+## License
+
+[MIT](LICENSE)

chunkguard-0.1.0/pyproject.toml

@@ -0,0 +1,50 @@
+[project]
+name = "chunkguard"
+version = "0.1.0"
+description = "PIC document chunking for RAG with any LLM and embedding model"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Zakariae Fakhri" }]
+requires-python = ">=3.11"
+keywords = ["rag", "chunking", "text-splitter", "retrieval", "llm", "embeddings"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Text Processing :: Linguistic",
+    "Typing :: Typed",
+]
+dependencies = ["pydantic>=2.0", "sentencex>=1.0.30"]
+
+[project.optional-dependencies]
+# .env configuration for any provider. 1.82.7/1.82.8 were compromised on PyPI.
+litellm = ["litellm>=1.83.0", "pydantic-settings>=2.2"]
+
+[project.urls]
+Homepage = "https://github.com/zakariaefakhri/chunkguard"
+Issues = "https://github.com/zakariaefakhri/chunkguard/issues"
+Paper = "https://aclanthology.org/2025.findings-acl.422/"
+
+[dependency-groups]
+dev = ["pytest>=8", "ruff>=0.6", "litellm>=1.83.0", "pydantic-settings>=2.2"]
+
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]
+
+[tool.pytest.ini_options]
+# LiteLLM's own types trigger this Pydantic warning on import.
+filterwarnings = ["ignore:.*ReadOnly.*:UserWarning"]

chunkguard-0.1.0/src/chunkguard/__init__.py

@@ -0,0 +1,23 @@
+"""PIC document chunking for RAG: text in, chunks out, with any LLM and embedding model."""
+
+from chunkguard.pic import SUMMARY_PROMPT, AsyncPICChunker, Chunk, PICChunker
+from chunkguard.sentences import split_sentences
+
+__all__ = [
+    "SUMMARY_PROMPT",
+    "AsyncPICChunker",
+    "Chunk",
+    "PICChunker",
+    "async_chunker_from_env",
+    "chunker_from_env",
+    "split_sentences",
+]
+
+
+def __getattr__(name: str):
+    # The .env helpers need the optional litellm extra; import them only on use.
+    if name in ("chunker_from_env", "async_chunker_from_env"):
+        from chunkguard import backend
+
+        return getattr(backend, name)
+    raise AttributeError(f"module 'chunkguard' has no attribute {name!r}")