chunkguard 0.1.0__py3-none-any.whl

chunkguard/__init__.py

@@ -0,0 +1,23 @@
+"""PIC document chunking for RAG: text in, chunks out, with any LLM and embedding model."""
+
+from chunkguard.pic import SUMMARY_PROMPT, AsyncPICChunker, Chunk, PICChunker
+from chunkguard.sentences import split_sentences
+
+__all__ = [
+    "SUMMARY_PROMPT",
+    "AsyncPICChunker",
+    "Chunk",
+    "PICChunker",
+    "async_chunker_from_env",
+    "chunker_from_env",
+    "split_sentences",
+]
+
+
+def __getattr__(name: str):
+    # The .env helpers need the optional litellm extra; import them only on use.
+    if name in ("chunker_from_env", "async_chunker_from_env"):
+        from chunkguard import backend
+
+        return getattr(backend, name)
+    raise AttributeError(f"module 'chunkguard' has no attribute {name!r}")

chunkguard/backend.py

@@ -0,0 +1,155 @@
+"""Any LLM and embedding provider via LiteLLM, configured from .env.
+
+Needs the extra: pip install "chunkguard[litellm]". Models are LiteLLM strings
+such as "openai/gpt-4.1-mini", "anthropic/claude-sonnet-5-5", "gemini/gemini-2.5-flash",
+"azure/<deployment>", "bedrock/...", "ollama/llama3.1" or "mistral/mistral-embed".
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+try:
+    import litellm
+    from pydantic_settings import BaseSettings, SettingsConfigDict
+except ImportError as error:
+    raise ImportError(
+        'chunker_from_env needs the litellm extra: pip install "chunkguard[litellm]"'
+    ) from error
+
+from chunkguard.pic import AsyncComplete, AsyncEmbed, AsyncPICChunker, Complete, Embed, PICChunker
+
+# Most providers accept far more, but Cohere caps a request at 96 inputs.
+DEFAULT_EMBED_BATCH_SIZE = 64
+# Keeps a request well below the common 300k-token limit, even for dense scripts.
+EMBED_BATCH_CHARS = 400_000
+# Only the embedding input is cut; the chunk text always keeps the whole sentence.
+EMBED_INPUT_MAX_CHARS = 8_000
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_prefix="CHUNKGUARD_", env_file=".env", env_ignore_empty=True, extra="ignore"
+    )
+
+    # Unset keys fall back to the provider's own variables (OPENAI_API_KEY, AWS_*, ...).
+    llm_model: str
+    llm_api_key: str | None = None
+    llm_api_base: str | None = None
+    llm_api_version: str | None = None
+
+    embedding_model: str
+    embedding_api_key: str | None = None
+    embedding_api_base: str | None = None
+    embedding_api_version: str | None = None
+    embedding_batch_size: int = DEFAULT_EMBED_BATCH_SIZE
+
+    # Per request. LiteLLM's own defaults are 600 s and no retries.
+    request_timeout: float = 120
+    max_retries: int = 2
+
+    max_chunk_chars: int = 2000  # 0 = no limit
+    min_chunk_chars: int = 200  # 0 = no merging
+    summary_max_chars: int = 100_000  # 0 = send the whole document
+
+
+def _connection(
+    api_key: str | None, api_base: str | None, api_version: str | None, s: Settings
+) -> dict:
+    options = {"api_key": api_key, "api_base": api_base, "api_version": api_version}
+    return {key: value for key, value in options.items() if value is not None} | {
+        "timeout": s.request_timeout,
+        "num_retries": s.max_retries,
+    }
+
+
+def _batches(texts: list[str], size: int) -> list[list[str]]:
+    batches: list[list[str]] = []
+    chars = 0
+    for text in (t[:EMBED_INPUT_MAX_CHARS] for t in texts):
+        if not batches or len(batches[-1]) == size or chars + len(text) > EMBED_BATCH_CHARS:
+            batches.append([])
+            chars = 0
+        batches[-1].append(text)
+        chars += len(text)
+    return batches
+
+
+def _vectors(response: Any) -> list[list[float]]:
+    items = sorted(response.data, key=lambda item: item["index"])
+    return [item["embedding"] for item in items]
+
+
+def _messages(prompt: str) -> list[dict[str, str]]:
+    return [{"role": "user", "content": prompt}]
+
+
+def litellm_complete(model: str, **connection: Any) -> Complete:
+    def complete(prompt: str) -> str:
+        response = litellm.completion(model=model, messages=_messages(prompt), **connection)
+        return response.choices[0].message.content or ""
+
+    return complete
+
+
+def litellm_embed(
+    model: str, batch_size: int = DEFAULT_EMBED_BATCH_SIZE, **connection: Any
+) -> Embed:
+    def embed(texts: list[str]) -> list[list[float]]:
+        vectors: list[list[float]] = []
+        for batch in _batches(texts, batch_size):
+            vectors.extend(_vectors(litellm.embedding(model=model, input=batch, **connection)))
+        return vectors
+
+    return embed
+
+
+def async_litellm_complete(model: str, **connection: Any) -> AsyncComplete:
+    async def complete(prompt: str) -> str:
+        response = await litellm.acompletion(model=model, messages=_messages(prompt), **connection)
+        return response.choices[0].message.content or ""
+
+    return complete
+
+
+def async_litellm_embed(
+    model: str, batch_size: int = DEFAULT_EMBED_BATCH_SIZE, **connection: Any
+) -> AsyncEmbed:
+    async def embed(texts: list[str]) -> list[list[float]]:
+        vectors: list[list[float]] = []
+        for batch in _batches(texts, batch_size):
+            response = await litellm.aembedding(model=model, input=batch, **connection)
+            vectors.extend(_vectors(response))
+        return vectors
+
+    return embed
+
+
+def _parts(env_file: str) -> tuple[Settings, dict, dict, dict]:
+    s = Settings(_env_file=env_file)
+    llm = _connection(s.llm_api_key, s.llm_api_base, s.llm_api_version, s)
+    embedder = _connection(s.embedding_api_key, s.embedding_api_base, s.embedding_api_version, s)
+    options = {
+        "max_chars": s.max_chunk_chars or None,
+        "min_chars": s.min_chunk_chars,
+        "summary_max_chars": s.summary_max_chars or None,
+    }
+    return s, llm, embedder, options
+
+
+def chunker_from_env(env_file: str = ".env") -> PICChunker:
+    s, llm, embedder, options = _parts(env_file)
+    return PICChunker(
+        litellm_complete(s.llm_model, **llm),
+        litellm_embed(s.embedding_model, s.embedding_batch_size, **embedder),
+        **options,
+    )
+
+
+def async_chunker_from_env(env_file: str = ".env") -> AsyncPICChunker:
+    s, llm, embedder, options = _parts(env_file)
+    return AsyncPICChunker(
+        async_litellm_complete(s.llm_model, **llm),
+        async_litellm_embed(s.embedding_model, s.embedding_batch_size, **embedder),
+        **options,
+    )

chunkguard/pic.py

@@ -0,0 +1,261 @@
+"""PIC: Pseudo-Instruction for document Chunking (Wang et al., Findings of ACL 2025).
+
+An LLM summary of the document stands in for the unknown user queries. Every
+sentence is compared with that summary; the mean similarity is the threshold.
+Runs of neighbouring sentences on the same side of the threshold form a chunk,
+so boundaries sit exactly where the text turns towards or away from the topic.
+
+Splitting long and merging short chunks is not part of the paper; both can be
+switched off (max_chars=None, min_chars=0).
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Awaitable, Callable
+from statistics import fmean
+
+from pydantic import BaseModel
+
+from chunkguard.sentences import split_sentences
+
+Span = tuple[int, int]
+# (text, language) -> sentence spans, see chunkguard.sentences
+SentenceSplitter = Callable[[str, str], list[Span]]
+# Any LLM: prompt -> completion text
+Complete = Callable[[str], str]
+AsyncComplete = Callable[[str], Awaitable[str]]
+# Any embedding model: texts -> one vector per text, same order
+Embed = Callable[[list[str]], list[list[float]]]
+AsyncEmbed = Callable[[list[str]], Awaitable[list[list[float]]]]
+
+# Summarizer prompt from the PIC paper (Appendix C), plus the output language.
+SUMMARY_PROMPT = (
+    "Read this passage:\n{text}\n"
+    "Write a summary for this passage. Your summary should be concise and informative. "
+    "Give the summary directly, without any other text. "
+    "Write it in the language of the passage (language code: {language})."
+)
+
+# A sentence range [first, end) within the document.
+Group = tuple[int, int]
+
+# Pieces of a long document the summarizer sees, spread evenly over the text.
+_EXCERPT_PIECES = 8
+
+
+class Chunk(BaseModel):
+    doc_id: str
+    index: int
+    text: str
+    start: int  # character offset in the input text, inclusive
+    end: int  # character offset in the input text, exclusive
+    sentence_start: int  # inclusive
+    sentence_end: int  # exclusive
+    # Sentences at or above the document mean make up at least half the text.
+    # Before merging every chunk is uniform; a merge may absorb a short opposite run.
+    relevant: bool
+    similarity: float  # mean cosine similarity of the chunk's sentences to the summary
+    summary: str
+
+
+class _Chunker:
+    def __init__(
+        self,
+        *,
+        max_chars: int | None = 2000,
+        min_chars: int = 200,
+        summary_max_chars: int | None = 100_000,
+        summary_prompt: str = SUMMARY_PROMPT,
+        sentence_splitter: SentenceSplitter = split_sentences,
+    ) -> None:
+        if max_chars is not None and max_chars < 1:
+            raise ValueError("max_chars must be positive or None")
+        if min_chars < 0 or (max_chars is not None and min_chars > max_chars):
+            raise ValueError("min_chars must be between 0 and max_chars")
+        if summary_max_chars is not None and summary_max_chars < _EXCERPT_PIECES:
+            raise ValueError(f"summary_max_chars must be at least {_EXCERPT_PIECES} or None")
+        self.max_chars = max_chars
+        self.min_chars = min_chars
+        self.summary_max_chars = summary_max_chars
+        self.summary_prompt = summary_prompt
+        self.sentence_splitter = sentence_splitter
+
+    def _prompt(self, text: str, language: str) -> str:
+        excerpt = _excerpt(text, self.summary_max_chars)
+        return self.summary_prompt.format(text=excerpt, language=language)
+
+    def _chunks(
+        self,
+        doc_id: str,
+        text: str,
+        spans: list[Span],
+        summary: str,
+        vectors: list[list[float]],
+    ) -> list[Chunk]:
+        if not summary:
+            raise ValueError(f"LLM returned an empty summary for {doc_id!r}")
+        if len(vectors) != len(spans) + 1:
+            raise ValueError(f"expected {len(spans) + 1} embeddings, got {len(vectors)}")
+
+        reference, *sentence_vectors = vectors
+        similarities = [_cosine(reference, vector) for vector in sentence_vectors]
+        threshold = fmean(similarities)
+        max_chars = math.inf if self.max_chars is None else self.max_chars
+
+        groups = _runs([s >= threshold for s in similarities])
+        groups = [p for g in groups for p in _split_long(g, spans, max_chars)]
+        groups = _merge_short(groups, spans, similarities, self.min_chars, max_chars)
+
+        chunks = []
+        for index, (first, end) in enumerate(groups):
+            start, stop = spans[first][0], spans[end - 1][1]
+            sizes = [e - s for s, e in spans[first:end]]
+            relevant_chars = sum(
+                n for n, sim in zip(sizes, similarities[first:end], strict=True) if sim >= threshold
+            )
+            chunks.append(
+                Chunk(
+                    doc_id=doc_id,
+                    index=index,
+                    text=text[start:stop],
+                    start=start,
+                    end=stop,
+                    sentence_start=first,
+                    sentence_end=end,
+                    relevant=2 * relevant_chars >= sum(sizes),
+                    similarity=fmean(similarities[first:end]),
+                    summary=summary,
+                )
+            )
+        return chunks
+
+
+class PICChunker(_Chunker):
+    """Text in, chunks out. Swappable with any splitter of the same signature.
+
+    Costs one LLM call and one embed call (for all sentences) per document.
+    """
+
+    def __init__(self, complete: Complete, embed: Embed, **options) -> None:
+        super().__init__(**options)
+        self.complete = complete
+        self.embed = embed
+
+    def split(self, doc_id: str, text: str, language: str) -> list[Chunk]:
+        spans = self.sentence_splitter(text, language)
+        if not spans:
+            return []
+        summary = self.complete(self._prompt(text, language)).strip()
+        vectors = self.embed([summary, *(text[s:e] for s, e in spans)])
+        return self._chunks(doc_id, text, spans, summary, vectors)
+
+
+class AsyncPICChunker(_Chunker):
+    """PICChunker with async complete/embed for asyncio indexing pipelines."""
+
+    def __init__(self, complete: AsyncComplete, embed: AsyncEmbed, **options) -> None:
+        super().__init__(**options)
+        self.complete = complete
+        self.embed = embed
+
+    async def split(self, doc_id: str, text: str, language: str) -> list[Chunk]:
+        spans = self.sentence_splitter(text, language)
+        if not spans:
+            return []
+        summary = (await self.complete(self._prompt(text, language))).strip()
+        vectors = await self.embed([summary, *(text[s:e] for s, e in spans)])
+        return self._chunks(doc_id, text, spans, summary, vectors)
+
+
+def _excerpt(text: str, limit: int | None) -> str:
+    """Shorten text to about limit characters, taking evenly spread pieces.
+
+    Keeps the summary within the LLM's context window while every part of the
+    document still contributes to it.
+    """
+    if limit is None or len(text) <= limit:
+        return text
+    size = limit // _EXCERPT_PIECES
+    step = (len(text) - size) / (_EXCERPT_PIECES - 1)
+    return "\n[…]\n".join(
+        text[round(i * step) : round(i * step) + size] for i in range(_EXCERPT_PIECES)
+    )
+
+
+def _cosine(a: list[float], b: list[float]) -> float:
+    norm = math.sqrt(sum(x * x for x in a)) * math.sqrt(sum(y * y for y in b))
+    return sum(x * y for x, y in zip(a, b, strict=True)) / norm if norm else 0.0
+
+
+def _runs(flags: list[bool]) -> list[Group]:
+    """Split the sentence sequence wherever the flag flips."""
+    groups, first = [], 0
+    for i in range(1, len(flags)):
+        if flags[i] != flags[i - 1]:
+            groups.append((first, i))
+            first = i
+    groups.append((first, len(flags)))
+    return groups
+
+
+def _length(spans: list[Span], group: Group) -> int:
+    first, end = group
+    return spans[end - 1][1] - spans[first][0]
+
+
+def _split_long(group: Group, spans: list[Span], max_chars: float) -> list[Group]:
+    """Split at sentence boundaries into near-equal pieces of at most max_chars.
+
+    A single sentence longer than max_chars stays whole.
+    """
+    total = _length(spans, group)
+    if total <= max_chars:
+        return [group]
+    target = total / math.ceil(total / max_chars)
+    first, end = group
+    pieces, start = [], first
+    for i in range(first + 1, end):
+        without, with_next = _length(spans, (start, i)), _length(spans, (start, i + 1))
+        if with_next > max_chars or with_next - target > target - without:
+            pieces.append((start, i))
+            start = i
+    pieces.append((start, end))
+    return pieces
+
+
+def _merge_short(
+    groups: list[Group],
+    spans: list[Span],
+    similarities: list[float],
+    min_chars: int,
+    max_chars: float,
+) -> list[Group]:
+    """Merge chunks below min_chars into the neighbour with the closest mean similarity.
+
+    max_chars wins over min_chars: a merge that would exceed it is skipped.
+    """
+    groups = list(groups)
+
+    def score(group: Group) -> float:
+        return fmean(similarities[group[0] : group[1]])
+
+    while True:
+        for k in sorted(range(len(groups)), key=lambda k: _length(spans, groups[k])):
+            if _length(spans, groups[k]) >= min_chars:
+                return groups
+            # Merging with neighbour k-1 or k+1 means merging the pair starting at lo.
+            pairs = [
+                lo
+                for lo in (k - 1, k)
+                if 0 <= lo < len(groups) - 1
+                and _length(spans, (groups[lo][0], groups[lo + 1][1])) <= max_chars
+            ]
+            if pairs:
+                lo = min(
+                    pairs, key=lambda lo: abs(score(groups[2 * lo + 1 - k]) - score(groups[k]))
+                )
+                groups[lo : lo + 2] = [(groups[lo][0], groups[lo + 1][1])]
+                break
+        else:
+            return groups

chunkguard/sentences.py

@@ -0,0 +1,61 @@
+"""Sentence splitting that returns character spans, so no text is lost.
+
+Boundaries come from sentencex (Wikimedia), which carries abbreviation and
+punctuation rules for many languages and falls back to related languages for
+the rest. On top of that, every line break ends a sentence, so headings and
+list items stand alone.
+"""
+
+import re
+
+import sentencex
+
+_SHORT_NUMBER = re.compile(r"(?:^|(?<=\s))\d{1,3}\.$")
+# Characters looked at around a boundary; slicing the whole text per boundary is quadratic.
+_WINDOW = 16
+
+
+def split_sentences(text: str, language: str) -> list[tuple[int, int]]:
+    """Return the (start, end) character spans of the sentences in ``text``.
+
+    ``language`` is a BCP 47 code such as "en" or "pt-BR"; only the primary
+    subtag is used. Spans exclude surrounding whitespace, so ``text[start:end]``
+    is the sentence, and whitespace-only pieces are dropped.
+    """
+    primary = language.split("-")[0].lower()
+    cuts = {i + 1 for i, char in enumerate(text) if char == "\n"}
+    for boundary in sentencex.get_sentence_boundaries(primary, text):
+        end = boundary["end_index"]
+        if not _continues_lowercase(text, end) and not _ends_with_short_number(text, end):
+            cuts.add(end)
+
+    spans: list[tuple[int, int]] = []
+    start = 0
+    for cut in [*sorted(cuts), len(text)]:
+        piece = text[start:cut]
+        left = len(piece) - len(piece.lstrip())
+        right = len(piece.rstrip())
+        if right > left:
+            spans.append((start + left, start + right))
+        start = cut
+    return spans
+
+
+def _continues_lowercase(text: str, position: int) -> bool:
+    """True if the same line goes on with a lowercase letter, i.e. after an abbreviation.
+
+    sentencex misses this for some abbreviations and for non-ASCII letters such
+    as "ü". Merging two sentences costs less than cutting one in half.
+    """
+    rest = text[position : position + _WINDOW].lstrip(" \t")
+    return bool(rest) and rest[0].islower()
+
+
+def _ends_with_short_number(text: str, position: int) -> bool:
+    """True if the piece ends in a number of up to three digits plus a period.
+
+    That is an ordinal in German, Hungarian, Danish, Czech and others ("im 18.
+    Jahrhundert"); sentencex only knows the case before month names. The cost
+    is a merged sentence in the rare English "We sold 3. Then ..." case.
+    """
+    return _SHORT_NUMBER.search(text[max(0, position - _WINDOW) : position].rstrip()) is not None

chunkguard-0.1.0.dist-info/METADATA

@@ -0,0 +1,254 @@
+Metadata-Version: 2.5
+Name: chunkguard
+Version: 0.1.0
+Summary: PIC document chunking for RAG with any LLM and embedding model
+Project-URL: Homepage, https://github.com/zakariaefakhri/chunkguard
+Project-URL: Issues, https://github.com/zakariaefakhri/chunkguard/issues
+Project-URL: Paper, https://aclanthology.org/2025.findings-acl.422/
+Author: Zakariae Fakhri
+License-Expression: MIT
+License-File: LICENSE
+Keywords: chunking,embeddings,llm,rag,retrieval,text-splitter
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: pydantic>=2.0
+Requires-Dist: sentencex>=1.0.30
+Provides-Extra: litellm
+Requires-Dist: litellm>=1.83.0; extra == 'litellm'
+Requires-Dist: pydantic-settings>=2.2; extra == 'litellm'
+Description-Content-Type: text/markdown
+
+# chunkguard
+
+[![CI](https://github.com/zakariaefakhri/chunkguard/actions/workflows/ci.yml/badge.svg)](https://github.com/zakariaefakhri/chunkguard/actions/workflows/ci.yml)
+
+Document chunking for RAG with **PIC** (Pseudo-Instruction for document Chunking), with any
+LLM and any embedding model. From
+[*Document Segmentation Matters for Retrieval-Augmented Generation*](https://aclanthology.org/2025.findings-acl.422/)
+(Wang et al., Findings of ACL 2025). This is an independent implementation, not the authors' code.
+
+When you build a knowledge base, you don't know the questions users will ask. PIC stands in
+for them with an LLM summary of the document. Every sentence is embedded and compared with
+the summary; the threshold is the mean similarity. Neighbouring sentences at or above it
+form a chunk of core content, neighbouring sentences below it a chunk of side details. So
+chunk boundaries sit exactly where the text turns towards or away from its topic, and every
+sentence is judged against the whole document rather than just its neighbour, as in
+semantic chunking.
+
+Order is kept, no sentence is cut, nothing is dropped. Cost per document: one LLM call and
+one embedding pass over its sentences. No training.
+
+## Install
+
+```bash
+pip install chunkguard              # core: bring your own model functions (Pydantic + sentencex)
+pip install "chunkguard[litellm]"   # plus .env configuration for 100+ providers via LiteLLM
+```
+
+## Choose your models
+
+chunkguard needs two things, and you pick both: an LLM that writes one summary per document,
+and an embedding model. They can come from different providers.
+
+**Option 1: plain functions.** `PICChunker` takes any `complete(prompt) -> str` and any
+`embed(texts) -> list[vector]`. No provider SDK is involved:
+
+```python
+from chunkguard import PICChunker
+from sentence_transformers import SentenceTransformer  # local embeddings, as an example
+import anthropic  # any LLM, as an example
+
+claude = anthropic.Anthropic()
+encoder = SentenceTransformer("BAAI/bge-m3")
+
+
+def complete(prompt: str) -> str:
+    message = claude.messages.create(
+        model="claude-sonnet-5-5", max_tokens=512, messages=[{"role": "user", "content": prompt}]
+    )
+    return message.content[0].text
+
+
+chunker = PICChunker(complete=complete, embed=lambda texts: encoder.encode(texts).tolist())
+chunks = chunker.split(doc_id="report-7", text=cleaned_text, language="en")
+```
+
+**Option 2: `.env` with LiteLLM.** Install the extra, copy [`.env.example`](.env.example) to
+`.env` and set the two models as [LiteLLM strings](https://docs.litellm.ai/docs/providers):
+
+```bash
+CHUNKGUARD_LLM_MODEL=anthropic/claude-sonnet-5-5
+CHUNKGUARD_EMBEDDING_MODEL=cohere/embed-multilingual-v3.0
+```
+
+Put keys in `.env` as `CHUNKGUARD_LLM_API_KEY` / `CHUNKGUARD_EMBEDDING_API_KEY`. If those are
+unset, LiteLLM uses the provider's usual variables (`ANTHROPIC_API_KEY`, `COHERE_API_KEY`,
+`AWS_*`, ...) from the process environment. Azure, Bedrock, Vertex, Ollama, vLLM and any
+OpenAI-compatible server work the same way. Then:
+
+```python
+from chunkguard import chunker_from_env
+
+chunker = chunker_from_env()
+chunks = chunker.split(doc_id="report-7", text=cleaned_text, language="en")
+
+for chunk in chunks:
+    print(chunk.index, chunk.relevant, round(chunk.similarity, 3), chunk.text[:60])
+```
+
+`split` takes plain text that extraction has already cleaned (line breaks inside sentences
+joined). Each `Chunk` carries:
+
+| Field | Meaning |
+|---|---|
+| `doc_id`, `index` | Document and position of the chunk in it |
+| `text` | Exact slice `input[start:end]` |
+| `start`, `end` | Character offsets into the input (end exclusive) |
+| `sentence_start`, `sentence_end` | Sentence range (end exclusive) |
+| `relevant` | Core content (`True`) or side details (`False`) |
+| `similarity` | Mean cosine similarity of the chunk's sentences to the summary |
+| `summary` | The document summary, for use as metadata in the index |
+
+`Chunk` is a Pydantic model; `chunk.model_dump()` gives a dict for your vector store.
+
+### Async
+
+```python
+from chunkguard import async_chunker_from_env
+
+chunker = async_chunker_from_env()
+chunks = await chunker.split("report-7", cleaned_text, "en")
+```
+
+`AsyncPICChunker` takes async `complete` and `embed` functions in the same way.
+
+### Your own summary prompt
+
+The default is the paper's prompt (`chunkguard.SUMMARY_PROMPT`) plus an instruction to answer
+in the document's language. Pass `summary_prompt` with `{text}` and `{language}` placeholders
+to change it.
+
+### Your own sentence splitter
+
+Sentences come from [sentencex](https://github.com/wikimedia/sentencex) (see [Limits](#limits)).
+To use another splitter, pass a function
+that returns `(start, end)` character spans. For example, NLTK's Punkt, which the paper uses
+for sentence splitting:
+
+```python
+from nltk.tokenize.punkt import PunktSentenceTokenizer
+
+punkt = PunktSentenceTokenizer()
+
+
+def nltk_sentences(text, language):
+    return list(punkt.span_tokenize(text))
+
+
+chunker = PICChunker(complete, embed, sentence_splitter=nltk_sentences)
+```
+
+## How it works
+
+1. **Sentences.** Split into sentences with sentencex, keeping character offsets. Every line
+   break also ends a sentence, so headings and list items stand alone.
+2. **Summary.** One LLM call with the summarizer prompt from the paper (Appendix C), plus an
+   instruction to answer in the document's language.
+3. **Similarity.** Embed the summary and every sentence; cosine similarity of each sentence
+   to the summary.
+4. **Threshold.** The mean of those similarities. A sentence is relevant at or above it.
+5. **Grouping.** Each run of neighbouring sentences with the same flag becomes one chunk.
+
+Steps 1 to 5 are the method as published. chunkguard adds three things for production use;
+set `max_chars=None, min_chars=0, summary_max_chars=None` to switch them off:
+
+6. **Long chunks** (over `max_chars`) are split at sentence boundaries into near-equal pieces.
+7. **Short chunks** (under `min_chars`) merge into the neighbour with the closest mean
+   similarity, unless that would exceed `max_chars`. A merged chunk is `relevant` when its
+   relevant sentences make up at least half of its text.
+8. **Long documents** reach the summarizer as eight evenly spread excerpts totalling
+   `summary_max_chars`, so the call fits the context window and every part of the document
+   still shapes the summary.
+
+## Configuration
+
+With plain functions, pass the options to `PICChunker` (`max_chars`, `min_chars`,
+`summary_max_chars`, `summary_prompt`, `sentence_splitter`). With the LiteLLM extra, they come
+from environment variables or `.env`, prefixed `CHUNKGUARD_`:
+
+| Variable | Default | Meaning |
+|---|---|---|
+| `LLM_MODEL` | required | LiteLLM model string, e.g. `gemini/gemini-2.5-flash` |
+| `LLM_API_KEY` | provider variable | API key |
+| `LLM_API_BASE` | – | Endpoint, e.g. Azure resource, Ollama or any OpenAI-compatible server |
+| `LLM_API_VERSION` | – | Azure only |
+| `EMBEDDING_MODEL`, `EMBEDDING_API_*` | | Same settings for the embedding model |
+| `EMBEDDING_BATCH_SIZE` | `64` | Texts per embedding request |
+| `REQUEST_TIMEOUT` | `120` | Seconds per request |
+| `MAX_RETRIES` | `2` | Retries per request on provider errors |
+| `MAX_CHUNK_CHARS` | `2000` | Split longer chunks; `0` = no limit |
+| `MIN_CHUNK_CHARS` | `200` | Merge shorter chunks; `0` = no merging |
+| `SUMMARY_MAX_CHARS` | `100000` | Excerpt longer documents for the summary; `0` = send all |
+
+Embedding requests stay below 400,000 characters each. A sentence longer than 8,000
+characters is truncated for embedding only; its chunk keeps the full text.
+
+LiteLLM versions 1.82.7 and 1.82.8 were compromised on PyPI in March 2026; the extra requires
+1.83.0 or later.
+
+## Limits
+
+- **Sentence splitting is rule-based.** sentencex carries punctuation and abbreviation rules
+  for many languages and falls back to related ones. chunkguard adds two language-neutral
+  rules: no cut before a lowercase letter, and no cut after a number of up to three digits
+  ("im 18. Jahrhundert"), which merges the rare English "We sold 3. Then ..." into one
+  sentence. Some boundaries are still missed (a sentence ending in "etc."), and languages
+  without sentence punctuation (Thai) are not split. Pass your own splitter if that matters.
+- **Sizes are in characters, not tokens.** Pick `MAX_CHUNK_CHARS` with your embedding model's
+  token limit in mind.
+- **The relevance flag is relative to the document.** Nearly every document has relevant and
+  irrelevant chunks, however on-topic it is overall. Use it to weight or filter retrieval,
+  not as an absolute quality score.
+- **Retrieval gains are the paper's results** on English Wikipedia and open-domain QA. This
+  repository does not reproduce them.
+
+## Development
+
+```bash
+uv sync
+uv run pytest
+uv run ruff check . && uv run ruff format --check .
+```
+
+Releases: bump `version` in `pyproject.toml`, then push a matching tag (`v0.1.0`). The
+release workflow publishes to PyPI via trusted publishing.
+
+## Citation
+
+If you use PIC in research, cite the paper:
+
+```bibtex
+@inproceedings{wang-etal-2025-document,
+  title     = {Document Segmentation Matters for Retrieval-Augmented Generation},
+  author    = {Wang, Zhitong and Gao, Cheng and Xiao, Chaojun and Huang, Yufei and Si, Shuzheng and
+               Luo, Kangyang and Bai, Yuzhuo and Li, Wenhao and Duan, Tangjian and Lv, Chuancheng and
+               Lu, Guoshan and Chen, Gang and Qi, Fanchao and Sun, Maosong},
+  booktitle = {Findings of the Association for Computational Linguistics: ACL 2025},
+  pages     = {8063--8075},
+  year      = {2025},
+  url       = {https://aclanthology.org/2025.findings-acl.422/}
+}
+```
+
+## License
+
+[MIT](LICENSE)

chunkguard-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+chunkguard/__init__.py,sha256=bh7xWGcRwkmf9mW0R7_3F-Ko0c9wPz4Kfpc_JQcOTcs,715
+chunkguard/backend.py,sha256=begTsJSEz8wf_SisXe2LRS2ey7R2tzUCE1z4V7rlEyE,5471
+chunkguard/pic.py,sha256=iFgG_D2pXcoKtWLEMFjaFSAHV85a4woWCLLha9P4gAQ,9796
+chunkguard/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+chunkguard/sentences.py,sha256=sDzjevqo3JhhGIq-micaxoGc1m03Vs0fubCjgUdtPok,2460
+chunkguard-0.1.0.dist-info/METADATA,sha256=eNCS2pEidmxDA9qCTpNLDWjZ8N4-37Qdxf49RstuMPk,10899
+chunkguard-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+chunkguard-0.1.0.dist-info/licenses/LICENSE,sha256=N8zGq3bv7D1IYT9PAbCzVgjR1VVpzmChWVkV3QDojb4,1072
+chunkguard-0.1.0.dist-info/RECORD,,

chunkguard-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

chunkguard-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Zakariae Fakhri
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.