chunkwise 0.1.0__tar.gz

chunkwise-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,32 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Lint (ruff)
+        run: ruff check .
+
+      - name: Type check (mypy)
+        run: mypy src/chunkwise
+
+      - name: Test (pytest)
+        run: pytest

chunkwise-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,28 @@
+name: Publish
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build tool
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

chunkwise-0.1.0/.gitignore

@@ -0,0 +1,15 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.eggs/
+.venv/
+venv/
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+.hypothesis/
+.coverage
+htmlcov/
+.DS_Store

chunkwise-0.1.0/CHANGELOG.md

@@ -0,0 +1,16 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - 2026-11-04
+
+### Added
+
+- Initial release.
+- `chunk()` public API with `fixed`, `recursive`, and `markdown` strategies.
+- `Chunk` dataclass enforcing `document[chunk.start:chunk.end] == chunk.text`.
+- Tokenizer protocol with a character-counting default; zero runtime dependencies.
+- Hypothesis-based property tests for the offset invariant.

chunkwise-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,32 @@
+# Contributing
+
+Thanks for considering a contribution to chunkwise.
+
+## Setup
+
+```bash
+git clone https://github.com/amin-albooyeh/chunkwise.git
+cd chunkwise
+pip install -e ".[dev]"
+```
+
+## Running checks
+
+```bash
+ruff check .
+mypy src/chunkwise
+pytest
+```
+
+All three run in CI on every pull request, across Python 3.10–3.13.
+
+## Guidelines
+
+- The offset invariant (`document[chunk.start:chunk.end] == chunk.text`) must hold for every strategy on every input. Any new strategy needs a corresponding Hypothesis property test in `tests/test_invariants.py`.
+- Strategies operate on `(start, end)` spans internally and only materialize text via `_spans.materialize`. Don't build chunks by slicing and reassembling strings inside a strategy.
+- Keep the base install dependency-free. Anything that requires a third-party package (e.g. `semantic`) ships as an optional extra.
+- Add tests for new behavior in `tests/test_strategies.py` or `tests/test_tokenizers.py` as appropriate.
+
+## Reporting issues
+
+Open a GitHub issue with a minimal reproducible example — ideally the document text, the strategy, and the parameters used.

chunkwise-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Amin Albooyeh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

chunkwise-0.1.0/PKG-INFO

@@ -0,0 +1,168 @@
+Metadata-Version: 2.5
+Name: chunkwise
+Version: 0.1.0
+Summary: Text chunking for RAG with reversible offsets — every chunk is an exact slice of the source document.
+Project-URL: Homepage, https://github.com/amin-albooyeh/chunkwise
+Project-URL: Repository, https://github.com/amin-albooyeh/chunkwise
+Project-URL: Documentation, https://github.com/amin-albooyeh/chunkwise/tree/main/docs
+Project-URL: Changelog, https://github.com/amin-albooyeh/chunkwise/blob/main/CHANGELOG.md
+Author-email: Amin Albooyeh <amin.albooyeh@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: chunking,embeddings,nlp,rag,retrieval
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: hypothesis>=6.0; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: semantic
+Requires-Dist: numpy; extra == 'semantic'
+Description-Content-Type: text/markdown
+
+# chunkwise
+
+**Text chunking for RAG with reversible offsets — every chunk is an exact slice of the source document.**
+
+[![CI](https://github.com/amin-albooyeh/chunkwise/actions/workflows/ci.yml/badge.svg)](https://github.com/amin-albooyeh/chunkwise/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/chunkwise.svg)](https://pypi.org/project/chunkwise/)
+[![Python versions](https://img.shields.io/pypi/pyversions/chunkwise.svg)](https://pypi.org/project/chunkwise/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+
+chunkwise splits documents into embeddable chunks for retrieval-augmented generation. Unlike most splitters, which return detached strings, every chunk carries the exact character range it came from, so `document[chunk.start:chunk.end] == chunk.text` always holds — making source highlighting, citation, and context expansion possible. Zero runtime dependencies, bring-your-own tokenizer, and structure-aware strategies for plain text and Markdown.
+
+## Why chunkwise
+
+Most chunking libraries hand back a `list[str]`. The moment text is detached from its origin, three things become impossible:
+
+- You can't **highlight** the passage a retrieval hit came from in the original document.
+- You can't **expand** a result to show the surrounding paragraph.
+- You can't tell **which section or heading** a passage lived under.
+
+Worse, splitters that strip or normalize whitespace silently drop characters — a failure mode that shows up only as retrieval quietly missing content, which is hard to notice and harder to debug.
+
+## Install
+
+```bash
+pip install chunkwise
+```
+
+No dependencies are pulled in. (An optional `semantic` extra will be added in v0.3 for embedding-based chunking.)
+
+## Quickstart
+
+```python
+from chunkwise import chunk
+
+document = open("report.md").read()
+chunks = chunk(document, strategy="recursive", max_tokens=512, overlap=50)
+
+for c in chunks:
+    assert document[c.start:c.end] == c.text  # always true, for every chunk
+    print(c.index, c.start, c.end, c.heading_path)
+```
+
+## The guarantee
+
+chunkwise holds one invariant across every strategy and every input:
+
+```python
+document[chunk.start:chunk.end] == chunk.text
+```
+
+This is enforced three ways:
+
+1. **Structurally** — internal logic operates on `(start, end)` integer spans and only materializes text at the final step, so no strategy (present or future) can break the invariant even by accident.
+2. **At runtime** — the `Chunk` type validates its own span on construction.
+3. **In CI** — a Hypothesis property test runs against generated documents on every commit, alongside a test that concatenating non-overlapping chunks reproduces the source byte for byte.
+
+The consequences are deliberate: chunks are never stripped, normalized, or rewritten, and overlap widens a chunk backwards into real preceding text rather than gluing on a copied prefix.
+
+See [docs/design.md](docs/design.md) for the full rationale.
+
+## Strategies
+
+| Strategy | Behavior |
+|---|---|
+| `fixed` | Hard windows of N tokens with configurable overlap. Fast, boundary-blind. Suited to logs and unstructured text. |
+| `recursive` | Falls back through a separator chain — paragraphs, then lines, then sentences, then words — breaking on the coarsest boundary that fits the budget. **The default.** |
+| `markdown` | Keeps fenced code blocks intact and tags every chunk with its heading path (e.g. `["Guide", "Install"]`), giving retrieval results a sense of place. |
+
+Planned: `code` (v0.2 — AST / tree-sitter boundaries) and `semantic` (v0.3 — embedding similarity cuts), the latter shipping as an optional extra so the base install stays dependency-free.
+
+See [docs/strategies.md](docs/strategies.md) for guidance on choosing one.
+
+## Tokenizer independence
+
+Token budgets depend on which model you're embedding for, so chunkwise doesn't hard-code one. The `tokenizer` parameter accepts any callable from `str` to `int`:
+
+```python
+from chunkwise import chunk
+import tiktoken
+
+enc = tiktoken.get_encoding("cl100k_base")
+chunks = chunk(document, max_tokens=512, tokenizer=lambda text: len(enc.encode(text)))
+```
+
+The default counts characters, keeping the base install at zero dependencies. The only requirement on a tokenizer is that its count is monotonic in string length — every real tokenizer satisfies this. See [docs/tokenizers.md](docs/tokenizers.md).
+
+## API
+
+```python
+def chunk(
+    document: str,
+    strategy: str = "recursive",   # "fixed" | "recursive" | "markdown"
+    max_tokens: int = 512,
+    overlap: int = 0,
+    tokenizer: Tokenizer = ...,    # str -> int, defaults to len
+) -> list[Chunk]: ...
+```
+
+Each `Chunk` is a frozen dataclass:
+
+```python
+@dataclass(frozen=True, slots=True)
+class Chunk:
+    text: str
+    start: int
+    end: int
+    index: int
+    heading_path: tuple[str, ...]  # populated by the markdown strategy
+```
+
+## Status
+
+v0.1.0 · Python 3.10+ · MIT licensed · zero runtime dependencies · tested on CI across Python 3.10–3.13.
+
+## Documentation
+
+- [docs/index.md](docs/index.md) — overview and quickstart
+- [docs/strategies.md](docs/strategies.md) — strategy reference and how to choose
+- [docs/tokenizers.md](docs/tokenizers.md) — bringing your own tokenizer
+- [docs/design.md](docs/design.md) — why spans, not strings
+
+## Benchmarks
+
+```bash
+python benchmarks/throughput.py
+```
+
+Compares chunkwise strategies against a naive fixed-width splitter on a synthetic document.
+
+## Contributing
+
+Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md) for setup, checks, and guidelines (in particular, the offset invariant that any new strategy must preserve).
+
+## License
+
+MIT — see [LICENSE](LICENSE).

chunkwise-0.1.0/README.md

@@ -0,0 +1,136 @@
+# chunkwise
+
+**Text chunking for RAG with reversible offsets — every chunk is an exact slice of the source document.**
+
+[![CI](https://github.com/amin-albooyeh/chunkwise/actions/workflows/ci.yml/badge.svg)](https://github.com/amin-albooyeh/chunkwise/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/chunkwise.svg)](https://pypi.org/project/chunkwise/)
+[![Python versions](https://img.shields.io/pypi/pyversions/chunkwise.svg)](https://pypi.org/project/chunkwise/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+
+chunkwise splits documents into embeddable chunks for retrieval-augmented generation. Unlike most splitters, which return detached strings, every chunk carries the exact character range it came from, so `document[chunk.start:chunk.end] == chunk.text` always holds — making source highlighting, citation, and context expansion possible. Zero runtime dependencies, bring-your-own tokenizer, and structure-aware strategies for plain text and Markdown.
+
+## Why chunkwise
+
+Most chunking libraries hand back a `list[str]`. The moment text is detached from its origin, three things become impossible:
+
+- You can't **highlight** the passage a retrieval hit came from in the original document.
+- You can't **expand** a result to show the surrounding paragraph.
+- You can't tell **which section or heading** a passage lived under.
+
+Worse, splitters that strip or normalize whitespace silently drop characters — a failure mode that shows up only as retrieval quietly missing content, which is hard to notice and harder to debug.
+
+## Install
+
+```bash
+pip install chunkwise
+```
+
+No dependencies are pulled in. (An optional `semantic` extra will be added in v0.3 for embedding-based chunking.)
+
+## Quickstart
+
+```python
+from chunkwise import chunk
+
+document = open("report.md").read()
+chunks = chunk(document, strategy="recursive", max_tokens=512, overlap=50)
+
+for c in chunks:
+    assert document[c.start:c.end] == c.text  # always true, for every chunk
+    print(c.index, c.start, c.end, c.heading_path)
+```
+
+## The guarantee
+
+chunkwise holds one invariant across every strategy and every input:
+
+```python
+document[chunk.start:chunk.end] == chunk.text
+```
+
+This is enforced three ways:
+
+1. **Structurally** — internal logic operates on `(start, end)` integer spans and only materializes text at the final step, so no strategy (present or future) can break the invariant even by accident.
+2. **At runtime** — the `Chunk` type validates its own span on construction.
+3. **In CI** — a Hypothesis property test runs against generated documents on every commit, alongside a test that concatenating non-overlapping chunks reproduces the source byte for byte.
+
+The consequences are deliberate: chunks are never stripped, normalized, or rewritten, and overlap widens a chunk backwards into real preceding text rather than gluing on a copied prefix.
+
+See [docs/design.md](docs/design.md) for the full rationale.
+
+## Strategies
+
+| Strategy | Behavior |
+|---|---|
+| `fixed` | Hard windows of N tokens with configurable overlap. Fast, boundary-blind. Suited to logs and unstructured text. |
+| `recursive` | Falls back through a separator chain — paragraphs, then lines, then sentences, then words — breaking on the coarsest boundary that fits the budget. **The default.** |
+| `markdown` | Keeps fenced code blocks intact and tags every chunk with its heading path (e.g. `["Guide", "Install"]`), giving retrieval results a sense of place. |
+
+Planned: `code` (v0.2 — AST / tree-sitter boundaries) and `semantic` (v0.3 — embedding similarity cuts), the latter shipping as an optional extra so the base install stays dependency-free.
+
+See [docs/strategies.md](docs/strategies.md) for guidance on choosing one.
+
+## Tokenizer independence
+
+Token budgets depend on which model you're embedding for, so chunkwise doesn't hard-code one. The `tokenizer` parameter accepts any callable from `str` to `int`:
+
+```python
+from chunkwise import chunk
+import tiktoken
+
+enc = tiktoken.get_encoding("cl100k_base")
+chunks = chunk(document, max_tokens=512, tokenizer=lambda text: len(enc.encode(text)))
+```
+
+The default counts characters, keeping the base install at zero dependencies. The only requirement on a tokenizer is that its count is monotonic in string length — every real tokenizer satisfies this. See [docs/tokenizers.md](docs/tokenizers.md).
+
+## API
+
+```python
+def chunk(
+    document: str,
+    strategy: str = "recursive",   # "fixed" | "recursive" | "markdown"
+    max_tokens: int = 512,
+    overlap: int = 0,
+    tokenizer: Tokenizer = ...,    # str -> int, defaults to len
+) -> list[Chunk]: ...
+```
+
+Each `Chunk` is a frozen dataclass:
+
+```python
+@dataclass(frozen=True, slots=True)
+class Chunk:
+    text: str
+    start: int
+    end: int
+    index: int
+    heading_path: tuple[str, ...]  # populated by the markdown strategy
+```
+
+## Status
+
+v0.1.0 · Python 3.10+ · MIT licensed · zero runtime dependencies · tested on CI across Python 3.10–3.13.
+
+## Documentation
+
+- [docs/index.md](docs/index.md) — overview and quickstart
+- [docs/strategies.md](docs/strategies.md) — strategy reference and how to choose
+- [docs/tokenizers.md](docs/tokenizers.md) — bringing your own tokenizer
+- [docs/design.md](docs/design.md) — why spans, not strings
+
+## Benchmarks
+
+```bash
+python benchmarks/throughput.py
+```
+
+Compares chunkwise strategies against a naive fixed-width splitter on a synthetic document.
+
+## Contributing
+
+Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md) for setup, checks, and guidelines (in particular, the offset invariant that any new strategy must preserve).
+
+## License
+
+MIT — see [LICENSE](LICENSE).

chunkwise-0.1.0/benchmarks/throughput.py

@@ -0,0 +1,47 @@
+"""Throughput benchmark: chunkwise strategies vs. a naive splitter.
+
+Run with: python benchmarks/throughput.py
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+
+from chunkwise import chunk
+
+SAMPLE_SIZE_CHARS = 200_000
+
+
+def naive_split(document: str, max_tokens: int) -> list[str]:
+    return [document[i : i + max_tokens] for i in range(0, len(document), max_tokens)]
+
+
+def make_sample() -> str:
+    paragraph = (
+        "This is a benchmark paragraph used to measure chunking throughput. "
+        "It repeats to simulate a long document.\n\n"
+    )
+    return paragraph * (SAMPLE_SIZE_CHARS // len(paragraph) + 1)
+
+
+def time_it(label: str, fn: Callable[[], object]) -> None:
+    start = time.perf_counter()
+    result = fn()
+    elapsed = time.perf_counter() - start
+    count = len(result) if isinstance(result, list) else -1
+    print(f"{label:24s} {elapsed*1000:8.2f} ms  ({count} chunks)")
+
+
+def main() -> None:
+    document = make_sample()
+    print(f"document size: {len(document):,} chars\n")
+
+    time_it("naive_split", lambda: naive_split(document, 512))
+    time_it("chunkwise fixed", lambda: chunk(document, strategy="fixed", max_tokens=512))
+    time_it("chunkwise recursive", lambda: chunk(document, strategy="recursive", max_tokens=512))
+    time_it("chunkwise markdown", lambda: chunk(document, strategy="markdown", max_tokens=512))
+
+
+if __name__ == "__main__":
+    main()

chunkwise-0.1.0/docs/design.md

@@ -0,0 +1,33 @@
+# Design notes: why spans, not strings
+
+## The problem with returning strings
+
+Most chunking libraries return `list[str]`. Once text is detached from its origin:
+
+- You can't highlight the passage a retrieval hit came from in the original document.
+- You can't expand a result to show the surrounding paragraph.
+- You can't tell which section or heading a passage lived under.
+
+Worse, splitters that strip or normalize whitespace silently drop characters — a failure mode that shows up only as retrieval quietly missing content, which is hard to notice and harder to debug.
+
+## The guarantee
+
+chunkwise holds one invariant across every strategy and every input:
+
+```python
+document[chunk.start:chunk.end] == chunk.text
+```
+
+This is enforced three ways:
+
+1. **Structurally** — internal logic operates on `(start, end)` integer pairs ([`_spans.py`](../src/chunkwise/_spans.py)) and only materializes text at the final step. No strategy, present or future, can break the invariant even by accident.
+2. **At runtime** — the `Chunk` type validates its own span on construction (see `Chunk.__post_init__` in [`types.py`](../src/chunkwise/types.py)).
+3. **In CI** — a Hypothesis property test ([`test_invariants.py`](../tests/test_invariants.py)) runs against generated documents on every commit, alongside a test that concatenating non-overlapping chunks reproduces the source byte for byte.
+
+## Consequences
+
+These are deliberate, not accidental:
+
+- Chunks are never stripped, normalized, or rewritten.
+- Overlap widens a chunk backwards into real preceding text rather than gluing on a copied prefix.
+- Window boundaries are found by binary search over character offsets (`find_boundary` in `_spans.py`), so the library works with any opaque tokenizer rather than assuming a fixed characters-to-tokens ratio.

chunkwise-0.1.0/docs/index.md

@@ -0,0 +1,32 @@
+# chunkwise
+
+Text chunking for RAG with reversible offsets — every chunk is an exact slice of the source document.
+
+## Why chunkwise
+
+Most chunking libraries hand back a `list[str]`. The moment the text is detached from its origin, you can't highlight the passage a retrieval hit came from, expand a result to show surrounding context, or tell which section a passage lived under.
+
+chunkwise holds one invariant across every strategy and every input:
+
+```python
+document[chunk.start:chunk.end] == chunk.text
+```
+
+## Quickstart
+
+```python
+from chunkwise import chunk
+
+document = open("report.md").read()
+chunks = chunk(document, strategy="markdown", max_tokens=512)
+
+for c in chunks:
+    assert document[c.start:c.end] == c.text
+    print(c.heading_path, len(c.text))
+```
+
+## Contents
+
+- [Strategies](strategies.md) — fixed, recursive, markdown, and what's planned
+- [Tokenizers](tokenizers.md) — bringing your own token counter
+- [Design](design.md) — why spans, not strings

chunkwise-0.1.0/docs/strategies.md

@@ -0,0 +1,26 @@
+# Strategies
+
+| Strategy | Behavior |
+|---|---|
+| `fixed` | Hard windows of N tokens with configurable overlap. Fast, boundary-blind. Suited to logs and unstructured text. |
+| `recursive` | Falls back through a separator chain — paragraphs, then lines, then sentences, then words — breaking on the coarsest boundary that fits the budget. The default. |
+| `markdown` | Keeps fenced code blocks intact and tags every chunk with its heading path (e.g. `["Guide", "Install"]`), giving retrieval results a sense of place. |
+
+## Planned
+
+- `code` (v0.2) — splits on function and class boundaries via `ast` and `tree-sitter`.
+- `semantic` (v0.3) — cuts where sentence-embedding similarity drops. Ships as an optional extra (`pip install chunkwise[semantic]`) so the base install stays dependency-free.
+
+## Choosing a strategy
+
+- Use `fixed` when the input has no structure to exploit (logs, transcripts) or when you need predictable, uniform chunk sizes.
+- Use `recursive` as the default for prose and general text.
+- Use `markdown` for documentation, READMEs, or any Markdown source where heading context and code fences matter.
+
+## Common parameters
+
+Every strategy takes the same three parameters:
+
+- `max_tokens` — the token budget per chunk, as measured by the tokenizer.
+- `overlap` — how many tokens of overlap between consecutive chunks. Overlap widens a chunk backwards into real preceding text; it never duplicates a copied prefix.
+- `tokenizer` — see [tokenizers.md](tokenizers.md).

chunkwise-0.1.0/docs/tokenizers.md

@@ -0,0 +1,46 @@
+# Tokenizers
+
+Token budgets depend on which model you're embedding for, so chunkwise doesn't hard-code one.
+
+The `tokenizer` parameter accepts any callable from `str` to `int`:
+
+```python
+class Tokenizer(Protocol):
+    def __call__(self, text: str) -> int: ...
+```
+
+## Default
+
+The default counts characters, keeping the base install at zero dependencies:
+
+```python
+DEFAULT_TOKENIZER = len
+```
+
+## Bring your own
+
+Pass a real token counter when you need accurate budgets for a specific model.
+
+### tiktoken
+
+```python
+import tiktoken
+from chunkwise import chunk
+
+enc = tiktoken.get_encoding("cl100k_base")
+chunks = chunk(document, max_tokens=512, tokenizer=lambda text: len(enc.encode(text)))
+```
+
+### transformers
+
+```python
+from transformers import AutoTokenizer
+from chunkwise import chunk
+
+tok = AutoTokenizer.from_pretrained("bert-base-uncased")
+chunks = chunk(document, max_tokens=512, tokenizer=lambda text: len(tok.encode(text)))
+```
+
+## The only requirement
+
+The token count must be monotonic in string length — a longer string never produces fewer tokens. Every real tokenizer satisfies this, and it's what lets chunkwise find chunk boundaries with binary search instead of assuming a fixed characters-to-tokens ratio.

chunkwise-0.1.0/pyproject.toml

@@ -0,0 +1,61 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "chunkwise"
+version = "0.1.0"
+description = "Text chunking for RAG with reversible offsets — every chunk is an exact slice of the source document."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [
+    { name = "Amin Albooyeh", email = "amin.albooyeh@gmail.com" },
+]
+keywords = ["rag", "chunking", "nlp", "embeddings", "retrieval"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Text Processing :: Linguistic",
+    "Typing :: Typed",
+]
+dependencies = []
+
+[project.optional-dependencies]
+semantic = ["numpy"]
+dev = [
+    "pytest>=8.0",
+    "hypothesis>=6.0",
+    "ruff>=0.6",
+    "mypy>=1.10",
+]
+
+[project.urls]
+Homepage = "https://github.com/amin-albooyeh/chunkwise"
+Repository = "https://github.com/amin-albooyeh/chunkwise"
+Documentation = "https://github.com/amin-albooyeh/chunkwise/tree/main/docs"
+Changelog = "https://github.com/amin-albooyeh/chunkwise/blob/main/CHANGELOG.md"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/chunkwise"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]
+
+[tool.mypy]
+python_version = "3.10"
+strict = true
+packages = ["chunkwise"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

chunkwise-0.1.0/src/chunkwise/__init__.py

@@ -0,0 +1,48 @@
+"""chunkwise: text chunking for RAG with reversible offsets.
+
+Every chunk carries the exact character range it came from, so
+``document[chunk.start:chunk.end] == chunk.text`` always holds.
+"""
+
+from __future__ import annotations
+
+from ._spans import DEFAULT_TOKENIZER
+from .strategies import chunk_fixed, chunk_markdown, chunk_recursive
+from .types import Chunk, Tokenizer
+
+__version__ = "0.1.0"
+
+_STRATEGIES = {
+    "fixed": chunk_fixed,
+    "recursive": chunk_recursive,
+    "markdown": chunk_markdown,
+}
+
+
+def chunk(
+    document: str,
+    strategy: str = "recursive",
+    max_tokens: int = 512,
+    overlap: int = 0,
+    tokenizer: Tokenizer = DEFAULT_TOKENIZER,
+) -> list[Chunk]:
+    """Split `document` into a list of Chunk objects.
+
+    Args:
+        document: the source text to split.
+        strategy: one of "fixed", "recursive" (default), "markdown".
+        max_tokens: the token budget per chunk, as measured by `tokenizer`.
+        overlap: how many tokens of overlap between consecutive chunks.
+        tokenizer: a callable from str to int token count. Defaults to
+            character counting, keeping the base install dependency-free.
+    """
+    try:
+        strategy_fn = _STRATEGIES[strategy]
+    except KeyError:
+        raise ValueError(
+            f"Unknown strategy {strategy!r}. Available: {sorted(_STRATEGIES)}"
+        ) from None
+    return strategy_fn(document, max_tokens, overlap, tokenizer)
+
+
+__all__ = ["chunk", "Chunk", "Tokenizer", "__version__"]

chunkwise-0.1.0/src/chunkwise/_spans.py

@@ -0,0 +1,47 @@
+"""Internal offset arithmetic shared by strategies.
+
+Strategies operate on (start, end) integer spans, not strings, until the
+final materialization step. This is what makes the offset invariant
+structurally impossible to break.
+"""
+
+from __future__ import annotations
+
+from .types import Chunk, Tokenizer
+
+def _char_count(text: str) -> int:
+    return len(text)
+
+
+DEFAULT_TOKENIZER: Tokenizer = _char_count
+
+
+def materialize(document: str, spans: list[tuple[int, int]]) -> list[Chunk]:
+    """Turn (start, end) spans into Chunk objects with text sliced from document."""
+    return [
+        Chunk(text=document[start:end], start=start, end=end, index=i)
+        for i, (start, end) in enumerate(spans)
+    ]
+
+
+def find_boundary(
+    document: str,
+    start: int,
+    budget: int,
+    tokenizer: Tokenizer,
+) -> int:
+    """Binary search for the largest end offset such that tokenizer(document[start:end]) <= budget.
+
+    Works with any opaque tokenizer rather than assuming a fixed characters-to-tokens ratio.
+    """
+    lo, hi = start, len(document)
+    if tokenizer(document[start:hi]) <= budget:
+        return hi
+
+    while hi - lo > 1:
+        mid = (lo + hi) // 2
+        if tokenizer(document[start:mid]) <= budget:
+            lo = mid
+        else:
+            hi = mid
+    return lo if lo > start else min(start + 1, len(document))

chunkwise-0.1.0/src/chunkwise/strategies/__init__.py

@@ -0,0 +1,7 @@
+"""Chunking strategies."""
+
+from .fixed import chunk_fixed
+from .markdown import chunk_markdown
+from .recursive import chunk_recursive
+
+__all__ = ["chunk_fixed", "chunk_recursive", "chunk_markdown"]

chunkwise-0.1.0/src/chunkwise/strategies/code.py

@@ -0,0 +1,19 @@
+"""Code-aware strategy: splits on function and class boundaries.
+
+Planned for v0.2. Will use `ast` for Python and `tree-sitter` for other
+languages to find boundaries, then chunk on those spans the same way
+`recursive` chunks on separators.
+"""
+
+from __future__ import annotations
+
+from ..types import Chunk, Tokenizer
+
+
+def chunk_code(
+    document: str,
+    max_tokens: int,
+    overlap: int,
+    tokenizer: Tokenizer,
+) -> list[Chunk]:
+    raise NotImplementedError("chunk_code is planned for v0.2")

chunkwise-0.1.0/src/chunkwise/strategies/fixed.py

@@ -0,0 +1,36 @@
+"""Fixed-window strategy: hard windows of N tokens with configurable overlap."""
+
+from __future__ import annotations
+
+from .._spans import find_boundary, materialize
+from ..types import Chunk, Tokenizer
+
+
+def chunk_fixed(
+    document: str,
+    max_tokens: int,
+    overlap: int,
+    tokenizer: Tokenizer,
+) -> list[Chunk]:
+    """Split document into hard windows of at most max_tokens, overlapping by `overlap` tokens.
+
+    Overlap widens a chunk backwards into real preceding text rather than
+    gluing on a copied prefix, preserving the offset invariant.
+    """
+    if max_tokens <= 0:
+        raise ValueError("max_tokens must be positive")
+    if overlap < 0 or overlap >= max_tokens:
+        raise ValueError("overlap must be in [0, max_tokens)")
+
+    spans: list[tuple[int, int]] = []
+    start = 0
+    n = len(document)
+    while start < n:
+        end = find_boundary(document, start, max_tokens, tokenizer)
+        spans.append((start, end))
+        if end >= n:
+            break
+        back_start = find_boundary(document, start, max_tokens - overlap, tokenizer)
+        start = max(back_start, start + 1)
+
+    return materialize(document, spans)

chunkwise-0.1.0/src/chunkwise/strategies/markdown.py

@@ -0,0 +1,75 @@
+"""Markdown-aware strategy: keeps fenced code blocks intact, tags chunks with heading paths."""
+
+from __future__ import annotations
+
+import re
+
+from .._spans import find_boundary
+from ..types import Chunk, Tokenizer
+
+HEADING_RE = re.compile(r"^(#{1,6})\s+(.*)$", re.MULTILINE)
+FENCE_RE = re.compile(r"^```.*?$.*?^```$", re.MULTILINE | re.DOTALL)
+
+
+def _code_fence_spans(document: str) -> list[tuple[int, int]]:
+    return [(m.start(), m.end()) for m in FENCE_RE.finditer(document)]
+
+
+def _heading_path_at(document: str, offset: int) -> tuple[str, ...]:
+    stack: list[tuple[int, str]] = []
+    for match in HEADING_RE.finditer(document):
+        if match.start() > offset:
+            break
+        level = len(match.group(1))
+        title = match.group(2).strip()
+        stack = [(lvl, t) for lvl, t in stack if lvl < level]
+        stack.append((level, title))
+    return tuple(title for _, title in stack)
+
+
+def chunk_markdown(
+    document: str,
+    max_tokens: int,
+    overlap: int,
+    tokenizer: Tokenizer,
+) -> list[Chunk]:
+    """Split markdown document, never breaking a fenced code block, tagging heading paths."""
+    if max_tokens <= 0:
+        raise ValueError("max_tokens must be positive")
+    if overlap < 0 or overlap >= max_tokens:
+        raise ValueError("overlap must be in [0, max_tokens)")
+
+    fences = _code_fence_spans(document)
+    spans: list[tuple[int, int]] = []
+    start = 0
+    n = len(document)
+
+    while start < n:
+        hard_limit = find_boundary(document, start, max_tokens, tokenizer)
+
+        for fence_start, fence_end in fences:
+            if fence_start < hard_limit < fence_end:
+                hard_limit = fence_end
+                break
+
+        end = max(hard_limit, start + 1)
+        end = min(end, n)
+        spans.append((start, end))
+        if end >= n:
+            break
+        if overlap == 0:
+            start = end
+        else:
+            back_start = find_boundary(document, start, max_tokens - overlap, tokenizer)
+            start = max(min(back_start, end), start + 1)
+
+    return [
+        Chunk(
+            text=document[s:e],
+            start=s,
+            end=e,
+            index=i,
+            heading_path=_heading_path_at(document, s),
+        )
+        for i, (s, e) in enumerate(spans)
+    ]

chunkwise-0.1.0/src/chunkwise/strategies/recursive.py

@@ -0,0 +1,61 @@
+"""Recursive strategy: falls back through a separator chain to find natural breaks.
+
+Breaks on the coarsest boundary that fits the budget — paragraphs, then lines,
+then sentences, then words. This is the default strategy.
+"""
+
+from __future__ import annotations
+
+import re
+
+from .._spans import find_boundary, materialize
+from ..types import Chunk, Tokenizer
+
+SEPARATOR_CHAIN = [
+    re.compile(r"\n\s*\n"),  # paragraphs
+    re.compile(r"\n"),  # lines
+    re.compile(r"(?<=[.!?])\s+"),  # sentences
+    re.compile(r"\s+"),  # words
+]
+
+
+def _best_split_point(document: str, start: int, limit: int) -> int:
+    """Find the best separator-aligned split point within (start, limit]."""
+    window = document[start:limit]
+    for separator in SEPARATOR_CHAIN:
+        matches = list(separator.finditer(window))
+        if matches:
+            last = matches[-1]
+            split = start + last.end()
+            if split > start:
+                return split
+    return limit
+
+
+def chunk_recursive(
+    document: str,
+    max_tokens: int,
+    overlap: int,
+    tokenizer: Tokenizer,
+) -> list[Chunk]:
+    """Split document on the coarsest separator that keeps each chunk within budget."""
+    if max_tokens <= 0:
+        raise ValueError("max_tokens must be positive")
+    if overlap < 0 or overlap >= max_tokens:
+        raise ValueError("overlap must be in [0, max_tokens)")
+
+    spans: list[tuple[int, int]] = []
+    start = 0
+    n = len(document)
+    while start < n:
+        hard_limit = find_boundary(document, start, max_tokens, tokenizer)
+        end = _best_split_point(document, start, hard_limit) if hard_limit < n else hard_limit
+        if end <= start:
+            end = hard_limit
+        spans.append((start, end))
+        if end >= n:
+            break
+        back_start = find_boundary(document, start, max_tokens - overlap, tokenizer)
+        start = max(min(back_start, end), start + 1)
+
+    return materialize(document, spans)

chunkwise-0.1.0/src/chunkwise/strategies/semantic.py

@@ -0,0 +1,20 @@
+"""Semantic strategy: cuts where sentence-embedding similarity drops.
+
+Planned for v0.3, shipping as an optional extra (`pip install chunkwise[semantic]`)
+so the base install stays dependency-free.
+"""
+
+from __future__ import annotations
+
+from ..types import Chunk, Tokenizer
+
+
+def chunk_semantic(
+    document: str,
+    max_tokens: int,
+    overlap: int,
+    tokenizer: Tokenizer,
+) -> list[Chunk]:
+    raise NotImplementedError(
+        "chunk_semantic is planned for v0.3 and requires the 'semantic' extra"
+    )

chunkwise-0.1.0/src/chunkwise/types.py

@@ -0,0 +1,41 @@
+"""Core types: the Chunk dataclass and the Tokenizer protocol."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class Tokenizer(Protocol):
+    """Any callable from str to an integer token count.
+
+    The only requirement is that the count is monotonic in string length —
+    every real tokenizer (character counting, tiktoken, transformers) satisfies this.
+    """
+
+    def __call__(self, text: str) -> int: ...
+
+
+@dataclass(frozen=True, slots=True)
+class Chunk:
+    """A chunk of a source document.
+
+    The invariant ``document[chunk.start:chunk.end] == chunk.text`` is enforced
+    on construction: a Chunk cannot exist with a span that doesn't match its text.
+    """
+
+    text: str
+    start: int
+    end: int
+    index: int = 0
+    heading_path: tuple[str, ...] = field(default_factory=tuple)
+
+    def __post_init__(self) -> None:
+        if self.end - self.start != len(self.text):
+            raise ValueError(
+                f"Chunk span ({self.start}, {self.end}) does not match "
+                f"text length {len(self.text)}"
+            )
+        if self.start < 0 or self.end < self.start:
+            raise ValueError(f"Invalid span: ({self.start}, {self.end})")

chunkwise-0.1.0/tests/fixtures/sample.md

@@ -0,0 +1,26 @@
+# Guide
+
+Welcome to the guide. This section introduces the project.
+
+## Install
+
+Run the following command to install the package:
+
+```bash
+pip install chunkwise
+```
+
+That's all you need for the base install.
+
+## Usage
+
+```python
+from chunkwise import chunk
+
+chunks = chunk("some long document", strategy="markdown")
+```
+
+### Advanced
+
+Heading paths let you tag each chunk with its place in the document, such
+as `["Guide", "Usage", "Advanced"]`.

chunkwise-0.1.0/tests/fixtures/sample.txt

@@ -0,0 +1,8 @@
+This is the first paragraph of the sample text fixture. It contains a few
+sentences to give chunking strategies something realistic to split.
+
+This is the second paragraph. It is separated from the first by a blank
+line, which the recursive strategy should treat as the coarsest boundary.
+
+The third and final paragraph is here to make sure multi-chunk splits have
+at least three natural breakpoints to work with during tests.

chunkwise-0.1.0/tests/test_invariants.py

@@ -0,0 +1,49 @@
+"""Hypothesis property tests for the core offset invariant.
+
+The invariant: document[chunk.start:chunk.end] == chunk.text, for every
+chunk, from every strategy, on every input.
+"""
+
+from __future__ import annotations
+
+from hypothesis import given
+from hypothesis import strategies as st
+
+from chunkwise import chunk
+
+STRATEGIES = ["fixed", "recursive", "markdown"]
+
+
+@given(
+    document=st.text(min_size=0, max_size=500),
+    strategy=st.sampled_from(STRATEGIES),
+    max_tokens=st.integers(min_value=1, max_value=200),
+)
+def test_offset_invariant_holds(document: str, strategy: str, max_tokens: int) -> None:
+    chunks = chunk(document, strategy=strategy, max_tokens=max_tokens, overlap=0)
+    for c in chunks:
+        assert document[c.start : c.end] == c.text
+
+
+@given(
+    document=st.text(min_size=1, max_size=500),
+    max_tokens=st.integers(min_value=1, max_value=200),
+)
+def test_non_overlapping_chunks_reconstruct_source(document: str, max_tokens: int) -> None:
+    chunks = chunk(document, strategy="fixed", max_tokens=max_tokens, overlap=0)
+    reconstructed = "".join(c.text for c in chunks)
+    assert reconstructed == document
+
+
+@given(
+    document=st.text(min_size=0, max_size=300),
+    max_tokens=st.integers(min_value=1, max_value=100),
+)
+def test_chunks_are_never_empty_except_for_empty_document(
+    document: str, max_tokens: int
+) -> None:
+    chunks = chunk(document, strategy="recursive", max_tokens=max_tokens, overlap=0)
+    if document == "":
+        assert chunks == []
+    else:
+        assert all(len(c.text) > 0 for c in chunks)

chunkwise-0.1.0/tests/test_strategies.py

@@ -0,0 +1,59 @@
+"""Per-strategy behaviour tests."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from chunkwise import chunk
+
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def test_fixed_respects_max_tokens() -> None:
+    document = "a" * 1000
+    chunks = chunk(document, strategy="fixed", max_tokens=100, overlap=0)
+    assert all(len(c.text) <= 100 for c in chunks)
+    assert "".join(c.text for c in chunks) == document
+
+
+def test_fixed_overlap_widens_backwards() -> None:
+    document = "abcdefghijklmnopqrstuvwxyz" * 10
+    chunks = chunk(document, strategy="fixed", max_tokens=20, overlap=5)
+    assert len(chunks) > 1
+    for prev, nxt in zip(chunks, chunks[1:], strict=False):
+        assert nxt.start < prev.end
+
+
+def test_fixed_rejects_invalid_overlap() -> None:
+    with pytest.raises(ValueError):
+        chunk("some text", strategy="fixed", max_tokens=10, overlap=10)
+
+
+def test_recursive_prefers_paragraph_breaks() -> None:
+    document = (FIXTURES / "sample.txt").read_text()
+    chunks = chunk(document, strategy="recursive", max_tokens=200, overlap=0)
+    assert len(chunks) >= 1
+    for c in chunks:
+        assert document[c.start : c.end] == c.text
+
+
+def test_markdown_tags_heading_path() -> None:
+    document = (FIXTURES / "sample.md").read_text()
+    chunks = chunk(document, strategy="markdown", max_tokens=80, overlap=0)
+    assert any(c.heading_path for c in chunks)
+    install_chunks = [c for c in chunks if c.heading_path and c.heading_path[-1] == "Install"]
+    assert install_chunks
+
+
+def test_markdown_keeps_code_fence_intact() -> None:
+    document = (FIXTURES / "sample.md").read_text()
+    chunks = chunk(document, strategy="markdown", max_tokens=15, overlap=0)
+    for c in chunks:
+        assert c.text.count("```") % 2 == 0
+
+
+def test_unknown_strategy_raises() -> None:
+    with pytest.raises(ValueError):
+        chunk("text", strategy="nonexistent")

chunkwise-0.1.0/tests/test_tokenizers.py

@@ -0,0 +1,42 @@
+"""Custom tokenizer contract tests.
+
+The only requirement on a tokenizer is that it's a callable from str to int,
+monotonic in string length. These tests check chunkwise honors whatever
+tokenizer it's given rather than assuming a characters-to-tokens ratio.
+"""
+
+from __future__ import annotations
+
+from chunkwise import chunk
+
+
+def word_count_tokenizer(text: str) -> int:
+    return len(text.split())
+
+
+def test_default_tokenizer_counts_characters() -> None:
+    document = "x" * 50
+    chunks = chunk(document, strategy="fixed", max_tokens=10, overlap=0)
+    assert all(len(c.text) <= 10 for c in chunks)
+
+
+def test_custom_tokenizer_is_respected() -> None:
+    document = " ".join(["word"] * 100)
+    chunks = chunk(
+        document,
+        strategy="fixed",
+        max_tokens=10,
+        overlap=0,
+        tokenizer=word_count_tokenizer,
+    )
+    for c in chunks:
+        assert word_count_tokenizer(c.text) <= 10
+
+
+def test_tokenizer_affects_chunk_count() -> None:
+    document = " ".join(["word"] * 100)
+    char_chunks = chunk(document, strategy="fixed", max_tokens=50, overlap=0)
+    word_chunks = chunk(
+        document, strategy="fixed", max_tokens=50, overlap=0, tokenizer=word_count_tokenizer
+    )
+    assert len(char_chunks) != len(word_chunks) or len(document) == 0