tokenfit 1.0.0__tar.gz

tokenfit-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shubham Divakar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tokenfit-1.0.0/PKG-INFO

@@ -0,0 +1,223 @@
+Metadata-Version: 2.4
+Name: tokenfit
+Version: 1.0.0
+Summary: Fit your whole repo into any small model's token window — context selection for free/small LLMs.
+Home-page: https://github.com/shubham10divakar/tokenfit
+Author: Shubham Divakar
+Author-email: shubham.divakar@gmail.com
+License: MIT
+Project-URL: Source, https://github.com/shubham10divakar/tokenfit
+Project-URL: Issues, https://github.com/shubham10divakar/tokenfit/issues
+Project-URL: Examples, https://github.com/shubham10divakar/tokenfit/blob/main/EXAMPLES.md
+Keywords: llm,rag,context,huggingface,coding-agent,retrieval
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: huggingface_hub>=0.25.0
+Requires-Dist: transformers>=4.44.0
+Requires-Dist: sentence-transformers>=3.0.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: hybrid
+Requires-Dist: rank-bm25>=0.2.2; extra == "hybrid"
+Provides-Extra: chroma
+Requires-Dist: chromadb>=0.5.0; extra == "chroma"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: build>=1.2; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# tokenfit
+
+> **Fit your whole repo into any small model's token window.**
+
+`tokenfit` is a **context-selection pre-processor** for free / small LLMs. Point it at
+your project's markdown + code, ask a question, and it returns the *most relevant* slice
+of your codebase — packed to fit a tight token budget — so a 7B model with an 8k window
+answers as if it read the whole repo.
+
+[![PyPI version](https://img.shields.io/pypi/v/tokenfit)](https://pypi.org/project/tokenfit/)
+[![Downloads](https://static.pepy.tech/badge/tokenfit)](https://pepy.tech/project/tokenfit)
+[![Downloads/month](https://img.shields.io/pypi/dm/tokenfit)](https://pypi.org/project/tokenfit/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](./LICENSE)
+[![Python](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/)
+
+---
+
+## Why
+
+GitHub Copilot moved to usage-based token billing (June 2026), pushing developers toward
+cheap open-source models on HuggingFace. But free/small models have **tiny context
+windows** — dump your whole repo at them and they choke or truncate.
+
+Existing tools (`tiny-agents`, `AGENTS.md`, `SKILL.md`) inject context *raw*. tokenfit is
+the missing **retrieval layer** that makes those models punch above their weight. It's a
+pre-processor: it builds the optimal prompt, then hands it to your model or agent
+framework — it does **not** trust a weak model to call a retrieval tool correctly.
+
+## How it works
+
+```
+query
+  │
+  ▼
+1. INGEST    load AGENTS.md / SKILL.md / docs / code  →  chunk
+2. INDEX     embed chunks (BAAI/bge-small, local)     →  persist
+3. RETRIEVE  cosine top-k semantic search
+4. BUDGET    tokenizer-aware fit to N tokens + citations
+  │
+  ▼
+optimal prompt  →  any HuggingFace model
+```
+
+## Does it actually beat just dumping the files? Yes.
+
+We ran the free **Qwen2.5-Coder-7B** against [`psf/requests`](https://github.com/psf/requests)
+— **~150,000 tokens** of code, ~19× bigger than an 8000-token budget — comparing two ways
+of feeding the model, across 10 questions (`tokenfit eval --compare`):
+
+- **Naive** — concatenate the files and truncate to 8000 tokens.
+- **Retrieved** — let tokenfit pick the relevant ~2000 tokens.
+
+| | Naive (8000 tok) | **tokenfit retrieved (~2000 tok)** |
+|---|---|---|
+| Wins (of 10) | 1 (1 tie) | **9** |
+| Cites the right source file | rarely | **almost always** |
+| Tokens per call | 8000 | **~2000 (≈4× cheaper)** |
+| Failure modes | "context doesn't provide info", quoted the changelog, once **answered in Chinese**, once **invented a class that doesn't exist** | accurate, code-grounded answers |
+
+**Why naive collapses:** the whole 8000-token budget filled up with `HISTORY.md` (the
+changelog) and never reached a single source file. tokenfit semantically skips the noise
+and fetches the right module — so it's **both more accurate _and_ ~4× cheaper per call.**
+
+> 📂 Full side-by-side transcripts in **[EXAMPLES.md](./EXAMPLES.md)**.
+
+## Install
+
+```bash
+pip install tokenfit
+```
+
+Set a HuggingFace token with **"Make calls to Inference Providers"** permission:
+
+```bash
+export HF_TOKEN=hf_your_token_here      # bash
+$env:HF_TOKEN = "hf_your_token_here"    # PowerShell
+```
+
+Verify it before you run anything:
+
+```bash
+tokenfit auth            # checks the token is set and valid
+tokenfit auth --ping     # also makes a 1-token call to confirm inference access
+```
+
+## Quickstart (CLI)
+
+The fastest way — no Python required:
+
+```bash
+# Ask a question: tokenfit retrieves the right context AND gets the model's answer
+tokenfit ask "How does the auth flow work?" --repo ./my-project
+
+# Just print the selected context (no model call, pipe it anywhere)
+tokenfit context "auth flow" --repo ./my-project
+
+# Pre-build / refresh the index for a repo
+tokenfit index --repo ./my-project --rebuild
+```
+
+Useful flags: `--budget 8000` (token budget), `--top-k 12` (chunks retrieved),
+`--model Qwen/Qwen2.5-Coder-7B-Instruct` (any HF model), `--rebuild` (re-index).
+Progress prints to stderr, so the answer/context on stdout stays clean for piping.
+
+tokenfit indexes common source + doc file types out of the box (Python, JS/TS, Go,
+Rust, Java, C#, C/C++, Ruby, PHP, Swift, GDScript, shell, plus md/yaml/toml/json…).
+Indexing a different language? Add globs with `--include`:
+
+```bash
+tokenfit ask "How does combat work?" --repo ./my-godot-game --include "*.gd" --rebuild
+```
+
+📂 **See [EXAMPLES.md](./EXAMPLES.md)** for real output — a free 7B model explaining a
+Godot game's movement code, grounded in the actual source.
+
+## Quickstart (Python)
+
+```python
+from tokenfit import pack
+from tokenfit.models import TokenfitModel
+
+# Select the best ~8k tokens of context for a question
+context = pack.build(
+    query="How does the auth flow work?",
+    repo="./my-project",
+    budget=8000,
+)
+
+# Feed it to any small HF model
+model = TokenfitModel(model="Qwen/Qwen2.5-Coder-7B-Instruct")
+answer = model.chat(
+    system="You are a coding assistant for THIS project. Use only the provided context.",
+    user=f"{context}\n\nQUESTION: How does the auth flow work?",
+)
+print(answer)
+```
+
+## Validation harness
+
+tokenfit ships with an eval harness that compares **naive truncation** vs **retrieved
+context** on your own repo — the experiment that proves the approach is worth it:
+
+```bash
+tokenfit eval --repo ./my-project --mode naive
+tokenfit eval --repo ./my-project --mode retrieved
+```
+
+Each run writes a graded comparison sheet to `tokenfit/eval/results/`. Score the answers
+1–5 and compare. Edit `tokenfit/eval/dataset/questions.yaml` to fit your project.
+
+## Roadmap
+
+- [x] **Phase 0** — eval harness + naive baseline
+- [x] **Phase 1** — semantic retrieval (chunk → embed → retrieve → budget)
+- [ ] **Phase 2** — hybrid BM25 + rerank + summarization for oversized chunks
+- [ ] **Phase 3** — `tiny-agents` / `smolagents` adapters, optional Chroma backend
+
+See [`idea.md`](./idea.md) for the rationale and [`plan.md`](./plan.md) for the full plan.
+
+## Development
+
+```bash
+git clone https://github.com/shubham10divakar/tokenfit
+cd tokenfit
+pip install -e ".[dev]"
+python -m tests.test_pipeline   # dep-free regression test
+```
+
+## License
+
+MIT — see [LICENSE](./LICENSE).

tokenfit-1.0.0/README.md

@@ -0,0 +1,171 @@
+# tokenfit
+
+> **Fit your whole repo into any small model's token window.**
+
+`tokenfit` is a **context-selection pre-processor** for free / small LLMs. Point it at
+your project's markdown + code, ask a question, and it returns the *most relevant* slice
+of your codebase — packed to fit a tight token budget — so a 7B model with an 8k window
+answers as if it read the whole repo.
+
+[![PyPI version](https://img.shields.io/pypi/v/tokenfit)](https://pypi.org/project/tokenfit/)
+[![Downloads](https://static.pepy.tech/badge/tokenfit)](https://pepy.tech/project/tokenfit)
+[![Downloads/month](https://img.shields.io/pypi/dm/tokenfit)](https://pypi.org/project/tokenfit/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](./LICENSE)
+[![Python](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/)
+
+---
+
+## Why
+
+GitHub Copilot moved to usage-based token billing (June 2026), pushing developers toward
+cheap open-source models on HuggingFace. But free/small models have **tiny context
+windows** — dump your whole repo at them and they choke or truncate.
+
+Existing tools (`tiny-agents`, `AGENTS.md`, `SKILL.md`) inject context *raw*. tokenfit is
+the missing **retrieval layer** that makes those models punch above their weight. It's a
+pre-processor: it builds the optimal prompt, then hands it to your model or agent
+framework — it does **not** trust a weak model to call a retrieval tool correctly.
+
+## How it works
+
+```
+query
+  │
+  ▼
+1. INGEST    load AGENTS.md / SKILL.md / docs / code  →  chunk
+2. INDEX     embed chunks (BAAI/bge-small, local)     →  persist
+3. RETRIEVE  cosine top-k semantic search
+4. BUDGET    tokenizer-aware fit to N tokens + citations
+  │
+  ▼
+optimal prompt  →  any HuggingFace model
+```
+
+## Does it actually beat just dumping the files? Yes.
+
+We ran the free **Qwen2.5-Coder-7B** against [`psf/requests`](https://github.com/psf/requests)
+— **~150,000 tokens** of code, ~19× bigger than an 8000-token budget — comparing two ways
+of feeding the model, across 10 questions (`tokenfit eval --compare`):
+
+- **Naive** — concatenate the files and truncate to 8000 tokens.
+- **Retrieved** — let tokenfit pick the relevant ~2000 tokens.
+
+| | Naive (8000 tok) | **tokenfit retrieved (~2000 tok)** |
+|---|---|---|
+| Wins (of 10) | 1 (1 tie) | **9** |
+| Cites the right source file | rarely | **almost always** |
+| Tokens per call | 8000 | **~2000 (≈4× cheaper)** |
+| Failure modes | "context doesn't provide info", quoted the changelog, once **answered in Chinese**, once **invented a class that doesn't exist** | accurate, code-grounded answers |
+
+**Why naive collapses:** the whole 8000-token budget filled up with `HISTORY.md` (the
+changelog) and never reached a single source file. tokenfit semantically skips the noise
+and fetches the right module — so it's **both more accurate _and_ ~4× cheaper per call.**
+
+> 📂 Full side-by-side transcripts in **[EXAMPLES.md](./EXAMPLES.md)**.
+
+## Install
+
+```bash
+pip install tokenfit
+```
+
+Set a HuggingFace token with **"Make calls to Inference Providers"** permission:
+
+```bash
+export HF_TOKEN=hf_your_token_here      # bash
+$env:HF_TOKEN = "hf_your_token_here"    # PowerShell
+```
+
+Verify it before you run anything:
+
+```bash
+tokenfit auth            # checks the token is set and valid
+tokenfit auth --ping     # also makes a 1-token call to confirm inference access
+```
+
+## Quickstart (CLI)
+
+The fastest way — no Python required:
+
+```bash
+# Ask a question: tokenfit retrieves the right context AND gets the model's answer
+tokenfit ask "How does the auth flow work?" --repo ./my-project
+
+# Just print the selected context (no model call, pipe it anywhere)
+tokenfit context "auth flow" --repo ./my-project
+
+# Pre-build / refresh the index for a repo
+tokenfit index --repo ./my-project --rebuild
+```
+
+Useful flags: `--budget 8000` (token budget), `--top-k 12` (chunks retrieved),
+`--model Qwen/Qwen2.5-Coder-7B-Instruct` (any HF model), `--rebuild` (re-index).
+Progress prints to stderr, so the answer/context on stdout stays clean for piping.
+
+tokenfit indexes common source + doc file types out of the box (Python, JS/TS, Go,
+Rust, Java, C#, C/C++, Ruby, PHP, Swift, GDScript, shell, plus md/yaml/toml/json…).
+Indexing a different language? Add globs with `--include`:
+
+```bash
+tokenfit ask "How does combat work?" --repo ./my-godot-game --include "*.gd" --rebuild
+```
+
+📂 **See [EXAMPLES.md](./EXAMPLES.md)** for real output — a free 7B model explaining a
+Godot game's movement code, grounded in the actual source.
+
+## Quickstart (Python)
+
+```python
+from tokenfit import pack
+from tokenfit.models import TokenfitModel
+
+# Select the best ~8k tokens of context for a question
+context = pack.build(
+    query="How does the auth flow work?",
+    repo="./my-project",
+    budget=8000,
+)
+
+# Feed it to any small HF model
+model = TokenfitModel(model="Qwen/Qwen2.5-Coder-7B-Instruct")
+answer = model.chat(
+    system="You are a coding assistant for THIS project. Use only the provided context.",
+    user=f"{context}\n\nQUESTION: How does the auth flow work?",
+)
+print(answer)
+```
+
+## Validation harness
+
+tokenfit ships with an eval harness that compares **naive truncation** vs **retrieved
+context** on your own repo — the experiment that proves the approach is worth it:
+
+```bash
+tokenfit eval --repo ./my-project --mode naive
+tokenfit eval --repo ./my-project --mode retrieved
+```
+
+Each run writes a graded comparison sheet to `tokenfit/eval/results/`. Score the answers
+1–5 and compare. Edit `tokenfit/eval/dataset/questions.yaml` to fit your project.
+
+## Roadmap
+
+- [x] **Phase 0** — eval harness + naive baseline
+- [x] **Phase 1** — semantic retrieval (chunk → embed → retrieve → budget)
+- [ ] **Phase 2** — hybrid BM25 + rerank + summarization for oversized chunks
+- [ ] **Phase 3** — `tiny-agents` / `smolagents` adapters, optional Chroma backend
+
+See [`idea.md`](./idea.md) for the rationale and [`plan.md`](./plan.md) for the full plan.
+
+## Development
+
+```bash
+git clone https://github.com/shubham10divakar/tokenfit
+cd tokenfit
+pip install -e ".[dev]"
+python -m tests.test_pipeline   # dep-free regression test
+```
+
+## License
+
+MIT — see [LICENSE](./LICENSE).

tokenfit-1.0.0/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"

tokenfit-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tokenfit-1.0.0/setup.py

@@ -0,0 +1,66 @@
+"""Packaging for tokenfit.
+
+Build:   python -m build         (produces wheel + sdist in dist/)
+Install: pip install .           (or `pip install -e .` for development)
+"""
+
+from pathlib import Path
+
+from setuptools import find_packages, setup
+
+ROOT = Path(__file__).parent
+LONG_DESCRIPTION = (ROOT / "README.md").read_text(encoding="utf-8")
+
+setup(
+    name="tokenfit",
+    version="1.0.0",
+    description="Fit your whole repo into any small model's token window — "
+    "context selection for free/small LLMs.",
+    long_description=LONG_DESCRIPTION,
+    long_description_content_type="text/markdown",
+    author="Shubham Divakar",
+    author_email="shubham.divakar@gmail.com",
+    url="https://github.com/shubham10divakar/tokenfit",
+    project_urls={
+        "Source": "https://github.com/shubham10divakar/tokenfit",
+        "Issues": "https://github.com/shubham10divakar/tokenfit/issues",
+        "Examples": "https://github.com/shubham10divakar/tokenfit/blob/main/EXAMPLES.md",
+    },
+    license="MIT",
+    packages=find_packages(include=["tokenfit", "tokenfit.*"]),
+    include_package_data=True,
+    package_data={"tokenfit": ["eval/dataset/*.yaml"]},
+    python_requires=">=3.9",
+    install_requires=[
+        "huggingface_hub>=0.25.0",
+        "transformers>=4.44.0",
+        "sentence-transformers>=3.0.0",
+        "numpy>=1.24.0",
+        "pyyaml>=6.0",
+    ],
+    extras_require={
+        # Phase 2 features (hybrid retrieval / scalable store)
+        "hybrid": ["rank-bm25>=0.2.2"],
+        "chroma": ["chromadb>=0.5.0"],
+        "dev": ["pytest>=8.0", "build>=1.2"],
+    },
+    entry_points={
+        "console_scripts": [
+            "tokenfit=tokenfit.cli:main",
+            "tokenfit-eval=tokenfit.eval.harness:main",
+        ],
+    },
+    keywords=["llm", "rag", "context", "huggingface", "coding-agent", "retrieval"],
+    classifiers=[
+        "Development Status :: 5 - Production/Stable",
+        "Intended Audience :: Developers",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    ],
+)

tokenfit-1.0.0/tests/test_pipeline.py

@@ -0,0 +1,63 @@
+"""Dependency-free regression test for the Phase 1 pipeline.
+
+Uses a fake keyword-based embedder so it runs without torch / sentence-transformers /
+network, yet still exercises chunk -> index -> retrieve -> budget end to end.
+
+Run:  python -m tests.test_pipeline   (or: pytest tests/)
+"""
+
+from __future__ import annotations
+
+import tempfile
+
+import numpy as np
+
+from tokenfit import budget, index, retrieve
+from tokenfit.ingest import Document, chunk_documents
+
+_VOCAB = ["auth", "login", "token", "database", "persist", "config", "test", "endpoint"]
+
+
+def _fake_embed(texts, model=index.EMBED_MODEL):
+    out = []
+    for t in texts:
+        tl = t.lower()
+        v = np.array([tl.count(w) for w in _VOCAB], dtype=np.float32)
+        n = np.linalg.norm(v)
+        out.append(v / n if n else v)
+    return np.vstack(out).astype(np.float32)
+
+
+class _FakeModel:
+    def count_tokens(self, text: str) -> int:
+        return max(1, len(text) // 4)
+
+
+def test_pipeline(monkeypatch=None):
+    # patch the embedder in both modules that reference it
+    index.embed_texts = _fake_embed
+    retrieve.embed_texts = _fake_embed
+
+    docs = [
+        Document("auth.py", "def login(user):\n    # validate auth token\n    return token"),
+        Document("db.py", "def save(rec):\n    # persist to database\n    database.write(rec)"),
+        Document("conf.py", "CONFIG = {}\n# load config values here"),
+    ]
+    chunks = chunk_documents(docs, target_chars=400)
+    assert len(chunks) == 3
+
+    with tempfile.TemporaryDirectory() as d:
+        index.build_index(chunks, d)
+        assert index.index_exists(d)
+
+        hits = retrieve.retrieve("how does login auth token work", d, top_k=3)
+        assert hits[0].doc_path == "auth.py"  # semantic ranking works
+
+        packed = budget.fit_to_budget(hits, _FakeModel(), budget=40)
+        assert _FakeModel().count_tokens(packed) <= 40  # budget respected
+        assert "### FILE: auth.py@" in packed  # citations present
+
+
+if __name__ == "__main__":
+    test_pipeline()
+    print("PASSED")

tokenfit-1.0.0/tokenfit/__init__.py

@@ -0,0 +1,9 @@
+"""tokenfit — context-selection pre-processor for free/small LLMs.
+
+Given a developer query + a project corpus (md files, code, vector DB), build the
+optimal token-budgeted context to feed a small HuggingFace model.
+
+See plan.md for the phased roadmap.
+"""
+
+__version__ = "1.0.0"

tokenfit-1.0.0/tokenfit/budget.py

@@ -0,0 +1,34 @@
+"""Budgeting — pack the highest-value chunks into a token window. (Phase 1)
+
+Greedy fit by relevance order (chunks arrive already ranked). Phase 2 adds
+summarization of oversized chunks instead of dropping them.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from tokenfit.ingest import Chunk
+
+if TYPE_CHECKING:  # only for type hints; avoids pulling the inference SDK
+    from tokenfit.models import TokenfitModel
+
+_HEADER = "### FILE: {label}\n{text}"
+
+
+def fit_to_budget(chunks: list[Chunk], model: "TokenfitModel", budget: int) -> str:
+    """Concatenate ranked chunks (with file citations) up to `budget` tokens.
+
+    Each chunk carries a `FILE: path@offset` header so the model can cite sources.
+    A chunk that doesn't fit is skipped (a later, smaller chunk may still fit).
+    """
+    parts: list[str] = []
+    used = 0
+    for c in chunks:
+        block = _HEADER.format(label=c.label, text=c.text)
+        cost = model.count_tokens(block) + 2  # +2 for the joining newlines
+        if used + cost > budget:
+            continue
+        parts.append(block)
+        used += cost
+    return "\n\n".join(parts)