doclighter 0.1.0__tar.gz

doclighter-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Pratyush
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

doclighter-0.1.0/PKG-INFO

@@ -0,0 +1,245 @@
+Metadata-Version: 2.4
+Name: doclighter
+Version: 0.1.0
+Summary: A semantic Ctrl+F that paints your document with a relevance gradient.
+Author-email: Pratyush <pratyush272@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/pratyush272/doclighter
+Project-URL: Repository, https://github.com/pratyush272/doclighter
+Project-URL: Issues, https://github.com/pratyush272/doclighter/issues
+Keywords: semantic-search,embeddings,visualization,rag,nlp,pdf
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Text Processing :: Indexing
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.21
+Requires-Dist: sentence-transformers>=2.2
+Requires-Dist: pypdf>=4.0
+Requires-Dist: requests>=2.25
+Provides-Extra: quantize
+Requires-Dist: faiss-cpu>=1.7; extra == "quantize"
+Provides-Extra: streamlit
+Requires-Dist: streamlit>=1.28; extra == "streamlit"
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: pytest-cov>=4; extra == "dev"
+Requires-Dist: ruff>=0.1; extra == "dev"
+Provides-Extra: all
+Requires-Dist: doclighter[dev,quantize,streamlit]; extra == "all"
+Dynamic: license-file
+
+# doclighter
+
+[![tests](https://github.com/pratyush272/doclighter/actions/workflows/test.yml/badge.svg)](https://github.com/pratyush272/doclighter/actions/workflows/test.yml)
+[![PyPI](https://img.shields.io/pypi/v/doclighter.svg)](https://pypi.org/project/doclighter/)
+[![Python](https://img.shields.io/pypi/pyversions/doclighter.svg)](https://pypi.org/project/doclighter/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+
+**A semantic Ctrl+F that paints your document with a relevance gradient.**
+
+`doclighter` is what you reach for when you need to *see* where a topic lives in a document — not be told an answer. It embeds your document at fine granularity, then projects query relevance back onto every word as a heatmap. No LLM, no hallucination, no top-K cliff.
+
+```python
+from doclighter import Doclighter
+
+doc = Doclighter.from_pdf("contract.pdf")
+result = doc.search("termination clauses")
+result  # In Jupyter: renders the whole document, color-coded by relevance
+```
+
+## Why this exists
+
+Traditional RAG hands an LLM the top 3–10 chunks and asks it to generate an answer. That's great when you trust the answer and don't need to read the source. But sometimes you *need to read the source* — legal review, paper skimming, contract diffing, due diligence — and you want a tool that helps you *navigate* a long document, not summarize it away.
+
+`doclighter` is for that. It treats the whole document as the output, and re-colors it by semantic relevance to your query. Hot regions deserve your attention; cold regions you can skim past.
+
+It's deterministic, fast (sub-100ms per query after indexing), and shows you the long tail — including the case where your query *doesn't* match anything (everything stays cold blue, which is itself useful information that RAG hides).
+
+## Install
+
+```bash
+pip install doclighter
+```
+
+Optional extras:
+```bash
+pip install "doclighter[quantize]"   # FAISS SQ8 index for very large docs
+pip install "doclighter[streamlit]"  # for the interactive demo app
+pip install "doclighter[dev]"        # for contributors
+```
+
+## Quickstart
+
+### Load a document
+
+```python
+from doclighter import Doclighter
+
+# From a PDF on disk
+doc = Doclighter.from_pdf("contract.pdf")
+
+# From a PDF URL
+doc = Doclighter.from_url("https://example.com/contract.pdf")
+
+# From raw text
+doc = Doclighter.from_text(open("paper.txt").read())
+```
+
+The first call downloads the default embedding model (~80 MB MiniLM) and embeds your document. For a ~10K word doc this takes ~25 seconds. Subsequent searches reuse the index.
+
+### Search
+
+```python
+result = doc.search("termination clauses")
+
+result.word_scores      # numpy array, shape (n_words,), values in [0, 1]
+result.top_chunks(k=10) # list of (chunk_text, score, (start, end))
+result.elapsed_ms       # ~10-50ms for typical docs
+result.to_html()        # HTML string for display anywhere
+```
+
+In Jupyter, just put `result` on the last line of a cell — it renders the heatmap inline.
+
+### Zoom: the `decay_sigma` knob
+
+The differentiating feature. `decay_sigma` controls how far semantic warmth spreads from a matched region:
+
+```python
+narrow = doc.search("termination", decay_sigma=5.0)   # sharp word-level highlights
+broad  = doc.search("termination", decay_sigma=80.0)  # broad thematic regions
+```
+
+Same index, no re-embedding. Drag the σ slider in the Streamlit demo to feel what this does.
+
+### Multi-query
+
+```python
+result = doc.search(
+    ["termination", "indemnification", "labour wages"],
+    multi_query_aggregate="max",  # or "sum" to favor regions matching multiple
+)
+```
+
+### Save / load the index
+
+Embedding is the slow step. Save once, reuse:
+
+```python
+doc.save("contract.idx")
+doc = Doclighter.load("contract.idx")
+```
+
+### Bring your own embedder
+
+Any callable mapping `list[str] -> np.ndarray` of shape `(N, dim)` works:
+
+```python
+from sentence_transformers import SentenceTransformer
+
+bge = SentenceTransformer("BAAI/bge-small-en-v1.5")
+doc = Doclighter.from_text(text, embedder=bge.encode)
+```
+
+## Streamlit demo
+
+```bash
+pip install "doclighter[streamlit]"
+streamlit run examples/streamlit_app.py
+```
+
+A working UI with PDF upload, query box, σ slider, and live re-rendering.
+
+## How it works
+
+1. **Chunk** the document into small rolling windows (default: 12 words, 50% overlap).
+2. **Embed** each chunk with sentence-transformers (default: `all-MiniLM-L6-v2`).
+3. **Score** each chunk against your query via cosine similarity.
+4. **Project** chunk scores back onto every word via exponential proximity decay:
+
+   ```
+   word_score[w] = max over chunks c of  raw[c] × exp(-distance(w, c) / sigma)
+   ```
+
+5. **Render** the document as colored HTML — words inherit warmth from their nearest semantically matched chunk.
+
+Step 4 is the interesting one. Max-aggregation (rather than sum) means a word's color reflects its single strongest semantic neighbor — visually intuitive and resistant to "many lukewarm chunks add up to red" noise.
+
+## How this compares to RAG
+
+`doclighter` and RAG solve different problems:
+
+| | RAG | doclighter |
+|---|---|---|
+| Output | LLM-generated answer | Document, recolored |
+| Best for | "What's the answer?" | "Where in the doc?" |
+| When query has no answer | LLM hedges / hallucinates | Document stays cold (honest) |
+| Hides chunk boundaries | No — top-K cliff | Yes — gradient smooths over |
+| Cost per query | LLM tokens | Free (one matmul) |
+| Determinism | Sampling-dependent | Fully deterministic |
+
+They're complementary. Use RAG when you trust the LLM with the question; use `doclighter` when you need to read the source yourself.
+
+The algorithmic kernel is conceptually related to [ColBERT](https://github.com/stanford-futuredata/ColBERT)'s MaxSim late-interaction, but applied to *visualization* rather than ranking, and at the word-level rather than the token-level.
+
+## API reference
+
+### `Doclighter(text, **kwargs)`
+
+| Parameter | Default | Description |
+|---|---|---|
+| `text` | required | The document as a string |
+| `chunk_size` | `12` | Words per rolling window |
+| `chunk_overlap` | `0.5` | Fraction overlap between windows |
+| `embedder` | `None` | Custom embedder callable (default: MiniLM) |
+| `embedding_model` | `"all-MiniLM-L6-v2"` | sentence-transformers model name |
+| `decay_sigma` | `20.0` | Default proximity decay scale (words) |
+| `quantize` | `False` | Use SQ8 FAISS index instead of flat exact |
+| `quantize_rerank_k` | `200` | If quantize=True, rerank top-K with exact |
+
+Alternate constructors: `Doclighter.from_text(...)`, `Doclighter.from_pdf(path, ...)`, `Doclighter.from_url(url, ...)`.
+
+### `doc.search(query, **kwargs) -> SearchResult`
+
+| Parameter | Default | Description |
+|---|---|---|
+| `query` | required | A string, or list of strings for multi-query |
+| `decay_sigma` | `None` | Override the doc's default sigma |
+| `multi_query_aggregate` | `"max"` | `"max"`, `"mean"`, or `"sum"` for multi-query |
+
+### `SearchResult`
+
+| Attribute / method | Returns |
+|---|---|
+| `.word_scores` | `np.ndarray` of shape `(n_words,)`, in `[0, 1]` |
+| `.chunk_scores` | `np.ndarray` of raw per-chunk cosine scores |
+| `.top_chunks(k=10)` | `list[(text, score, (start, end))]` |
+| `.to_html(**kwargs)` | HTML string of the heatmap-colored document |
+| `.elapsed_ms` | Search latency |
+
+## Development
+
+```bash
+git clone https://github.com/pratyush272/doclighter
+cd doclighter
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+MIT. See [LICENSE](LICENSE).
+
+## Citation / acknowledgement
+
+If you use `doclighter` in research, a link back is appreciated. The proximity-decay scoring idea borrows from passage-retrieval literature; max-aggregation over fine-grained matches is in spirit closest to ColBERT.

doclighter-0.1.0/README.md

@@ -0,0 +1,204 @@
+# doclighter
+
+[![tests](https://github.com/pratyush272/doclighter/actions/workflows/test.yml/badge.svg)](https://github.com/pratyush272/doclighter/actions/workflows/test.yml)
+[![PyPI](https://img.shields.io/pypi/v/doclighter.svg)](https://pypi.org/project/doclighter/)
+[![Python](https://img.shields.io/pypi/pyversions/doclighter.svg)](https://pypi.org/project/doclighter/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+
+**A semantic Ctrl+F that paints your document with a relevance gradient.**
+
+`doclighter` is what you reach for when you need to *see* where a topic lives in a document — not be told an answer. It embeds your document at fine granularity, then projects query relevance back onto every word as a heatmap. No LLM, no hallucination, no top-K cliff.
+
+```python
+from doclighter import Doclighter
+
+doc = Doclighter.from_pdf("contract.pdf")
+result = doc.search("termination clauses")
+result  # In Jupyter: renders the whole document, color-coded by relevance
+```
+
+## Why this exists
+
+Traditional RAG hands an LLM the top 3–10 chunks and asks it to generate an answer. That's great when you trust the answer and don't need to read the source. But sometimes you *need to read the source* — legal review, paper skimming, contract diffing, due diligence — and you want a tool that helps you *navigate* a long document, not summarize it away.
+
+`doclighter` is for that. It treats the whole document as the output, and re-colors it by semantic relevance to your query. Hot regions deserve your attention; cold regions you can skim past.
+
+It's deterministic, fast (sub-100ms per query after indexing), and shows you the long tail — including the case where your query *doesn't* match anything (everything stays cold blue, which is itself useful information that RAG hides).
+
+## Install
+
+```bash
+pip install doclighter
+```
+
+Optional extras:
+```bash
+pip install "doclighter[quantize]"   # FAISS SQ8 index for very large docs
+pip install "doclighter[streamlit]"  # for the interactive demo app
+pip install "doclighter[dev]"        # for contributors
+```
+
+## Quickstart
+
+### Load a document
+
+```python
+from doclighter import Doclighter
+
+# From a PDF on disk
+doc = Doclighter.from_pdf("contract.pdf")
+
+# From a PDF URL
+doc = Doclighter.from_url("https://example.com/contract.pdf")
+
+# From raw text
+doc = Doclighter.from_text(open("paper.txt").read())
+```
+
+The first call downloads the default embedding model (~80 MB MiniLM) and embeds your document. For a ~10K word doc this takes ~25 seconds. Subsequent searches reuse the index.
+
+### Search
+
+```python
+result = doc.search("termination clauses")
+
+result.word_scores      # numpy array, shape (n_words,), values in [0, 1]
+result.top_chunks(k=10) # list of (chunk_text, score, (start, end))
+result.elapsed_ms       # ~10-50ms for typical docs
+result.to_html()        # HTML string for display anywhere
+```
+
+In Jupyter, just put `result` on the last line of a cell — it renders the heatmap inline.
+
+### Zoom: the `decay_sigma` knob
+
+The differentiating feature. `decay_sigma` controls how far semantic warmth spreads from a matched region:
+
+```python
+narrow = doc.search("termination", decay_sigma=5.0)   # sharp word-level highlights
+broad  = doc.search("termination", decay_sigma=80.0)  # broad thematic regions
+```
+
+Same index, no re-embedding. Drag the σ slider in the Streamlit demo to feel what this does.
+
+### Multi-query
+
+```python
+result = doc.search(
+    ["termination", "indemnification", "labour wages"],
+    multi_query_aggregate="max",  # or "sum" to favor regions matching multiple
+)
+```
+
+### Save / load the index
+
+Embedding is the slow step. Save once, reuse:
+
+```python
+doc.save("contract.idx")
+doc = Doclighter.load("contract.idx")
+```
+
+### Bring your own embedder
+
+Any callable mapping `list[str] -> np.ndarray` of shape `(N, dim)` works:
+
+```python
+from sentence_transformers import SentenceTransformer
+
+bge = SentenceTransformer("BAAI/bge-small-en-v1.5")
+doc = Doclighter.from_text(text, embedder=bge.encode)
+```
+
+## Streamlit demo
+
+```bash
+pip install "doclighter[streamlit]"
+streamlit run examples/streamlit_app.py
+```
+
+A working UI with PDF upload, query box, σ slider, and live re-rendering.
+
+## How it works
+
+1. **Chunk** the document into small rolling windows (default: 12 words, 50% overlap).
+2. **Embed** each chunk with sentence-transformers (default: `all-MiniLM-L6-v2`).
+3. **Score** each chunk against your query via cosine similarity.
+4. **Project** chunk scores back onto every word via exponential proximity decay:
+
+   ```
+   word_score[w] = max over chunks c of  raw[c] × exp(-distance(w, c) / sigma)
+   ```
+
+5. **Render** the document as colored HTML — words inherit warmth from their nearest semantically matched chunk.
+
+Step 4 is the interesting one. Max-aggregation (rather than sum) means a word's color reflects its single strongest semantic neighbor — visually intuitive and resistant to "many lukewarm chunks add up to red" noise.
+
+## How this compares to RAG
+
+`doclighter` and RAG solve different problems:
+
+| | RAG | doclighter |
+|---|---|---|
+| Output | LLM-generated answer | Document, recolored |
+| Best for | "What's the answer?" | "Where in the doc?" |
+| When query has no answer | LLM hedges / hallucinates | Document stays cold (honest) |
+| Hides chunk boundaries | No — top-K cliff | Yes — gradient smooths over |
+| Cost per query | LLM tokens | Free (one matmul) |
+| Determinism | Sampling-dependent | Fully deterministic |
+
+They're complementary. Use RAG when you trust the LLM with the question; use `doclighter` when you need to read the source yourself.
+
+The algorithmic kernel is conceptually related to [ColBERT](https://github.com/stanford-futuredata/ColBERT)'s MaxSim late-interaction, but applied to *visualization* rather than ranking, and at the word-level rather than the token-level.
+
+## API reference
+
+### `Doclighter(text, **kwargs)`
+
+| Parameter | Default | Description |
+|---|---|---|
+| `text` | required | The document as a string |
+| `chunk_size` | `12` | Words per rolling window |
+| `chunk_overlap` | `0.5` | Fraction overlap between windows |
+| `embedder` | `None` | Custom embedder callable (default: MiniLM) |
+| `embedding_model` | `"all-MiniLM-L6-v2"` | sentence-transformers model name |
+| `decay_sigma` | `20.0` | Default proximity decay scale (words) |
+| `quantize` | `False` | Use SQ8 FAISS index instead of flat exact |
+| `quantize_rerank_k` | `200` | If quantize=True, rerank top-K with exact |
+
+Alternate constructors: `Doclighter.from_text(...)`, `Doclighter.from_pdf(path, ...)`, `Doclighter.from_url(url, ...)`.
+
+### `doc.search(query, **kwargs) -> SearchResult`
+
+| Parameter | Default | Description |
+|---|---|---|
+| `query` | required | A string, or list of strings for multi-query |
+| `decay_sigma` | `None` | Override the doc's default sigma |
+| `multi_query_aggregate` | `"max"` | `"max"`, `"mean"`, or `"sum"` for multi-query |
+
+### `SearchResult`
+
+| Attribute / method | Returns |
+|---|---|
+| `.word_scores` | `np.ndarray` of shape `(n_words,)`, in `[0, 1]` |
+| `.chunk_scores` | `np.ndarray` of raw per-chunk cosine scores |
+| `.top_chunks(k=10)` | `list[(text, score, (start, end))]` |
+| `.to_html(**kwargs)` | HTML string of the heatmap-colored document |
+| `.elapsed_ms` | Search latency |
+
+## Development
+
+```bash
+git clone https://github.com/pratyush272/doclighter
+cd doclighter
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+MIT. See [LICENSE](LICENSE).
+
+## Citation / acknowledgement
+
+If you use `doclighter` in research, a link back is appreciated. The proximity-decay scoring idea borrows from passage-retrieval literature; max-aggregation over fine-grained matches is in spirit closest to ColBERT.

doclighter-0.1.0/doclighter/__init__.py

@@ -0,0 +1,21 @@
+"""Doclighter — a semantic Ctrl+F that paints your document with relevance.
+
+See https://github.com/pratyush272/doclighter for docs.
+"""
+from .core import Doclighter, SearchResult
+from .chunking import Chunk, make_chunks
+from .scoring import word_heatmap, aggregate_multi_query
+from .render import render_html, score_to_hex
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Doclighter",
+    "SearchResult",
+    "Chunk",
+    "make_chunks",
+    "word_heatmap",
+    "aggregate_multi_query",
+    "render_html",
+    "score_to_hex",
+]

doclighter-0.1.0/doclighter/chunking.py

@@ -0,0 +1,58 @@
+"""Rolling window chunking.
+
+Small word-window chunks (default 12 words, 50% overlap) are the unit of
+semantic match. This is deliberately finer than typical RAG chunking
+(256-1024 tokens) because Doclighter is a visualization tool, not a
+context-window filler — fine chunks give fine spatial resolution.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import List
+
+
+@dataclass(frozen=True)
+class Chunk:
+    """A single rolling window over the document word list."""
+
+    text: str
+    start: int   # inclusive word index
+    end: int     # exclusive word index
+
+
+def make_chunks(
+    words: List[str],
+    chunk_size: int = 12,
+    overlap: float = 0.5,
+) -> List[Chunk]:
+    """Split a word list into rolling windows.
+
+    Parameters
+    ----------
+    words : list of str
+        Tokenized document (typically ``text.split()``).
+    chunk_size : int
+        Words per window. Default 12 — small enough that semantic units rarely
+        get cut, large enough that MiniLM produces a useful embedding.
+    overlap : float
+        Fraction of overlap between consecutive windows, in [0, 1).
+        Default 0.5 means 50%% overlap (step = chunk_size / 2).
+
+    Returns
+    -------
+    list of Chunk
+    """
+    if not 0 <= overlap < 1:
+        raise ValueError(f"overlap must be in [0, 1), got {overlap}")
+    if chunk_size < 1:
+        raise ValueError(f"chunk_size must be >= 1, got {chunk_size}")
+
+    step = max(1, int(chunk_size * (1 - overlap)))
+    chunks: List[Chunk] = []
+    for i in range(0, len(words), step):
+        window = words[i : i + chunk_size]
+        if window:
+            chunks.append(Chunk(text=" ".join(window), start=i, end=i + len(window)))
+        if i + chunk_size >= len(words):
+            break  # last window already covers tail
+    return chunks