ragless 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. {ragless-0.1.0 → ragless-0.2.0}/.github/workflows/ci.yml +1 -1
  2. ragless-0.2.0/CHANGELOG.md +52 -0
  3. ragless-0.2.0/DECISIONS.md +109 -0
  4. {ragless-0.1.0 → ragless-0.2.0}/PKG-INFO +108 -2
  5. ragless-0.2.0/PROGRESS.md +23 -0
  6. {ragless-0.1.0 → ragless-0.2.0}/README.md +105 -1
  7. ragless-0.2.0/examples/context_for_llm.py +32 -0
  8. ragless-0.2.0/examples/prompt_templates.py +35 -0
  9. {ragless-0.1.0 → ragless-0.2.0}/pyproject.toml +2 -1
  10. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/__init__.py +31 -1
  11. ragless-0.2.0/src/ragless/context/__init__.py +153 -0
  12. ragless-0.2.0/src/ragless/context/_lexical.py +84 -0
  13. ragless-0.2.0/src/ragless/context/passage.py +177 -0
  14. ragless-0.2.0/src/ragless/context/pipeline.py +318 -0
  15. ragless-0.2.0/src/ragless/context/query_expansion.py +39 -0
  16. ragless-0.2.0/src/ragless/context/tokens.py +33 -0
  17. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/core.py +154 -0
  18. ragless-0.2.0/tests/test_context_integration.py +205 -0
  19. ragless-0.2.0/tests/test_context_output.py +132 -0
  20. ragless-0.2.0/tests/test_context_pipeline.py +273 -0
  21. ragless-0.2.0/tests/test_context_tokens.py +36 -0
  22. ragless-0.2.0/tests/test_examples.py +41 -0
  23. ragless-0.1.0/CHANGELOG.md +0 -25
  24. ragless-0.1.0/DECISIONS.md +0 -105
  25. ragless-0.1.0/PROGRESS.md +0 -86
  26. {ragless-0.1.0 → ragless-0.2.0}/.github/workflows/publish.yml +0 -0
  27. {ragless-0.1.0 → ragless-0.2.0}/.gitignore +0 -0
  28. {ragless-0.1.0 → ragless-0.2.0}/LICENSE +0 -0
  29. {ragless-0.1.0 → ragless-0.2.0}/examples/basic_search.py +0 -0
  30. {ragless-0.1.0 → ragless-0.2.0}/examples/filters_and_metadata.py +0 -0
  31. {ragless-0.1.0 → ragless-0.2.0}/examples/in_memory_text.py +0 -0
  32. {ragless-0.1.0 → ragless-0.2.0}/examples/sample_docs/guides/caching.md +0 -0
  33. {ragless-0.1.0 → ragless-0.2.0}/examples/sample_docs/installation.md +0 -0
  34. {ragless-0.1.0 → ragless-0.2.0}/examples/sample_docs/searching.md +0 -0
  35. {ragless-0.1.0 → ragless-0.2.0}/examples/semantic_and_rerank.py +0 -0
  36. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/cache.py +0 -0
  37. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/chunking.py +0 -0
  38. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/embedders/__init__.py +0 -0
  39. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/embedders/base.py +0 -0
  40. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/embedders/fastembed_embedder.py +0 -0
  41. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/exceptions.py +0 -0
  42. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/fusion.py +0 -0
  43. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/rerank.py +0 -0
  44. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/retrievers/__init__.py +0 -0
  45. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/retrievers/base.py +0 -0
  46. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/retrievers/bm25.py +0 -0
  47. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/retrievers/fuzzy.py +0 -0
  48. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/retrievers/semantic.py +0 -0
  49. {ragless-0.1.0 → ragless-0.2.0}/src/ragless/types.py +0 -0
  50. {ragless-0.1.0 → ragless-0.2.0}/tests/conftest.py +0 -0
  51. {ragless-0.1.0 → ragless-0.2.0}/tests/test_bm25.py +0 -0
  52. {ragless-0.1.0 → ragless-0.2.0}/tests/test_cache.py +0 -0
  53. {ragless-0.1.0 → ragless-0.2.0}/tests/test_chunking.py +0 -0
  54. {ragless-0.1.0 → ragless-0.2.0}/tests/test_core.py +0 -0
  55. {ragless-0.1.0 → ragless-0.2.0}/tests/test_fusion.py +0 -0
  56. {ragless-0.1.0 → ragless-0.2.0}/tests/test_fuzzy.py +0 -0
  57. {ragless-0.1.0 → ragless-0.2.0}/tests/test_missing_extra.py +0 -0
  58. {ragless-0.1.0 → ragless-0.2.0}/tests/test_rerank.py +0 -0
  59. {ragless-0.1.0 → ragless-0.2.0}/tests/test_semantic.py +0 -0
  60. {ragless-0.1.0 → ragless-0.2.0}/tests/test_semantic_real_model.py +0 -0
@@ -45,7 +45,7 @@ jobs:
45
45
  - uses: actions/setup-python@v5
46
46
  with:
47
47
  python-version: "3.12"
48
- - run: pip install -e ".[dev,semantic]"
48
+ - run: pip install -e ".[dev,semantic,tokens]"
49
49
  - run: pytest -q --run-slow -m slow
50
50
 
51
51
  build:
@@ -0,0 +1,52 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/).
7
+
8
+ ## [0.2.0] - 2026-11-12
9
+
10
+ ### Added
11
+
12
+ - `Retriever.context()`: turns search results into a `Context` fitted to a
13
+ token budget, with citations, inclusion `reasons`, and matched-term
14
+ highlight offsets.
15
+ - `ragless.context` pipeline: composable, swappable steps --
16
+ `WindowExpand` (neighboring-line expansion), `MergeAdjacent` (combines
17
+ overlapping/adjacent hits from the same file with correct line ranges),
18
+ `Dedup` (fuzzy near-duplicate removal), `Diversify` (Maximal Marginal
19
+ Relevance, using the semantic embedder when available or a lexical
20
+ fallback otherwise), `Compress` (extractive sentence/line selection --
21
+ never rewrites text), and `Pack` (greedy token-budget fitting that never
22
+ exceeds the budget).
23
+ - `Context.to_text()/.to_markdown()/.to_xml()/.to_json()` output formats.
24
+ - `Retriever.prompt()`: renders a `Context` into a prompt string via a
25
+ default template, a custom `str.format` template, or a custom callable.
26
+ `ragless` still never calls an LLM anywhere.
27
+ - Pseudo-relevance feedback query expansion (`query_expansion=True`,
28
+ off by default): RM3-style term expansion from the first pass's top
29
+ results, re-searched and merged.
30
+ - Dependency-free token estimation by default; exact counts via
31
+ `ragless.context.TiktokenCounter` under the new `tokens` extra
32
+ (`pip install 'ragless[tokens]'`).
33
+ - New examples: `context_for_llm.py`, `prompt_templates.py`.
34
+
35
+ ## [0.1.0] - 2026-11-12
36
+
37
+ ### Added
38
+
39
+ - `Retriever`: library-first API indexing a directory or in-memory text.
40
+ - `LineChunker`: overlapping line-window chunking that preserves file path
41
+ and 1-indexed start/end line numbers.
42
+ - `BM25Retriever`: Okapi BM25 keyword search (pure NumPy, no extra deps).
43
+ - `FuzzyRetriever`: fuzzy string matching via `rapidfuzz`.
44
+ - `SemanticRetriever`: local embedding search (`fastembed`, optional
45
+ `semantic` extra) with NumPy cosine similarity.
46
+ - Reciprocal Rank Fusion (`reciprocal_rank_fusion`) to combine engine
47
+ rankings, with configurable weights and `k`.
48
+ - `CrossEncoderReranker`: optional cross-encoder reranking of fused results.
49
+ - On-disk embedding cache (`EmbeddingCache`) as plain `.npy`/`.json` sidecar
50
+ files, keyed by file mtime/size, so only changed files are re-embedded.
51
+ - Metadata and path filters on `search()`.
52
+ - Pluggable chunkers, retrievers, and embedders.
@@ -0,0 +1,109 @@
1
+ # v0.2 design decisions
2
+
3
+ Working notes for the `ragless.context` feature (the "A" in RAG). Not shipped
4
+ to PyPI as documentation; this is an internal log of choices made while
5
+ building autonomously, so the reasoning doesn't have to be reconstructed later.
6
+
7
+ ## Module layout
8
+
9
+ `ragless/context/`:
10
+ - `tokens.py` -- `estimate_tokens` (chars/4 heuristic, no deps) and
11
+ `TiktokenCounter` (lazy-imports `tiktoken`, requires the new `tokens` extra).
12
+ - `passage.py` -- `Passage`, `Citation`, `Context` (with `to_text/to_markdown/
13
+ to_xml/to_json`) and the default prompt template.
14
+ - `pipeline.py` -- `ContextStep` base class and the six built-in steps
15
+ (`WindowExpand`, `MergeAdjacent`, `Dedup`, `Diversify`, `Compress`, `Pack`),
16
+ plus `build_default_pipeline()`.
17
+ - `query_expansion.py` -- RM3-style pseudo-relevance feedback, off by default.
18
+ - `__init__.py` -- `build_context()` orchestration + public re-exports.
19
+
20
+ `Retriever.context()` and `Retriever.prompt()` live in `core.py` (lazy-import
21
+ `ragless.context` to avoid a cycle) so the public entry point stays
22
+ `r.context(...)` / `r.prompt(...)`, matching the existing `r.search(...)`
23
+ style instead of a separate builder object the user has to construct.
24
+
25
+ ## Why a step-based pipeline instead of one big function
26
+
27
+ Each step is a small class implementing `run(passages, state) -> passages`.
28
+ `state` (`PipelineState`) carries the query, token budget/counter, and a
29
+ document-text/embedder accessor so steps don't each need their own
30
+ constructor wiring for things that are per-call, not per-step. Steps only
31
+ take constructor args for their own tunables (window size, dedup threshold,
32
+ MMR lambda, ...). This lets `pipeline=[...]` fully replace the default list,
33
+ and a future graph-expansion step (v0.3) only needs to append/prepend another
34
+ step that adds candidate passages with its own `reasons` entry -- it doesn't
35
+ need to know about merge/dedup/pack internals.
36
+
37
+ ## Passage identity and merging across chunk boundaries
38
+
39
+ Merging adjacent/overlapping hits needs the *original* document text, not
40
+ just concatenated chunk text, because `LineChunker` chunks overlap by
41
+ default -- naively joining two chunks would duplicate the overlapping lines.
42
+ So `Retriever` now also keeps `self._doc_text: dict[str, str]` (doc key ->
43
+ full original text) and `self._chunk_doc_id: dict[str, str]` (chunk id -> doc
44
+ key), built during the existing chunking loop in `__init__`. Both are
45
+ private/internal; chunking itself and the public `Chunk` dataclass are
46
+ unchanged, so this doesn't touch the frozen `Chunk` API or any existing test
47
+ that constructs `Chunk(...)` directly. Doc keys are the same `path` or
48
+ `f"mem:{i}"` prefixes already used for chunk ids, so merged/expanded passages
49
+ from in-memory text get a citation id like `mem:0` instead of a path.
50
+
51
+ `MergeAdjacent` and `WindowExpand` re-slice from `self._doc_text[doc_id]`
52
+ by line range rather than editing chunk text, so a merged passage's text is
53
+ always an exact line-range slice of the source, and `start_line`/`end_line`
54
+ stay accurate for citations.
55
+
56
+ ## Diversity (MMR)
57
+
58
+ Uses the retriever's own semantic embedder when `engines` includes
59
+ `"semantic"` (found via `retriever._retrievers.get("semantic").embedder`);
60
+ otherwise falls back to a lexical similarity matrix from
61
+ `rapidfuzz.fuzz.token_sort_ratio`. Either way `Diversify` only needs a
62
+ `similarity(i, j) -> float in [0, 1]` function, so the embedder dependency is
63
+ optional and never required at import time.
64
+
65
+ ## Extractive compression
66
+
67
+ Sentence-level for prose, line/block-level for recognized code file
68
+ extensions (`_CODE_EXTS`). Scoring is lexical token overlap with the query
69
+ (no embeddings required, since `compress` has to work without the `semantic`
70
+ extra too). Compression only *selects* a subsequence of the original
71
+ sentences/lines -- it never rewrites text -- and joins non-contiguous kept
72
+ spans with a literal `"\n[...]\n"` marker so the output is still clearly
73
+ attributable to the source. Tests assert every span between markers is an
74
+ exact substring of the original passage text.
75
+
76
+ ## Token budget (`Pack`)
77
+
78
+ Greedy: steps through passages in their current order and keeps a passage
79
+ only if it still fits the remaining budget, so the total is a hard max, not a
80
+ best-effort trim -- "budget smaller than one passage" yields an empty
81
+ `Context` rather than a truncated one, since clipping mid-passage would also
82
+ clip mid-citation. Default token counter is a dependency-free chars/4
83
+ estimate; `TiktokenCounter` (needs `pip install ragless[tokens]`) or a fully
84
+ custom `Callable[[str], int]` can replace it via `token_counter=`.
85
+
86
+ ## Query expansion (RM3-style)
87
+
88
+ Off by default (`query_expansion=False` on `context()`). When enabled: run
89
+ the initial search, pull term frequencies from the top few hits' text
90
+ (excluding a small stopword list and terms already in the query), append the
91
+ top N new terms to the query, re-search, and merge candidate pools by chunk
92
+ id (keep the max score). If no new terms clear the frequency threshold
93
+ (small corpus, all terms already in the query, etc.) it falls back to the
94
+ unexpanded candidates untouched rather than erroring.
95
+
96
+ ## Output formats and prompts
97
+
98
+ `Context.to_text/to_markdown/to_xml/to_json()` are plain string builders, no
99
+ templating engine. `r.prompt(query, template=..., **context_kwargs)` accepts
100
+ either a `str.format`-style template with `{query}`/`{context}` placeholders
101
+ (context rendered via `context_format=` before being formatted in) or a
102
+ callable `(query, Context) -> str` for full control. `ragless` itself never
103
+ imports an LLM client or makes a network call anywhere in this module.
104
+
105
+ ## No new required dependencies
106
+
107
+ `ragless.context` only imports `numpy` and `rapidfuzz` (already required by
108
+ v0.1) plus the stdlib. `tiktoken` lives behind the new `tokens` extra and is
109
+ only imported inside `TiktokenCounter.__init__`.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: ragless
3
- Version: 0.1.0
3
+ Version: 0.2.0
4
4
  Summary: File-based retrieval for RAG, no vector database required
5
5
  Project-URL: Homepage, https://github.com/mohmdqasim/ragless
6
6
  Project-URL: Repository, https://github.com/mohmdqasim/ragless
@@ -31,6 +31,8 @@ Requires-Dist: pytest>=7.4; extra == 'dev'
31
31
  Requires-Dist: ruff>=0.6; extra == 'dev'
32
32
  Provides-Extra: semantic
33
33
  Requires-Dist: fastembed>=0.3; extra == 'semantic'
34
+ Provides-Extra: tokens
35
+ Requires-Dist: tiktoken>=0.7; extra == 'tokens'
34
36
  Description-Content-Type: text/markdown
35
37
 
36
38
  # ragless
@@ -151,6 +153,98 @@ r = Retriever("./docs", engines={"bm25": MyRetriever(), "fuzzy": FuzzyRetriever(
151
153
  Implement `ragless.chunking.Chunker`, `ragless.retrievers.base.BaseRetriever`,
152
154
  or `ragless.embedders.base.Embedder` to plug in your own.
153
155
 
156
+ ## Context for your LLM
157
+
158
+ `ragless` handles everything up to the LLM call: `r.context()` turns a query
159
+ into ready-to-use, budget-fitted context. There's no LLM inside `ragless` --
160
+ it never makes a network call or imports an LLM client; it just hands you the
161
+ material to send to whichever model you call yourself.
162
+
163
+ ```python
164
+ from ragless import Retriever
165
+
166
+ r = Retriever("./docs")
167
+ ctx = r.context("how do I configure retries", max_tokens=2000)
168
+
169
+ print(ctx.token_count) # never exceeds max_tokens
170
+ for citation in ctx.citations:
171
+ print(citation) # [1] docs/retries.md:12-34
172
+ ```
173
+
174
+ Under the hood, `context()` runs a pipeline of small, swappable steps:
175
+ expand hits with neighboring lines, merge adjacent hits from the same file,
176
+ drop near-duplicates, diversify with MMR, optionally compress to the most
177
+ relevant sentences/lines, then pack whole passages into the token budget
178
+ (never truncating mid-passage). Every passage keeps a `reasons` list (e.g.
179
+ `["direct match"]`, `["direct match", "neighbor window"]`) explaining why it
180
+ was included.
181
+
182
+ ### Output formats
183
+
184
+ ```python
185
+ ctx.to_text() # plain text with trailing "[n] path:start-end" citations
186
+ ctx.to_markdown() # fenced code blocks for code paths, blockquotes for prose
187
+ ctx.to_xml() # <context><passage index="1" ref="...">...</passage></context>
188
+ ctx.to_json() # full structure, including matched-term highlight offsets
189
+ ctx.to_text(mark=True) # wraps matched query terms, e.g. with >>term<<
190
+ ```
191
+
192
+ ### Prompts -- still no LLM call
193
+
194
+ ```python
195
+ prompt = r.prompt(
196
+ "how do I configure retries",
197
+ max_tokens=2000,
198
+ template="Context:\n{context}\n\nQuestion: {query}\nAnswer:", # optional
199
+ )
200
+
201
+ # Send it to any client yourself:
202
+ # response = my_llm_client.complete(prompt)
203
+ ```
204
+
205
+ ### Tuning the pipeline
206
+
207
+ ```python
208
+ r.context(
209
+ "query",
210
+ max_tokens=2000,
211
+ window=3, # lines of neighboring context to pull in
212
+ dedup_threshold=90, # fuzzy-ratio cutoff for near-duplicates
213
+ diversify=True, # reorder by Maximal Marginal Relevance
214
+ compress=True, # keep only the most query-relevant sentences/lines
215
+ query_expansion=True, # pseudo-relevance feedback (RM3-style), off by default
216
+ )
217
+ ```
218
+
219
+ Exact token counts (instead of the dependency-free default estimate) are
220
+ available via `pip install 'ragless[tokens]'`:
221
+
222
+ ```python
223
+ from ragless import Retriever
224
+ from ragless.context import TiktokenCounter
225
+
226
+ r = Retriever("./docs")
227
+ r.context("query", max_tokens=2000, token_counter=TiktokenCounter())
228
+ ```
229
+
230
+ ### Custom pipeline steps
231
+
232
+ ```python
233
+ from ragless.context import ContextStep, Pack
234
+
235
+
236
+ class DropShortPassages(ContextStep):
237
+ def run(self, passages, state):
238
+ return [p for p in passages if len(p.text) > 40]
239
+
240
+
241
+ r.context("query", max_tokens=2000, pipeline=[DropShortPassages(), Pack()])
242
+ ```
243
+
244
+ A pipeline step only needs `run(passages, state) -> passages`, so a future
245
+ graph-expansion step can plug in the same way: add candidate passages with
246
+ their own `reasons` entry, without touching merge/dedup/pack.
247
+
154
248
  ## Why
155
249
 
156
250
  Most "RAG" tutorials reach for a vector database before checking whether they
@@ -195,10 +289,22 @@ docs folder sitting next to your code, `ragless` is simpler.
195
289
  [Reciprocal Rank Fusion](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf),
196
290
  apply metadata/path filters, and optionally rerank the top candidates with
197
291
  a cross-encoder.
292
+ 5. **Context** -- `r.context()` runs search results through a pipeline (expand
293
+ -> merge -> dedup -> diversify -> compress -> pack) and returns a
294
+ `Context` fitted to a token budget, with citations and highlight offsets.
295
+ `r.prompt()` renders that into a prompt string. No LLM call happens
296
+ anywhere in `ragless`.
297
+
298
+ ## Roadmap
299
+
300
+ - **Graph-based retrieval** (planned next): expand candidates by following
301
+ references between chunks (e.g. imports, links, call graphs) as another
302
+ pluggable `context` pipeline step, so related-but-not-directly-matched
303
+ passages can be pulled in with their own `reasons` entry.
198
304
 
199
305
  ## Status
200
306
 
201
- 0.1.0 -- the API may still shift. Feedback and issues welcome.
307
+ 0.2.0 -- the API may still shift. Feedback and issues welcome.
202
308
 
203
309
  ## License
204
310
 
@@ -0,0 +1,23 @@
1
+ # v0.2 progress
2
+
3
+ ## Status: in progress
4
+
5
+ - [x] Read existing v0.1 code/tests/README/CI, confirm no prior v0.2 work.
6
+ - [x] Write DECISIONS.md with the v0.2 architecture.
7
+ - [ ] Implement `ragless.context` package (tokens, passage/context, pipeline
8
+ steps, query expansion, orchestration).
9
+ - [ ] Wire `Retriever.context()` / `Retriever.prompt()` into `core.py`, add
10
+ the internal `_doc_text` / `_chunk_doc_id` bookkeeping.
11
+ - [ ] Update `ragless/__init__.py` exports + bump version to 0.2.0.
12
+ - [ ] Tests for every pipeline step + integration tests + edge cases.
13
+ - [ ] Run v0.1 tests (must still pass), ruff, coverage >= 85%.
14
+ - [ ] Build wheel, `twine check`, install + smoke test in a fresh venv.
15
+ - [ ] README "Context for your LLM" section, CHANGELOG, new examples,
16
+ Roadmap section.
17
+ - [ ] Commit in small logical commits, push, green CI.
18
+ - [ ] Tag v0.2.0, push tag, watch publish workflow.
19
+ - [ ] Verify `pip install ragless==0.2.0` + smoke test `r.context(...)`.
20
+ - [ ] GitHub release for v0.2.0.
21
+
22
+ Will fill in the final summary (what was built, test/coverage numbers,
23
+ release links) at the end.
@@ -116,6 +116,98 @@ r = Retriever("./docs", engines={"bm25": MyRetriever(), "fuzzy": FuzzyRetriever(
116
116
  Implement `ragless.chunking.Chunker`, `ragless.retrievers.base.BaseRetriever`,
117
117
  or `ragless.embedders.base.Embedder` to plug in your own.
118
118
 
119
+ ## Context for your LLM
120
+
121
+ `ragless` handles everything up to the LLM call: `r.context()` turns a query
122
+ into ready-to-use, budget-fitted context. There's no LLM inside `ragless` --
123
+ it never makes a network call or imports an LLM client; it just hands you the
124
+ material to send to whichever model you call yourself.
125
+
126
+ ```python
127
+ from ragless import Retriever
128
+
129
+ r = Retriever("./docs")
130
+ ctx = r.context("how do I configure retries", max_tokens=2000)
131
+
132
+ print(ctx.token_count) # never exceeds max_tokens
133
+ for citation in ctx.citations:
134
+ print(citation) # [1] docs/retries.md:12-34
135
+ ```
136
+
137
+ Under the hood, `context()` runs a pipeline of small, swappable steps:
138
+ expand hits with neighboring lines, merge adjacent hits from the same file,
139
+ drop near-duplicates, diversify with MMR, optionally compress to the most
140
+ relevant sentences/lines, then pack whole passages into the token budget
141
+ (never truncating mid-passage). Every passage keeps a `reasons` list (e.g.
142
+ `["direct match"]`, `["direct match", "neighbor window"]`) explaining why it
143
+ was included.
144
+
145
+ ### Output formats
146
+
147
+ ```python
148
+ ctx.to_text() # plain text with trailing "[n] path:start-end" citations
149
+ ctx.to_markdown() # fenced code blocks for code paths, blockquotes for prose
150
+ ctx.to_xml() # <context><passage index="1" ref="...">...</passage></context>
151
+ ctx.to_json() # full structure, including matched-term highlight offsets
152
+ ctx.to_text(mark=True) # wraps matched query terms, e.g. with >>term<<
153
+ ```
154
+
155
+ ### Prompts -- still no LLM call
156
+
157
+ ```python
158
+ prompt = r.prompt(
159
+ "how do I configure retries",
160
+ max_tokens=2000,
161
+ template="Context:\n{context}\n\nQuestion: {query}\nAnswer:", # optional
162
+ )
163
+
164
+ # Send it to any client yourself:
165
+ # response = my_llm_client.complete(prompt)
166
+ ```
167
+
168
+ ### Tuning the pipeline
169
+
170
+ ```python
171
+ r.context(
172
+ "query",
173
+ max_tokens=2000,
174
+ window=3, # lines of neighboring context to pull in
175
+ dedup_threshold=90, # fuzzy-ratio cutoff for near-duplicates
176
+ diversify=True, # reorder by Maximal Marginal Relevance
177
+ compress=True, # keep only the most query-relevant sentences/lines
178
+ query_expansion=True, # pseudo-relevance feedback (RM3-style), off by default
179
+ )
180
+ ```
181
+
182
+ Exact token counts (instead of the dependency-free default estimate) are
183
+ available via `pip install 'ragless[tokens]'`:
184
+
185
+ ```python
186
+ from ragless import Retriever
187
+ from ragless.context import TiktokenCounter
188
+
189
+ r = Retriever("./docs")
190
+ r.context("query", max_tokens=2000, token_counter=TiktokenCounter())
191
+ ```
192
+
193
+ ### Custom pipeline steps
194
+
195
+ ```python
196
+ from ragless.context import ContextStep, Pack
197
+
198
+
199
+ class DropShortPassages(ContextStep):
200
+ def run(self, passages, state):
201
+ return [p for p in passages if len(p.text) > 40]
202
+
203
+
204
+ r.context("query", max_tokens=2000, pipeline=[DropShortPassages(), Pack()])
205
+ ```
206
+
207
+ A pipeline step only needs `run(passages, state) -> passages`, so a future
208
+ graph-expansion step can plug in the same way: add candidate passages with
209
+ their own `reasons` entry, without touching merge/dedup/pack.
210
+
119
211
  ## Why
120
212
 
121
213
  Most "RAG" tutorials reach for a vector database before checking whether they
@@ -160,10 +252,22 @@ docs folder sitting next to your code, `ragless` is simpler.
160
252
  [Reciprocal Rank Fusion](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf),
161
253
  apply metadata/path filters, and optionally rerank the top candidates with
162
254
  a cross-encoder.
255
+ 5. **Context** -- `r.context()` runs search results through a pipeline (expand
256
+ -> merge -> dedup -> diversify -> compress -> pack) and returns a
257
+ `Context` fitted to a token budget, with citations and highlight offsets.
258
+ `r.prompt()` renders that into a prompt string. No LLM call happens
259
+ anywhere in `ragless`.
260
+
261
+ ## Roadmap
262
+
263
+ - **Graph-based retrieval** (planned next): expand candidates by following
264
+ references between chunks (e.g. imports, links, call graphs) as another
265
+ pluggable `context` pipeline step, so related-but-not-directly-matched
266
+ passages can be pulled in with their own `reasons` entry.
163
267
 
164
268
  ## Status
165
269
 
166
- 0.1.0 -- the API may still shift. Feedback and issues welcome.
270
+ 0.2.0 -- the API may still shift. Feedback and issues welcome.
167
271
 
168
272
  ## License
169
273
 
@@ -0,0 +1,32 @@
1
+ """Build ready-to-use context for any LLM -- no vector database, no LLM call here.
2
+
3
+ Run from the repo root: `python examples/context_for_llm.py`
4
+ """
5
+
6
+ from pathlib import Path
7
+
8
+ from ragless import Retriever
9
+
10
+ DOCS_DIR = Path(__file__).parent / "sample_docs"
11
+
12
+
13
+ def main() -> None:
14
+ retriever = Retriever(str(DOCS_DIR))
15
+ query = "how are embeddings cached"
16
+
17
+ ctx = retriever.context(query, max_tokens=300)
18
+ print(f"Packed {len(ctx.passages)} passage(s), ~{ctx.token_count} tokens\n")
19
+
20
+ print("-- markdown --")
21
+ print(ctx.to_markdown())
22
+
23
+ print("\n-- citations --")
24
+ for citation in ctx.citations:
25
+ print(f" {citation} score={citation.score:.4f} reasons={citation.reasons}")
26
+
27
+ print("\n-- plain text with matched terms marked --")
28
+ print(ctx.to_text(mark=True))
29
+
30
+
31
+ if __name__ == "__main__":
32
+ main()
@@ -0,0 +1,35 @@
1
+ """Render a ready-to-send prompt string -- ragless never calls an LLM itself.
2
+
3
+ Run from the repo root: `python examples/prompt_templates.py`
4
+ """
5
+
6
+ from pathlib import Path
7
+
8
+ from ragless import Retriever
9
+
10
+ DOCS_DIR = Path(__file__).parent / "sample_docs"
11
+
12
+
13
+ def main() -> None:
14
+ retriever = Retriever(str(DOCS_DIR))
15
+ query = "how do I install the semantic extra"
16
+
17
+ print("-- default template --")
18
+ print(retriever.prompt(query, max_tokens=300))
19
+
20
+ print("\n-- custom string template --")
21
+ custom = "Context:\n{context}\n\nUser question: {query}\nRespond concisely:"
22
+ print(retriever.prompt(query, template=custom, max_tokens=300))
23
+
24
+ print("\n-- custom callable template --")
25
+ print(
26
+ retriever.prompt(
27
+ query,
28
+ template=lambda q, ctx: f"[{len(ctx.passages)} passage(s)] {q}",
29
+ max_tokens=300,
30
+ )
31
+ )
32
+
33
+
34
+ if __name__ == "__main__":
35
+ main()
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "ragless"
7
- version = "0.1.0"
7
+ version = "0.2.0"
8
8
  description = "File-based retrieval for RAG, no vector database required"
9
9
  readme = { file = "README.md", content-type = "text/markdown" }
10
10
  license = "MIT"
@@ -31,6 +31,7 @@ dependencies = [
31
31
 
32
32
  [project.optional-dependencies]
33
33
  semantic = ["fastembed>=0.3"]
34
+ tokens = ["tiktoken>=0.7"]
34
35
  dev = [
35
36
  "pytest>=7.4",
36
37
  "pytest-cov>=4.1",
@@ -1,6 +1,22 @@
1
1
  """ragless: file-based retrieval for RAG, no vector database required."""
2
2
 
3
3
  from ragless.chunking import Chunker, LineChunker
4
+ from ragless.context import (
5
+ Citation,
6
+ Compress,
7
+ Context,
8
+ ContextStep,
9
+ Dedup,
10
+ Diversify,
11
+ MergeAdjacent,
12
+ Pack,
13
+ Passage,
14
+ PipelineState,
15
+ TiktokenCounter,
16
+ WindowExpand,
17
+ build_default_pipeline,
18
+ estimate_tokens,
19
+ )
4
20
  from ragless.core import Retriever
5
21
  from ragless.embedders.base import Embedder
6
22
  from ragless.exceptions import MissingDependencyError, RaglessError, UnknownRetrieverError
@@ -9,7 +25,7 @@ from ragless.rerank import CrossEncoderReranker, Reranker
9
25
  from ragless.retrievers import BaseRetriever, BM25Retriever, FuzzyRetriever, SemanticRetriever
10
26
  from ragless.types import Chunk, SearchResult
11
27
 
12
- __version__ = "0.1.0"
28
+ __version__ = "0.2.0"
13
29
 
14
30
  __all__ = [
15
31
  "Retriever",
@@ -28,5 +44,19 @@ __all__ = [
28
44
  "RaglessError",
29
45
  "MissingDependencyError",
30
46
  "UnknownRetrieverError",
47
+ "Context",
48
+ "Citation",
49
+ "Passage",
50
+ "ContextStep",
51
+ "WindowExpand",
52
+ "MergeAdjacent",
53
+ "Dedup",
54
+ "Diversify",
55
+ "Compress",
56
+ "Pack",
57
+ "PipelineState",
58
+ "build_default_pipeline",
59
+ "estimate_tokens",
60
+ "TiktokenCounter",
31
61
  "__version__",
32
62
  ]