ragless 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ragless-0.1.0 → ragless-0.2.0}/.github/workflows/ci.yml +1 -1
- ragless-0.2.0/CHANGELOG.md +52 -0
- ragless-0.2.0/DECISIONS.md +109 -0
- {ragless-0.1.0 → ragless-0.2.0}/PKG-INFO +108 -2
- ragless-0.2.0/PROGRESS.md +23 -0
- {ragless-0.1.0 → ragless-0.2.0}/README.md +105 -1
- ragless-0.2.0/examples/context_for_llm.py +32 -0
- ragless-0.2.0/examples/prompt_templates.py +35 -0
- {ragless-0.1.0 → ragless-0.2.0}/pyproject.toml +2 -1
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/__init__.py +31 -1
- ragless-0.2.0/src/ragless/context/__init__.py +153 -0
- ragless-0.2.0/src/ragless/context/_lexical.py +84 -0
- ragless-0.2.0/src/ragless/context/passage.py +177 -0
- ragless-0.2.0/src/ragless/context/pipeline.py +318 -0
- ragless-0.2.0/src/ragless/context/query_expansion.py +39 -0
- ragless-0.2.0/src/ragless/context/tokens.py +33 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/core.py +154 -0
- ragless-0.2.0/tests/test_context_integration.py +205 -0
- ragless-0.2.0/tests/test_context_output.py +132 -0
- ragless-0.2.0/tests/test_context_pipeline.py +273 -0
- ragless-0.2.0/tests/test_context_tokens.py +36 -0
- ragless-0.2.0/tests/test_examples.py +41 -0
- ragless-0.1.0/CHANGELOG.md +0 -25
- ragless-0.1.0/DECISIONS.md +0 -105
- ragless-0.1.0/PROGRESS.md +0 -86
- {ragless-0.1.0 → ragless-0.2.0}/.github/workflows/publish.yml +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/.gitignore +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/LICENSE +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/examples/basic_search.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/examples/filters_and_metadata.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/examples/in_memory_text.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/examples/sample_docs/guides/caching.md +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/examples/sample_docs/installation.md +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/examples/sample_docs/searching.md +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/examples/semantic_and_rerank.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/cache.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/chunking.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/embedders/__init__.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/embedders/base.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/embedders/fastembed_embedder.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/exceptions.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/fusion.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/rerank.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/retrievers/__init__.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/retrievers/base.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/retrievers/bm25.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/retrievers/fuzzy.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/retrievers/semantic.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/src/ragless/types.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/tests/conftest.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/tests/test_bm25.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/tests/test_cache.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/tests/test_chunking.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/tests/test_core.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/tests/test_fusion.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/tests/test_fuzzy.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/tests/test_missing_extra.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/tests/test_rerank.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/tests/test_semantic.py +0 -0
- {ragless-0.1.0 → ragless-0.2.0}/tests/test_semantic_real_model.py +0 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/).
|
|
7
|
+
|
|
8
|
+
## [0.2.0] - 2026-11-12
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- `Retriever.context()`: turns search results into a `Context` fitted to a
|
|
13
|
+
token budget, with citations, inclusion `reasons`, and matched-term
|
|
14
|
+
highlight offsets.
|
|
15
|
+
- `ragless.context` pipeline: composable, swappable steps --
|
|
16
|
+
`WindowExpand` (neighboring-line expansion), `MergeAdjacent` (combines
|
|
17
|
+
overlapping/adjacent hits from the same file with correct line ranges),
|
|
18
|
+
`Dedup` (fuzzy near-duplicate removal), `Diversify` (Maximal Marginal
|
|
19
|
+
Relevance, using the semantic embedder when available or a lexical
|
|
20
|
+
fallback otherwise), `Compress` (extractive sentence/line selection --
|
|
21
|
+
never rewrites text), and `Pack` (greedy token-budget fitting that never
|
|
22
|
+
exceeds the budget).
|
|
23
|
+
- `Context.to_text()/.to_markdown()/.to_xml()/.to_json()` output formats.
|
|
24
|
+
- `Retriever.prompt()`: renders a `Context` into a prompt string via a
|
|
25
|
+
default template, a custom `str.format` template, or a custom callable.
|
|
26
|
+
`ragless` still never calls an LLM anywhere.
|
|
27
|
+
- Pseudo-relevance feedback query expansion (`query_expansion=True`,
|
|
28
|
+
off by default): RM3-style term expansion from the first pass's top
|
|
29
|
+
results, re-searched and merged.
|
|
30
|
+
- Dependency-free token estimation by default; exact counts via
|
|
31
|
+
`ragless.context.TiktokenCounter` under the new `tokens` extra
|
|
32
|
+
(`pip install 'ragless[tokens]'`).
|
|
33
|
+
- New examples: `context_for_llm.py`, `prompt_templates.py`.
|
|
34
|
+
|
|
35
|
+
## [0.1.0] - 2026-11-12
|
|
36
|
+
|
|
37
|
+
### Added
|
|
38
|
+
|
|
39
|
+
- `Retriever`: library-first API indexing a directory or in-memory text.
|
|
40
|
+
- `LineChunker`: overlapping line-window chunking that preserves file path
|
|
41
|
+
and 1-indexed start/end line numbers.
|
|
42
|
+
- `BM25Retriever`: Okapi BM25 keyword search (pure NumPy, no extra deps).
|
|
43
|
+
- `FuzzyRetriever`: fuzzy string matching via `rapidfuzz`.
|
|
44
|
+
- `SemanticRetriever`: local embedding search (`fastembed`, optional
|
|
45
|
+
`semantic` extra) with NumPy cosine similarity.
|
|
46
|
+
- Reciprocal Rank Fusion (`reciprocal_rank_fusion`) to combine engine
|
|
47
|
+
rankings, with configurable weights and `k`.
|
|
48
|
+
- `CrossEncoderReranker`: optional cross-encoder reranking of fused results.
|
|
49
|
+
- On-disk embedding cache (`EmbeddingCache`) as plain `.npy`/`.json` sidecar
|
|
50
|
+
files, keyed by file mtime/size, so only changed files are re-embedded.
|
|
51
|
+
- Metadata and path filters on `search()`.
|
|
52
|
+
- Pluggable chunkers, retrievers, and embedders.
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# v0.2 design decisions
|
|
2
|
+
|
|
3
|
+
Working notes for the `ragless.context` feature (the "A" in RAG). Not shipped
|
|
4
|
+
to PyPI as documentation; this is an internal log of choices made while
|
|
5
|
+
building autonomously, so the reasoning doesn't have to be reconstructed later.
|
|
6
|
+
|
|
7
|
+
## Module layout
|
|
8
|
+
|
|
9
|
+
`ragless/context/`:
|
|
10
|
+
- `tokens.py` -- `estimate_tokens` (chars/4 heuristic, no deps) and
|
|
11
|
+
`TiktokenCounter` (lazy-imports `tiktoken`, requires the new `tokens` extra).
|
|
12
|
+
- `passage.py` -- `Passage`, `Citation`, `Context` (with `to_text/to_markdown/
|
|
13
|
+
to_xml/to_json`) and the default prompt template.
|
|
14
|
+
- `pipeline.py` -- `ContextStep` base class and the six built-in steps
|
|
15
|
+
(`WindowExpand`, `MergeAdjacent`, `Dedup`, `Diversify`, `Compress`, `Pack`),
|
|
16
|
+
plus `build_default_pipeline()`.
|
|
17
|
+
- `query_expansion.py` -- RM3-style pseudo-relevance feedback, off by default.
|
|
18
|
+
- `__init__.py` -- `build_context()` orchestration + public re-exports.
|
|
19
|
+
|
|
20
|
+
`Retriever.context()` and `Retriever.prompt()` live in `core.py` (lazy-import
|
|
21
|
+
`ragless.context` to avoid a cycle) so the public entry point stays
|
|
22
|
+
`r.context(...)` / `r.prompt(...)`, matching the existing `r.search(...)`
|
|
23
|
+
style instead of a separate builder object the user has to construct.
|
|
24
|
+
|
|
25
|
+
## Why a step-based pipeline instead of one big function
|
|
26
|
+
|
|
27
|
+
Each step is a small class implementing `run(passages, state) -> passages`.
|
|
28
|
+
`state` (`PipelineState`) carries the query, token budget/counter, and a
|
|
29
|
+
document-text/embedder accessor so steps don't each need their own
|
|
30
|
+
constructor wiring for things that are per-call, not per-step. Steps only
|
|
31
|
+
take constructor args for their own tunables (window size, dedup threshold,
|
|
32
|
+
MMR lambda, ...). This lets `pipeline=[...]` fully replace the default list,
|
|
33
|
+
and a future graph-expansion step (v0.3) only needs to append/prepend another
|
|
34
|
+
step that adds candidate passages with its own `reasons` entry -- it doesn't
|
|
35
|
+
need to know about merge/dedup/pack internals.
|
|
36
|
+
|
|
37
|
+
## Passage identity and merging across chunk boundaries
|
|
38
|
+
|
|
39
|
+
Merging adjacent/overlapping hits needs the *original* document text, not
|
|
40
|
+
just concatenated chunk text, because `LineChunker` chunks overlap by
|
|
41
|
+
default -- naively joining two chunks would duplicate the overlapping lines.
|
|
42
|
+
So `Retriever` now also keeps `self._doc_text: dict[str, str]` (doc key ->
|
|
43
|
+
full original text) and `self._chunk_doc_id: dict[str, str]` (chunk id -> doc
|
|
44
|
+
key), built during the existing chunking loop in `__init__`. Both are
|
|
45
|
+
private/internal; chunking itself and the public `Chunk` dataclass are
|
|
46
|
+
unchanged, so this doesn't touch the frozen `Chunk` API or any existing test
|
|
47
|
+
that constructs `Chunk(...)` directly. Doc keys are the same `path` or
|
|
48
|
+
`f"mem:{i}"` prefixes already used for chunk ids, so merged/expanded passages
|
|
49
|
+
from in-memory text get a citation id like `mem:0` instead of a path.
|
|
50
|
+
|
|
51
|
+
`MergeAdjacent` and `WindowExpand` re-slice from `self._doc_text[doc_id]`
|
|
52
|
+
by line range rather than editing chunk text, so a merged passage's text is
|
|
53
|
+
always an exact line-range slice of the source, and `start_line`/`end_line`
|
|
54
|
+
stay accurate for citations.
|
|
55
|
+
|
|
56
|
+
## Diversity (MMR)
|
|
57
|
+
|
|
58
|
+
Uses the retriever's own semantic embedder when `engines` includes
|
|
59
|
+
`"semantic"` (found via `retriever._retrievers.get("semantic").embedder`);
|
|
60
|
+
otherwise falls back to a lexical similarity matrix from
|
|
61
|
+
`rapidfuzz.fuzz.token_sort_ratio`. Either way `Diversify` only needs a
|
|
62
|
+
`similarity(i, j) -> float in [0, 1]` function, so the embedder dependency is
|
|
63
|
+
optional and never required at import time.
|
|
64
|
+
|
|
65
|
+
## Extractive compression
|
|
66
|
+
|
|
67
|
+
Sentence-level for prose, line/block-level for recognized code file
|
|
68
|
+
extensions (`_CODE_EXTS`). Scoring is lexical token overlap with the query
|
|
69
|
+
(no embeddings required, since `compress` has to work without the `semantic`
|
|
70
|
+
extra too). Compression only *selects* a subsequence of the original
|
|
71
|
+
sentences/lines -- it never rewrites text -- and joins non-contiguous kept
|
|
72
|
+
spans with a literal `"\n[...]\n"` marker so the output is still clearly
|
|
73
|
+
attributable to the source. Tests assert every span between markers is an
|
|
74
|
+
exact substring of the original passage text.
|
|
75
|
+
|
|
76
|
+
## Token budget (`Pack`)
|
|
77
|
+
|
|
78
|
+
Greedy: steps through passages in their current order and keeps a passage
|
|
79
|
+
only if it still fits the remaining budget, so the total is a hard max, not a
|
|
80
|
+
best-effort trim -- "budget smaller than one passage" yields an empty
|
|
81
|
+
`Context` rather than a truncated one, since clipping mid-passage would also
|
|
82
|
+
clip mid-citation. Default token counter is a dependency-free chars/4
|
|
83
|
+
estimate; `TiktokenCounter` (needs `pip install ragless[tokens]`) or a fully
|
|
84
|
+
custom `Callable[[str], int]` can replace it via `token_counter=`.
|
|
85
|
+
|
|
86
|
+
## Query expansion (RM3-style)
|
|
87
|
+
|
|
88
|
+
Off by default (`query_expansion=False` on `context()`). When enabled: run
|
|
89
|
+
the initial search, pull term frequencies from the top few hits' text
|
|
90
|
+
(excluding a small stopword list and terms already in the query), append the
|
|
91
|
+
top N new terms to the query, re-search, and merge candidate pools by chunk
|
|
92
|
+
id (keep the max score). If no new terms clear the frequency threshold
|
|
93
|
+
(small corpus, all terms already in the query, etc.) it falls back to the
|
|
94
|
+
unexpanded candidates untouched rather than erroring.
|
|
95
|
+
|
|
96
|
+
## Output formats and prompts
|
|
97
|
+
|
|
98
|
+
`Context.to_text/to_markdown/to_xml/to_json()` are plain string builders, no
|
|
99
|
+
templating engine. `r.prompt(query, template=..., **context_kwargs)` accepts
|
|
100
|
+
either a `str.format`-style template with `{query}`/`{context}` placeholders
|
|
101
|
+
(context rendered via `context_format=` before being formatted in) or a
|
|
102
|
+
callable `(query, Context) -> str` for full control. `ragless` itself never
|
|
103
|
+
imports an LLM client or makes a network call anywhere in this module.
|
|
104
|
+
|
|
105
|
+
## No new required dependencies
|
|
106
|
+
|
|
107
|
+
`ragless.context` only imports `numpy` and `rapidfuzz` (already required by
|
|
108
|
+
v0.1) plus the stdlib. `tiktoken` lives behind the new `tokens` extra and is
|
|
109
|
+
only imported inside `TiktokenCounter.__init__`.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ragless
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: File-based retrieval for RAG, no vector database required
|
|
5
5
|
Project-URL: Homepage, https://github.com/mohmdqasim/ragless
|
|
6
6
|
Project-URL: Repository, https://github.com/mohmdqasim/ragless
|
|
@@ -31,6 +31,8 @@ Requires-Dist: pytest>=7.4; extra == 'dev'
|
|
|
31
31
|
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
32
32
|
Provides-Extra: semantic
|
|
33
33
|
Requires-Dist: fastembed>=0.3; extra == 'semantic'
|
|
34
|
+
Provides-Extra: tokens
|
|
35
|
+
Requires-Dist: tiktoken>=0.7; extra == 'tokens'
|
|
34
36
|
Description-Content-Type: text/markdown
|
|
35
37
|
|
|
36
38
|
# ragless
|
|
@@ -151,6 +153,98 @@ r = Retriever("./docs", engines={"bm25": MyRetriever(), "fuzzy": FuzzyRetriever(
|
|
|
151
153
|
Implement `ragless.chunking.Chunker`, `ragless.retrievers.base.BaseRetriever`,
|
|
152
154
|
or `ragless.embedders.base.Embedder` to plug in your own.
|
|
153
155
|
|
|
156
|
+
## Context for your LLM
|
|
157
|
+
|
|
158
|
+
`ragless` handles everything up to the LLM call: `r.context()` turns a query
|
|
159
|
+
into ready-to-use, budget-fitted context. There's no LLM inside `ragless` --
|
|
160
|
+
it never makes a network call or imports an LLM client; it just hands you the
|
|
161
|
+
material to send to whichever model you call yourself.
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
from ragless import Retriever
|
|
165
|
+
|
|
166
|
+
r = Retriever("./docs")
|
|
167
|
+
ctx = r.context("how do I configure retries", max_tokens=2000)
|
|
168
|
+
|
|
169
|
+
print(ctx.token_count) # never exceeds max_tokens
|
|
170
|
+
for citation in ctx.citations:
|
|
171
|
+
print(citation) # [1] docs/retries.md:12-34
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
Under the hood, `context()` runs a pipeline of small, swappable steps:
|
|
175
|
+
expand hits with neighboring lines, merge adjacent hits from the same file,
|
|
176
|
+
drop near-duplicates, diversify with MMR, optionally compress to the most
|
|
177
|
+
relevant sentences/lines, then pack whole passages into the token budget
|
|
178
|
+
(never truncating mid-passage). Every passage keeps a `reasons` list (e.g.
|
|
179
|
+
`["direct match"]`, `["direct match", "neighbor window"]`) explaining why it
|
|
180
|
+
was included.
|
|
181
|
+
|
|
182
|
+
### Output formats
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
ctx.to_text() # plain text with trailing "[n] path:start-end" citations
|
|
186
|
+
ctx.to_markdown() # fenced code blocks for code paths, blockquotes for prose
|
|
187
|
+
ctx.to_xml() # <context><passage index="1" ref="...">...</passage></context>
|
|
188
|
+
ctx.to_json() # full structure, including matched-term highlight offsets
|
|
189
|
+
ctx.to_text(mark=True) # wraps matched query terms, e.g. with >>term<<
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
### Prompts -- still no LLM call
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
prompt = r.prompt(
|
|
196
|
+
"how do I configure retries",
|
|
197
|
+
max_tokens=2000,
|
|
198
|
+
template="Context:\n{context}\n\nQuestion: {query}\nAnswer:", # optional
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
# Send it to any client yourself:
|
|
202
|
+
# response = my_llm_client.complete(prompt)
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
### Tuning the pipeline
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
r.context(
|
|
209
|
+
"query",
|
|
210
|
+
max_tokens=2000,
|
|
211
|
+
window=3, # lines of neighboring context to pull in
|
|
212
|
+
dedup_threshold=90, # fuzzy-ratio cutoff for near-duplicates
|
|
213
|
+
diversify=True, # reorder by Maximal Marginal Relevance
|
|
214
|
+
compress=True, # keep only the most query-relevant sentences/lines
|
|
215
|
+
query_expansion=True, # pseudo-relevance feedback (RM3-style), off by default
|
|
216
|
+
)
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
Exact token counts (instead of the dependency-free default estimate) are
|
|
220
|
+
available via `pip install 'ragless[tokens]'`:
|
|
221
|
+
|
|
222
|
+
```python
|
|
223
|
+
from ragless import Retriever
|
|
224
|
+
from ragless.context import TiktokenCounter
|
|
225
|
+
|
|
226
|
+
r = Retriever("./docs")
|
|
227
|
+
r.context("query", max_tokens=2000, token_counter=TiktokenCounter())
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
### Custom pipeline steps
|
|
231
|
+
|
|
232
|
+
```python
|
|
233
|
+
from ragless.context import ContextStep, Pack
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
class DropShortPassages(ContextStep):
|
|
237
|
+
def run(self, passages, state):
|
|
238
|
+
return [p for p in passages if len(p.text) > 40]
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
r.context("query", max_tokens=2000, pipeline=[DropShortPassages(), Pack()])
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
A pipeline step only needs `run(passages, state) -> passages`, so a future
|
|
245
|
+
graph-expansion step can plug in the same way: add candidate passages with
|
|
246
|
+
their own `reasons` entry, without touching merge/dedup/pack.
|
|
247
|
+
|
|
154
248
|
## Why
|
|
155
249
|
|
|
156
250
|
Most "RAG" tutorials reach for a vector database before checking whether they
|
|
@@ -195,10 +289,22 @@ docs folder sitting next to your code, `ragless` is simpler.
|
|
|
195
289
|
[Reciprocal Rank Fusion](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf),
|
|
196
290
|
apply metadata/path filters, and optionally rerank the top candidates with
|
|
197
291
|
a cross-encoder.
|
|
292
|
+
5. **Context** -- `r.context()` runs search results through a pipeline (expand
|
|
293
|
+
-> merge -> dedup -> diversify -> compress -> pack) and returns a
|
|
294
|
+
`Context` fitted to a token budget, with citations and highlight offsets.
|
|
295
|
+
`r.prompt()` renders that into a prompt string. No LLM call happens
|
|
296
|
+
anywhere in `ragless`.
|
|
297
|
+
|
|
298
|
+
## Roadmap
|
|
299
|
+
|
|
300
|
+
- **Graph-based retrieval** (planned next): expand candidates by following
|
|
301
|
+
references between chunks (e.g. imports, links, call graphs) as another
|
|
302
|
+
pluggable `context` pipeline step, so related-but-not-directly-matched
|
|
303
|
+
passages can be pulled in with their own `reasons` entry.
|
|
198
304
|
|
|
199
305
|
## Status
|
|
200
306
|
|
|
201
|
-
0.
|
|
307
|
+
0.2.0 -- the API may still shift. Feedback and issues welcome.
|
|
202
308
|
|
|
203
309
|
## License
|
|
204
310
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# v0.2 progress
|
|
2
|
+
|
|
3
|
+
## Status: in progress
|
|
4
|
+
|
|
5
|
+
- [x] Read existing v0.1 code/tests/README/CI, confirm no prior v0.2 work.
|
|
6
|
+
- [x] Write DECISIONS.md with the v0.2 architecture.
|
|
7
|
+
- [ ] Implement `ragless.context` package (tokens, passage/context, pipeline
|
|
8
|
+
steps, query expansion, orchestration).
|
|
9
|
+
- [ ] Wire `Retriever.context()` / `Retriever.prompt()` into `core.py`, add
|
|
10
|
+
the internal `_doc_text` / `_chunk_doc_id` bookkeeping.
|
|
11
|
+
- [ ] Update `ragless/__init__.py` exports + bump version to 0.2.0.
|
|
12
|
+
- [ ] Tests for every pipeline step + integration tests + edge cases.
|
|
13
|
+
- [ ] Run v0.1 tests (must still pass), ruff, coverage >= 85%.
|
|
14
|
+
- [ ] Build wheel, `twine check`, install + smoke test in a fresh venv.
|
|
15
|
+
- [ ] README "Context for your LLM" section, CHANGELOG, new examples,
|
|
16
|
+
Roadmap section.
|
|
17
|
+
- [ ] Commit in small logical commits, push, green CI.
|
|
18
|
+
- [ ] Tag v0.2.0, push tag, watch publish workflow.
|
|
19
|
+
- [ ] Verify `pip install ragless==0.2.0` + smoke test `r.context(...)`.
|
|
20
|
+
- [ ] GitHub release for v0.2.0.
|
|
21
|
+
|
|
22
|
+
Will fill in the final summary (what was built, test/coverage numbers,
|
|
23
|
+
release links) at the end.
|
|
@@ -116,6 +116,98 @@ r = Retriever("./docs", engines={"bm25": MyRetriever(), "fuzzy": FuzzyRetriever(
|
|
|
116
116
|
Implement `ragless.chunking.Chunker`, `ragless.retrievers.base.BaseRetriever`,
|
|
117
117
|
or `ragless.embedders.base.Embedder` to plug in your own.
|
|
118
118
|
|
|
119
|
+
## Context for your LLM
|
|
120
|
+
|
|
121
|
+
`ragless` handles everything up to the LLM call: `r.context()` turns a query
|
|
122
|
+
into ready-to-use, budget-fitted context. There's no LLM inside `ragless` --
|
|
123
|
+
it never makes a network call or imports an LLM client; it just hands you the
|
|
124
|
+
material to send to whichever model you call yourself.
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
from ragless import Retriever
|
|
128
|
+
|
|
129
|
+
r = Retriever("./docs")
|
|
130
|
+
ctx = r.context("how do I configure retries", max_tokens=2000)
|
|
131
|
+
|
|
132
|
+
print(ctx.token_count) # never exceeds max_tokens
|
|
133
|
+
for citation in ctx.citations:
|
|
134
|
+
print(citation) # [1] docs/retries.md:12-34
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Under the hood, `context()` runs a pipeline of small, swappable steps:
|
|
138
|
+
expand hits with neighboring lines, merge adjacent hits from the same file,
|
|
139
|
+
drop near-duplicates, diversify with MMR, optionally compress to the most
|
|
140
|
+
relevant sentences/lines, then pack whole passages into the token budget
|
|
141
|
+
(never truncating mid-passage). Every passage keeps a `reasons` list (e.g.
|
|
142
|
+
`["direct match"]`, `["direct match", "neighbor window"]`) explaining why it
|
|
143
|
+
was included.
|
|
144
|
+
|
|
145
|
+
### Output formats
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
ctx.to_text() # plain text with trailing "[n] path:start-end" citations
|
|
149
|
+
ctx.to_markdown() # fenced code blocks for code paths, blockquotes for prose
|
|
150
|
+
ctx.to_xml() # <context><passage index="1" ref="...">...</passage></context>
|
|
151
|
+
ctx.to_json() # full structure, including matched-term highlight offsets
|
|
152
|
+
ctx.to_text(mark=True) # wraps matched query terms, e.g. with >>term<<
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
### Prompts -- still no LLM call
|
|
156
|
+
|
|
157
|
+
```python
|
|
158
|
+
prompt = r.prompt(
|
|
159
|
+
"how do I configure retries",
|
|
160
|
+
max_tokens=2000,
|
|
161
|
+
template="Context:\n{context}\n\nQuestion: {query}\nAnswer:", # optional
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
# Send it to any client yourself:
|
|
165
|
+
# response = my_llm_client.complete(prompt)
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
### Tuning the pipeline
|
|
169
|
+
|
|
170
|
+
```python
|
|
171
|
+
r.context(
|
|
172
|
+
"query",
|
|
173
|
+
max_tokens=2000,
|
|
174
|
+
window=3, # lines of neighboring context to pull in
|
|
175
|
+
dedup_threshold=90, # fuzzy-ratio cutoff for near-duplicates
|
|
176
|
+
diversify=True, # reorder by Maximal Marginal Relevance
|
|
177
|
+
compress=True, # keep only the most query-relevant sentences/lines
|
|
178
|
+
query_expansion=True, # pseudo-relevance feedback (RM3-style), off by default
|
|
179
|
+
)
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
Exact token counts (instead of the dependency-free default estimate) are
|
|
183
|
+
available via `pip install 'ragless[tokens]'`:
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
from ragless import Retriever
|
|
187
|
+
from ragless.context import TiktokenCounter
|
|
188
|
+
|
|
189
|
+
r = Retriever("./docs")
|
|
190
|
+
r.context("query", max_tokens=2000, token_counter=TiktokenCounter())
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
### Custom pipeline steps
|
|
194
|
+
|
|
195
|
+
```python
|
|
196
|
+
from ragless.context import ContextStep, Pack
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class DropShortPassages(ContextStep):
|
|
200
|
+
def run(self, passages, state):
|
|
201
|
+
return [p for p in passages if len(p.text) > 40]
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
r.context("query", max_tokens=2000, pipeline=[DropShortPassages(), Pack()])
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
A pipeline step only needs `run(passages, state) -> passages`, so a future
|
|
208
|
+
graph-expansion step can plug in the same way: add candidate passages with
|
|
209
|
+
their own `reasons` entry, without touching merge/dedup/pack.
|
|
210
|
+
|
|
119
211
|
## Why
|
|
120
212
|
|
|
121
213
|
Most "RAG" tutorials reach for a vector database before checking whether they
|
|
@@ -160,10 +252,22 @@ docs folder sitting next to your code, `ragless` is simpler.
|
|
|
160
252
|
[Reciprocal Rank Fusion](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf),
|
|
161
253
|
apply metadata/path filters, and optionally rerank the top candidates with
|
|
162
254
|
a cross-encoder.
|
|
255
|
+
5. **Context** -- `r.context()` runs search results through a pipeline (expand
|
|
256
|
+
-> merge -> dedup -> diversify -> compress -> pack) and returns a
|
|
257
|
+
`Context` fitted to a token budget, with citations and highlight offsets.
|
|
258
|
+
`r.prompt()` renders that into a prompt string. No LLM call happens
|
|
259
|
+
anywhere in `ragless`.
|
|
260
|
+
|
|
261
|
+
## Roadmap
|
|
262
|
+
|
|
263
|
+
- **Graph-based retrieval** (planned next): expand candidates by following
|
|
264
|
+
references between chunks (e.g. imports, links, call graphs) as another
|
|
265
|
+
pluggable `context` pipeline step, so related-but-not-directly-matched
|
|
266
|
+
passages can be pulled in with their own `reasons` entry.
|
|
163
267
|
|
|
164
268
|
## Status
|
|
165
269
|
|
|
166
|
-
0.
|
|
270
|
+
0.2.0 -- the API may still shift. Feedback and issues welcome.
|
|
167
271
|
|
|
168
272
|
## License
|
|
169
273
|
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Build ready-to-use context for any LLM -- no vector database, no LLM call here.
|
|
2
|
+
|
|
3
|
+
Run from the repo root: `python examples/context_for_llm.py`
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from ragless import Retriever
|
|
9
|
+
|
|
10
|
+
DOCS_DIR = Path(__file__).parent / "sample_docs"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def main() -> None:
|
|
14
|
+
retriever = Retriever(str(DOCS_DIR))
|
|
15
|
+
query = "how are embeddings cached"
|
|
16
|
+
|
|
17
|
+
ctx = retriever.context(query, max_tokens=300)
|
|
18
|
+
print(f"Packed {len(ctx.passages)} passage(s), ~{ctx.token_count} tokens\n")
|
|
19
|
+
|
|
20
|
+
print("-- markdown --")
|
|
21
|
+
print(ctx.to_markdown())
|
|
22
|
+
|
|
23
|
+
print("\n-- citations --")
|
|
24
|
+
for citation in ctx.citations:
|
|
25
|
+
print(f" {citation} score={citation.score:.4f} reasons={citation.reasons}")
|
|
26
|
+
|
|
27
|
+
print("\n-- plain text with matched terms marked --")
|
|
28
|
+
print(ctx.to_text(mark=True))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
if __name__ == "__main__":
|
|
32
|
+
main()
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Render a ready-to-send prompt string -- ragless never calls an LLM itself.
|
|
2
|
+
|
|
3
|
+
Run from the repo root: `python examples/prompt_templates.py`
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from ragless import Retriever
|
|
9
|
+
|
|
10
|
+
DOCS_DIR = Path(__file__).parent / "sample_docs"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def main() -> None:
|
|
14
|
+
retriever = Retriever(str(DOCS_DIR))
|
|
15
|
+
query = "how do I install the semantic extra"
|
|
16
|
+
|
|
17
|
+
print("-- default template --")
|
|
18
|
+
print(retriever.prompt(query, max_tokens=300))
|
|
19
|
+
|
|
20
|
+
print("\n-- custom string template --")
|
|
21
|
+
custom = "Context:\n{context}\n\nUser question: {query}\nRespond concisely:"
|
|
22
|
+
print(retriever.prompt(query, template=custom, max_tokens=300))
|
|
23
|
+
|
|
24
|
+
print("\n-- custom callable template --")
|
|
25
|
+
print(
|
|
26
|
+
retriever.prompt(
|
|
27
|
+
query,
|
|
28
|
+
template=lambda q, ctx: f"[{len(ctx.passages)} passage(s)] {q}",
|
|
29
|
+
max_tokens=300,
|
|
30
|
+
)
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
if __name__ == "__main__":
|
|
35
|
+
main()
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "ragless"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "File-based retrieval for RAG, no vector database required"
|
|
9
9
|
readme = { file = "README.md", content-type = "text/markdown" }
|
|
10
10
|
license = "MIT"
|
|
@@ -31,6 +31,7 @@ dependencies = [
|
|
|
31
31
|
|
|
32
32
|
[project.optional-dependencies]
|
|
33
33
|
semantic = ["fastembed>=0.3"]
|
|
34
|
+
tokens = ["tiktoken>=0.7"]
|
|
34
35
|
dev = [
|
|
35
36
|
"pytest>=7.4",
|
|
36
37
|
"pytest-cov>=4.1",
|
|
@@ -1,6 +1,22 @@
|
|
|
1
1
|
"""ragless: file-based retrieval for RAG, no vector database required."""
|
|
2
2
|
|
|
3
3
|
from ragless.chunking import Chunker, LineChunker
|
|
4
|
+
from ragless.context import (
|
|
5
|
+
Citation,
|
|
6
|
+
Compress,
|
|
7
|
+
Context,
|
|
8
|
+
ContextStep,
|
|
9
|
+
Dedup,
|
|
10
|
+
Diversify,
|
|
11
|
+
MergeAdjacent,
|
|
12
|
+
Pack,
|
|
13
|
+
Passage,
|
|
14
|
+
PipelineState,
|
|
15
|
+
TiktokenCounter,
|
|
16
|
+
WindowExpand,
|
|
17
|
+
build_default_pipeline,
|
|
18
|
+
estimate_tokens,
|
|
19
|
+
)
|
|
4
20
|
from ragless.core import Retriever
|
|
5
21
|
from ragless.embedders.base import Embedder
|
|
6
22
|
from ragless.exceptions import MissingDependencyError, RaglessError, UnknownRetrieverError
|
|
@@ -9,7 +25,7 @@ from ragless.rerank import CrossEncoderReranker, Reranker
|
|
|
9
25
|
from ragless.retrievers import BaseRetriever, BM25Retriever, FuzzyRetriever, SemanticRetriever
|
|
10
26
|
from ragless.types import Chunk, SearchResult
|
|
11
27
|
|
|
12
|
-
__version__ = "0.
|
|
28
|
+
__version__ = "0.2.0"
|
|
13
29
|
|
|
14
30
|
__all__ = [
|
|
15
31
|
"Retriever",
|
|
@@ -28,5 +44,19 @@ __all__ = [
|
|
|
28
44
|
"RaglessError",
|
|
29
45
|
"MissingDependencyError",
|
|
30
46
|
"UnknownRetrieverError",
|
|
47
|
+
"Context",
|
|
48
|
+
"Citation",
|
|
49
|
+
"Passage",
|
|
50
|
+
"ContextStep",
|
|
51
|
+
"WindowExpand",
|
|
52
|
+
"MergeAdjacent",
|
|
53
|
+
"Dedup",
|
|
54
|
+
"Diversify",
|
|
55
|
+
"Compress",
|
|
56
|
+
"Pack",
|
|
57
|
+
"PipelineState",
|
|
58
|
+
"build_default_pipeline",
|
|
59
|
+
"estimate_tokens",
|
|
60
|
+
"TiktokenCounter",
|
|
31
61
|
"__version__",
|
|
32
62
|
]
|