ragless 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ragless-0.1.0/.github/workflows/ci.yml +74 -0
- ragless-0.1.0/.github/workflows/publish.yml +55 -0
- ragless-0.1.0/.gitignore +15 -0
- ragless-0.1.0/CHANGELOG.md +25 -0
- ragless-0.1.0/DECISIONS.md +105 -0
- ragless-0.1.0/LICENSE +21 -0
- ragless-0.1.0/PKG-INFO +205 -0
- ragless-0.1.0/PROGRESS.md +86 -0
- ragless-0.1.0/README.md +170 -0
- ragless-0.1.0/examples/basic_search.py +26 -0
- ragless-0.1.0/examples/filters_and_metadata.py +32 -0
- ragless-0.1.0/examples/in_memory_text.py +22 -0
- ragless-0.1.0/examples/sample_docs/guides/caching.md +10 -0
- ragless-0.1.0/examples/sample_docs/installation.md +13 -0
- ragless-0.1.0/examples/sample_docs/searching.md +8 -0
- ragless-0.1.0/examples/semantic_and_rerank.py +37 -0
- ragless-0.1.0/pyproject.toml +66 -0
- ragless-0.1.0/src/ragless/__init__.py +32 -0
- ragless-0.1.0/src/ragless/cache.py +100 -0
- ragless-0.1.0/src/ragless/chunking.py +88 -0
- ragless-0.1.0/src/ragless/core.py +365 -0
- ragless-0.1.0/src/ragless/embedders/__init__.py +3 -0
- ragless-0.1.0/src/ragless/embedders/base.py +26 -0
- ragless-0.1.0/src/ragless/embedders/fastembed_embedder.py +41 -0
- ragless-0.1.0/src/ragless/exceptions.py +21 -0
- ragless-0.1.0/src/ragless/fusion.py +39 -0
- ragless-0.1.0/src/ragless/rerank.py +51 -0
- ragless-0.1.0/src/ragless/retrievers/__init__.py +6 -0
- ragless-0.1.0/src/ragless/retrievers/base.py +34 -0
- ragless-0.1.0/src/ragless/retrievers/bm25.py +81 -0
- ragless-0.1.0/src/ragless/retrievers/fuzzy.py +35 -0
- ragless-0.1.0/src/ragless/retrievers/semantic.py +100 -0
- ragless-0.1.0/src/ragless/types.py +49 -0
- ragless-0.1.0/tests/conftest.py +89 -0
- ragless-0.1.0/tests/test_bm25.py +64 -0
- ragless-0.1.0/tests/test_cache.py +110 -0
- ragless-0.1.0/tests/test_chunking.py +64 -0
- ragless-0.1.0/tests/test_core.py +186 -0
- ragless-0.1.0/tests/test_fusion.py +57 -0
- ragless-0.1.0/tests/test_fuzzy.py +42 -0
- ragless-0.1.0/tests/test_missing_extra.py +50 -0
- ragless-0.1.0/tests/test_rerank.py +31 -0
- ragless-0.1.0/tests/test_semantic.py +114 -0
- ragless-0.1.0/tests/test_semantic_real_model.py +37 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
lint:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
- uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: "3.12"
|
|
20
|
+
- run: pip install -e ".[dev]"
|
|
21
|
+
- run: ruff check .
|
|
22
|
+
- run: ruff format --check .
|
|
23
|
+
|
|
24
|
+
test:
|
|
25
|
+
needs: lint
|
|
26
|
+
strategy:
|
|
27
|
+
fail-fast: false
|
|
28
|
+
matrix:
|
|
29
|
+
os: [ubuntu-latest, macos-latest]
|
|
30
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
31
|
+
runs-on: ${{ matrix.os }}
|
|
32
|
+
steps:
|
|
33
|
+
- uses: actions/checkout@v4
|
|
34
|
+
- uses: actions/setup-python@v5
|
|
35
|
+
with:
|
|
36
|
+
python-version: ${{ matrix.python-version }}
|
|
37
|
+
- run: pip install -e ".[dev]"
|
|
38
|
+
- run: pytest -q --cov=ragless --cov-report=term-missing --cov-fail-under=85
|
|
39
|
+
|
|
40
|
+
test-semantic-slow:
|
|
41
|
+
needs: lint
|
|
42
|
+
runs-on: ubuntu-latest
|
|
43
|
+
steps:
|
|
44
|
+
- uses: actions/checkout@v4
|
|
45
|
+
- uses: actions/setup-python@v5
|
|
46
|
+
with:
|
|
47
|
+
python-version: "3.12"
|
|
48
|
+
- run: pip install -e ".[dev,semantic]"
|
|
49
|
+
- run: pytest -q --run-slow -m slow
|
|
50
|
+
|
|
51
|
+
build:
|
|
52
|
+
needs: [test, test-semantic-slow]
|
|
53
|
+
runs-on: ubuntu-latest
|
|
54
|
+
steps:
|
|
55
|
+
- uses: actions/checkout@v4
|
|
56
|
+
- uses: actions/setup-python@v5
|
|
57
|
+
with:
|
|
58
|
+
python-version: "3.12"
|
|
59
|
+
- run: pip install build
|
|
60
|
+
- run: python -m build
|
|
61
|
+
- run: pip install dist/*.whl
|
|
62
|
+
- name: Smoke test the built wheel
|
|
63
|
+
run: |
|
|
64
|
+
cd /tmp
|
|
65
|
+
python -c "
|
|
66
|
+
import ragless
|
|
67
|
+
r = ragless.Retriever(['hello world', 'goodbye world'])
|
|
68
|
+
assert r.search('hello')
|
|
69
|
+
print('ragless', ragless.__version__, 'smoke test OK')
|
|
70
|
+
"
|
|
71
|
+
- uses: actions/upload-artifact@v4
|
|
72
|
+
with:
|
|
73
|
+
name: dist
|
|
74
|
+
path: dist/
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
test:
|
|
13
|
+
strategy:
|
|
14
|
+
fail-fast: false
|
|
15
|
+
matrix:
|
|
16
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
- uses: actions/setup-python@v5
|
|
21
|
+
with:
|
|
22
|
+
python-version: ${{ matrix.python-version }}
|
|
23
|
+
- run: pip install -e ".[dev]"
|
|
24
|
+
- run: pytest -q --cov=ragless --cov-fail-under=85
|
|
25
|
+
|
|
26
|
+
build:
|
|
27
|
+
needs: test
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
steps:
|
|
30
|
+
- uses: actions/checkout@v4
|
|
31
|
+
- uses: actions/setup-python@v5
|
|
32
|
+
with:
|
|
33
|
+
python-version: "3.12"
|
|
34
|
+
- run: pip install build
|
|
35
|
+
- run: python -m build
|
|
36
|
+
- uses: actions/upload-artifact@v4
|
|
37
|
+
with:
|
|
38
|
+
name: dist
|
|
39
|
+
path: dist/
|
|
40
|
+
|
|
41
|
+
publish:
|
|
42
|
+
needs: build
|
|
43
|
+
runs-on: ubuntu-latest
|
|
44
|
+
environment:
|
|
45
|
+
name: pypi
|
|
46
|
+
url: https://pypi.org/project/ragless/
|
|
47
|
+
permissions:
|
|
48
|
+
id-token: write
|
|
49
|
+
steps:
|
|
50
|
+
- uses: actions/download-artifact@v4
|
|
51
|
+
with:
|
|
52
|
+
name: dist
|
|
53
|
+
path: dist/
|
|
54
|
+
- name: Publish to PyPI
|
|
55
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
ragless-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/).
|
|
7
|
+
|
|
8
|
+
## [0.1.0] - 2026-11-12
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- `Retriever`: library-first API indexing a directory or in-memory text.
|
|
13
|
+
- `LineChunker`: overlapping line-window chunking that preserves file path
|
|
14
|
+
and 1-indexed start/end line numbers.
|
|
15
|
+
- `BM25Retriever`: Okapi BM25 keyword search (pure NumPy, no extra deps).
|
|
16
|
+
- `FuzzyRetriever`: fuzzy string matching via `rapidfuzz`.
|
|
17
|
+
- `SemanticRetriever`: local embedding search (`fastembed`, optional
|
|
18
|
+
`semantic` extra) with NumPy cosine similarity.
|
|
19
|
+
- Reciprocal Rank Fusion (`reciprocal_rank_fusion`) to combine engine
|
|
20
|
+
rankings, with configurable weights and `k`.
|
|
21
|
+
- `CrossEncoderReranker`: optional cross-encoder reranking of fused results.
|
|
22
|
+
- On-disk embedding cache (`EmbeddingCache`) as plain `.npy`/`.json` sidecar
|
|
23
|
+
files, keyed by file mtime/size, so only changed files are re-embedded.
|
|
24
|
+
- Metadata and path filters on `search()`.
|
|
25
|
+
- Pluggable chunkers, retrievers, and embedders.
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# Autonomous build decisions
|
|
2
|
+
|
|
3
|
+
Decisions made while building `ragless` without the user available to ask.
|
|
4
|
+
|
|
5
|
+
## Packaging / tooling
|
|
6
|
+
|
|
7
|
+
- **Build backend**: hatchling with a `src/` layout. Simple, no `setup.py`,
|
|
8
|
+
plays well with editable installs and PyPI trusted publishing.
|
|
9
|
+
- **Python support**: `>=3.10`. Chosen as a modern-but-not-bleeding-edge
|
|
10
|
+
floor; CI covers 3.10-3.13 on Ubuntu and macOS.
|
|
11
|
+
- **GitHub owner/repo**: created under the authenticated `gh` account
|
|
12
|
+
`mohmdqasim`, repo name `ragless` (public). `pyproject.toml` URLs point
|
|
13
|
+
at `https://github.com/mohmdqasim/ragless`.
|
|
14
|
+
- **Linting**: `ruff` (check + format), configured in `pyproject.toml`
|
|
15
|
+
(`[tool.ruff]`). CI fails on any lint or formatting diff.
|
|
16
|
+
- **Coverage gate**: `--cov-fail-under=85` in CI, matching the 85%+ target.
|
|
17
|
+
Actual achieved coverage on the fast suite is ~90%.
|
|
18
|
+
|
|
19
|
+
## Dependencies
|
|
20
|
+
|
|
21
|
+
- **BM25**: implemented from scratch (`ragless/retrievers/bm25.py`) using
|
|
22
|
+
only NumPy, rather than pulling in `rank-bm25` or a search engine. Keeps
|
|
23
|
+
the core dependency surface to just `numpy` + `rapidfuzz`.
|
|
24
|
+
- **Fuzzy**: `rapidfuzz` (explicitly requested in the spec), used via
|
|
25
|
+
`rapidfuzz.process.extract`.
|
|
26
|
+
- **Semantic embeddings**: `fastembed` (ONNX runtime-based), not
|
|
27
|
+
`sentence-transformers`/`torch`. Much lighter install for an optional
|
|
28
|
+
extra, still fully local with no API keys. Default model:
|
|
29
|
+
`BAAI/bge-small-en-v1.5` (verified available in fastembed's supported
|
|
30
|
+
model list at build time).
|
|
31
|
+
- **Cross-encoder rerank**: also via `fastembed` (`fastembed.rerank.
|
|
32
|
+
cross_encoder.TextCrossEncoder`), under the same `semantic` extra rather
|
|
33
|
+
than a separate `rerank` extra, since it's the same dependency. Default
|
|
34
|
+
model: `Xenova/ms-marco-MiniLM-L-6-v2`.
|
|
35
|
+
- Both default model names were confirmed against the installed fastembed
|
|
36
|
+
version's `list_supported_models()` before being hardcoded.
|
|
37
|
+
|
|
38
|
+
## API shape
|
|
39
|
+
|
|
40
|
+
- `Retriever(source, *, texts=None, chunker=None, engines=None,
|
|
41
|
+
embedder=None, cache_dir=None, include=None, exclude=None, metadata=None,
|
|
42
|
+
encoding="utf-8", rrf_k=60, weights=None)`.
|
|
43
|
+
- `source` accepts a directory path, a single file path, or a sequence of
|
|
44
|
+
raw strings (in-memory documents) directly as the first positional arg,
|
|
45
|
+
per the spec's "should also accept in-memory text". `texts=` is an
|
|
46
|
+
additional kwarg so in-memory notes can be mixed in alongside a
|
|
47
|
+
directory source.
|
|
48
|
+
- `engines` defaults to `("bm25", "fuzzy")` only -- semantic is opt-in via
|
|
49
|
+
`engines=[..., "semantic"]`, keeping the core install light as required.
|
|
50
|
+
It also accepts a `Mapping[str, BaseRetriever]` for full pluggability
|
|
51
|
+
(bring your own retriever implementation under any engine name).
|
|
52
|
+
- `metadata` can be a constant dict or a `path -> dict` callable, so
|
|
53
|
+
per-file metadata tagging doesn't require a separate data structure.
|
|
54
|
+
- **Filters** are two separate mechanisms, matching the spec's "metadata and
|
|
55
|
+
path filters": `filters={"key": value_or_predicate}` against chunk
|
|
56
|
+
metadata, and `path_filter="*glob*"` (or a list of globs) against the
|
|
57
|
+
chunk's file path. In-memory chunks (`path is None`) never match a
|
|
58
|
+
`path_filter`.
|
|
59
|
+
- **Fusion**: Reciprocal Rank Fusion with `k=60` (the constant from the
|
|
60
|
+
original Cormack et al. RRF paper), configurable per-`Retriever` via
|
|
61
|
+
`rrf_k=` and per-engine `weights=`.
|
|
62
|
+
- **Caching**: one `.npy` + `.json` sidecar pair per *source file* (not per
|
|
63
|
+
chunk), keyed by `sha256(path :: embedder_name)[:24]` to keep filenames
|
|
64
|
+
short and collision-resistant, with the original path stored inside the
|
|
65
|
+
JSON for debuggability. Invalidated on `(mtime, size)` signature change,
|
|
66
|
+
embedder name/dim change, or chunk id list change (covers chunker/config
|
|
67
|
+
changes too, not just file edits). In-memory text is never cached (no
|
|
68
|
+
stable file identity to invalidate against).
|
|
69
|
+
- Binary files and files that fail to decode under the given `encoding` are
|
|
70
|
+
silently skipped rather than raising, matching "works on folders" robustly
|
|
71
|
+
without per-file configuration. (Detected via a null-byte check in the
|
|
72
|
+
first 8KB, then a decode attempt.)
|
|
73
|
+
|
|
74
|
+
## Testing
|
|
75
|
+
|
|
76
|
+
- `tests/conftest.py` provides a dependency-free `FakeEmbedder` (hashed
|
|
77
|
+
bag-of-words) so semantic-retriever logic, caching, and fusion can all be
|
|
78
|
+
tested without downloading a real model.
|
|
79
|
+
- Real-model tests (`tests/test_semantic_real_model.py`) are marked
|
|
80
|
+
`@pytest.mark.slow` and additionally gated by a `--run-slow` CLI flag
|
|
81
|
+
(added in `conftest.py`); they're skipped by default and run in a
|
|
82
|
+
dedicated CI job (`test-semantic-slow`) that installs the `semantic`
|
|
83
|
+
extra. The fast test suite never needs network access or fastembed
|
|
84
|
+
installed (verified by running it in a venv with no semantic extra).
|
|
85
|
+
- "Missing optional extra" behavior is tested by monkeypatching
|
|
86
|
+
`sys.modules["fastembed"] = None` rather than requiring a real
|
|
87
|
+
extra-less environment for every test run -- `FastEmbedEmbedder` and
|
|
88
|
+
`CrossEncoderReranker` both import `fastembed` lazily inside `__init__`,
|
|
89
|
+
so this reliably triggers the same `ImportError` path.
|
|
90
|
+
|
|
91
|
+
## CI / publishing
|
|
92
|
+
|
|
93
|
+
- `ci.yml`: `lint` (ruff) -> `test` (matrix: ubuntu/macos x 3.10-3.13,
|
|
94
|
+
`--cov-fail-under=85`) + `test-semantic-slow` (ubuntu/3.12 only, installs
|
|
95
|
+
`semantic` extra, runs `--run-slow -m slow`) -> `build` (sdist/wheel +
|
|
96
|
+
wheel-install smoke test).
|
|
97
|
+
- `publish.yml`: triggers on `v*` tags; re-runs the test matrix and a build
|
|
98
|
+
job *inside this workflow* (a separate workflow run can't `needs:` another
|
|
99
|
+
workflow's jobs), then `publish` (environment `pypi`, `permissions:
|
|
100
|
+
id-token: write`) uses `pypa/gh-action-pypi-publish` with OIDC trusted
|
|
101
|
+
publishing -- no API token stored in the repo.
|
|
102
|
+
- **PyPI trusted publisher registration is a manual, credential-gated step**
|
|
103
|
+
that this session cannot perform (it requires logging into pypi.org as
|
|
104
|
+
the account that will own the `ragless` project). See `PROGRESS.md` for
|
|
105
|
+
exactly what to do.
|
ragless-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ragless contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
ragless-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ragless
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: File-based retrieval for RAG, no vector database required
|
|
5
|
+
Project-URL: Homepage, https://github.com/mohmdqasim/ragless
|
|
6
|
+
Project-URL: Repository, https://github.com/mohmdqasim/ragless
|
|
7
|
+
Project-URL: Issues, https://github.com/mohmdqasim/ragless/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/mohmdqasim/ragless/blob/main/CHANGELOG.md
|
|
9
|
+
Author-email: Qasim Arshad <mohmdqasim0@gmail.com>
|
|
10
|
+
Maintainer-email: Qasim Arshad <mohmdqasim0@gmail.com>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: bm25,embeddings,information-retrieval,rag,retrieval,search
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Classifier: Topic :: Text Processing :: Indexing
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Requires-Dist: numpy>=1.24
|
|
26
|
+
Requires-Dist: rapidfuzz>=3.0
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest-cov>=4.1; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest>=7.4; extra == 'dev'
|
|
31
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
32
|
+
Provides-Extra: semantic
|
|
33
|
+
Requires-Dist: fastembed>=0.3; extra == 'semantic'
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
# ragless
|
|
37
|
+
|
|
38
|
+
File-based retrieval for RAG, with no vector database.
|
|
39
|
+
|
|
40
|
+
`ragless` indexes a folder of text files (or in-memory strings) and lets you
|
|
41
|
+
search them with BM25 keyword search, fuzzy string matching, and optional
|
|
42
|
+
local semantic embeddings -- combined with Reciprocal Rank Fusion. Everything
|
|
43
|
+
lives on disk as plain files (or in memory): no Postgres/pgvector, no
|
|
44
|
+
Pinecone/Weaviate/Chroma server, nothing to run.
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from ragless import Retriever
|
|
48
|
+
|
|
49
|
+
r = Retriever("./docs")
|
|
50
|
+
for hit in r.search("how do I configure retries", mode="hybrid"):
|
|
51
|
+
print(f"{hit.path}:{hit.start_line}-{hit.end_line} score={hit.score:.3f}")
|
|
52
|
+
print(hit.text)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Install
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install ragless # BM25 + fuzzy (no heavy deps)
|
|
59
|
+
pip install 'ragless[semantic]' # + local embeddings and cross-encoder rerank
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Quickstart
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
from ragless import Retriever
|
|
66
|
+
|
|
67
|
+
# Index a directory of files.
|
|
68
|
+
r = Retriever("./docs")
|
|
69
|
+
results = r.search("installation steps", mode="hybrid", top_k=5)
|
|
70
|
+
|
|
71
|
+
# Or index in-memory text directly -- no files required.
|
|
72
|
+
r = Retriever(
|
|
73
|
+
[
|
|
74
|
+
"ragless indexes folders of text files for retrieval.",
|
|
75
|
+
"BM25, fuzzy matching, and semantic search are combined with RRF.",
|
|
76
|
+
]
|
|
77
|
+
)
|
|
78
|
+
results = r.search("how are results combined?")
|
|
79
|
+
|
|
80
|
+
for hit in results:
|
|
81
|
+
print(hit.path, hit.start_line, hit.end_line, hit.score)
|
|
82
|
+
print(hit.text)
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
### Search modes
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
r.search("query", mode="hybrid") # fuse every configured engine (default)
|
|
89
|
+
r.search("query", mode="bm25") # keyword search only
|
|
90
|
+
r.search("query", mode="fuzzy") # fuzzy string matching only
|
|
91
|
+
r.search("query", mode="semantic") # embeddings only (requires the extra)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### Semantic search and caching
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from ragless import Retriever
|
|
98
|
+
|
|
99
|
+
r = Retriever(
|
|
100
|
+
"./docs",
|
|
101
|
+
engines=["bm25", "fuzzy", "semantic"],
|
|
102
|
+
cache_dir="./.ragless_cache", # embeddings cached here as .npy/.json
|
|
103
|
+
)
|
|
104
|
+
r.search("query", mode="hybrid")
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Embeddings are cached per source file, keyed by the file's mtime/size and the
|
|
108
|
+
embedder's name. Editing one file only re-embeds that file's chunks on the
|
|
109
|
+
next run; untouched files are loaded straight from the cache.
|
|
110
|
+
|
|
111
|
+
### Cross-encoder rerank
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
r.search("query", mode="hybrid", rerank=True)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Reranking takes the top fused candidates and rescoring them with a small
|
|
118
|
+
local cross-encoder (also via `fastembed`, under the `semantic` extra) for
|
|
119
|
+
higher precision at the top of the list.
|
|
120
|
+
|
|
121
|
+
### Filters
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
# Only chunks whose metadata says category="guide"
|
|
125
|
+
r.search("query", filters={"category": "guide"})
|
|
126
|
+
|
|
127
|
+
# Only chunks from markdown files
|
|
128
|
+
r.search("query", path_filter="*.md")
|
|
129
|
+
|
|
130
|
+
# Per-document metadata, assigned when building the index
|
|
131
|
+
r = Retriever(
|
|
132
|
+
"./docs",
|
|
133
|
+
metadata=lambda path: {"category": "guide"} if path and "guide" in path else {},
|
|
134
|
+
)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
### Pluggable components
|
|
138
|
+
|
|
139
|
+
Chunkers, retrievers, and embedders are all swappable:
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
from ragless import Retriever, Chunker, BaseRetriever
|
|
143
|
+
from ragless.chunking import LineChunker
|
|
144
|
+
|
|
145
|
+
r = Retriever("./docs", chunker=LineChunker(lines_per_chunk=60, overlap=10))
|
|
146
|
+
|
|
147
|
+
# Bring your own retriever/embedder implementation:
|
|
148
|
+
r = Retriever("./docs", engines={"bm25": MyRetriever(), "fuzzy": FuzzyRetriever()})
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Implement `ragless.chunking.Chunker`, `ragless.retrievers.base.BaseRetriever`,
|
|
152
|
+
or `ragless.embedders.base.Embedder` to plug in your own.
|
|
153
|
+
|
|
154
|
+
## Why
|
|
155
|
+
|
|
156
|
+
Most "RAG" tutorials reach for a vector database before checking whether they
|
|
157
|
+
need one. For a folder of a few thousand files, plain keyword search (BM25),
|
|
158
|
+
fuzzy matching, and a small local embedding model get you most of the way
|
|
159
|
+
there, with:
|
|
160
|
+
|
|
161
|
+
- No server to run, no API keys, no network calls (semantic search runs a
|
|
162
|
+
small model locally via [fastembed](https://github.com/qdrant/fastembed)).
|
|
163
|
+
- No ingestion pipeline: point it at a directory and it chunks and indexes on
|
|
164
|
+
construction.
|
|
165
|
+
- Caching as plain sidecar files (`.npy` + `.json`) next to your index, not a
|
|
166
|
+
database you have to back up and migrate.
|
|
167
|
+
- Every result keeps the file path and line range it came from.
|
|
168
|
+
|
|
169
|
+
| | ragless | vector DB (Pinecone/Chroma/pgvector) |
|
|
170
|
+
|---|---|---|
|
|
171
|
+
| Infra | none | a running database/service |
|
|
172
|
+
| Setup | `Retriever("./docs")` | provision, configure, ingest |
|
|
173
|
+
| Keyword search | built in (BM25) | usually bolted on separately |
|
|
174
|
+
| Fuzzy/typo tolerance | built in | usually absent |
|
|
175
|
+
| Good for | a few thousand files, local tools, CLIs, CI | large, constantly-changing, multi-tenant corpora |
|
|
176
|
+
| Cold start | instant | needs the service up and reachable |
|
|
177
|
+
|
|
178
|
+
If you need to search tens of millions of chunks with concurrent writers
|
|
179
|
+
across a fleet of machines, use a vector database. If you need to search the
|
|
180
|
+
docs folder sitting next to your code, `ragless` is simpler.
|
|
181
|
+
|
|
182
|
+
## How it works
|
|
183
|
+
|
|
184
|
+
1. **Load** -- read files from a directory (or accept raw in-memory strings).
|
|
185
|
+
Binary files and undecodable text are skipped automatically.
|
|
186
|
+
2. **Chunk** -- split each document into overlapping line-based windows,
|
|
187
|
+
keeping the file path and 1-indexed `start_line`/`end_line` for every
|
|
188
|
+
chunk (`ragless.chunking.LineChunker`, pluggable).
|
|
189
|
+
3. **Index** -- each configured engine builds its own index over the chunks:
|
|
190
|
+
- `BM25Retriever`: classic Okapi BM25 over tokenized text (NumPy).
|
|
191
|
+
- `FuzzyRetriever`: `rapidfuzz` string similarity.
|
|
192
|
+
- `SemanticRetriever`: dense embeddings (`fastembed`) with cosine
|
|
193
|
+
similarity (NumPy), cached to disk as `.npy`/`.json` sidecar files.
|
|
194
|
+
4. **Search** -- query every configured engine, fuse their rankings with
|
|
195
|
+
[Reciprocal Rank Fusion](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf),
|
|
196
|
+
apply metadata/path filters, and optionally rerank the top candidates with
|
|
197
|
+
a cross-encoder.
|
|
198
|
+
|
|
199
|
+
## Status
|
|
200
|
+
|
|
201
|
+
0.1.0 -- the API may still shift. Feedback and issues welcome.
|
|
202
|
+
|
|
203
|
+
## License
|
|
204
|
+
|
|
205
|
+
MIT
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# Progress
|
|
2
|
+
|
|
3
|
+
Autonomous build log for `ragless`. Updated as work proceeds.
|
|
4
|
+
|
|
5
|
+
## Status: build complete, CI green on GitHub, publishing blocked on a manual PyPI step
|
|
6
|
+
|
|
7
|
+
### Done
|
|
8
|
+
|
|
9
|
+
- [x] Full package implemented under `src/ragless/` (chunking, BM25, fuzzy,
|
|
10
|
+
semantic, fusion, rerank, cache, core `Retriever`), pluggable
|
|
11
|
+
throughout, with type hints and docstrings.
|
|
12
|
+
- [x] Test suite: 61 fast tests (chunker, each retriever, fusion, cache
|
|
13
|
+
invalidation, filters, directory/in-memory loading, binary/empty/
|
|
14
|
+
unicode/large-file edge cases, missing-extra error messages, custom
|
|
15
|
+
reranker) + 2 slow tests gated behind `--run-slow`/`-m slow`.
|
|
16
|
+
- [x] Fast suite passes in a venv **without** the `semantic` extra installed
|
|
17
|
+
(confirms the core install is genuinely light).
|
|
18
|
+
- [x] Slow suite passes with the real `fastembed` model
|
|
19
|
+
(`BAAI/bge-small-en-v1.5` + `Xenova/ms-marco-MiniLM-L-6-v2`).
|
|
20
|
+
- [x] `ruff check .` and `ruff format --check .` both clean.
|
|
21
|
+
- [x] Coverage: 90% (gate set at 85% in CI).
|
|
22
|
+
- [x] `python -m build` produces sdist + wheel; installed the wheel into a
|
|
23
|
+
fresh venv and ran an import + search smoke test successfully.
|
|
24
|
+
- [x] README (quickstart, API examples, how it works, comparison table),
|
|
25
|
+
CHANGELOG.md, `examples/` (4 runnable scripts, all manually run and
|
|
26
|
+
verified, including the semantic+rerank one).
|
|
27
|
+
- [x] `.github/workflows/ci.yml` (lint, test matrix ubuntu/macos x
|
|
28
|
+
3.10-3.13, slow-model job, build+smoke-test job).
|
|
29
|
+
- [x] `.github/workflows/publish.yml` (triggers on `v*` tags, reruns tests,
|
|
30
|
+
builds, then publishes via OIDC trusted publishing to the `pypi`
|
|
31
|
+
environment).
|
|
32
|
+
|
|
33
|
+
- [x] `git init` + initial commit, history kept to clean, correctly-authored
|
|
34
|
+
commits (`Qasim Arshad <mohmdqasim0@gmail.com>`), no AI/trailer
|
|
35
|
+
mentions anywhere.
|
|
36
|
+
- [x] `twine check dist/*` passes (sdist + wheel); PyPI page will render the
|
|
37
|
+
full README as `text/markdown` with correct author/maintainer and
|
|
38
|
+
project URLs (Homepage/Repository/Issues/Changelog).
|
|
39
|
+
- [x] Public GitHub repo created at https://github.com/mohmdqasim/ragless
|
|
40
|
+
(the repo was first created under the wrong, auto-selected `gh`
|
|
41
|
+
account and was recreated here after the user logged `gh` into their
|
|
42
|
+
personal account; the old repo was deleted by the user).
|
|
43
|
+
- [x] `pypi` GitHub Environment created on the repo via `gh api`.
|
|
44
|
+
- [x] CI green on `main`: lint, test matrix (ubuntu/macos x 3.10-3.13),
|
|
45
|
+
slow-model job, and build+smoke-test job all passed
|
|
46
|
+
(https://github.com/mohmdqasim/ragless/actions).
|
|
47
|
+
|
|
48
|
+
### Next
|
|
49
|
+
|
|
50
|
+
- [ ] Tag `v0.1.0`, push the tag, watch the publish workflow run.
|
|
51
|
+
- [ ] Verify with `pip install ragless` in a fresh venv + a quick search.
|
|
52
|
+
|
|
53
|
+
## Blocking item: PyPI trusted publisher registration
|
|
54
|
+
|
|
55
|
+
**This cannot be completed autonomously** -- it requires logging into
|
|
56
|
+
pypi.org with human credentials, which this session does not have and
|
|
57
|
+
should not attempt to obtain.
|
|
58
|
+
|
|
59
|
+
`publish.yml` is wired for PyPI's OIDC "trusted publishing": no API token is
|
|
60
|
+
stored anywhere, but PyPI must be told in advance which GitHub repo/workflow
|
|
61
|
+
is allowed to publish the `ragless` project. For a project that has never
|
|
62
|
+
been published before, this is done once via PyPI's web UI as a **pending
|
|
63
|
+
publisher**, which reserves the name and links it to this workflow ahead of
|
|
64
|
+
the first publish.
|
|
65
|
+
|
|
66
|
+
### What you need to do (takes ~1 minute)
|
|
67
|
+
|
|
68
|
+
1. Log into https://pypi.org (create an account first if you don't have
|
|
69
|
+
one).
|
|
70
|
+
2. Go to https://pypi.org/manage/account/publishing/
|
|
71
|
+
3. Under "Add a new pending publisher", fill in:
|
|
72
|
+
- **PyPI project name**: `ragless`
|
|
73
|
+
- **Owner**: `mohmdqasim`
|
|
74
|
+
- **Repository name**: `ragless`
|
|
75
|
+
- **Workflow name**: `publish.yml`
|
|
76
|
+
- **Environment name**: `pypi`
|
|
77
|
+
4. Submit. That's it -- no token to copy anywhere.
|
|
78
|
+
|
|
79
|
+
Once that's set up, the next tag push (or a re-run of a failed `publish.yml`
|
|
80
|
+
run) will publish successfully. If this step hasn't been done yet when this
|
|
81
|
+
session tags `v0.1.0` and pushes it, the `publish` job in the workflow run
|
|
82
|
+
will fail at the PyPI upload step with an authentication error -- that is
|
|
83
|
+
expected and not a bug in the workflow; it just means the pending publisher
|
|
84
|
+
above still needs to be registered, after which re-running that same
|
|
85
|
+
workflow run (via the GitHub Actions UI, or pushing a new tag like `v0.1.1`)
|
|
86
|
+
will succeed.
|