ragless 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ragless/__init__.py +32 -0
- ragless/cache.py +100 -0
- ragless/chunking.py +88 -0
- ragless/core.py +365 -0
- ragless/embedders/__init__.py +3 -0
- ragless/embedders/base.py +26 -0
- ragless/embedders/fastembed_embedder.py +41 -0
- ragless/exceptions.py +21 -0
- ragless/fusion.py +39 -0
- ragless/rerank.py +51 -0
- ragless/retrievers/__init__.py +6 -0
- ragless/retrievers/base.py +34 -0
- ragless/retrievers/bm25.py +81 -0
- ragless/retrievers/fuzzy.py +35 -0
- ragless/retrievers/semantic.py +100 -0
- ragless/types.py +49 -0
- ragless-0.1.0.dist-info/METADATA +205 -0
- ragless-0.1.0.dist-info/RECORD +20 -0
- ragless-0.1.0.dist-info/WHEEL +4 -0
- ragless-0.1.0.dist-info/licenses/LICENSE +21 -0
ragless/__init__.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""ragless: file-based retrieval for RAG, no vector database required."""
|
|
2
|
+
|
|
3
|
+
from ragless.chunking import Chunker, LineChunker
|
|
4
|
+
from ragless.core import Retriever
|
|
5
|
+
from ragless.embedders.base import Embedder
|
|
6
|
+
from ragless.exceptions import MissingDependencyError, RaglessError, UnknownRetrieverError
|
|
7
|
+
from ragless.fusion import reciprocal_rank_fusion
|
|
8
|
+
from ragless.rerank import CrossEncoderReranker, Reranker
|
|
9
|
+
from ragless.retrievers import BaseRetriever, BM25Retriever, FuzzyRetriever, SemanticRetriever
|
|
10
|
+
from ragless.types import Chunk, SearchResult
|
|
11
|
+
|
|
12
|
+
__version__ = "0.1.0"
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"Retriever",
|
|
16
|
+
"Chunk",
|
|
17
|
+
"SearchResult",
|
|
18
|
+
"Chunker",
|
|
19
|
+
"LineChunker",
|
|
20
|
+
"BaseRetriever",
|
|
21
|
+
"BM25Retriever",
|
|
22
|
+
"FuzzyRetriever",
|
|
23
|
+
"SemanticRetriever",
|
|
24
|
+
"Embedder",
|
|
25
|
+
"reciprocal_rank_fusion",
|
|
26
|
+
"CrossEncoderReranker",
|
|
27
|
+
"Reranker",
|
|
28
|
+
"RaglessError",
|
|
29
|
+
"MissingDependencyError",
|
|
30
|
+
"UnknownRetrieverError",
|
|
31
|
+
"__version__",
|
|
32
|
+
]
|
ragless/cache.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""On-disk embedding cache: plain sidecar files, no database.
|
|
2
|
+
|
|
3
|
+
Each cached source file gets two sidecar files inside the cache directory:
|
|
4
|
+
a ``.npy`` holding the embedding matrix and a ``.json`` holding the metadata
|
|
5
|
+
needed to validate the cache (which embedder produced it, the source file's
|
|
6
|
+
mtime/size, and the exact chunk ids it covers). There is no cache index or
|
|
7
|
+
manifest to keep in sync -- each file's validity is self-contained, and a
|
|
8
|
+
changed source file simply gets new sidecar files on its next embed.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import hashlib
|
|
14
|
+
import json
|
|
15
|
+
from collections.abc import Sequence
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import TYPE_CHECKING
|
|
18
|
+
|
|
19
|
+
import numpy as np
|
|
20
|
+
|
|
21
|
+
from ragless.types import Chunk
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from ragless.embedders.base import Embedder
|
|
25
|
+
|
|
26
|
+
FileSignature = tuple[float, int]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def file_signature(path: Path) -> FileSignature:
|
|
30
|
+
"""A cheap (mtime, size) signature used to detect that a file changed."""
|
|
31
|
+
stat = path.stat()
|
|
32
|
+
return stat.st_mtime, stat.st_size
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _cache_key(source_path: str, embedder_name: str) -> str:
|
|
36
|
+
digest = hashlib.sha256(f"{source_path}::{embedder_name}".encode()).hexdigest()
|
|
37
|
+
return digest[:24]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class EmbeddingCache:
|
|
41
|
+
"""Caches per-file chunk embeddings on disk as ``.npy`` + ``.json`` pairs."""
|
|
42
|
+
|
|
43
|
+
def __init__(self, cache_dir: str | Path) -> None:
|
|
44
|
+
self.cache_dir = Path(cache_dir)
|
|
45
|
+
|
|
46
|
+
def _paths_for(self, source_path: str, embedder_name: str) -> tuple[Path, Path]:
|
|
47
|
+
key = _cache_key(source_path, embedder_name)
|
|
48
|
+
return self.cache_dir / f"{key}.npy", self.cache_dir / f"{key}.json"
|
|
49
|
+
|
|
50
|
+
def get(
|
|
51
|
+
self,
|
|
52
|
+
source_path: str,
|
|
53
|
+
signature: FileSignature,
|
|
54
|
+
embedder: Embedder,
|
|
55
|
+
chunks: Sequence[Chunk],
|
|
56
|
+
) -> np.ndarray | None:
|
|
57
|
+
"""Return cached embeddings for ``chunks`` if the cache entry is still valid."""
|
|
58
|
+
npy_path, json_path = self._paths_for(source_path, embedder.name)
|
|
59
|
+
if not npy_path.exists() or not json_path.exists():
|
|
60
|
+
return None
|
|
61
|
+
try:
|
|
62
|
+
meta = json.loads(json_path.read_text())
|
|
63
|
+
except (OSError, json.JSONDecodeError):
|
|
64
|
+
return None
|
|
65
|
+
|
|
66
|
+
if meta.get("embedder") != embedder.name or meta.get("dim") != embedder.dim:
|
|
67
|
+
return None
|
|
68
|
+
if tuple(meta.get("signature", ())) != tuple(signature):
|
|
69
|
+
return None
|
|
70
|
+
if meta.get("chunk_ids") != [c.id for c in chunks]:
|
|
71
|
+
return None
|
|
72
|
+
|
|
73
|
+
try:
|
|
74
|
+
vectors = np.load(npy_path)
|
|
75
|
+
except (OSError, ValueError):
|
|
76
|
+
return None
|
|
77
|
+
if vectors.shape != (len(chunks), embedder.dim):
|
|
78
|
+
return None
|
|
79
|
+
return vectors
|
|
80
|
+
|
|
81
|
+
def put(
|
|
82
|
+
self,
|
|
83
|
+
source_path: str,
|
|
84
|
+
signature: FileSignature,
|
|
85
|
+
embedder: Embedder,
|
|
86
|
+
chunks: Sequence[Chunk],
|
|
87
|
+
vectors: np.ndarray,
|
|
88
|
+
) -> None:
|
|
89
|
+
"""Persist ``vectors`` (aligned with ``chunks``) for ``source_path``."""
|
|
90
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
npy_path, json_path = self._paths_for(source_path, embedder.name)
|
|
92
|
+
np.save(npy_path, vectors)
|
|
93
|
+
meta = {
|
|
94
|
+
"path": source_path,
|
|
95
|
+
"embedder": embedder.name,
|
|
96
|
+
"dim": embedder.dim,
|
|
97
|
+
"signature": list(signature),
|
|
98
|
+
"chunk_ids": [c.id for c in chunks],
|
|
99
|
+
}
|
|
100
|
+
json_path.write_text(json.dumps(meta))
|
ragless/chunking.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Pluggable chunkers that split document text into retrievable :class:`Chunk` objects."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from ragless.types import Chunk
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Chunker(ABC):
|
|
12
|
+
"""Base class for splitting a document's text into chunks.
|
|
13
|
+
|
|
14
|
+
Implementations must preserve enough information to report the original
|
|
15
|
+
``start_line``/``end_line`` span for every chunk they produce.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
@abstractmethod
|
|
19
|
+
def chunk(
|
|
20
|
+
self,
|
|
21
|
+
text: str,
|
|
22
|
+
*,
|
|
23
|
+
path: str | None,
|
|
24
|
+
chunk_id_prefix: str,
|
|
25
|
+
metadata: dict[str, Any] | None = None,
|
|
26
|
+
) -> list[Chunk]:
|
|
27
|
+
"""Split ``text`` into a list of :class:`Chunk`.
|
|
28
|
+
|
|
29
|
+
``chunk_id_prefix`` should be used to build globally unique chunk ids,
|
|
30
|
+
e.g. ``f"{chunk_id_prefix}:{index}"``.
|
|
31
|
+
"""
|
|
32
|
+
raise NotImplementedError
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class LineChunker(Chunker):
|
|
36
|
+
"""Splits text into fixed-size, overlapping windows of lines.
|
|
37
|
+
|
|
38
|
+
This is the default chunker. It is format-agnostic (works for source code,
|
|
39
|
+
markdown, plain text, ...) and always keeps track of the 1-indexed
|
|
40
|
+
``start_line``/``end_line`` range each chunk came from.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
def __init__(self, lines_per_chunk: int = 40, overlap: int = 5) -> None:
|
|
44
|
+
if lines_per_chunk <= 0:
|
|
45
|
+
raise ValueError("lines_per_chunk must be positive")
|
|
46
|
+
if overlap < 0 or overlap >= lines_per_chunk:
|
|
47
|
+
raise ValueError("overlap must be in [0, lines_per_chunk)")
|
|
48
|
+
self.lines_per_chunk = lines_per_chunk
|
|
49
|
+
self.overlap = overlap
|
|
50
|
+
|
|
51
|
+
def chunk(
|
|
52
|
+
self,
|
|
53
|
+
text: str,
|
|
54
|
+
*,
|
|
55
|
+
path: str | None,
|
|
56
|
+
chunk_id_prefix: str,
|
|
57
|
+
metadata: dict[str, Any] | None = None,
|
|
58
|
+
) -> list[Chunk]:
|
|
59
|
+
lines = text.splitlines()
|
|
60
|
+
if not lines:
|
|
61
|
+
return []
|
|
62
|
+
|
|
63
|
+
meta = dict(metadata or {})
|
|
64
|
+
step = self.lines_per_chunk - self.overlap
|
|
65
|
+
chunks: list[Chunk] = []
|
|
66
|
+
index = 0
|
|
67
|
+
start = 0
|
|
68
|
+
n = len(lines)
|
|
69
|
+
while start < n:
|
|
70
|
+
end = min(start + self.lines_per_chunk, n)
|
|
71
|
+
chunk_lines = lines[start:end]
|
|
72
|
+
chunk_text = "\n".join(chunk_lines)
|
|
73
|
+
if chunk_text.strip():
|
|
74
|
+
chunks.append(
|
|
75
|
+
Chunk(
|
|
76
|
+
id=f"{chunk_id_prefix}:{index}",
|
|
77
|
+
text=chunk_text,
|
|
78
|
+
path=path,
|
|
79
|
+
start_line=start + 1,
|
|
80
|
+
end_line=end,
|
|
81
|
+
metadata=meta,
|
|
82
|
+
)
|
|
83
|
+
)
|
|
84
|
+
index += 1
|
|
85
|
+
if end == n:
|
|
86
|
+
break
|
|
87
|
+
start += step
|
|
88
|
+
return chunks
|
ragless/core.py
ADDED
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
"""The library's main entry point: :class:`Retriever`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import fnmatch
|
|
6
|
+
import os
|
|
7
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from ragless.cache import EmbeddingCache, FileSignature, file_signature
|
|
12
|
+
from ragless.chunking import Chunker, LineChunker
|
|
13
|
+
from ragless.embedders.base import Embedder
|
|
14
|
+
from ragless.exceptions import UnknownRetrieverError
|
|
15
|
+
from ragless.fusion import reciprocal_rank_fusion
|
|
16
|
+
from ragless.rerank import Reranker
|
|
17
|
+
from ragless.retrievers.base import BaseRetriever
|
|
18
|
+
from ragless.retrievers.bm25 import BM25Retriever
|
|
19
|
+
from ragless.retrievers.fuzzy import FuzzyRetriever
|
|
20
|
+
from ragless.retrievers.semantic import SemanticRetriever
|
|
21
|
+
from ragless.types import Chunk, SearchResult
|
|
22
|
+
|
|
23
|
+
DEFAULT_ENGINES: tuple[str, ...] = ("bm25", "fuzzy")
|
|
24
|
+
|
|
25
|
+
_SKIP_DIRS = {
|
|
26
|
+
".git",
|
|
27
|
+
"__pycache__",
|
|
28
|
+
".ragless_cache",
|
|
29
|
+
"node_modules",
|
|
30
|
+
".venv",
|
|
31
|
+
"venv",
|
|
32
|
+
".mypy_cache",
|
|
33
|
+
".pytest_cache",
|
|
34
|
+
".ruff_cache",
|
|
35
|
+
".tox",
|
|
36
|
+
".eggs",
|
|
37
|
+
"dist",
|
|
38
|
+
"build",
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
MetadataSource = Mapping[str, Any] | Callable[[str | None], Mapping[str, Any]] | None
|
|
42
|
+
EngineSpec = Sequence[str] | Mapping[str, BaseRetriever] | None
|
|
43
|
+
Filters = Mapping[str, Any | Callable[[Any], bool]] | None
|
|
44
|
+
PathFilter = str | Sequence[str] | None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class Retriever:
|
|
48
|
+
"""Index a directory (or in-memory text) and search it with no vector DB.
|
|
49
|
+
|
|
50
|
+
Three retrieval strategies -- BM25 keyword search, fuzzy string matching,
|
|
51
|
+
and (optionally) semantic embedding search -- are combined with
|
|
52
|
+
Reciprocal Rank Fusion. Everything is pluggable: chunker, retrievers,
|
|
53
|
+
and embedder can all be swapped out.
|
|
54
|
+
|
|
55
|
+
Examples:
|
|
56
|
+
>>> r = Retriever("./docs")
|
|
57
|
+
>>> results = r.search("installation steps", mode="hybrid")
|
|
58
|
+
|
|
59
|
+
>>> r = Retriever(["first note", "second note about pandas"])
|
|
60
|
+
>>> results = r.search("pandas")
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
def __init__(
|
|
64
|
+
self,
|
|
65
|
+
source: str | Path | Sequence[str] | None = None,
|
|
66
|
+
*,
|
|
67
|
+
texts: Sequence[str] | None = None,
|
|
68
|
+
chunker: Chunker | None = None,
|
|
69
|
+
engines: EngineSpec = None,
|
|
70
|
+
embedder: Embedder | None = None,
|
|
71
|
+
cache_dir: str | Path | None = None,
|
|
72
|
+
include: Sequence[str] | None = None,
|
|
73
|
+
exclude: Sequence[str] | None = None,
|
|
74
|
+
metadata: MetadataSource = None,
|
|
75
|
+
encoding: str = "utf-8",
|
|
76
|
+
rrf_k: int = 60,
|
|
77
|
+
weights: Mapping[str, float] | None = None,
|
|
78
|
+
) -> None:
|
|
79
|
+
"""
|
|
80
|
+
Args:
|
|
81
|
+
source: A directory path, a single file path, or a sequence of
|
|
82
|
+
raw in-memory strings (each treated as one document).
|
|
83
|
+
texts: Additional in-memory strings to index alongside ``source``.
|
|
84
|
+
chunker: Splits document text into :class:`~ragless.types.Chunk`.
|
|
85
|
+
Defaults to :class:`~ragless.chunking.LineChunker`.
|
|
86
|
+
engines: Which retrievers to build and query. Either a sequence
|
|
87
|
+
of names from ``{"bm25", "fuzzy", "semantic"}`` (default
|
|
88
|
+
``("bm25", "fuzzy")``), or a mapping of name to a pre-built
|
|
89
|
+
:class:`~ragless.retrievers.base.BaseRetriever` for full
|
|
90
|
+
control over pluggable retrievers.
|
|
91
|
+
embedder: Embedding backend used by the ``"semantic"`` engine.
|
|
92
|
+
Defaults to :class:`~ragless.embedders.fastembed_embedder.FastEmbedEmbedder`,
|
|
93
|
+
which requires ``pip install ragless[semantic]``.
|
|
94
|
+
cache_dir: Directory to store the semantic embedding cache
|
|
95
|
+
(sidecar ``.npy``/``.json`` files). Only files on disk are
|
|
96
|
+
cached; in-memory text is always re-embedded.
|
|
97
|
+
include: Glob patterns (matched against each file's relative
|
|
98
|
+
POSIX path); only matching files are indexed.
|
|
99
|
+
exclude: Glob patterns for files to skip.
|
|
100
|
+
metadata: Either a constant mapping applied to every chunk, or a
|
|
101
|
+
callable ``path -> mapping`` for per-document metadata (used
|
|
102
|
+
by ``filters=`` in :meth:`search`). ``path`` is ``None`` for
|
|
103
|
+
in-memory text.
|
|
104
|
+
encoding: Text encoding used to read files. Files that fail to
|
|
105
|
+
decode, or contain null bytes, are treated as binary and
|
|
106
|
+
skipped.
|
|
107
|
+
rrf_k: The ``k`` constant in Reciprocal Rank Fusion.
|
|
108
|
+
weights: Optional per-engine weights used during fusion, e.g.
|
|
109
|
+
``{"bm25": 1.0, "semantic": 2.0}``.
|
|
110
|
+
"""
|
|
111
|
+
self.chunker = chunker or LineChunker()
|
|
112
|
+
self.rrf_k = rrf_k
|
|
113
|
+
self.weights = weights
|
|
114
|
+
self._cache_dir = cache_dir
|
|
115
|
+
self._reranker: Reranker | None = None
|
|
116
|
+
|
|
117
|
+
documents = _load_documents(
|
|
118
|
+
source, texts, include=include, exclude=exclude, encoding=encoding
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
self._chunks: list[Chunk] = []
|
|
122
|
+
self._chunk_by_id: dict[str, Chunk] = {}
|
|
123
|
+
self._file_signatures: dict[str, FileSignature] = {}
|
|
124
|
+
|
|
125
|
+
mem_index = 0
|
|
126
|
+
for path, text in documents:
|
|
127
|
+
if path is None:
|
|
128
|
+
prefix = f"mem:{mem_index}"
|
|
129
|
+
mem_index += 1
|
|
130
|
+
else:
|
|
131
|
+
prefix = path
|
|
132
|
+
self._file_signatures[path] = file_signature(Path(path))
|
|
133
|
+
|
|
134
|
+
chunk_metadata = _resolve_metadata(metadata, path)
|
|
135
|
+
for chunk in self.chunker.chunk(
|
|
136
|
+
text, path=path, chunk_id_prefix=prefix, metadata=chunk_metadata
|
|
137
|
+
):
|
|
138
|
+
self._chunks.append(chunk)
|
|
139
|
+
self._chunk_by_id[chunk.id] = chunk
|
|
140
|
+
|
|
141
|
+
self._retrievers: dict[str, BaseRetriever] = {}
|
|
142
|
+
if isinstance(engines, Mapping):
|
|
143
|
+
self._retrievers.update(engines)
|
|
144
|
+
else:
|
|
145
|
+
for name in engines if engines is not None else DEFAULT_ENGINES:
|
|
146
|
+
self._retrievers[name] = self._build_engine(name, embedder, cache_dir)
|
|
147
|
+
|
|
148
|
+
for _name, retriever in self._retrievers.items():
|
|
149
|
+
if isinstance(retriever, SemanticRetriever):
|
|
150
|
+
retriever.index_with_signatures(self._chunks, self._file_signatures)
|
|
151
|
+
else:
|
|
152
|
+
retriever.index(self._chunks)
|
|
153
|
+
|
|
154
|
+
def _build_engine(
|
|
155
|
+
self, name: str, embedder: Embedder | None, cache_dir: str | Path | None
|
|
156
|
+
) -> BaseRetriever:
|
|
157
|
+
if name == "bm25":
|
|
158
|
+
return BM25Retriever()
|
|
159
|
+
if name == "fuzzy":
|
|
160
|
+
return FuzzyRetriever()
|
|
161
|
+
if name == "semantic":
|
|
162
|
+
resolved_embedder = embedder or _default_embedder()
|
|
163
|
+
cache = EmbeddingCache(cache_dir) if cache_dir is not None else None
|
|
164
|
+
return SemanticRetriever(resolved_embedder, cache=cache)
|
|
165
|
+
raise UnknownRetrieverError(
|
|
166
|
+
f"Unknown retriever engine {name!r}; expected one of "
|
|
167
|
+
f"'bm25', 'fuzzy', 'semantic', or an engines= mapping with a custom retriever."
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
def search(
|
|
171
|
+
self,
|
|
172
|
+
query: str,
|
|
173
|
+
*,
|
|
174
|
+
mode: str = "hybrid",
|
|
175
|
+
top_k: int = 10,
|
|
176
|
+
filters: Filters = None,
|
|
177
|
+
path_filter: PathFilter = None,
|
|
178
|
+
rerank: bool | Reranker = False,
|
|
179
|
+
fetch_k: int | None = None,
|
|
180
|
+
) -> list[SearchResult]:
|
|
181
|
+
"""Search the indexed corpus.
|
|
182
|
+
|
|
183
|
+
Args:
|
|
184
|
+
query: The search query.
|
|
185
|
+
mode: ``"hybrid"`` fuses every configured engine; or the name of
|
|
186
|
+
a single configured engine (e.g. ``"bm25"``).
|
|
187
|
+
top_k: Maximum number of results to return.
|
|
188
|
+
filters: Mapping of metadata key to either an exact value or a
|
|
189
|
+
predicate ``value -> bool``; a chunk must satisfy all of them.
|
|
190
|
+
path_filter: One or more glob patterns; a chunk's path must match
|
|
191
|
+
at least one. In-memory chunks (``path is None``) never match.
|
|
192
|
+
rerank: ``True`` to rerank fused candidates with the default
|
|
193
|
+
:class:`~ragless.rerank.CrossEncoderReranker` (requires
|
|
194
|
+
``pip install ragless[semantic]``), or a custom object
|
|
195
|
+
implementing the :class:`~ragless.rerank.Reranker` protocol.
|
|
196
|
+
fetch_k: How many results to fetch from each engine before fusion
|
|
197
|
+
and filtering. Defaults to ``max(top_k * 5, 25)``.
|
|
198
|
+
|
|
199
|
+
Returns:
|
|
200
|
+
Up to ``top_k`` :class:`~ragless.types.SearchResult`, best first.
|
|
201
|
+
"""
|
|
202
|
+
if not self._chunks or not query or not query.strip():
|
|
203
|
+
return []
|
|
204
|
+
|
|
205
|
+
engine_names = self._resolve_mode(mode)
|
|
206
|
+
effective_fetch_k = fetch_k if fetch_k is not None else max(top_k * 5, 25)
|
|
207
|
+
|
|
208
|
+
rankings = {}
|
|
209
|
+
for name in engine_names:
|
|
210
|
+
rankings[name] = self._retrievers[name].query(query, effective_fetch_k)
|
|
211
|
+
|
|
212
|
+
fused = reciprocal_rank_fusion(rankings, k=self.rrf_k, weights=self.weights)
|
|
213
|
+
candidates = [
|
|
214
|
+
(chunk_id, score, raw)
|
|
215
|
+
for chunk_id, score, raw in fused
|
|
216
|
+
if self._passes_filters(self._chunk_by_id[chunk_id], filters, path_filter)
|
|
217
|
+
]
|
|
218
|
+
|
|
219
|
+
if rerank:
|
|
220
|
+
return self._rerank(query, candidates, rerank, top_k)
|
|
221
|
+
|
|
222
|
+
return [
|
|
223
|
+
SearchResult(chunk=self._chunk_by_id[chunk_id], score=score, scores=raw)
|
|
224
|
+
for chunk_id, score, raw in candidates[:top_k]
|
|
225
|
+
]
|
|
226
|
+
|
|
227
|
+
def _rerank(
|
|
228
|
+
self,
|
|
229
|
+
query: str,
|
|
230
|
+
candidates: list[tuple[str, float, dict[str, float]]],
|
|
231
|
+
rerank: bool | Reranker,
|
|
232
|
+
top_k: int,
|
|
233
|
+
) -> list[SearchResult]:
|
|
234
|
+
reranker = rerank if hasattr(rerank, "rerank") else self._get_default_reranker()
|
|
235
|
+
pool = candidates[: max(top_k * 3, 20)]
|
|
236
|
+
pool_chunks = [self._chunk_by_id[chunk_id] for chunk_id, _, _ in pool]
|
|
237
|
+
raw_by_id = {chunk_id: raw for chunk_id, _, raw in pool}
|
|
238
|
+
|
|
239
|
+
reranked = reranker.rerank(query, pool_chunks, top_k=top_k)
|
|
240
|
+
return [
|
|
241
|
+
SearchResult(
|
|
242
|
+
chunk=self._chunk_by_id[chunk_id],
|
|
243
|
+
score=score,
|
|
244
|
+
scores={**raw_by_id.get(chunk_id, {}), "rerank": score},
|
|
245
|
+
)
|
|
246
|
+
for chunk_id, score in reranked
|
|
247
|
+
]
|
|
248
|
+
|
|
249
|
+
def _get_default_reranker(self) -> Reranker:
|
|
250
|
+
if self._reranker is None:
|
|
251
|
+
from ragless.rerank import CrossEncoderReranker
|
|
252
|
+
|
|
253
|
+
self._reranker = CrossEncoderReranker()
|
|
254
|
+
return self._reranker
|
|
255
|
+
|
|
256
|
+
def _resolve_mode(self, mode: str) -> list[str]:
|
|
257
|
+
if mode == "hybrid":
|
|
258
|
+
return list(self._retrievers.keys())
|
|
259
|
+
if mode in self._retrievers:
|
|
260
|
+
return [mode]
|
|
261
|
+
raise UnknownRetrieverError(
|
|
262
|
+
f"Unknown search mode {mode!r}; configured engines are "
|
|
263
|
+
f"{list(self._retrievers.keys())} (plus 'hybrid')."
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
def _passes_filters(self, chunk: Chunk, filters: Filters, path_filter: PathFilter) -> bool:
|
|
267
|
+
if filters:
|
|
268
|
+
for key, expected in filters.items():
|
|
269
|
+
actual = chunk.metadata.get(key)
|
|
270
|
+
if callable(expected):
|
|
271
|
+
if not expected(actual):
|
|
272
|
+
return False
|
|
273
|
+
elif actual != expected:
|
|
274
|
+
return False
|
|
275
|
+
if path_filter:
|
|
276
|
+
if chunk.path is None:
|
|
277
|
+
return False
|
|
278
|
+
patterns = [path_filter] if isinstance(path_filter, str) else list(path_filter)
|
|
279
|
+
if not any(fnmatch.fnmatch(chunk.path, pattern) for pattern in patterns):
|
|
280
|
+
return False
|
|
281
|
+
return True
|
|
282
|
+
|
|
283
|
+
def __len__(self) -> int:
|
|
284
|
+
return len(self._chunks)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _resolve_metadata(metadata: MetadataSource, path: str | None) -> dict[str, Any]:
|
|
288
|
+
if metadata is None:
|
|
289
|
+
return {}
|
|
290
|
+
if callable(metadata):
|
|
291
|
+
return dict(metadata(path) or {})
|
|
292
|
+
return dict(metadata)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _default_embedder() -> Embedder:
|
|
296
|
+
from ragless.embedders.fastembed_embedder import FastEmbedEmbedder
|
|
297
|
+
|
|
298
|
+
return FastEmbedEmbedder()
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _is_text_sequence(source: Any) -> bool:
|
|
302
|
+
return isinstance(source, (list, tuple)) and not isinstance(source, (str, bytes))
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _read_text_file(path: Path, encoding: str) -> str | None:
|
|
306
|
+
try:
|
|
307
|
+
raw = path.read_bytes()
|
|
308
|
+
except OSError:
|
|
309
|
+
return None
|
|
310
|
+
if b"\x00" in raw[:8192]:
|
|
311
|
+
return None
|
|
312
|
+
try:
|
|
313
|
+
return raw.decode(encoding)
|
|
314
|
+
except UnicodeDecodeError:
|
|
315
|
+
return None
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _iter_files(
|
|
319
|
+
root: Path, include: Sequence[str] | None, exclude: Sequence[str] | None
|
|
320
|
+
) -> list[Path]:
|
|
321
|
+
matches: list[Path] = []
|
|
322
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
323
|
+
dirnames[:] = sorted(d for d in dirnames if d not in _SKIP_DIRS and not d.startswith("."))
|
|
324
|
+
for filename in sorted(filenames):
|
|
325
|
+
file_path = Path(dirpath) / filename
|
|
326
|
+
rel = file_path.relative_to(root).as_posix()
|
|
327
|
+
if include and not any(fnmatch.fnmatch(rel, pattern) for pattern in include):
|
|
328
|
+
continue
|
|
329
|
+
if exclude and any(fnmatch.fnmatch(rel, pattern) for pattern in exclude):
|
|
330
|
+
continue
|
|
331
|
+
matches.append(file_path)
|
|
332
|
+
return matches
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _load_documents(
|
|
336
|
+
source: str | Path | Sequence[str] | None,
|
|
337
|
+
texts: Sequence[str] | None,
|
|
338
|
+
*,
|
|
339
|
+
include: Sequence[str] | None,
|
|
340
|
+
exclude: Sequence[str] | None,
|
|
341
|
+
encoding: str,
|
|
342
|
+
) -> list[tuple[str | None, str]]:
|
|
343
|
+
documents: list[tuple[str | None, str]] = []
|
|
344
|
+
|
|
345
|
+
if source is not None:
|
|
346
|
+
if _is_text_sequence(source):
|
|
347
|
+
documents.extend((None, t) for t in source)
|
|
348
|
+
else:
|
|
349
|
+
path = Path(source)
|
|
350
|
+
if path.is_file():
|
|
351
|
+
content = _read_text_file(path, encoding)
|
|
352
|
+
if content is not None:
|
|
353
|
+
documents.append((str(path), content))
|
|
354
|
+
elif path.is_dir():
|
|
355
|
+
for file_path in _iter_files(path, include, exclude):
|
|
356
|
+
content = _read_text_file(file_path, encoding)
|
|
357
|
+
if content is not None:
|
|
358
|
+
documents.append((str(file_path), content))
|
|
359
|
+
else:
|
|
360
|
+
raise FileNotFoundError(f"Source path does not exist: {path}")
|
|
361
|
+
|
|
362
|
+
if texts:
|
|
363
|
+
documents.extend((None, t) for t in texts)
|
|
364
|
+
|
|
365
|
+
return documents
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Base interface implemented by every embedding backend."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Embedder(ABC):
|
|
12
|
+
"""A pluggable text-embedding backend.
|
|
13
|
+
|
|
14
|
+
``name`` must uniquely identify the model (and any settings that affect
|
|
15
|
+
its output vectors) since it is part of the embedding cache key: changing
|
|
16
|
+
models invalidates cached embeddings rather than silently reusing stale
|
|
17
|
+
ones.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
name: str
|
|
21
|
+
dim: int
|
|
22
|
+
|
|
23
|
+
@abstractmethod
|
|
24
|
+
def embed(self, texts: Sequence[str]) -> np.ndarray:
|
|
25
|
+
"""Embed ``texts`` into a ``(len(texts), self.dim)`` float32 array."""
|
|
26
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Default semantic embedder: a small local model served via fastembed (ONNX runtime).
|
|
2
|
+
|
|
3
|
+
Requires the ``semantic`` extra: ``pip install ragless[semantic]``.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from collections.abc import Sequence
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
from ragless.embedders.base import Embedder
|
|
13
|
+
from ragless.exceptions import MissingDependencyError
|
|
14
|
+
|
|
15
|
+
DEFAULT_MODEL = "BAAI/bge-small-en-v1.5"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class FastEmbedEmbedder(Embedder):
|
|
19
|
+
"""Wraps :mod:`fastembed` to produce dense vectors with a small local model.
|
|
20
|
+
|
|
21
|
+
No GPU or external service is required; models run locally via ONNX
|
|
22
|
+
runtime and are cached on disk by fastembed after the first download.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self, model_name: str = DEFAULT_MODEL) -> None:
|
|
26
|
+
try:
|
|
27
|
+
from fastembed import TextEmbedding
|
|
28
|
+
except ImportError as exc: # pragma: no cover - exercised without extra installed
|
|
29
|
+
raise MissingDependencyError("Semantic search", "semantic") from exc
|
|
30
|
+
|
|
31
|
+
self.model_name = model_name
|
|
32
|
+
self.name = f"fastembed:{model_name}"
|
|
33
|
+
self._model = TextEmbedding(model_name=model_name)
|
|
34
|
+
probe = next(iter(self._model.embed(["dimension probe"])))
|
|
35
|
+
self.dim = int(np.asarray(probe).shape[0])
|
|
36
|
+
|
|
37
|
+
def embed(self, texts: Sequence[str]) -> np.ndarray:
|
|
38
|
+
if not texts:
|
|
39
|
+
return np.zeros((0, self.dim), dtype=np.float32)
|
|
40
|
+
vectors = list(self._model.embed(list(texts)))
|
|
41
|
+
return np.asarray(vectors, dtype=np.float32)
|
ragless/exceptions.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Exceptions raised by ragless."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class RaglessError(Exception):
|
|
7
|
+
"""Base class for all ragless errors."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class MissingDependencyError(RaglessError):
|
|
11
|
+
"""Raised when an optional extra is required but not installed."""
|
|
12
|
+
|
|
13
|
+
def __init__(self, feature: str, extra: str) -> None:
|
|
14
|
+
super().__init__(
|
|
15
|
+
f"{feature} requires the '{extra}' extra. "
|
|
16
|
+
f"Install it with: pip install 'ragless[{extra}]'"
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class UnknownRetrieverError(RaglessError):
|
|
21
|
+
"""Raised when an unrecognised retriever name is requested."""
|
ragless/fusion.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Reciprocal Rank Fusion: combines multiple rankings using rank, not raw scores.
|
|
2
|
+
|
|
3
|
+
RRF sidesteps the problem that BM25, fuzzy, and cosine-similarity scores live
|
|
4
|
+
on incomparable scales -- it only looks at each result's position within its
|
|
5
|
+
own ranking.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Mapping, Sequence
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def reciprocal_rank_fusion(
|
|
14
|
+
rankings: Mapping[str, Sequence[tuple[str, float]]],
|
|
15
|
+
*,
|
|
16
|
+
k: int = 60,
|
|
17
|
+
weights: Mapping[str, float] | None = None,
|
|
18
|
+
) -> list[tuple[str, float, dict[str, float]]]:
|
|
19
|
+
"""Fuse named rankings of ``(chunk_id, score)`` pairs into one ranking.
|
|
20
|
+
|
|
21
|
+
Each ranking's contribution to item ``chunk_id`` is ``weight / (k + rank)``
|
|
22
|
+
where ``rank`` is its 1-indexed position in that ranking. Items absent
|
|
23
|
+
from a ranking simply get no contribution from it.
|
|
24
|
+
|
|
25
|
+
Returns ``(chunk_id, fused_score, raw_scores)`` tuples sorted by
|
|
26
|
+
``fused_score`` descending, where ``raw_scores`` maps retriever name to
|
|
27
|
+
that retriever's original (unfused) score for this chunk.
|
|
28
|
+
"""
|
|
29
|
+
fused: dict[str, float] = {}
|
|
30
|
+
raw_scores: dict[str, dict[str, float]] = {}
|
|
31
|
+
|
|
32
|
+
for retriever_name, ranking in rankings.items():
|
|
33
|
+
weight = 1.0 if weights is None else weights.get(retriever_name, 1.0)
|
|
34
|
+
for rank, (chunk_id, score) in enumerate(ranking, start=1):
|
|
35
|
+
fused[chunk_id] = fused.get(chunk_id, 0.0) + weight / (k + rank)
|
|
36
|
+
raw_scores.setdefault(chunk_id, {})[retriever_name] = score
|
|
37
|
+
|
|
38
|
+
ordered = sorted(fused.items(), key=lambda item: item[1], reverse=True)
|
|
39
|
+
return [(chunk_id, score, raw_scores.get(chunk_id, {})) for chunk_id, score in ordered]
|
ragless/rerank.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Optional cross-encoder reranking of fused search results."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from typing import Protocol
|
|
7
|
+
|
|
8
|
+
from ragless.exceptions import MissingDependencyError
|
|
9
|
+
from ragless.types import Chunk
|
|
10
|
+
|
|
11
|
+
DEFAULT_RERANK_MODEL = "Xenova/ms-marco-MiniLM-L-6-v2"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Reranker(Protocol):
|
|
15
|
+
"""Interface a reranker must satisfy to plug into :class:`ragless.Retriever`."""
|
|
16
|
+
|
|
17
|
+
def rerank(
|
|
18
|
+
self, query: str, chunks: Sequence[Chunk], top_k: int | None = None
|
|
19
|
+
) -> list[tuple[str, float]]:
|
|
20
|
+
"""Return ``(chunk_id, score)`` pairs for ``chunks``, best first."""
|
|
21
|
+
...
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class CrossEncoderReranker:
|
|
25
|
+
"""Reranks candidates with a small local cross-encoder via fastembed.
|
|
26
|
+
|
|
27
|
+
Unlike the bi-encoder used for semantic search, a cross-encoder scores
|
|
28
|
+
the query and each candidate jointly, which is slower but typically more
|
|
29
|
+
accurate -- it is meant to be applied only to the top handful of fused
|
|
30
|
+
candidates, not the whole corpus. Requires ``pip install ragless[semantic]``.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def __init__(self, model_name: str = DEFAULT_RERANK_MODEL) -> None:
|
|
34
|
+
try:
|
|
35
|
+
from fastembed.rerank.cross_encoder import TextCrossEncoder
|
|
36
|
+
except ImportError as exc: # pragma: no cover - exercised without extra installed
|
|
37
|
+
raise MissingDependencyError("Cross-encoder reranking", "semantic") from exc
|
|
38
|
+
|
|
39
|
+
self.model_name = model_name
|
|
40
|
+
self._model = TextCrossEncoder(model_name=model_name)
|
|
41
|
+
|
|
42
|
+
def rerank(
|
|
43
|
+
self, query: str, chunks: Sequence[Chunk], top_k: int | None = None
|
|
44
|
+
) -> list[tuple[str, float]]:
|
|
45
|
+
if not chunks:
|
|
46
|
+
return []
|
|
47
|
+
texts = [c.text for c in chunks]
|
|
48
|
+
scores = [float(s) for s in self._model.rerank(query, texts)]
|
|
49
|
+
pairs = list(zip((c.id for c in chunks), scores, strict=True))
|
|
50
|
+
pairs.sort(key=lambda pair: pair[1], reverse=True)
|
|
51
|
+
return pairs[:top_k] if top_k is not None else pairs
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
from ragless.retrievers.base import BaseRetriever
|
|
2
|
+
from ragless.retrievers.bm25 import BM25Retriever
|
|
3
|
+
from ragless.retrievers.fuzzy import FuzzyRetriever
|
|
4
|
+
from ragless.retrievers.semantic import SemanticRetriever
|
|
5
|
+
|
|
6
|
+
__all__ = ["BaseRetriever", "BM25Retriever", "FuzzyRetriever", "SemanticRetriever"]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Base interface implemented by every retrieval backend."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
|
|
8
|
+
from ragless.types import Chunk
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class BaseRetriever(ABC):
|
|
12
|
+
"""A pluggable retrieval strategy.
|
|
13
|
+
|
|
14
|
+
A retriever is given the full set of chunks once (:meth:`index`) and can
|
|
15
|
+
then be queried repeatedly (:meth:`query`). Implementations should not
|
|
16
|
+
mutate chunks; they only need to produce relevance rankings.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
name: str
|
|
20
|
+
|
|
21
|
+
@abstractmethod
|
|
22
|
+
def index(self, chunks: Sequence[Chunk]) -> None:
|
|
23
|
+
"""Build the retriever's internal index over ``chunks``."""
|
|
24
|
+
raise NotImplementedError
|
|
25
|
+
|
|
26
|
+
@abstractmethod
|
|
27
|
+
def query(self, query: str, top_k: int) -> list[tuple[str, float]]:
|
|
28
|
+
"""Return up to ``top_k`` ``(chunk_id, score)`` pairs, best first.
|
|
29
|
+
|
|
30
|
+
Scores are retriever-specific and not necessarily comparable across
|
|
31
|
+
retrievers; fusion (see :mod:`ragless.fusion`) only relies on rank
|
|
32
|
+
order, not on the raw magnitudes.
|
|
33
|
+
"""
|
|
34
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""BM25 (Okapi) keyword retriever."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
import re
|
|
7
|
+
from collections import Counter
|
|
8
|
+
from collections.abc import Sequence
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
from ragless.retrievers.base import BaseRetriever
|
|
13
|
+
from ragless.types import Chunk
|
|
14
|
+
|
|
15
|
+
_TOKEN_RE = re.compile(r"[A-Za-z0-9_]+")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _tokenize(text: str) -> list[str]:
|
|
19
|
+
return _TOKEN_RE.findall(text.lower())
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class BM25Retriever(BaseRetriever):
|
|
23
|
+
"""Classic Okapi BM25 ranking over tokenized chunk text.
|
|
24
|
+
|
|
25
|
+
Pure NumPy implementation; no external search engine or index server.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
name = "bm25"
|
|
29
|
+
|
|
30
|
+
def __init__(self, k1: float = 1.5, b: float = 0.75) -> None:
|
|
31
|
+
self.k1 = k1
|
|
32
|
+
self.b = b
|
|
33
|
+
self._chunk_ids: list[str] = []
|
|
34
|
+
self._doc_term_freqs: list[Counter[str]] = []
|
|
35
|
+
self._doc_lens = np.zeros(0, dtype=np.float64)
|
|
36
|
+
self._avgdl = 0.0
|
|
37
|
+
self._idf: dict[str, float] = {}
|
|
38
|
+
|
|
39
|
+
def index(self, chunks: Sequence[Chunk]) -> None:
|
|
40
|
+
self._chunk_ids = [c.id for c in chunks]
|
|
41
|
+
tokenized = [_tokenize(c.text) for c in chunks]
|
|
42
|
+
self._doc_term_freqs = [Counter(toks) for toks in tokenized]
|
|
43
|
+
self._doc_lens = np.array([len(toks) for toks in tokenized], dtype=np.float64)
|
|
44
|
+
self._avgdl = float(self._doc_lens.mean()) if len(self._doc_lens) else 0.0
|
|
45
|
+
|
|
46
|
+
n_docs = len(tokenized)
|
|
47
|
+
doc_freq: Counter[str] = Counter()
|
|
48
|
+
for toks in tokenized:
|
|
49
|
+
doc_freq.update(set(toks))
|
|
50
|
+
self._idf = {
|
|
51
|
+
term: math.log(1 + (n_docs - freq + 0.5) / (freq + 0.5))
|
|
52
|
+
for term, freq in doc_freq.items()
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
def query(self, query: str, top_k: int) -> list[tuple[str, float]]:
|
|
56
|
+
if top_k <= 0 or not self._chunk_ids:
|
|
57
|
+
return []
|
|
58
|
+
q_tokens = _tokenize(query)
|
|
59
|
+
if not q_tokens:
|
|
60
|
+
return []
|
|
61
|
+
|
|
62
|
+
n = len(self._chunk_ids)
|
|
63
|
+
scores = np.zeros(n, dtype=np.float64)
|
|
64
|
+
avgdl = self._avgdl or 1.0
|
|
65
|
+
norm_lens = self.k1 * (1 - self.b + self.b * self._doc_lens / avgdl)
|
|
66
|
+
|
|
67
|
+
for term in q_tokens:
|
|
68
|
+
idf = self._idf.get(term)
|
|
69
|
+
if idf is None:
|
|
70
|
+
continue
|
|
71
|
+
tf = np.array([freqs.get(term, 0) for freqs in self._doc_term_freqs], dtype=np.float64)
|
|
72
|
+
denom = tf + norm_lens
|
|
73
|
+
denom = np.where(denom == 0, 1.0, denom)
|
|
74
|
+
scores += idf * (tf * (self.k1 + 1)) / denom
|
|
75
|
+
|
|
76
|
+
hits = np.nonzero(scores > 0)[0]
|
|
77
|
+
if hits.size == 0:
|
|
78
|
+
return []
|
|
79
|
+
k = min(top_k, hits.size)
|
|
80
|
+
ranked = hits[np.argsort(-scores[hits])][:k]
|
|
81
|
+
return [(self._chunk_ids[i], float(scores[i])) for i in ranked]
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Fuzzy string-similarity retriever backed by rapidfuzz."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable, Sequence
|
|
6
|
+
|
|
7
|
+
from rapidfuzz import fuzz, process
|
|
8
|
+
|
|
9
|
+
from ragless.retrievers.base import BaseRetriever
|
|
10
|
+
from ragless.types import Chunk
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class FuzzyRetriever(BaseRetriever):
|
|
14
|
+
"""Ranks chunks by fuzzy string similarity to the query.
|
|
15
|
+
|
|
16
|
+
Good at catching typos, partial matches, and near-duplicate phrasing that
|
|
17
|
+
exact keyword search (BM25) would miss.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
name = "fuzzy"
|
|
21
|
+
|
|
22
|
+
def __init__(self, scorer: Callable[..., float] = fuzz.WRatio) -> None:
|
|
23
|
+
self.scorer = scorer
|
|
24
|
+
self._chunk_ids: list[str] = []
|
|
25
|
+
self._texts: list[str] = []
|
|
26
|
+
|
|
27
|
+
def index(self, chunks: Sequence[Chunk]) -> None:
|
|
28
|
+
self._chunk_ids = [c.id for c in chunks]
|
|
29
|
+
self._texts = [c.text for c in chunks]
|
|
30
|
+
|
|
31
|
+
def query(self, query: str, top_k: int) -> list[tuple[str, float]]:
|
|
32
|
+
if top_k <= 0 or not self._texts or not query.strip():
|
|
33
|
+
return []
|
|
34
|
+
matches = process.extract(query, self._texts, scorer=self.scorer, limit=top_k)
|
|
35
|
+
return [(self._chunk_ids[index], float(score)) for _, score, index in matches if score > 0]
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Semantic (embedding) retriever: NumPy cosine similarity, no vector DB."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from ragless.cache import EmbeddingCache, FileSignature
|
|
10
|
+
from ragless.embedders.base import Embedder
|
|
11
|
+
from ragless.retrievers.base import BaseRetriever
|
|
12
|
+
from ragless.types import Chunk
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class SemanticRetriever(BaseRetriever):
|
|
16
|
+
"""Ranks chunks by cosine similarity between dense embeddings.
|
|
17
|
+
|
|
18
|
+
Embeddings are plain NumPy arrays held in memory for search; there is no
|
|
19
|
+
vector database involved. When an :class:`~ragless.cache.EmbeddingCache`
|
|
20
|
+
is supplied, embeddings for unchanged files are loaded from disk instead
|
|
21
|
+
of being recomputed.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
name = "semantic"
|
|
25
|
+
|
|
26
|
+
def __init__(self, embedder: Embedder, cache: EmbeddingCache | None = None) -> None:
|
|
27
|
+
self.embedder = embedder
|
|
28
|
+
self.cache = cache
|
|
29
|
+
self._chunk_ids: list[str] = []
|
|
30
|
+
self._vectors = np.zeros((0, embedder.dim), dtype=np.float32)
|
|
31
|
+
|
|
32
|
+
def index(self, chunks: Sequence[Chunk]) -> None:
|
|
33
|
+
"""Index ``chunks`` without per-file cache invalidation signatures.
|
|
34
|
+
|
|
35
|
+
Used when a retriever is built standalone. :meth:`ragless.Retriever`
|
|
36
|
+
instead calls :meth:`index_with_signatures` so unchanged files reuse
|
|
37
|
+
cached embeddings.
|
|
38
|
+
"""
|
|
39
|
+
self.index_with_signatures(chunks, {})
|
|
40
|
+
|
|
41
|
+
def index_with_signatures(
|
|
42
|
+
self,
|
|
43
|
+
chunks: Sequence[Chunk],
|
|
44
|
+
signatures: dict[str, FileSignature],
|
|
45
|
+
) -> None:
|
|
46
|
+
if not chunks:
|
|
47
|
+
self._chunk_ids = []
|
|
48
|
+
self._vectors = np.zeros((0, self.embedder.dim), dtype=np.float32)
|
|
49
|
+
return
|
|
50
|
+
|
|
51
|
+
groups: dict[str | None, list[Chunk]] = {}
|
|
52
|
+
order: list[str | None] = []
|
|
53
|
+
for c in chunks:
|
|
54
|
+
if c.path not in groups:
|
|
55
|
+
groups[c.path] = []
|
|
56
|
+
order.append(c.path)
|
|
57
|
+
groups[c.path].append(c)
|
|
58
|
+
|
|
59
|
+
all_ids: list[str] = []
|
|
60
|
+
all_vectors: list[np.ndarray] = []
|
|
61
|
+
for path in order:
|
|
62
|
+
group_chunks = groups[path]
|
|
63
|
+
signature = signatures.get(path) if path is not None else None
|
|
64
|
+
vectors: np.ndarray | None = None
|
|
65
|
+
if self.cache is not None and path is not None and signature is not None:
|
|
66
|
+
vectors = self.cache.get(path, signature, self.embedder, group_chunks)
|
|
67
|
+
if vectors is None:
|
|
68
|
+
vectors = self.embedder.embed([c.text for c in group_chunks])
|
|
69
|
+
if self.cache is not None and path is not None and signature is not None:
|
|
70
|
+
self.cache.put(path, signature, self.embedder, group_chunks, vectors)
|
|
71
|
+
all_ids.extend(c.id for c in group_chunks)
|
|
72
|
+
all_vectors.append(vectors)
|
|
73
|
+
|
|
74
|
+
matrix = np.concatenate(all_vectors, axis=0).astype(np.float32)
|
|
75
|
+
self._vectors = _normalize_rows(matrix)
|
|
76
|
+
self._chunk_ids = all_ids
|
|
77
|
+
|
|
78
|
+
def query(self, query: str, top_k: int) -> list[tuple[str, float]]:
|
|
79
|
+
if top_k <= 0 or not self._chunk_ids or not query.strip():
|
|
80
|
+
return []
|
|
81
|
+
q_vec = self.embedder.embed([query])[0].astype(np.float32)
|
|
82
|
+
norm = float(np.linalg.norm(q_vec))
|
|
83
|
+
if norm == 0.0:
|
|
84
|
+
return []
|
|
85
|
+
q_vec = q_vec / norm
|
|
86
|
+
|
|
87
|
+
scores = self._vectors @ q_vec
|
|
88
|
+
k = min(top_k, scores.shape[0])
|
|
89
|
+
if k < scores.shape[0]:
|
|
90
|
+
idx = np.argpartition(-scores, k - 1)[:k]
|
|
91
|
+
else:
|
|
92
|
+
idx = np.arange(scores.shape[0])
|
|
93
|
+
idx = idx[np.argsort(-scores[idx])]
|
|
94
|
+
return [(self._chunk_ids[i], float(scores[i])) for i in idx]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _normalize_rows(matrix: np.ndarray) -> np.ndarray:
|
|
98
|
+
norms = np.linalg.norm(matrix, axis=1, keepdims=True)
|
|
99
|
+
norms[norms == 0] = 1.0
|
|
100
|
+
return matrix / norms
|
ragless/types.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Core data types shared across ragless."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(frozen=True)
|
|
10
|
+
class Chunk:
|
|
11
|
+
"""A unit of retrievable text, tied back to its source location."""
|
|
12
|
+
|
|
13
|
+
id: str
|
|
14
|
+
text: str
|
|
15
|
+
path: str | None
|
|
16
|
+
"""Source file path, or ``None`` for in-memory text that has no file."""
|
|
17
|
+
start_line: int
|
|
18
|
+
end_line: int
|
|
19
|
+
metadata: dict[str, Any] = field(default_factory=dict)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class SearchResult:
|
|
24
|
+
"""A single scored hit returned from :meth:`ragless.Retriever.search`."""
|
|
25
|
+
|
|
26
|
+
chunk: Chunk
|
|
27
|
+
score: float
|
|
28
|
+
scores: dict[str, float] = field(default_factory=dict)
|
|
29
|
+
"""Per-retriever contribution, e.g. ``{"bm25": 0.8, "fuzzy": 0.4}``."""
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
def path(self) -> str | None:
|
|
33
|
+
return self.chunk.path
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def text(self) -> str:
|
|
37
|
+
return self.chunk.text
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def start_line(self) -> int:
|
|
41
|
+
return self.chunk.start_line
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def end_line(self) -> int:
|
|
45
|
+
return self.chunk.end_line
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def metadata(self) -> dict[str, Any]:
|
|
49
|
+
return self.chunk.metadata
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ragless
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: File-based retrieval for RAG, no vector database required
|
|
5
|
+
Project-URL: Homepage, https://github.com/mohmdqasim/ragless
|
|
6
|
+
Project-URL: Repository, https://github.com/mohmdqasim/ragless
|
|
7
|
+
Project-URL: Issues, https://github.com/mohmdqasim/ragless/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/mohmdqasim/ragless/blob/main/CHANGELOG.md
|
|
9
|
+
Author-email: Qasim Arshad <mohmdqasim0@gmail.com>
|
|
10
|
+
Maintainer-email: Qasim Arshad <mohmdqasim0@gmail.com>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: bm25,embeddings,information-retrieval,rag,retrieval,search
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Classifier: Topic :: Text Processing :: Indexing
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Requires-Dist: numpy>=1.24
|
|
26
|
+
Requires-Dist: rapidfuzz>=3.0
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest-cov>=4.1; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest>=7.4; extra == 'dev'
|
|
31
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
32
|
+
Provides-Extra: semantic
|
|
33
|
+
Requires-Dist: fastembed>=0.3; extra == 'semantic'
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
# ragless
|
|
37
|
+
|
|
38
|
+
File-based retrieval for RAG, with no vector database.
|
|
39
|
+
|
|
40
|
+
`ragless` indexes a folder of text files (or in-memory strings) and lets you
|
|
41
|
+
search them with BM25 keyword search, fuzzy string matching, and optional
|
|
42
|
+
local semantic embeddings -- combined with Reciprocal Rank Fusion. Everything
|
|
43
|
+
lives on disk as plain files (or in memory): no Postgres/pgvector, no
|
|
44
|
+
Pinecone/Weaviate/Chroma server, nothing to run.
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from ragless import Retriever
|
|
48
|
+
|
|
49
|
+
r = Retriever("./docs")
|
|
50
|
+
for hit in r.search("how do I configure retries", mode="hybrid"):
|
|
51
|
+
print(f"{hit.path}:{hit.start_line}-{hit.end_line} score={hit.score:.3f}")
|
|
52
|
+
print(hit.text)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Install
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install ragless # BM25 + fuzzy (no heavy deps)
|
|
59
|
+
pip install 'ragless[semantic]' # + local embeddings and cross-encoder rerank
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Quickstart
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
from ragless import Retriever
|
|
66
|
+
|
|
67
|
+
# Index a directory of files.
|
|
68
|
+
r = Retriever("./docs")
|
|
69
|
+
results = r.search("installation steps", mode="hybrid", top_k=5)
|
|
70
|
+
|
|
71
|
+
# Or index in-memory text directly -- no files required.
|
|
72
|
+
r = Retriever(
|
|
73
|
+
[
|
|
74
|
+
"ragless indexes folders of text files for retrieval.",
|
|
75
|
+
"BM25, fuzzy matching, and semantic search are combined with RRF.",
|
|
76
|
+
]
|
|
77
|
+
)
|
|
78
|
+
results = r.search("how are results combined?")
|
|
79
|
+
|
|
80
|
+
for hit in results:
|
|
81
|
+
print(hit.path, hit.start_line, hit.end_line, hit.score)
|
|
82
|
+
print(hit.text)
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
### Search modes
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
r.search("query", mode="hybrid") # fuse every configured engine (default)
|
|
89
|
+
r.search("query", mode="bm25") # keyword search only
|
|
90
|
+
r.search("query", mode="fuzzy") # fuzzy string matching only
|
|
91
|
+
r.search("query", mode="semantic") # embeddings only (requires the extra)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### Semantic search and caching
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from ragless import Retriever
|
|
98
|
+
|
|
99
|
+
r = Retriever(
|
|
100
|
+
"./docs",
|
|
101
|
+
engines=["bm25", "fuzzy", "semantic"],
|
|
102
|
+
cache_dir="./.ragless_cache", # embeddings cached here as .npy/.json
|
|
103
|
+
)
|
|
104
|
+
r.search("query", mode="hybrid")
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Embeddings are cached per source file, keyed by the file's mtime/size and the
|
|
108
|
+
embedder's name. Editing one file only re-embeds that file's chunks on the
|
|
109
|
+
next run; untouched files are loaded straight from the cache.
|
|
110
|
+
|
|
111
|
+
### Cross-encoder rerank
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
r.search("query", mode="hybrid", rerank=True)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Reranking takes the top fused candidates and rescoring them with a small
|
|
118
|
+
local cross-encoder (also via `fastembed`, under the `semantic` extra) for
|
|
119
|
+
higher precision at the top of the list.
|
|
120
|
+
|
|
121
|
+
### Filters
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
# Only chunks whose metadata says category="guide"
|
|
125
|
+
r.search("query", filters={"category": "guide"})
|
|
126
|
+
|
|
127
|
+
# Only chunks from markdown files
|
|
128
|
+
r.search("query", path_filter="*.md")
|
|
129
|
+
|
|
130
|
+
# Per-document metadata, assigned when building the index
|
|
131
|
+
r = Retriever(
|
|
132
|
+
"./docs",
|
|
133
|
+
metadata=lambda path: {"category": "guide"} if path and "guide" in path else {},
|
|
134
|
+
)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
### Pluggable components
|
|
138
|
+
|
|
139
|
+
Chunkers, retrievers, and embedders are all swappable:
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
from ragless import Retriever, Chunker, BaseRetriever
|
|
143
|
+
from ragless.chunking import LineChunker
|
|
144
|
+
|
|
145
|
+
r = Retriever("./docs", chunker=LineChunker(lines_per_chunk=60, overlap=10))
|
|
146
|
+
|
|
147
|
+
# Bring your own retriever/embedder implementation:
|
|
148
|
+
r = Retriever("./docs", engines={"bm25": MyRetriever(), "fuzzy": FuzzyRetriever()})
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Implement `ragless.chunking.Chunker`, `ragless.retrievers.base.BaseRetriever`,
|
|
152
|
+
or `ragless.embedders.base.Embedder` to plug in your own.
|
|
153
|
+
|
|
154
|
+
## Why
|
|
155
|
+
|
|
156
|
+
Most "RAG" tutorials reach for a vector database before checking whether they
|
|
157
|
+
need one. For a folder of a few thousand files, plain keyword search (BM25),
|
|
158
|
+
fuzzy matching, and a small local embedding model get you most of the way
|
|
159
|
+
there, with:
|
|
160
|
+
|
|
161
|
+
- No server to run, no API keys, no network calls (semantic search runs a
|
|
162
|
+
small model locally via [fastembed](https://github.com/qdrant/fastembed)).
|
|
163
|
+
- No ingestion pipeline: point it at a directory and it chunks and indexes on
|
|
164
|
+
construction.
|
|
165
|
+
- Caching as plain sidecar files (`.npy` + `.json`) next to your index, not a
|
|
166
|
+
database you have to back up and migrate.
|
|
167
|
+
- Every result keeps the file path and line range it came from.
|
|
168
|
+
|
|
169
|
+
| | ragless | vector DB (Pinecone/Chroma/pgvector) |
|
|
170
|
+
|---|---|---|
|
|
171
|
+
| Infra | none | a running database/service |
|
|
172
|
+
| Setup | `Retriever("./docs")` | provision, configure, ingest |
|
|
173
|
+
| Keyword search | built in (BM25) | usually bolted on separately |
|
|
174
|
+
| Fuzzy/typo tolerance | built in | usually absent |
|
|
175
|
+
| Good for | a few thousand files, local tools, CLIs, CI | large, constantly-changing, multi-tenant corpora |
|
|
176
|
+
| Cold start | instant | needs the service up and reachable |
|
|
177
|
+
|
|
178
|
+
If you need to search tens of millions of chunks with concurrent writers
|
|
179
|
+
across a fleet of machines, use a vector database. If you need to search the
|
|
180
|
+
docs folder sitting next to your code, `ragless` is simpler.
|
|
181
|
+
|
|
182
|
+
## How it works
|
|
183
|
+
|
|
184
|
+
1. **Load** -- read files from a directory (or accept raw in-memory strings).
|
|
185
|
+
Binary files and undecodable text are skipped automatically.
|
|
186
|
+
2. **Chunk** -- split each document into overlapping line-based windows,
|
|
187
|
+
keeping the file path and 1-indexed `start_line`/`end_line` for every
|
|
188
|
+
chunk (`ragless.chunking.LineChunker`, pluggable).
|
|
189
|
+
3. **Index** -- each configured engine builds its own index over the chunks:
|
|
190
|
+
- `BM25Retriever`: classic Okapi BM25 over tokenized text (NumPy).
|
|
191
|
+
- `FuzzyRetriever`: `rapidfuzz` string similarity.
|
|
192
|
+
- `SemanticRetriever`: dense embeddings (`fastembed`) with cosine
|
|
193
|
+
similarity (NumPy), cached to disk as `.npy`/`.json` sidecar files.
|
|
194
|
+
4. **Search** -- query every configured engine, fuse their rankings with
|
|
195
|
+
[Reciprocal Rank Fusion](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf),
|
|
196
|
+
apply metadata/path filters, and optionally rerank the top candidates with
|
|
197
|
+
a cross-encoder.
|
|
198
|
+
|
|
199
|
+
## Status
|
|
200
|
+
|
|
201
|
+
0.1.0 -- the API may still shift. Feedback and issues welcome.
|
|
202
|
+
|
|
203
|
+
## License
|
|
204
|
+
|
|
205
|
+
MIT
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
ragless/__init__.py,sha256=69ndwdapG4IBFMTHae5xt1hVKwJjRUyAvCswnJ_1v5o,943
|
|
2
|
+
ragless/cache.py,sha256=f90UYUnv9d4R-sWOzxM_MDt_bRlvcKbfGlZpqOhYfN8,3383
|
|
3
|
+
ragless/chunking.py,sha256=n4-_rM897oee9hznpuF8cHQW0oDGsv0zKKk_gerkF9M,2714
|
|
4
|
+
ragless/core.py,sha256=0DzGdxTdOorEEwiRLxupaRZznLxI9BTrBQmfNIK5Lc8,14007
|
|
5
|
+
ragless/exceptions.py,sha256=jVW3yvBIRTwJj8n51l05UzHXlEz7tJm20dI67gkh_g8,593
|
|
6
|
+
ragless/fusion.py,sha256=_DoYMcMQCavqoiYaeCeCerYI30ocECiGdqO-z4mtQeg,1648
|
|
7
|
+
ragless/rerank.py,sha256=YAZ6ffZR1X5SZ2azN2HulBbyiljza-OhaLR3XmoXenc,1972
|
|
8
|
+
ragless/types.py,sha256=exU4pm517LF1DP3xefgth3a5TUFAckMD1-3XOlnKjS0,1184
|
|
9
|
+
ragless/embedders/__init__.py,sha256=lSDIW967LnKrarDrihNx-k-JJI0Qy1SKXS41bo_O8rM,68
|
|
10
|
+
ragless/embedders/base.py,sha256=QSbr56iMFiW9yHeXkBTAYCdBJfM0KCmLjsHWzrkfVpU,725
|
|
11
|
+
ragless/embedders/fastembed_embedder.py,sha256=omxnoxryb-EctLsXcn2akVho86xhAQZv1Faj7SSrWDU,1477
|
|
12
|
+
ragless/retrievers/__init__.py,sha256=UItt2W1I-iJDJWFvP7O9Ts1tDJ0i3NaVGWkSOlMTQbo,295
|
|
13
|
+
ragless/retrievers/base.py,sha256=DHGeBH0P5DhnT4taRLWrVxt1GaUfSHFDoJimuLKXn68,1097
|
|
14
|
+
ragless/retrievers/bm25.py,sha256=-4BXYKCn5dJdgg1TbRtQcdusiRh8nj6AHSWKWzUF2qQ,2646
|
|
15
|
+
ragless/retrievers/fuzzy.py,sha256=-lFeKDnOOU1FEXiz9FNaVPZDWmwVx7jCEJFDSKt77pg,1194
|
|
16
|
+
ragless/retrievers/semantic.py,sha256=4pZO-LyDLgE7BwmHHGfjYFgRZxQBfDFhqZ_5u_qgP3Q,3770
|
|
17
|
+
ragless-0.1.0.dist-info/METADATA,sha256=IiEnyYIPMjvFBwxJ9CkPMIB_zue82xNBfw96Ho6tC90,7292
|
|
18
|
+
ragless-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
19
|
+
ragless-0.1.0.dist-info/licenses/LICENSE,sha256=5MI5V3ka3qwyD7Pcub8_OUihTgWnrMzYjvzai4bVbzM,1077
|
|
20
|
+
ragless-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ragless contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|