simlar 1.0.0__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {simlar-1.0.0/src/simlar.egg-info → simlar-1.2.0}/PKG-INFO +62 -50
- {simlar-1.0.0 → simlar-1.2.0}/README.md +57 -46
- {simlar-1.0.0 → simlar-1.2.0}/pyproject.toml +24 -6
- simlar-1.2.0/src/simlar/__init__.py +74 -0
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/contracts.py +75 -19
- simlar-1.2.0/src/simlar/embeddings.py +147 -0
- simlar-1.2.0/src/simlar/indexes/__init__.py +17 -0
- simlar-1.2.0/src/simlar/indexes/_filtered.pyi +86 -0
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/_helix.pyi +20 -8
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/_relevance.pyi +17 -7
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/_simlar.pyi +18 -8
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/_streaming.pyi +14 -12
- simlar-1.2.0/src/simlar/indexes/bm25c_index.py +137 -0
- simlar-1.2.0/src/simlar/indexes/filtered_index.py +54 -0
- simlar-1.2.0/src/simlar/indexes/helix_index.py +200 -0
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/lookup_index.py +27 -14
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/registry.py +12 -5
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/relevance_index.py +30 -20
- simlar-1.2.0/src/simlar/indexes/simlar_engine.py +233 -0
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/streaming_index.py +50 -17
- simlar-1.2.0/src/simlar/integrations/_filtering.py +74 -0
- simlar-1.2.0/src/simlar/integrations/haystack/_filters.py +253 -0
- simlar-1.2.0/src/simlar/integrations/haystack/simlar_document_store.py +579 -0
- simlar-1.2.0/src/simlar/integrations/haystack/simlar_retriever.py +209 -0
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/integrations/langchain/langchain_retriever.py +7 -5
- simlar-1.2.0/src/simlar/integrations/langchain/simlar_vector_store.py +428 -0
- simlar-1.2.0/src/simlar/integrations/llama_index/simlar_vector_store.py +470 -0
- {simlar-1.0.0 → simlar-1.2.0/src/simlar.egg-info}/PKG-INFO +62 -50
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar.egg-info/SOURCES.txt +10 -2
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar.egg-info/requires.txt +4 -3
- simlar-1.2.0/tests/test_bm25c_index.py +300 -0
- simlar-1.2.0/tests/test_embeddings.py +351 -0
- {simlar-1.0.0 → simlar-1.2.0}/tests/test_helix_index.py +10 -1
- {simlar-1.0.0 → simlar-1.2.0}/tests/test_lookup_index.py +20 -3
- {simlar-1.0.0 → simlar-1.2.0}/tests/test_registry.py +16 -1
- {simlar-1.0.0 → simlar-1.2.0}/tests/test_relevance_index.py +17 -0
- simlar-1.2.0/tests/test_tmp_parallel_probe.py +104 -0
- simlar-1.0.0/src/simlar/__init__.py +0 -24
- simlar-1.0.0/src/simlar/indexes/__init__.py +0 -7
- simlar-1.0.0/src/simlar/indexes/helix_index.py +0 -126
- simlar-1.0.0/src/simlar/indexes/simlar_engine.py +0 -113
- simlar-1.0.0/src/simlar/integrations/haystack/simlar_document_store.py +0 -401
- simlar-1.0.0/src/simlar/integrations/haystack/simlar_retriever.py +0 -64
- simlar-1.0.0/src/simlar/integrations/langchain/simlar_vector_store.py +0 -268
- simlar-1.0.0/src/simlar/integrations/llama_index/simlar_retriever.py +0 -186
- simlar-1.0.0/src/simlar/integrations/llama_index/simlar_vector_store.py +0 -248
- {simlar-1.0.0 → simlar-1.2.0}/LICENSE +0 -0
- {simlar-1.0.0 → simlar-1.2.0}/NOTICE +0 -0
- {simlar-1.0.0 → simlar-1.2.0}/setup.cfg +0 -0
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/fusion/__init__.py +0 -0
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/fusion/_rrf.pyi +0 -0
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/fusion/rrf.py +0 -0
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/integrations/__init__.py +0 -0
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/persistence.py +0 -0
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar/py.typed +0 -0
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar.egg-info/dependency_links.txt +0 -0
- {simlar-1.0.0 → simlar-1.2.0}/src/simlar.egg-info/top_level.txt +0 -0
- {simlar-1.0.0 → simlar-1.2.0}/tests/test_contracts.py +0 -0
- {simlar-1.0.0 → simlar-1.2.0}/tests/test_persistence.py +0 -0
- {simlar-1.0.0 → simlar-1.2.0}/tests/test_simlar_engine.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: simlar
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: High-performance similarity search — open-source Python wrapper
|
|
5
5
|
Author-email: TekDatum <hello@tekdatum.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -16,12 +16,13 @@ Description-Content-Type: text/markdown
|
|
|
16
16
|
License-File: LICENSE
|
|
17
17
|
License-File: NOTICE
|
|
18
18
|
Requires-Dist: numpy<2.5,>=1.24
|
|
19
|
+
Requires-Dist: PyStemmer>=2.0
|
|
19
20
|
Provides-Extra: langchain
|
|
20
|
-
Requires-Dist: langchain-core>=0
|
|
21
|
+
Requires-Dist: langchain-core>=1.0; extra == "langchain"
|
|
21
22
|
Provides-Extra: haystack
|
|
22
|
-
Requires-Dist: haystack-ai
|
|
23
|
+
Requires-Dist: haystack-ai<4,>=3.0; extra == "haystack"
|
|
23
24
|
Provides-Extra: llama-index
|
|
24
|
-
Requires-Dist: llama-index-core>=0.
|
|
25
|
+
Requires-Dist: llama-index-core>=0.14; extra == "llama-index"
|
|
25
26
|
Provides-Extra: dev
|
|
26
27
|
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
27
28
|
Requires-Dist: numpy<2.5,>=1.24; extra == "dev"
|
|
@@ -32,22 +33,65 @@ Dynamic: license-file
|
|
|
32
33
|
# sim_LAR
|
|
33
34
|
|
|
34
35
|
Python search library combining keyword, semantic, and hybrid search into a single ranked result list — with first-class LangChain, LlamaIndex, and Haystack support.
|
|
36
|
+
|
|
35
37
|
---
|
|
36
38
|
|
|
37
39
|
## Installation
|
|
38
40
|
|
|
39
|
-
|
|
41
|
+
Both the open wrapper and the engine are on public PyPI:
|
|
40
42
|
|
|
41
43
|
```bash
|
|
42
44
|
pip install simlar
|
|
45
|
+
pip install simlar-engine
|
|
43
46
|
```
|
|
44
47
|
|
|
48
|
+
`simlar-engine` is TekDatum's proprietary binary, licensed under a [Commercial EULA](./EULA.md) — installing it means accepting those terms. See [docs/installation.md](docs/installation.md) for the compatible version matrix between `simlar` and `simlar-engine`.
|
|
45
49
|
|
|
46
|
-
|
|
47
|
-
|
|
50
|
+
## Quick start
|
|
51
|
+
|
|
52
|
+
### Hybrid search
|
|
53
|
+
|
|
54
|
+
Combine keyword relevance and semantic similarity into one ranked list. The easiest way to get started is to give the index an embedder so Simlar can generate document and query embeddings for you.
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from sentence_transformers import SentenceTransformer
|
|
58
|
+
from simlar import HelixIndex
|
|
59
|
+
|
|
60
|
+
corpus = [
|
|
61
|
+
"cancer treatment with immunotherapy",
|
|
62
|
+
"machine learning transformers",
|
|
63
|
+
"climate change renewable energy",
|
|
64
|
+
]
|
|
65
|
+
ids = [f"doc_{i}" for i in range(len(corpus))]
|
|
66
|
+
|
|
67
|
+
model = SentenceTransformer("WhereIsAI/UAE-Large-V1")
|
|
68
|
+
|
|
69
|
+
index = HelixIndex(embedder=model.encode)
|
|
70
|
+
index.add(ids=ids, texts=corpus)
|
|
71
|
+
|
|
72
|
+
results = index.search("immunotherapy clinical trial", k=10)
|
|
73
|
+
for r in results:
|
|
74
|
+
print(r.rank, r.id, f"{r.score:.4f}")
|
|
48
75
|
```
|
|
49
76
|
|
|
50
|
-
|
|
77
|
+
The embedder is used automatically when vectors are not supplied. You can also pass pre-computed vectors when you already have them.
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
import numpy as np
|
|
81
|
+
from simlar import HelixIndex
|
|
82
|
+
|
|
83
|
+
vectors = np.load("corpus_embeddings.npy") # shape (n, dim)
|
|
84
|
+
|
|
85
|
+
index = HelixIndex(embedder=model.encode)
|
|
86
|
+
index.add(ids=ids, texts=corpus, vectors=vectors)
|
|
87
|
+
|
|
88
|
+
query_vec = np.load("query_embedding.npy") # shape (1, dim)
|
|
89
|
+
results = index.search(
|
|
90
|
+
query_text="immunotherapy clinical trial",
|
|
91
|
+
query_vector=query_vec,
|
|
92
|
+
k=10,
|
|
93
|
+
)
|
|
94
|
+
```
|
|
51
95
|
|
|
52
96
|
### Keyword search
|
|
53
97
|
|
|
@@ -92,34 +136,17 @@ for r in results:
|
|
|
92
136
|
print(r.rank, r.id, f"{r.score:.4f}")
|
|
93
137
|
```
|
|
94
138
|
|
|
95
|
-
###
|
|
139
|
+
### Exact-term matching
|
|
96
140
|
|
|
97
|
-
|
|
141
|
+
Like keyword search, but scores by term overlap alone, with no relevance ranking — a lighter-weight option when you don't need BM25's IDF/length-normalization.
|
|
98
142
|
|
|
99
143
|
```python
|
|
100
|
-
|
|
101
|
-
from simlar import HelixIndex, RelevanceIndex, SimlarEngine, ReciprocalRankFusion
|
|
144
|
+
from simlar import LookupIndex
|
|
102
145
|
|
|
103
|
-
|
|
104
|
-
ids
|
|
105
|
-
vectors = np.load("corpus_embeddings.npy") # shape (n, dim)
|
|
106
|
-
|
|
107
|
-
index = HelixIndex(
|
|
108
|
-
text_index=RelevanceIndex(),
|
|
109
|
-
vector_index=SimlarEngine(),
|
|
110
|
-
fusion=ReciprocalRankFusion(),
|
|
111
|
-
top_k=20,
|
|
112
|
-
)
|
|
113
|
-
index.add(ids=ids, texts=corpus, vectors=vectors)
|
|
146
|
+
idx = LookupIndex()
|
|
147
|
+
idx.add(ids=ids, texts=corpus)
|
|
114
148
|
|
|
115
|
-
|
|
116
|
-
results = index.search(
|
|
117
|
-
query_text="immunotherapy clinical trial",
|
|
118
|
-
query_vector=query_vec,
|
|
119
|
-
k=10,
|
|
120
|
-
)
|
|
121
|
-
for r in results:
|
|
122
|
-
print(r.rank, r.id, f"{r.score:.4f}")
|
|
149
|
+
results = idx.search("immunotherapy clinical trial", k=1)
|
|
123
150
|
```
|
|
124
151
|
|
|
125
152
|
## Framework integrations
|
|
@@ -135,27 +162,11 @@ sim_LAR works as a drop-in component in:
|
|
|
135
162
|
| Index | Best for |
|
|
136
163
|
|-------|----------|
|
|
137
164
|
| `RelevanceIndex` | Keyword search over text — no embeddings required |
|
|
165
|
+
| `LookupIndex` | Fast exact-term matching over text, no relevance ranking |
|
|
138
166
|
| `SimlarEngine` | Semantic search over pre-computed embedding vectors |
|
|
139
167
|
| `HelixIndex` | Both signals combined; corpus fits in memory |
|
|
140
168
|
| `StreamingHybridIndex` | Both signals; very large corpora added in batches |
|
|
141
169
|
|
|
142
|
-
## Producing embeddings
|
|
143
|
-
|
|
144
|
-
sim_LAR is model-agnostic — it accepts any `(n, dim)` float32 NumPy array. Use whichever embedding library fits your project. Example with [sentence-transformers](https://sbert.net/):
|
|
145
|
-
|
|
146
|
-
```bash
|
|
147
|
-
pip install sentence-transformers
|
|
148
|
-
```
|
|
149
|
-
|
|
150
|
-
```python
|
|
151
|
-
from sentence_transformers import SentenceTransformer
|
|
152
|
-
|
|
153
|
-
model = SentenceTransformer("WhereIsAI/UAE-Large-V1")
|
|
154
|
-
|
|
155
|
-
corpus_vectors = model.encode(corpus, normalize_embeddings=True) # (n, dim) float32
|
|
156
|
-
query_vec = model.encode(["my search query"], normalize_embeddings=True) # (1, dim)
|
|
157
|
-
```
|
|
158
|
-
|
|
159
170
|
## Saving and loading indexes
|
|
160
171
|
|
|
161
172
|
Every index type shares the same save/load interface.
|
|
@@ -196,7 +207,8 @@ We do not accept contributions to the proprietary engine.
|
|
|
196
207
|
| Document | Description |
|
|
197
208
|
|----------|-------------|
|
|
198
209
|
| [Concepts](docs/concepts.md) | Architecture and design decisions |
|
|
199
|
-
| [API Reference](docs/
|
|
210
|
+
| [API Reference](docs/api-reference.md) | Full public API |
|
|
211
|
+
| [Installation](docs/installation.md) | Setup and the simlar/simlar-engine compatibility matrix |
|
|
200
212
|
| [Examples](docs/examples/) | Runnable scripts |
|
|
201
213
|
|
|
202
214
|
## License
|
|
@@ -225,7 +237,7 @@ If you only want to read or contribute to the open layer, you never need a licen
|
|
|
225
237
|
## What's open and what's not
|
|
226
238
|
|
|
227
239
|
**Open (Apache 2.0, in this repo):**
|
|
228
|
-
- The SDK and public API surface (`RelevanceIndex`, `HelixIndex`, `StreamingHybridIndex`, `ReciprocalRankFusion`)
|
|
240
|
+
- The SDK and public API surface (`RelevanceIndex`, `LookupIndex`, `HelixIndex`, `StreamingHybridIndex`, `ReciprocalRankFusion`)
|
|
229
241
|
- Framework adapters (LangChain / LlamaIndex / Haystack)
|
|
230
242
|
- The `load_from_directory()` loader and `@register` extension API
|
|
231
243
|
- Configuration schema, type stubs, examples, and docs
|
|
@@ -1,22 +1,65 @@
|
|
|
1
1
|
# sim_LAR
|
|
2
2
|
|
|
3
3
|
Python search library combining keyword, semantic, and hybrid search into a single ranked result list — with first-class LangChain, LlamaIndex, and Haystack support.
|
|
4
|
+
|
|
4
5
|
---
|
|
5
6
|
|
|
6
7
|
## Installation
|
|
7
8
|
|
|
8
|
-
|
|
9
|
+
Both the open wrapper and the engine are on public PyPI:
|
|
9
10
|
|
|
10
11
|
```bash
|
|
11
12
|
pip install simlar
|
|
13
|
+
pip install simlar-engine
|
|
12
14
|
```
|
|
13
15
|
|
|
16
|
+
`simlar-engine` is TekDatum's proprietary binary, licensed under a [Commercial EULA](./EULA.md) — installing it means accepting those terms. See [docs/installation.md](docs/installation.md) for the compatible version matrix between `simlar` and `simlar-engine`.
|
|
14
17
|
|
|
15
|
-
|
|
16
|
-
|
|
18
|
+
## Quick start
|
|
19
|
+
|
|
20
|
+
### Hybrid search
|
|
21
|
+
|
|
22
|
+
Combine keyword relevance and semantic similarity into one ranked list. The easiest way to get started is to give the index an embedder so Simlar can generate document and query embeddings for you.
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
from sentence_transformers import SentenceTransformer
|
|
26
|
+
from simlar import HelixIndex
|
|
27
|
+
|
|
28
|
+
corpus = [
|
|
29
|
+
"cancer treatment with immunotherapy",
|
|
30
|
+
"machine learning transformers",
|
|
31
|
+
"climate change renewable energy",
|
|
32
|
+
]
|
|
33
|
+
ids = [f"doc_{i}" for i in range(len(corpus))]
|
|
34
|
+
|
|
35
|
+
model = SentenceTransformer("WhereIsAI/UAE-Large-V1")
|
|
36
|
+
|
|
37
|
+
index = HelixIndex(embedder=model.encode)
|
|
38
|
+
index.add(ids=ids, texts=corpus)
|
|
39
|
+
|
|
40
|
+
results = index.search("immunotherapy clinical trial", k=10)
|
|
41
|
+
for r in results:
|
|
42
|
+
print(r.rank, r.id, f"{r.score:.4f}")
|
|
17
43
|
```
|
|
18
44
|
|
|
19
|
-
|
|
45
|
+
The embedder is used automatically when vectors are not supplied. You can also pass pre-computed vectors when you already have them.
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
import numpy as np
|
|
49
|
+
from simlar import HelixIndex
|
|
50
|
+
|
|
51
|
+
vectors = np.load("corpus_embeddings.npy") # shape (n, dim)
|
|
52
|
+
|
|
53
|
+
index = HelixIndex(embedder=model.encode)
|
|
54
|
+
index.add(ids=ids, texts=corpus, vectors=vectors)
|
|
55
|
+
|
|
56
|
+
query_vec = np.load("query_embedding.npy") # shape (1, dim)
|
|
57
|
+
results = index.search(
|
|
58
|
+
query_text="immunotherapy clinical trial",
|
|
59
|
+
query_vector=query_vec,
|
|
60
|
+
k=10,
|
|
61
|
+
)
|
|
62
|
+
```
|
|
20
63
|
|
|
21
64
|
### Keyword search
|
|
22
65
|
|
|
@@ -61,34 +104,17 @@ for r in results:
|
|
|
61
104
|
print(r.rank, r.id, f"{r.score:.4f}")
|
|
62
105
|
```
|
|
63
106
|
|
|
64
|
-
###
|
|
107
|
+
### Exact-term matching
|
|
65
108
|
|
|
66
|
-
|
|
109
|
+
Like keyword search, but scores by term overlap alone, with no relevance ranking — a lighter-weight option when you don't need BM25's IDF/length-normalization.
|
|
67
110
|
|
|
68
111
|
```python
|
|
69
|
-
|
|
70
|
-
from simlar import HelixIndex, RelevanceIndex, SimlarEngine, ReciprocalRankFusion
|
|
112
|
+
from simlar import LookupIndex
|
|
71
113
|
|
|
72
|
-
|
|
73
|
-
ids
|
|
74
|
-
vectors = np.load("corpus_embeddings.npy") # shape (n, dim)
|
|
75
|
-
|
|
76
|
-
index = HelixIndex(
|
|
77
|
-
text_index=RelevanceIndex(),
|
|
78
|
-
vector_index=SimlarEngine(),
|
|
79
|
-
fusion=ReciprocalRankFusion(),
|
|
80
|
-
top_k=20,
|
|
81
|
-
)
|
|
82
|
-
index.add(ids=ids, texts=corpus, vectors=vectors)
|
|
114
|
+
idx = LookupIndex()
|
|
115
|
+
idx.add(ids=ids, texts=corpus)
|
|
83
116
|
|
|
84
|
-
|
|
85
|
-
results = index.search(
|
|
86
|
-
query_text="immunotherapy clinical trial",
|
|
87
|
-
query_vector=query_vec,
|
|
88
|
-
k=10,
|
|
89
|
-
)
|
|
90
|
-
for r in results:
|
|
91
|
-
print(r.rank, r.id, f"{r.score:.4f}")
|
|
117
|
+
results = idx.search("immunotherapy clinical trial", k=1)
|
|
92
118
|
```
|
|
93
119
|
|
|
94
120
|
## Framework integrations
|
|
@@ -104,27 +130,11 @@ sim_LAR works as a drop-in component in:
|
|
|
104
130
|
| Index | Best for |
|
|
105
131
|
|-------|----------|
|
|
106
132
|
| `RelevanceIndex` | Keyword search over text — no embeddings required |
|
|
133
|
+
| `LookupIndex` | Fast exact-term matching over text, no relevance ranking |
|
|
107
134
|
| `SimlarEngine` | Semantic search over pre-computed embedding vectors |
|
|
108
135
|
| `HelixIndex` | Both signals combined; corpus fits in memory |
|
|
109
136
|
| `StreamingHybridIndex` | Both signals; very large corpora added in batches |
|
|
110
137
|
|
|
111
|
-
## Producing embeddings
|
|
112
|
-
|
|
113
|
-
sim_LAR is model-agnostic — it accepts any `(n, dim)` float32 NumPy array. Use whichever embedding library fits your project. Example with [sentence-transformers](https://sbert.net/):
|
|
114
|
-
|
|
115
|
-
```bash
|
|
116
|
-
pip install sentence-transformers
|
|
117
|
-
```
|
|
118
|
-
|
|
119
|
-
```python
|
|
120
|
-
from sentence_transformers import SentenceTransformer
|
|
121
|
-
|
|
122
|
-
model = SentenceTransformer("WhereIsAI/UAE-Large-V1")
|
|
123
|
-
|
|
124
|
-
corpus_vectors = model.encode(corpus, normalize_embeddings=True) # (n, dim) float32
|
|
125
|
-
query_vec = model.encode(["my search query"], normalize_embeddings=True) # (1, dim)
|
|
126
|
-
```
|
|
127
|
-
|
|
128
138
|
## Saving and loading indexes
|
|
129
139
|
|
|
130
140
|
Every index type shares the same save/load interface.
|
|
@@ -165,7 +175,8 @@ We do not accept contributions to the proprietary engine.
|
|
|
165
175
|
| Document | Description |
|
|
166
176
|
|----------|-------------|
|
|
167
177
|
| [Concepts](docs/concepts.md) | Architecture and design decisions |
|
|
168
|
-
| [API Reference](docs/
|
|
178
|
+
| [API Reference](docs/api-reference.md) | Full public API |
|
|
179
|
+
| [Installation](docs/installation.md) | Setup and the simlar/simlar-engine compatibility matrix |
|
|
169
180
|
| [Examples](docs/examples/) | Runnable scripts |
|
|
170
181
|
|
|
171
182
|
## License
|
|
@@ -194,7 +205,7 @@ If you only want to read or contribute to the open layer, you never need a licen
|
|
|
194
205
|
## What's open and what's not
|
|
195
206
|
|
|
196
207
|
**Open (Apache 2.0, in this repo):**
|
|
197
|
-
- The SDK and public API surface (`RelevanceIndex`, `HelixIndex`, `StreamingHybridIndex`, `ReciprocalRankFusion`)
|
|
208
|
+
- The SDK and public API surface (`RelevanceIndex`, `LookupIndex`, `HelixIndex`, `StreamingHybridIndex`, `ReciprocalRankFusion`)
|
|
198
209
|
- Framework adapters (LangChain / LlamaIndex / Haystack)
|
|
199
210
|
- The `load_from_directory()` loader and `@register` extension API
|
|
200
211
|
- Configuration schema, type stubs, examples, and docs
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "simlar"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.2.0"
|
|
8
8
|
description = "High-performance similarity search — open-source Python wrapper"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -17,16 +17,18 @@ authors = [
|
|
|
17
17
|
{name = "TekDatum", email = "hello@tekdatum.com"},
|
|
18
18
|
]
|
|
19
19
|
|
|
20
|
-
# simlar-engine is the proprietary binary
|
|
21
|
-
#
|
|
20
|
+
# simlar-engine is the proprietary binary (Commercial EULA, not Apache 2.0),
|
|
21
|
+
# so it's not declared here as a normal dependency; install it separately.
|
|
22
|
+
# See docs/installation.md for instructions and the compatible version matrix.
|
|
22
23
|
dependencies = [
|
|
23
24
|
"numpy>=1.24,<2.5",
|
|
25
|
+
"PyStemmer>=2.0",
|
|
24
26
|
]
|
|
25
27
|
|
|
26
28
|
[project.optional-dependencies]
|
|
27
|
-
langchain = ["langchain-core>=0
|
|
28
|
-
haystack = ["haystack-ai>=
|
|
29
|
-
llama_index = ["llama-index-core>=0.
|
|
29
|
+
langchain = ["langchain-core>=1.0"]
|
|
30
|
+
haystack = ["haystack-ai>=3.0,<4"]
|
|
31
|
+
llama_index = ["llama-index-core>=0.14"]
|
|
30
32
|
dev = [
|
|
31
33
|
"pytest>=8.0",
|
|
32
34
|
"numpy>=1.24,<2.5",
|
|
@@ -76,6 +78,18 @@ testpaths = ["tests"]
|
|
|
76
78
|
pythonpath = ["src"]
|
|
77
79
|
addopts = "-ra -q"
|
|
78
80
|
|
|
81
|
+
# ── Coverage ──────────────────────────────────────────────────────────────────
|
|
82
|
+
|
|
83
|
+
# CI runs against the engine stub, where the tests for these modules are
|
|
84
|
+
# skipped (they need SIMLAR_REAL_ENGINE=1). Leave them out so the coverage
|
|
85
|
+
# total only measures code the stub run can exercise.
|
|
86
|
+
[tool.coverage.run]
|
|
87
|
+
omit = [
|
|
88
|
+
"*/simlar/integrations/haystack/_filters.py",
|
|
89
|
+
"*/simlar/integrations/haystack/simlar_document_store.py",
|
|
90
|
+
"*/simlar/integrations/llama_index/simlar_vector_store.py",
|
|
91
|
+
]
|
|
92
|
+
|
|
79
93
|
# ── Mypy ─────────────────────────────────────────────────────────────────────
|
|
80
94
|
|
|
81
95
|
[tool.mypy]
|
|
@@ -108,3 +122,7 @@ ignore_missing_imports = true
|
|
|
108
122
|
[[tool.mypy.overrides]]
|
|
109
123
|
module = ["fsspec", "fsspec.*"]
|
|
110
124
|
ignore_missing_imports = true
|
|
125
|
+
|
|
126
|
+
[[tool.mypy.overrides]]
|
|
127
|
+
module = ["bm25c", "bm25c.*"]
|
|
128
|
+
ignore_missing_imports = true
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
from simlar_engine import FilterError, MetadataFilter, SQLFilter
|
|
2
|
+
|
|
3
|
+
from simlar.contracts import SearchResult, TextIndex, VectorIndex
|
|
4
|
+
from simlar.embeddings import CallableEmbedder, Embedder
|
|
5
|
+
from simlar.fusion import ReciprocalRankFusion
|
|
6
|
+
from simlar.indexes.bm25c_index import BM25CIndex
|
|
7
|
+
from simlar.indexes.filtered_index import FilteredIndex
|
|
8
|
+
from simlar.indexes.helix_index import HelixIndex
|
|
9
|
+
from simlar.indexes.lookup_index import LookupIndex
|
|
10
|
+
from simlar.indexes.registry import load_from_directory, register
|
|
11
|
+
from simlar.indexes.relevance_index import RelevanceIndex
|
|
12
|
+
from simlar.indexes.simlar_engine import SimlarEngine
|
|
13
|
+
from simlar.indexes.streaming_index import StreamingHelixIndex as StreamingHybridIndex
|
|
14
|
+
|
|
15
|
+
_MIN_ENGINE_VERSION = (1, 1, 0)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _check_engine_version() -> None:
|
|
19
|
+
"""Best-effort compatibility guard between this wrapper and simlar-engine.
|
|
20
|
+
|
|
21
|
+
simlar-engine is a proprietary binary installed as a separate package
|
|
22
|
+
(see docs/installation.md) — there is no packaging-level dependency pin
|
|
23
|
+
tying the two together, so a too-old engine would otherwise fail with a
|
|
24
|
+
confusing TypeError deep inside a search() call instead of a clear error
|
|
25
|
+
at import time. Silently skipped when the installed engine doesn't
|
|
26
|
+
expose __version__ (e.g. a test double, or a build that predates this
|
|
27
|
+
check) rather than treated as a failure — this is a floor, not a full
|
|
28
|
+
dependency resolver.
|
|
29
|
+
"""
|
|
30
|
+
import simlar_engine
|
|
31
|
+
|
|
32
|
+
engine_version = getattr(simlar_engine, "__version__", None)
|
|
33
|
+
if engine_version is None:
|
|
34
|
+
return
|
|
35
|
+
try:
|
|
36
|
+
parsed = tuple(int(p) for p in engine_version.split(".")[:3])
|
|
37
|
+
except ValueError:
|
|
38
|
+
return
|
|
39
|
+
if parsed < _MIN_ENGINE_VERSION:
|
|
40
|
+
raise RuntimeError(
|
|
41
|
+
f"simlar requires simlar-engine >= {'.'.join(map(str, _MIN_ENGINE_VERSION))}, "
|
|
42
|
+
f"but {engine_version} is installed. See docs/installation.md for the "
|
|
43
|
+
"compatible version matrix."
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
_check_engine_version()
|
|
48
|
+
|
|
49
|
+
__all__ = [
|
|
50
|
+
# Contracts — extension points
|
|
51
|
+
"SearchResult",
|
|
52
|
+
"TextIndex",
|
|
53
|
+
"VectorIndex",
|
|
54
|
+
# Embeddings
|
|
55
|
+
"Embedder",
|
|
56
|
+
"CallableEmbedder",
|
|
57
|
+
# Fusion
|
|
58
|
+
"ReciprocalRankFusion",
|
|
59
|
+
# Indexes
|
|
60
|
+
"RelevanceIndex",
|
|
61
|
+
"SimlarEngine",
|
|
62
|
+
"BM25CIndex",
|
|
63
|
+
"HelixIndex",
|
|
64
|
+
"StreamingHybridIndex",
|
|
65
|
+
"LookupIndex",
|
|
66
|
+
# Filtering
|
|
67
|
+
"FilteredIndex",
|
|
68
|
+
"SQLFilter",
|
|
69
|
+
"MetadataFilter",
|
|
70
|
+
"FilterError",
|
|
71
|
+
# Registry
|
|
72
|
+
"register",
|
|
73
|
+
"load_from_directory",
|
|
74
|
+
]
|
|
@@ -26,11 +26,11 @@ class Index(ABC):
|
|
|
26
26
|
"""Minimal contract: every index can persist and report its type and size."""
|
|
27
27
|
|
|
28
28
|
@abstractmethod
|
|
29
|
-
def save(self, path: str) -> None: ...
|
|
29
|
+
def save(self, path: str, base_dir: str | None = None) -> None: ...
|
|
30
30
|
|
|
31
31
|
@classmethod
|
|
32
32
|
@abstractmethod
|
|
33
|
-
def load(cls, path: str) -> Index: ...
|
|
33
|
+
def load(cls, path: str, base_dir: str | None = None) -> Index: ...
|
|
34
34
|
|
|
35
35
|
@property
|
|
36
36
|
@abstractmethod
|
|
@@ -44,6 +44,10 @@ class Index(ABC):
|
|
|
44
44
|
@abstractmethod
|
|
45
45
|
def is_trained(self) -> bool: ...
|
|
46
46
|
|
|
47
|
+
@property
|
|
48
|
+
@abstractmethod
|
|
49
|
+
def ids(self) -> list[str]: ...
|
|
50
|
+
|
|
47
51
|
|
|
48
52
|
# ── Text index ────────────────────────────────────────────────────────────────
|
|
49
53
|
|
|
@@ -54,8 +58,12 @@ class TextIndex(Index):
|
|
|
54
58
|
# ── Public API ─────────────────────────────────────────────────────────────
|
|
55
59
|
|
|
56
60
|
@abstractmethod
|
|
57
|
-
def add(self, ids: list[str], texts: list[str]) -> None:
|
|
58
|
-
"""Append new documents. Raises ValueError on duplicate IDs; use update() to replace.
|
|
61
|
+
def add(self, ids: list[str], texts: list[str], parallel: bool = True) -> None:
|
|
62
|
+
"""Append new documents. Raises ValueError on duplicate IDs; use update() to replace.
|
|
63
|
+
|
|
64
|
+
`parallel` is accepted for parity with the vector side; text indexing
|
|
65
|
+
is single-threaded in every backend.
|
|
66
|
+
"""
|
|
59
67
|
|
|
60
68
|
@abstractmethod
|
|
61
69
|
def update(self, ids: list[str], texts: list[str]) -> None:
|
|
@@ -66,12 +74,29 @@ class TextIndex(Index):
|
|
|
66
74
|
"""Remove documents by ID, rebuilding internal structures."""
|
|
67
75
|
|
|
68
76
|
@abstractmethod
|
|
69
|
-
def search(
|
|
77
|
+
def search(
|
|
78
|
+
self,
|
|
79
|
+
query: str | list[str],
|
|
80
|
+
k: int,
|
|
81
|
+
parallel: bool = True,
|
|
82
|
+
batch_size: int | None = None,
|
|
83
|
+
candidates: np.ndarray | None = None,
|
|
84
|
+
) -> list[SearchResult] | list[list[SearchResult]]:
|
|
85
|
+
"""Rank documents against query. `parallel` threads a batch of queries.
|
|
86
|
+
|
|
87
|
+
`candidates`, if given, is a single flat array of positions applied
|
|
88
|
+
to EVERY query in a batch call -- not a per-query restriction like
|
|
89
|
+
`VectorIndex.search_raw`'s `candidates` (used for Helix's per-query
|
|
90
|
+
text->vector cascade). A backend that can restrict a search this
|
|
91
|
+
cheaply doesn't need a per-query shape here: the one caller that
|
|
92
|
+
needs it (SQLFilter-style pre-filtering) always applies the same
|
|
93
|
+
filter to every query in a batch.
|
|
94
|
+
"""
|
|
70
95
|
|
|
71
96
|
# ── Internal ───────────────────────────────────────────────────────────────
|
|
72
97
|
|
|
73
98
|
@abstractmethod
|
|
74
|
-
def fit(self, corpus: list[str], parallel: bool =
|
|
99
|
+
def fit(self, corpus: list[str], parallel: bool = True, **kwargs) -> None:
|
|
75
100
|
"""Build the model from a raw corpus list. Called by add() and HelixIndex."""
|
|
76
101
|
|
|
77
102
|
@abstractmethod
|
|
@@ -79,10 +104,12 @@ class TextIndex(Index):
|
|
|
79
104
|
self,
|
|
80
105
|
queries: str | list[str],
|
|
81
106
|
k: int,
|
|
82
|
-
parallel: bool =
|
|
107
|
+
parallel: bool = True,
|
|
108
|
+
candidates: np.ndarray | None = None,
|
|
83
109
|
) -> tuple[np.ndarray, np.ndarray]:
|
|
84
110
|
"""Return (ids, scores) shaped (n_queries, k), int64/float64.
|
|
85
|
-
Used internally by HelixIndex._search_raw().
|
|
111
|
+
Used internally by HelixIndex._search_raw(). See `search`'s
|
|
112
|
+
docstring for what `candidates` does.
|
|
86
113
|
"""
|
|
87
114
|
|
|
88
115
|
|
|
@@ -95,10 +122,25 @@ class VectorIndex(Index):
|
|
|
95
122
|
# ── Public API ─────────────────────────────────────────────────────────────
|
|
96
123
|
|
|
97
124
|
@abstractmethod
|
|
98
|
-
def add(self, ids: list[str], vectors: np.ndarray) -> None:
|
|
125
|
+
def add(self, ids: list[str], vectors: np.ndarray, parallel: bool = True) -> None:
|
|
126
|
+
"""Append new vectors. `parallel` threads their quantization."""
|
|
99
127
|
|
|
100
128
|
@abstractmethod
|
|
101
|
-
def search(
|
|
129
|
+
def search(
|
|
130
|
+
self,
|
|
131
|
+
query: np.ndarray,
|
|
132
|
+
k: int,
|
|
133
|
+
parallel: bool = True,
|
|
134
|
+
batch_size: int | None = None,
|
|
135
|
+
candidates: np.ndarray | None = None,
|
|
136
|
+
) -> list[SearchResult]:
|
|
137
|
+
"""Rank documents against query. `parallel` threads a batch of queries.
|
|
138
|
+
|
|
139
|
+
`candidates`, if given, is a single flat array of positions applied
|
|
140
|
+
to every query -- the same pre-filter contract as `TextIndex.search`
|
|
141
|
+
(used by FilteredIndex). Fewer allowed documents than `k` means
|
|
142
|
+
fewer results.
|
|
143
|
+
"""
|
|
102
144
|
|
|
103
145
|
@abstractmethod
|
|
104
146
|
def update(self, ids: list[str], vectors: np.ndarray) -> None:
|
|
@@ -110,7 +152,7 @@ class VectorIndex(Index):
|
|
|
110
152
|
def fit(
|
|
111
153
|
self,
|
|
112
154
|
embeddings: np.ndarray,
|
|
113
|
-
parallel: bool =
|
|
155
|
+
parallel: bool = True,
|
|
114
156
|
params: _Parameters | None = None,
|
|
115
157
|
**kwargs,
|
|
116
158
|
) -> None:
|
|
@@ -122,11 +164,15 @@ class VectorIndex(Index):
|
|
|
122
164
|
vectors: np.ndarray,
|
|
123
165
|
k: int,
|
|
124
166
|
candidates: np.ndarray | None = None,
|
|
125
|
-
parallel: bool =
|
|
167
|
+
parallel: bool = True,
|
|
126
168
|
n_candidates: int | None = None,
|
|
127
169
|
) -> tuple[np.ndarray, np.ndarray]:
|
|
128
170
|
"""Return (ids, distances) shaped (n_queries, k), int64/float64.
|
|
129
171
|
Used internally by HelixIndex._search_raw().
|
|
172
|
+
|
|
173
|
+
`candidates` is either a 1D array of positions shared by every query
|
|
174
|
+
(the pre-filter, see `search`) or a 2D (n_queries, n_cand) array of
|
|
175
|
+
per-query rows (Helix's text->vector cascade).
|
|
130
176
|
"""
|
|
131
177
|
|
|
132
178
|
@abstractmethod
|
|
@@ -151,17 +197,29 @@ class CompositeIndex(Index):
|
|
|
151
197
|
@abstractmethod
|
|
152
198
|
def search(
|
|
153
199
|
self,
|
|
154
|
-
query_text: str | None = None,
|
|
200
|
+
query_text: str | list[str] | None = None,
|
|
155
201
|
query_vector: np.ndarray | None = None,
|
|
156
202
|
k: int = 10,
|
|
157
|
-
|
|
203
|
+
parallel: bool = True,
|
|
204
|
+
batch_size: int | None = None,
|
|
205
|
+
candidates: np.ndarray | None = None,
|
|
206
|
+
) -> list[SearchResult] | list[list[SearchResult]]:
|
|
207
|
+
"""Search every sub-index and fuse. `parallel` threads a batch of queries.
|
|
208
|
+
|
|
209
|
+
A single query returns `list[SearchResult]`; a batch (e.g. a list of
|
|
210
|
+
query texts, or a 2D array of query vectors) returns one such list
|
|
211
|
+
per query, in order — the same batch contract as `TextIndex.search`.
|
|
212
|
+
|
|
213
|
+
`candidates`, if given, is a single flat array of positions applied
|
|
214
|
+
to every query, restricting every sub-index (see FilteredIndex).
|
|
215
|
+
"""
|
|
158
216
|
|
|
159
217
|
@abstractmethod
|
|
160
218
|
def fit(
|
|
161
219
|
self,
|
|
162
220
|
corpus: list,
|
|
163
221
|
vectors: np.ndarray,
|
|
164
|
-
parallel: bool =
|
|
222
|
+
parallel: bool = True,
|
|
165
223
|
**kwargs,
|
|
166
224
|
) -> None:
|
|
167
225
|
"""Internal: build all sub-indexes. Used by StreamingHybridIndex shards."""
|
|
@@ -172,10 +230,8 @@ class CompositeIndex(Index):
|
|
|
172
230
|
|
|
173
231
|
@runtime_checkable
|
|
174
232
|
class FusionStrategy(Protocol):
|
|
175
|
-
"""Combines N result lists into a single ranked list."""
|
|
176
|
-
|
|
177
233
|
def __call__(
|
|
178
234
|
self,
|
|
179
|
-
results: list[
|
|
235
|
+
results: list[tuple[np.ndarray, np.ndarray]],
|
|
180
236
|
k: int,
|
|
181
|
-
) ->
|
|
237
|
+
) -> tuple[np.ndarray, np.ndarray]: ...
|