simlar 1.0.0__tar.gz → 1.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. {simlar-1.0.0/src/simlar.egg-info → simlar-1.2.0}/PKG-INFO +62 -50
  2. {simlar-1.0.0 → simlar-1.2.0}/README.md +57 -46
  3. {simlar-1.0.0 → simlar-1.2.0}/pyproject.toml +24 -6
  4. simlar-1.2.0/src/simlar/__init__.py +74 -0
  5. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/contracts.py +75 -19
  6. simlar-1.2.0/src/simlar/embeddings.py +147 -0
  7. simlar-1.2.0/src/simlar/indexes/__init__.py +17 -0
  8. simlar-1.2.0/src/simlar/indexes/_filtered.pyi +86 -0
  9. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/_helix.pyi +20 -8
  10. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/_relevance.pyi +17 -7
  11. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/_simlar.pyi +18 -8
  12. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/_streaming.pyi +14 -12
  13. simlar-1.2.0/src/simlar/indexes/bm25c_index.py +137 -0
  14. simlar-1.2.0/src/simlar/indexes/filtered_index.py +54 -0
  15. simlar-1.2.0/src/simlar/indexes/helix_index.py +200 -0
  16. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/lookup_index.py +27 -14
  17. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/registry.py +12 -5
  18. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/relevance_index.py +30 -20
  19. simlar-1.2.0/src/simlar/indexes/simlar_engine.py +233 -0
  20. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/indexes/streaming_index.py +50 -17
  21. simlar-1.2.0/src/simlar/integrations/_filtering.py +74 -0
  22. simlar-1.2.0/src/simlar/integrations/haystack/_filters.py +253 -0
  23. simlar-1.2.0/src/simlar/integrations/haystack/simlar_document_store.py +579 -0
  24. simlar-1.2.0/src/simlar/integrations/haystack/simlar_retriever.py +209 -0
  25. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/integrations/langchain/langchain_retriever.py +7 -5
  26. simlar-1.2.0/src/simlar/integrations/langchain/simlar_vector_store.py +428 -0
  27. simlar-1.2.0/src/simlar/integrations/llama_index/simlar_vector_store.py +470 -0
  28. {simlar-1.0.0 → simlar-1.2.0/src/simlar.egg-info}/PKG-INFO +62 -50
  29. {simlar-1.0.0 → simlar-1.2.0}/src/simlar.egg-info/SOURCES.txt +10 -2
  30. {simlar-1.0.0 → simlar-1.2.0}/src/simlar.egg-info/requires.txt +4 -3
  31. simlar-1.2.0/tests/test_bm25c_index.py +300 -0
  32. simlar-1.2.0/tests/test_embeddings.py +351 -0
  33. {simlar-1.0.0 → simlar-1.2.0}/tests/test_helix_index.py +10 -1
  34. {simlar-1.0.0 → simlar-1.2.0}/tests/test_lookup_index.py +20 -3
  35. {simlar-1.0.0 → simlar-1.2.0}/tests/test_registry.py +16 -1
  36. {simlar-1.0.0 → simlar-1.2.0}/tests/test_relevance_index.py +17 -0
  37. simlar-1.2.0/tests/test_tmp_parallel_probe.py +104 -0
  38. simlar-1.0.0/src/simlar/__init__.py +0 -24
  39. simlar-1.0.0/src/simlar/indexes/__init__.py +0 -7
  40. simlar-1.0.0/src/simlar/indexes/helix_index.py +0 -126
  41. simlar-1.0.0/src/simlar/indexes/simlar_engine.py +0 -113
  42. simlar-1.0.0/src/simlar/integrations/haystack/simlar_document_store.py +0 -401
  43. simlar-1.0.0/src/simlar/integrations/haystack/simlar_retriever.py +0 -64
  44. simlar-1.0.0/src/simlar/integrations/langchain/simlar_vector_store.py +0 -268
  45. simlar-1.0.0/src/simlar/integrations/llama_index/simlar_retriever.py +0 -186
  46. simlar-1.0.0/src/simlar/integrations/llama_index/simlar_vector_store.py +0 -248
  47. {simlar-1.0.0 → simlar-1.2.0}/LICENSE +0 -0
  48. {simlar-1.0.0 → simlar-1.2.0}/NOTICE +0 -0
  49. {simlar-1.0.0 → simlar-1.2.0}/setup.cfg +0 -0
  50. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/fusion/__init__.py +0 -0
  51. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/fusion/_rrf.pyi +0 -0
  52. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/fusion/rrf.py +0 -0
  53. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/integrations/__init__.py +0 -0
  54. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/persistence.py +0 -0
  55. {simlar-1.0.0 → simlar-1.2.0}/src/simlar/py.typed +0 -0
  56. {simlar-1.0.0 → simlar-1.2.0}/src/simlar.egg-info/dependency_links.txt +0 -0
  57. {simlar-1.0.0 → simlar-1.2.0}/src/simlar.egg-info/top_level.txt +0 -0
  58. {simlar-1.0.0 → simlar-1.2.0}/tests/test_contracts.py +0 -0
  59. {simlar-1.0.0 → simlar-1.2.0}/tests/test_persistence.py +0 -0
  60. {simlar-1.0.0 → simlar-1.2.0}/tests/test_simlar_engine.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: simlar
3
- Version: 1.0.0
3
+ Version: 1.2.0
4
4
  Summary: High-performance similarity search — open-source Python wrapper
5
5
  Author-email: TekDatum <hello@tekdatum.com>
6
6
  License: Apache-2.0
@@ -16,12 +16,13 @@ Description-Content-Type: text/markdown
16
16
  License-File: LICENSE
17
17
  License-File: NOTICE
18
18
  Requires-Dist: numpy<2.5,>=1.24
19
+ Requires-Dist: PyStemmer>=2.0
19
20
  Provides-Extra: langchain
20
- Requires-Dist: langchain-core>=0.2; extra == "langchain"
21
+ Requires-Dist: langchain-core>=1.0; extra == "langchain"
21
22
  Provides-Extra: haystack
22
- Requires-Dist: haystack-ai>=2.0; extra == "haystack"
23
+ Requires-Dist: haystack-ai<4,>=3.0; extra == "haystack"
23
24
  Provides-Extra: llama-index
24
- Requires-Dist: llama-index-core>=0.10; extra == "llama-index"
25
+ Requires-Dist: llama-index-core>=0.14; extra == "llama-index"
25
26
  Provides-Extra: dev
26
27
  Requires-Dist: pytest>=8.0; extra == "dev"
27
28
  Requires-Dist: numpy<2.5,>=1.24; extra == "dev"
@@ -32,22 +33,65 @@ Dynamic: license-file
32
33
  # sim_LAR
33
34
 
34
35
  Python search library combining keyword, semantic, and hybrid search into a single ranked result list — with first-class LangChain, LlamaIndex, and Haystack support.
36
+
35
37
  ---
36
38
 
37
39
  ## Installation
38
40
 
39
- The open wrapper and the engine is on public PyPI:
41
+ Both the open wrapper and the engine are on public PyPI:
40
42
 
41
43
  ```bash
42
44
  pip install simlar
45
+ pip install simlar-engine
43
46
  ```
44
47
 
48
+ `simlar-engine` is TekDatum's proprietary binary, licensed under a [Commercial EULA](./EULA.md) — installing it means accepting those terms. See [docs/installation.md](docs/installation.md) for the compatible version matrix between `simlar` and `simlar-engine`.
45
49
 
46
- ```bash
47
- pip install simlar-engine
50
+ ## Quick start
51
+
52
+ ### Hybrid search
53
+
54
+ Combine keyword relevance and semantic similarity into one ranked list. The easiest way to get started is to give the index an embedder so Simlar can generate document and query embeddings for you.
55
+
56
+ ```python
57
+ from sentence_transformers import SentenceTransformer
58
+ from simlar import HelixIndex
59
+
60
+ corpus = [
61
+ "cancer treatment with immunotherapy",
62
+ "machine learning transformers",
63
+ "climate change renewable energy",
64
+ ]
65
+ ids = [f"doc_{i}" for i in range(len(corpus))]
66
+
67
+ model = SentenceTransformer("WhereIsAI/UAE-Large-V1")
68
+
69
+ index = HelixIndex(embedder=model.encode)
70
+ index.add(ids=ids, texts=corpus)
71
+
72
+ results = index.search("immunotherapy clinical trial", k=10)
73
+ for r in results:
74
+ print(r.rank, r.id, f"{r.score:.4f}")
48
75
  ```
49
76
 
50
- ## Quick start
77
+ The embedder is used automatically when vectors are not supplied. You can also pass pre-computed vectors when you already have them.
78
+
79
+ ```python
80
+ import numpy as np
81
+ from simlar import HelixIndex
82
+
83
+ vectors = np.load("corpus_embeddings.npy") # shape (n, dim)
84
+
85
+ index = HelixIndex(embedder=model.encode)
86
+ index.add(ids=ids, texts=corpus, vectors=vectors)
87
+
88
+ query_vec = np.load("query_embedding.npy") # shape (1, dim)
89
+ results = index.search(
90
+ query_text="immunotherapy clinical trial",
91
+ query_vector=query_vec,
92
+ k=10,
93
+ )
94
+ ```
51
95
 
52
96
  ### Keyword search
53
97
 
@@ -92,34 +136,17 @@ for r in results:
92
136
  print(r.rank, r.id, f"{r.score:.4f}")
93
137
  ```
94
138
 
95
- ### Hybrid search
139
+ ### Exact-term matching
96
140
 
97
- Combine keyword relevance and semantic similarity into one ranked list.
141
+ Like keyword search, but scores by term overlap alone, with no relevance ranking — a lighter-weight option when you don't need BM25's IDF/length-normalization.
98
142
 
99
143
  ```python
100
- import numpy as np
101
- from simlar import HelixIndex, RelevanceIndex, SimlarEngine, ReciprocalRankFusion
144
+ from simlar import LookupIndex
102
145
 
103
- corpus = ["cancer treatment with immunotherapy", "machine learning transformers", ...]
104
- ids = [f"doc_{i}" for i in range(len(corpus))]
105
- vectors = np.load("corpus_embeddings.npy") # shape (n, dim)
106
-
107
- index = HelixIndex(
108
- text_index=RelevanceIndex(),
109
- vector_index=SimlarEngine(),
110
- fusion=ReciprocalRankFusion(),
111
- top_k=20,
112
- )
113
- index.add(ids=ids, texts=corpus, vectors=vectors)
146
+ idx = LookupIndex()
147
+ idx.add(ids=ids, texts=corpus)
114
148
 
115
- query_vec = np.load("query_embedding.npy")
116
- results = index.search(
117
- query_text="immunotherapy clinical trial",
118
- query_vector=query_vec,
119
- k=10,
120
- )
121
- for r in results:
122
- print(r.rank, r.id, f"{r.score:.4f}")
149
+ results = idx.search("immunotherapy clinical trial", k=1)
123
150
  ```
124
151
 
125
152
  ## Framework integrations
@@ -135,27 +162,11 @@ sim_LAR works as a drop-in component in:
135
162
  | Index | Best for |
136
163
  |-------|----------|
137
164
  | `RelevanceIndex` | Keyword search over text — no embeddings required |
165
+ | `LookupIndex` | Fast exact-term matching over text, no relevance ranking |
138
166
  | `SimlarEngine` | Semantic search over pre-computed embedding vectors |
139
167
  | `HelixIndex` | Both signals combined; corpus fits in memory |
140
168
  | `StreamingHybridIndex` | Both signals; very large corpora added in batches |
141
169
 
142
- ## Producing embeddings
143
-
144
- sim_LAR is model-agnostic — it accepts any `(n, dim)` float32 NumPy array. Use whichever embedding library fits your project. Example with [sentence-transformers](https://sbert.net/):
145
-
146
- ```bash
147
- pip install sentence-transformers
148
- ```
149
-
150
- ```python
151
- from sentence_transformers import SentenceTransformer
152
-
153
- model = SentenceTransformer("WhereIsAI/UAE-Large-V1")
154
-
155
- corpus_vectors = model.encode(corpus, normalize_embeddings=True) # (n, dim) float32
156
- query_vec = model.encode(["my search query"], normalize_embeddings=True) # (1, dim)
157
- ```
158
-
159
170
  ## Saving and loading indexes
160
171
 
161
172
  Every index type shares the same save/load interface.
@@ -196,7 +207,8 @@ We do not accept contributions to the proprietary engine.
196
207
  | Document | Description |
197
208
  |----------|-------------|
198
209
  | [Concepts](docs/concepts.md) | Architecture and design decisions |
199
- | [API Reference](docs/concepts.md) | Full public API |
210
+ | [API Reference](docs/api-reference.md) | Full public API |
211
+ | [Installation](docs/installation.md) | Setup and the simlar/simlar-engine compatibility matrix |
200
212
  | [Examples](docs/examples/) | Runnable scripts |
201
213
 
202
214
  ## License
@@ -225,7 +237,7 @@ If you only want to read or contribute to the open layer, you never need a licen
225
237
  ## What's open and what's not
226
238
 
227
239
  **Open (Apache 2.0, in this repo):**
228
- - The SDK and public API surface (`RelevanceIndex`, `HelixIndex`, `StreamingHybridIndex`, `ReciprocalRankFusion`)
240
+ - The SDK and public API surface (`RelevanceIndex`, `LookupIndex`, `HelixIndex`, `StreamingHybridIndex`, `ReciprocalRankFusion`)
229
241
  - Framework adapters (LangChain / LlamaIndex / Haystack)
230
242
  - The `load_from_directory()` loader and `@register` extension API
231
243
  - Configuration schema, type stubs, examples, and docs
@@ -1,22 +1,65 @@
1
1
  # sim_LAR
2
2
 
3
3
  Python search library combining keyword, semantic, and hybrid search into a single ranked result list — with first-class LangChain, LlamaIndex, and Haystack support.
4
+
4
5
  ---
5
6
 
6
7
  ## Installation
7
8
 
8
- The open wrapper and the engine is on public PyPI:
9
+ Both the open wrapper and the engine are on public PyPI:
9
10
 
10
11
  ```bash
11
12
  pip install simlar
13
+ pip install simlar-engine
12
14
  ```
13
15
 
16
+ `simlar-engine` is TekDatum's proprietary binary, licensed under a [Commercial EULA](./EULA.md) — installing it means accepting those terms. See [docs/installation.md](docs/installation.md) for the compatible version matrix between `simlar` and `simlar-engine`.
14
17
 
15
- ```bash
16
- pip install simlar-engine
18
+ ## Quick start
19
+
20
+ ### Hybrid search
21
+
22
+ Combine keyword relevance and semantic similarity into one ranked list. The easiest way to get started is to give the index an embedder so Simlar can generate document and query embeddings for you.
23
+
24
+ ```python
25
+ from sentence_transformers import SentenceTransformer
26
+ from simlar import HelixIndex
27
+
28
+ corpus = [
29
+ "cancer treatment with immunotherapy",
30
+ "machine learning transformers",
31
+ "climate change renewable energy",
32
+ ]
33
+ ids = [f"doc_{i}" for i in range(len(corpus))]
34
+
35
+ model = SentenceTransformer("WhereIsAI/UAE-Large-V1")
36
+
37
+ index = HelixIndex(embedder=model.encode)
38
+ index.add(ids=ids, texts=corpus)
39
+
40
+ results = index.search("immunotherapy clinical trial", k=10)
41
+ for r in results:
42
+ print(r.rank, r.id, f"{r.score:.4f}")
17
43
  ```
18
44
 
19
- ## Quick start
45
+ The embedder is used automatically when vectors are not supplied. You can also pass pre-computed vectors when you already have them.
46
+
47
+ ```python
48
+ import numpy as np
49
+ from simlar import HelixIndex
50
+
51
+ vectors = np.load("corpus_embeddings.npy") # shape (n, dim)
52
+
53
+ index = HelixIndex(embedder=model.encode)
54
+ index.add(ids=ids, texts=corpus, vectors=vectors)
55
+
56
+ query_vec = np.load("query_embedding.npy") # shape (1, dim)
57
+ results = index.search(
58
+ query_text="immunotherapy clinical trial",
59
+ query_vector=query_vec,
60
+ k=10,
61
+ )
62
+ ```
20
63
 
21
64
  ### Keyword search
22
65
 
@@ -61,34 +104,17 @@ for r in results:
61
104
  print(r.rank, r.id, f"{r.score:.4f}")
62
105
  ```
63
106
 
64
- ### Hybrid search
107
+ ### Exact-term matching
65
108
 
66
- Combine keyword relevance and semantic similarity into one ranked list.
109
+ Like keyword search, but scores by term overlap alone, with no relevance ranking — a lighter-weight option when you don't need BM25's IDF/length-normalization.
67
110
 
68
111
  ```python
69
- import numpy as np
70
- from simlar import HelixIndex, RelevanceIndex, SimlarEngine, ReciprocalRankFusion
112
+ from simlar import LookupIndex
71
113
 
72
- corpus = ["cancer treatment with immunotherapy", "machine learning transformers", ...]
73
- ids = [f"doc_{i}" for i in range(len(corpus))]
74
- vectors = np.load("corpus_embeddings.npy") # shape (n, dim)
75
-
76
- index = HelixIndex(
77
- text_index=RelevanceIndex(),
78
- vector_index=SimlarEngine(),
79
- fusion=ReciprocalRankFusion(),
80
- top_k=20,
81
- )
82
- index.add(ids=ids, texts=corpus, vectors=vectors)
114
+ idx = LookupIndex()
115
+ idx.add(ids=ids, texts=corpus)
83
116
 
84
- query_vec = np.load("query_embedding.npy")
85
- results = index.search(
86
- query_text="immunotherapy clinical trial",
87
- query_vector=query_vec,
88
- k=10,
89
- )
90
- for r in results:
91
- print(r.rank, r.id, f"{r.score:.4f}")
117
+ results = idx.search("immunotherapy clinical trial", k=1)
92
118
  ```
93
119
 
94
120
  ## Framework integrations
@@ -104,27 +130,11 @@ sim_LAR works as a drop-in component in:
104
130
  | Index | Best for |
105
131
  |-------|----------|
106
132
  | `RelevanceIndex` | Keyword search over text — no embeddings required |
133
+ | `LookupIndex` | Fast exact-term matching over text, no relevance ranking |
107
134
  | `SimlarEngine` | Semantic search over pre-computed embedding vectors |
108
135
  | `HelixIndex` | Both signals combined; corpus fits in memory |
109
136
  | `StreamingHybridIndex` | Both signals; very large corpora added in batches |
110
137
 
111
- ## Producing embeddings
112
-
113
- sim_LAR is model-agnostic — it accepts any `(n, dim)` float32 NumPy array. Use whichever embedding library fits your project. Example with [sentence-transformers](https://sbert.net/):
114
-
115
- ```bash
116
- pip install sentence-transformers
117
- ```
118
-
119
- ```python
120
- from sentence_transformers import SentenceTransformer
121
-
122
- model = SentenceTransformer("WhereIsAI/UAE-Large-V1")
123
-
124
- corpus_vectors = model.encode(corpus, normalize_embeddings=True) # (n, dim) float32
125
- query_vec = model.encode(["my search query"], normalize_embeddings=True) # (1, dim)
126
- ```
127
-
128
138
  ## Saving and loading indexes
129
139
 
130
140
  Every index type shares the same save/load interface.
@@ -165,7 +175,8 @@ We do not accept contributions to the proprietary engine.
165
175
  | Document | Description |
166
176
  |----------|-------------|
167
177
  | [Concepts](docs/concepts.md) | Architecture and design decisions |
168
- | [API Reference](docs/concepts.md) | Full public API |
178
+ | [API Reference](docs/api-reference.md) | Full public API |
179
+ | [Installation](docs/installation.md) | Setup and the simlar/simlar-engine compatibility matrix |
169
180
  | [Examples](docs/examples/) | Runnable scripts |
170
181
 
171
182
  ## License
@@ -194,7 +205,7 @@ If you only want to read or contribute to the open layer, you never need a licen
194
205
  ## What's open and what's not
195
206
 
196
207
  **Open (Apache 2.0, in this repo):**
197
- - The SDK and public API surface (`RelevanceIndex`, `HelixIndex`, `StreamingHybridIndex`, `ReciprocalRankFusion`)
208
+ - The SDK and public API surface (`RelevanceIndex`, `LookupIndex`, `HelixIndex`, `StreamingHybridIndex`, `ReciprocalRankFusion`)
198
209
  - Framework adapters (LangChain / LlamaIndex / Haystack)
199
210
  - The `load_from_directory()` loader and `@register` extension API
200
211
  - Configuration schema, type stubs, examples, and docs
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "simlar"
7
- version = "1.0.0"
7
+ version = "1.2.0"
8
8
  description = "High-performance similarity search — open-source Python wrapper"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -17,16 +17,18 @@ authors = [
17
17
  {name = "TekDatum", email = "hello@tekdatum.com"},
18
18
  ]
19
19
 
20
- # simlar-engine is the proprietary binary; install from the private index first.
21
- # See installation.md for instructions.
20
+ # simlar-engine is the proprietary binary (Commercial EULA, not Apache 2.0),
21
+ # so it's not declared here as a normal dependency; install it separately.
22
+ # See docs/installation.md for instructions and the compatible version matrix.
22
23
  dependencies = [
23
24
  "numpy>=1.24,<2.5",
25
+ "PyStemmer>=2.0",
24
26
  ]
25
27
 
26
28
  [project.optional-dependencies]
27
- langchain = ["langchain-core>=0.2"]
28
- haystack = ["haystack-ai>=2.0"]
29
- llama_index = ["llama-index-core>=0.10"]
29
+ langchain = ["langchain-core>=1.0"]
30
+ haystack = ["haystack-ai>=3.0,<4"]
31
+ llama_index = ["llama-index-core>=0.14"]
30
32
  dev = [
31
33
  "pytest>=8.0",
32
34
  "numpy>=1.24,<2.5",
@@ -76,6 +78,18 @@ testpaths = ["tests"]
76
78
  pythonpath = ["src"]
77
79
  addopts = "-ra -q"
78
80
 
81
+ # ── Coverage ──────────────────────────────────────────────────────────────────
82
+
83
+ # CI runs against the engine stub, where the tests for these modules are
84
+ # skipped (they need SIMLAR_REAL_ENGINE=1). Leave them out so the coverage
85
+ # total only measures code the stub run can exercise.
86
+ [tool.coverage.run]
87
+ omit = [
88
+ "*/simlar/integrations/haystack/_filters.py",
89
+ "*/simlar/integrations/haystack/simlar_document_store.py",
90
+ "*/simlar/integrations/llama_index/simlar_vector_store.py",
91
+ ]
92
+
79
93
  # ── Mypy ─────────────────────────────────────────────────────────────────────
80
94
 
81
95
  [tool.mypy]
@@ -108,3 +122,7 @@ ignore_missing_imports = true
108
122
  [[tool.mypy.overrides]]
109
123
  module = ["fsspec", "fsspec.*"]
110
124
  ignore_missing_imports = true
125
+
126
+ [[tool.mypy.overrides]]
127
+ module = ["bm25c", "bm25c.*"]
128
+ ignore_missing_imports = true
@@ -0,0 +1,74 @@
1
+ from simlar_engine import FilterError, MetadataFilter, SQLFilter
2
+
3
+ from simlar.contracts import SearchResult, TextIndex, VectorIndex
4
+ from simlar.embeddings import CallableEmbedder, Embedder
5
+ from simlar.fusion import ReciprocalRankFusion
6
+ from simlar.indexes.bm25c_index import BM25CIndex
7
+ from simlar.indexes.filtered_index import FilteredIndex
8
+ from simlar.indexes.helix_index import HelixIndex
9
+ from simlar.indexes.lookup_index import LookupIndex
10
+ from simlar.indexes.registry import load_from_directory, register
11
+ from simlar.indexes.relevance_index import RelevanceIndex
12
+ from simlar.indexes.simlar_engine import SimlarEngine
13
+ from simlar.indexes.streaming_index import StreamingHelixIndex as StreamingHybridIndex
14
+
15
+ _MIN_ENGINE_VERSION = (1, 1, 0)
16
+
17
+
18
+ def _check_engine_version() -> None:
19
+ """Best-effort compatibility guard between this wrapper and simlar-engine.
20
+
21
+ simlar-engine is a proprietary binary installed as a separate package
22
+ (see docs/installation.md) — there is no packaging-level dependency pin
23
+ tying the two together, so a too-old engine would otherwise fail with a
24
+ confusing TypeError deep inside a search() call instead of a clear error
25
+ at import time. Silently skipped when the installed engine doesn't
26
+ expose __version__ (e.g. a test double, or a build that predates this
27
+ check) rather than treated as a failure — this is a floor, not a full
28
+ dependency resolver.
29
+ """
30
+ import simlar_engine
31
+
32
+ engine_version = getattr(simlar_engine, "__version__", None)
33
+ if engine_version is None:
34
+ return
35
+ try:
36
+ parsed = tuple(int(p) for p in engine_version.split(".")[:3])
37
+ except ValueError:
38
+ return
39
+ if parsed < _MIN_ENGINE_VERSION:
40
+ raise RuntimeError(
41
+ f"simlar requires simlar-engine >= {'.'.join(map(str, _MIN_ENGINE_VERSION))}, "
42
+ f"but {engine_version} is installed. See docs/installation.md for the "
43
+ "compatible version matrix."
44
+ )
45
+
46
+
47
+ _check_engine_version()
48
+
49
+ __all__ = [
50
+ # Contracts — extension points
51
+ "SearchResult",
52
+ "TextIndex",
53
+ "VectorIndex",
54
+ # Embeddings
55
+ "Embedder",
56
+ "CallableEmbedder",
57
+ # Fusion
58
+ "ReciprocalRankFusion",
59
+ # Indexes
60
+ "RelevanceIndex",
61
+ "SimlarEngine",
62
+ "BM25CIndex",
63
+ "HelixIndex",
64
+ "StreamingHybridIndex",
65
+ "LookupIndex",
66
+ # Filtering
67
+ "FilteredIndex",
68
+ "SQLFilter",
69
+ "MetadataFilter",
70
+ "FilterError",
71
+ # Registry
72
+ "register",
73
+ "load_from_directory",
74
+ ]
@@ -26,11 +26,11 @@ class Index(ABC):
26
26
  """Minimal contract: every index can persist and report its type and size."""
27
27
 
28
28
  @abstractmethod
29
- def save(self, path: str) -> None: ...
29
+ def save(self, path: str, base_dir: str | None = None) -> None: ...
30
30
 
31
31
  @classmethod
32
32
  @abstractmethod
33
- def load(cls, path: str) -> Index: ...
33
+ def load(cls, path: str, base_dir: str | None = None) -> Index: ...
34
34
 
35
35
  @property
36
36
  @abstractmethod
@@ -44,6 +44,10 @@ class Index(ABC):
44
44
  @abstractmethod
45
45
  def is_trained(self) -> bool: ...
46
46
 
47
+ @property
48
+ @abstractmethod
49
+ def ids(self) -> list[str]: ...
50
+
47
51
 
48
52
  # ── Text index ────────────────────────────────────────────────────────────────
49
53
 
@@ -54,8 +58,12 @@ class TextIndex(Index):
54
58
  # ── Public API ─────────────────────────────────────────────────────────────
55
59
 
56
60
  @abstractmethod
57
- def add(self, ids: list[str], texts: list[str]) -> None:
58
- """Append new documents. Raises ValueError on duplicate IDs; use update() to replace."""
61
+ def add(self, ids: list[str], texts: list[str], parallel: bool = True) -> None:
62
+ """Append new documents. Raises ValueError on duplicate IDs; use update() to replace.
63
+
64
+ `parallel` is accepted for parity with the vector side; text indexing
65
+ is single-threaded in every backend.
66
+ """
59
67
 
60
68
  @abstractmethod
61
69
  def update(self, ids: list[str], texts: list[str]) -> None:
@@ -66,12 +74,29 @@ class TextIndex(Index):
66
74
  """Remove documents by ID, rebuilding internal structures."""
67
75
 
68
76
  @abstractmethod
69
- def search(self, query: str, k: int) -> list[SearchResult]: ...
77
+ def search(
78
+ self,
79
+ query: str | list[str],
80
+ k: int,
81
+ parallel: bool = True,
82
+ batch_size: int | None = None,
83
+ candidates: np.ndarray | None = None,
84
+ ) -> list[SearchResult] | list[list[SearchResult]]:
85
+ """Rank documents against query. `parallel` threads a batch of queries.
86
+
87
+ `candidates`, if given, is a single flat array of positions applied
88
+ to EVERY query in a batch call -- not a per-query restriction like
89
+ `VectorIndex.search_raw`'s `candidates` (used for Helix's per-query
90
+ text->vector cascade). A backend that can restrict a search this
91
+ cheaply doesn't need a per-query shape here: the one caller that
92
+ needs it (SQLFilter-style pre-filtering) always applies the same
93
+ filter to every query in a batch.
94
+ """
70
95
 
71
96
  # ── Internal ───────────────────────────────────────────────────────────────
72
97
 
73
98
  @abstractmethod
74
- def fit(self, corpus: list[str], parallel: bool = False, **kwargs) -> None:
99
+ def fit(self, corpus: list[str], parallel: bool = True, **kwargs) -> None:
75
100
  """Build the model from a raw corpus list. Called by add() and HelixIndex."""
76
101
 
77
102
  @abstractmethod
@@ -79,10 +104,12 @@ class TextIndex(Index):
79
104
  self,
80
105
  queries: str | list[str],
81
106
  k: int,
82
- parallel: bool = False,
107
+ parallel: bool = True,
108
+ candidates: np.ndarray | None = None,
83
109
  ) -> tuple[np.ndarray, np.ndarray]:
84
110
  """Return (ids, scores) shaped (n_queries, k), int64/float64.
85
- Used internally by HelixIndex._search_raw().
111
+ Used internally by HelixIndex._search_raw(). See `search`'s
112
+ docstring for what `candidates` does.
86
113
  """
87
114
 
88
115
 
@@ -95,10 +122,25 @@ class VectorIndex(Index):
95
122
  # ── Public API ─────────────────────────────────────────────────────────────
96
123
 
97
124
  @abstractmethod
98
- def add(self, ids: list[str], vectors: np.ndarray) -> None: ...
125
+ def add(self, ids: list[str], vectors: np.ndarray, parallel: bool = True) -> None:
126
+ """Append new vectors. `parallel` threads their quantization."""
99
127
 
100
128
  @abstractmethod
101
- def search(self, query: np.ndarray, k: int) -> list[SearchResult]: ...
129
+ def search(
130
+ self,
131
+ query: np.ndarray,
132
+ k: int,
133
+ parallel: bool = True,
134
+ batch_size: int | None = None,
135
+ candidates: np.ndarray | None = None,
136
+ ) -> list[SearchResult]:
137
+ """Rank documents against query. `parallel` threads a batch of queries.
138
+
139
+ `candidates`, if given, is a single flat array of positions applied
140
+ to every query -- the same pre-filter contract as `TextIndex.search`
141
+ (used by FilteredIndex). Fewer allowed documents than `k` means
142
+ fewer results.
143
+ """
102
144
 
103
145
  @abstractmethod
104
146
  def update(self, ids: list[str], vectors: np.ndarray) -> None:
@@ -110,7 +152,7 @@ class VectorIndex(Index):
110
152
  def fit(
111
153
  self,
112
154
  embeddings: np.ndarray,
113
- parallel: bool = False,
155
+ parallel: bool = True,
114
156
  params: _Parameters | None = None,
115
157
  **kwargs,
116
158
  ) -> None:
@@ -122,11 +164,15 @@ class VectorIndex(Index):
122
164
  vectors: np.ndarray,
123
165
  k: int,
124
166
  candidates: np.ndarray | None = None,
125
- parallel: bool = False,
167
+ parallel: bool = True,
126
168
  n_candidates: int | None = None,
127
169
  ) -> tuple[np.ndarray, np.ndarray]:
128
170
  """Return (ids, distances) shaped (n_queries, k), int64/float64.
129
171
  Used internally by HelixIndex._search_raw().
172
+
173
+ `candidates` is either a 1D array of positions shared by every query
174
+ (the pre-filter, see `search`) or a 2D (n_queries, n_cand) array of
175
+ per-query rows (Helix's text->vector cascade).
130
176
  """
131
177
 
132
178
  @abstractmethod
@@ -151,17 +197,29 @@ class CompositeIndex(Index):
151
197
  @abstractmethod
152
198
  def search(
153
199
  self,
154
- query_text: str | None = None,
200
+ query_text: str | list[str] | None = None,
155
201
  query_vector: np.ndarray | None = None,
156
202
  k: int = 10,
157
- ) -> list[SearchResult]: ...
203
+ parallel: bool = True,
204
+ batch_size: int | None = None,
205
+ candidates: np.ndarray | None = None,
206
+ ) -> list[SearchResult] | list[list[SearchResult]]:
207
+ """Search every sub-index and fuse. `parallel` threads a batch of queries.
208
+
209
+ A single query returns `list[SearchResult]`; a batch (e.g. a list of
210
+ query texts, or a 2D array of query vectors) returns one such list
211
+ per query, in order — the same batch contract as `TextIndex.search`.
212
+
213
+ `candidates`, if given, is a single flat array of positions applied
214
+ to every query, restricting every sub-index (see FilteredIndex).
215
+ """
158
216
 
159
217
  @abstractmethod
160
218
  def fit(
161
219
  self,
162
220
  corpus: list,
163
221
  vectors: np.ndarray,
164
- parallel: bool = False,
222
+ parallel: bool = True,
165
223
  **kwargs,
166
224
  ) -> None:
167
225
  """Internal: build all sub-indexes. Used by StreamingHybridIndex shards."""
@@ -172,10 +230,8 @@ class CompositeIndex(Index):
172
230
 
173
231
  @runtime_checkable
174
232
  class FusionStrategy(Protocol):
175
- """Combines N result lists into a single ranked list."""
176
-
177
233
  def __call__(
178
234
  self,
179
- results: list[list[SearchResult]],
235
+ results: list[tuple[np.ndarray, np.ndarray]],
180
236
  k: int,
181
- ) -> list[SearchResult]: ...
237
+ ) -> tuple[np.ndarray, np.ndarray]: ...