vexor 0.22.0__tar.gz → 0.23.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vexor-0.22.0 → vexor-0.23.0rc1}/PKG-INFO +28 -5
- {vexor-0.22.0 → vexor-0.23.0rc1}/README.md +26 -3
- {vexor-0.22.0 → vexor-0.23.0rc1}/plugins/vexor/.claude-plugin/plugin.json +1 -1
- {vexor-0.22.0 → vexor-0.23.0rc1}/pyproject.toml +1 -1
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/__init__.py +1 -1
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/api.py +55 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/cache.py +45 -13
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/cli.py +59 -2
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/config.py +150 -3
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/providers/openai.py +14 -4
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/search.py +16 -1
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/services/config_service.py +30 -2
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/services/index_service.py +56 -4
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/services/init_service.py +12 -2
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/services/search_service.py +63 -6
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/text.py +17 -3
- {vexor-0.22.0 → vexor-0.23.0rc1}/.gitignore +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/LICENSE +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/gui/README.md +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/plugins/vexor/README.md +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/plugins/vexor/skills/vexor-cli/SKILL.md +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/plugins/vexor/skills/vexor-cli/references/install-vexor.md +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/__main__.py +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/modes.py +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/output.py +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/providers/__init__.py +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/providers/gemini.py +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/providers/local.py +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/services/__init__.py +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/services/cache_service.py +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/services/content_extract_service.py +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/services/js_parser.py +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/services/keyword_service.py +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/services/skill_service.py +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/services/system_service.py +0 -0
- {vexor-0.22.0 → vexor-0.23.0rc1}/vexor/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vexor
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.23.0rc1
|
|
4
4
|
Summary: A vector-powered CLI for semantic search over files.
|
|
5
5
|
Project-URL: Repository, https://github.com/scarletkc/vexor
|
|
6
6
|
Author: scarletkc
|
|
@@ -22,7 +22,7 @@ Classifier: Topic :: Text Processing :: Indexing
|
|
|
22
22
|
Classifier: Topic :: Utilities
|
|
23
23
|
Requires-Python: >=3.9
|
|
24
24
|
Requires-Dist: charset-normalizer>=3.3.0
|
|
25
|
-
Requires-Dist: google-genai>=
|
|
25
|
+
Requires-Dist: google-genai>=1.57.0
|
|
26
26
|
Requires-Dist: numpy>=1.23.0
|
|
27
27
|
Requires-Dist: openai>=1.0.0
|
|
28
28
|
Requires-Dist: pathspec>=0.12.1
|
|
@@ -64,6 +64,7 @@ Description-Content-Type: text/markdown
|
|
|
64
64
|
[](https://github.com/scarletkc/vexor/actions/workflows/publish.yml)
|
|
65
65
|
[](https://codecov.io/github/scarletkc/vexor)
|
|
66
66
|
[](https://github.com/scarletkc/vexor/blob/main/LICENSE)
|
|
67
|
+
[](https://deepwiki.com/scarletkc/vexor)
|
|
67
68
|
|
|
68
69
|
</div>
|
|
69
70
|
|
|
@@ -171,12 +172,15 @@ Skill source: [`plugins/vexor/skills/vexor-cli`](https://github.com/scarletkc/ve
|
|
|
171
172
|
## Configuration
|
|
172
173
|
|
|
173
174
|
```bash
|
|
174
|
-
vexor config --set-provider openai # default; also supports gemini/custom/local
|
|
175
|
+
vexor config --set-provider openai # default; also supports gemini/voyageai/custom/local
|
|
175
176
|
vexor config --set-model text-embedding-3-small
|
|
177
|
+
vexor config --set-provider voyageai # uses voyage defaults when model/base_url are unset
|
|
176
178
|
vexor config --set-batch-size 0 # 0 = single request
|
|
177
179
|
vexor config --set-embed-concurrency 4 # parallel embedding requests
|
|
178
180
|
vexor config --set-extract-concurrency 4 # parallel file extraction workers
|
|
179
181
|
vexor config --set-extract-backend auto # auto|thread|process (default: auto)
|
|
182
|
+
vexor config --set-embedding-dimensions 1024 # optional, model/provider dependent
|
|
183
|
+
vexor config --clear-embedding-dimensions # reset to model default dimension
|
|
180
184
|
vexor config --set-auto-index true # auto-index before search (default)
|
|
181
185
|
vexor config --rerank bm25 # optional BM25 rerank for top-k results
|
|
182
186
|
vexor config --rerank flashrank # FlashRank rerank (requires optional extra)
|
|
@@ -202,7 +206,7 @@ Config stored in `~/.vexor/config.json`.
|
|
|
202
206
|
```bash
|
|
203
207
|
vexor config --set-api-key "YOUR_KEY"
|
|
204
208
|
```
|
|
205
|
-
Or via environment: `VEXOR_API_KEY`, `OPENAI_API_KEY`, or `
|
|
209
|
+
Or via environment: `VEXOR_API_KEY`, `OPENAI_API_KEY`, `GOOGLE_GENAI_API_KEY`, or `VOYAGE_API_KEY`.
|
|
206
210
|
|
|
207
211
|
### Rerank
|
|
208
212
|
|
|
@@ -222,11 +226,30 @@ Recommended defaults:
|
|
|
222
226
|
|
|
223
227
|
### Providers: Remote vs Local
|
|
224
228
|
|
|
225
|
-
Vexor supports both remote API providers (`openai`, `gemini`, `custom`) and a local provider (`local`):
|
|
229
|
+
Vexor supports both remote API providers (`openai`, `gemini`, `voyageai`, `custom`) and a local provider (`local`):
|
|
226
230
|
- Remote providers use `api_key` and optional `base_url`.
|
|
231
|
+
- `voyageai` defaults to `https://api.voyageai.com/v1` when `base_url` is not set.
|
|
227
232
|
- `custom` is OpenAI-compatible and requires both `model` and `base_url`.
|
|
228
233
|
- Local provider ignores `api_key/base_url` and only uses `model` plus `local_cuda` (CPU/GPU switch).
|
|
229
234
|
|
|
235
|
+
### Embedding Dimensions
|
|
236
|
+
|
|
237
|
+
Embedding dimensions are optional. If unset, the provider/model default is used.
|
|
238
|
+
Custom dimensions are validated for:
|
|
239
|
+
- OpenAI `text-embedding-3-*`
|
|
240
|
+
- Voyage `voyage-3*` and `voyage-code-3*`
|
|
241
|
+
|
|
242
|
+
```bash
|
|
243
|
+
vexor config --set-embedding-dimensions 1024
|
|
244
|
+
vexor config --clear-embedding-dimensions
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
If you change dimensions after an index is built, rebuild the index:
|
|
248
|
+
|
|
249
|
+
```bash
|
|
250
|
+
vexor index --path .
|
|
251
|
+
```
|
|
252
|
+
|
|
230
253
|
### Local Model (Offline)
|
|
231
254
|
|
|
232
255
|
Install the lightweight local backend:
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
[](https://github.com/scarletkc/vexor/actions/workflows/publish.yml)
|
|
10
10
|
[](https://codecov.io/github/scarletkc/vexor)
|
|
11
11
|
[](https://github.com/scarletkc/vexor/blob/main/LICENSE)
|
|
12
|
+
[](https://deepwiki.com/scarletkc/vexor)
|
|
12
13
|
|
|
13
14
|
</div>
|
|
14
15
|
|
|
@@ -116,12 +117,15 @@ Skill source: [`plugins/vexor/skills/vexor-cli`](https://github.com/scarletkc/ve
|
|
|
116
117
|
## Configuration
|
|
117
118
|
|
|
118
119
|
```bash
|
|
119
|
-
vexor config --set-provider openai # default; also supports gemini/custom/local
|
|
120
|
+
vexor config --set-provider openai # default; also supports gemini/voyageai/custom/local
|
|
120
121
|
vexor config --set-model text-embedding-3-small
|
|
122
|
+
vexor config --set-provider voyageai # uses voyage defaults when model/base_url are unset
|
|
121
123
|
vexor config --set-batch-size 0 # 0 = single request
|
|
122
124
|
vexor config --set-embed-concurrency 4 # parallel embedding requests
|
|
123
125
|
vexor config --set-extract-concurrency 4 # parallel file extraction workers
|
|
124
126
|
vexor config --set-extract-backend auto # auto|thread|process (default: auto)
|
|
127
|
+
vexor config --set-embedding-dimensions 1024 # optional, model/provider dependent
|
|
128
|
+
vexor config --clear-embedding-dimensions # reset to model default dimension
|
|
125
129
|
vexor config --set-auto-index true # auto-index before search (default)
|
|
126
130
|
vexor config --rerank bm25 # optional BM25 rerank for top-k results
|
|
127
131
|
vexor config --rerank flashrank # FlashRank rerank (requires optional extra)
|
|
@@ -147,7 +151,7 @@ Config stored in `~/.vexor/config.json`.
|
|
|
147
151
|
```bash
|
|
148
152
|
vexor config --set-api-key "YOUR_KEY"
|
|
149
153
|
```
|
|
150
|
-
Or via environment: `VEXOR_API_KEY`, `OPENAI_API_KEY`, or `
|
|
154
|
+
Or via environment: `VEXOR_API_KEY`, `OPENAI_API_KEY`, `GOOGLE_GENAI_API_KEY`, or `VOYAGE_API_KEY`.
|
|
151
155
|
|
|
152
156
|
### Rerank
|
|
153
157
|
|
|
@@ -167,11 +171,30 @@ Recommended defaults:
|
|
|
167
171
|
|
|
168
172
|
### Providers: Remote vs Local
|
|
169
173
|
|
|
170
|
-
Vexor supports both remote API providers (`openai`, `gemini`, `custom`) and a local provider (`local`):
|
|
174
|
+
Vexor supports both remote API providers (`openai`, `gemini`, `voyageai`, `custom`) and a local provider (`local`):
|
|
171
175
|
- Remote providers use `api_key` and optional `base_url`.
|
|
176
|
+
- `voyageai` defaults to `https://api.voyageai.com/v1` when `base_url` is not set.
|
|
172
177
|
- `custom` is OpenAI-compatible and requires both `model` and `base_url`.
|
|
173
178
|
- Local provider ignores `api_key/base_url` and only uses `model` plus `local_cuda` (CPU/GPU switch).
|
|
174
179
|
|
|
180
|
+
### Embedding Dimensions
|
|
181
|
+
|
|
182
|
+
Embedding dimensions are optional. If unset, the provider/model default is used.
|
|
183
|
+
Custom dimensions are validated for:
|
|
184
|
+
- OpenAI `text-embedding-3-*`
|
|
185
|
+
- Voyage `voyage-3*` and `voyage-code-3*`
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
vexor config --set-embedding-dimensions 1024
|
|
189
|
+
vexor config --clear-embedding-dimensions
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
If you change dimensions after an index is built, rebuild the index:
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
vexor index --path .
|
|
196
|
+
```
|
|
197
|
+
|
|
175
198
|
### Local Model (Offline)
|
|
176
199
|
|
|
177
200
|
Install the lightweight local backend:
|
|
@@ -21,6 +21,7 @@ from .config import (
|
|
|
21
21
|
SUPPORTED_RERANKERS,
|
|
22
22
|
config_from_json,
|
|
23
23
|
config_dir_context,
|
|
24
|
+
validate_embedding_dimensions_for_model,
|
|
24
25
|
load_config,
|
|
25
26
|
resolve_default_model,
|
|
26
27
|
set_config_dir,
|
|
@@ -67,6 +68,7 @@ class RuntimeSettings:
|
|
|
67
68
|
rerank: str
|
|
68
69
|
flashrank_model: str | None
|
|
69
70
|
remote_rerank: RemoteRerankConfig | None
|
|
71
|
+
embedding_dimensions: int | None
|
|
70
72
|
|
|
71
73
|
|
|
72
74
|
@dataclass(slots=True)
|
|
@@ -82,6 +84,7 @@ class InMemoryIndex:
|
|
|
82
84
|
base_url: str | None
|
|
83
85
|
api_key: str | None
|
|
84
86
|
local_cuda: bool
|
|
87
|
+
embedding_dimensions: int | None = None
|
|
85
88
|
rerank: str = DEFAULT_RERANK
|
|
86
89
|
flashrank_model: str | None = None
|
|
87
90
|
remote_rerank: RemoteRerankConfig | None = None
|
|
@@ -140,6 +143,7 @@ class InMemoryIndex:
|
|
|
140
143
|
temporary_index=True,
|
|
141
144
|
no_cache=no_cache,
|
|
142
145
|
rerank=effective_rerank,
|
|
146
|
+
embedding_dimensions=self.embedding_dimensions,
|
|
143
147
|
flashrank_model=(
|
|
144
148
|
flashrank_model
|
|
145
149
|
if flashrank_model is not None
|
|
@@ -282,6 +286,7 @@ class VexorClient:
|
|
|
282
286
|
base_url: str | None = None,
|
|
283
287
|
api_key: str | None = None,
|
|
284
288
|
local_cuda: bool | None = None,
|
|
289
|
+
embedding_dimensions: int | None = None,
|
|
285
290
|
auto_index: bool | None = None,
|
|
286
291
|
use_config: bool | None = None,
|
|
287
292
|
config: Config | Mapping[str, object] | str | None = None,
|
|
@@ -316,6 +321,7 @@ class VexorClient:
|
|
|
316
321
|
base_url=base_url,
|
|
317
322
|
api_key=api_key,
|
|
318
323
|
local_cuda=local_cuda,
|
|
324
|
+
embedding_dimensions=embedding_dimensions,
|
|
319
325
|
auto_index=auto_index,
|
|
320
326
|
use_config=resolved_use_config,
|
|
321
327
|
config=config,
|
|
@@ -346,6 +352,7 @@ class VexorClient:
|
|
|
346
352
|
base_url: str | None = None,
|
|
347
353
|
api_key: str | None = None,
|
|
348
354
|
local_cuda: bool | None = None,
|
|
355
|
+
embedding_dimensions: int | None = None,
|
|
349
356
|
use_config: bool | None = None,
|
|
350
357
|
config: Config | Mapping[str, object] | str | None = None,
|
|
351
358
|
data_dir: Path | str | None = None,
|
|
@@ -375,6 +382,7 @@ class VexorClient:
|
|
|
375
382
|
base_url=base_url,
|
|
376
383
|
api_key=api_key,
|
|
377
384
|
local_cuda=local_cuda,
|
|
385
|
+
embedding_dimensions=embedding_dimensions,
|
|
378
386
|
use_config=resolved_use_config,
|
|
379
387
|
config=config,
|
|
380
388
|
runtime_config=self._runtime_config,
|
|
@@ -402,6 +410,7 @@ class VexorClient:
|
|
|
402
410
|
base_url: str | None = None,
|
|
403
411
|
api_key: str | None = None,
|
|
404
412
|
local_cuda: bool | None = None,
|
|
413
|
+
embedding_dimensions: int | None = None,
|
|
405
414
|
use_config: bool | None = None,
|
|
406
415
|
config: Config | Mapping[str, object] | str | None = None,
|
|
407
416
|
no_cache: bool = True,
|
|
@@ -432,6 +441,7 @@ class VexorClient:
|
|
|
432
441
|
base_url=base_url,
|
|
433
442
|
api_key=api_key,
|
|
434
443
|
local_cuda=local_cuda,
|
|
444
|
+
embedding_dimensions=embedding_dimensions,
|
|
435
445
|
use_config=resolved_use_config,
|
|
436
446
|
config=config,
|
|
437
447
|
no_cache=no_cache,
|
|
@@ -518,6 +528,7 @@ def search(
|
|
|
518
528
|
base_url: str | None = None,
|
|
519
529
|
api_key: str | None = None,
|
|
520
530
|
local_cuda: bool | None = None,
|
|
531
|
+
embedding_dimensions: int | None = None,
|
|
521
532
|
auto_index: bool | None = None,
|
|
522
533
|
use_config: bool = True,
|
|
523
534
|
config: Config | Mapping[str, object] | str | None = None,
|
|
@@ -547,6 +558,7 @@ def search(
|
|
|
547
558
|
base_url=base_url,
|
|
548
559
|
api_key=api_key,
|
|
549
560
|
local_cuda=local_cuda,
|
|
561
|
+
embedding_dimensions=embedding_dimensions,
|
|
550
562
|
auto_index=auto_index,
|
|
551
563
|
use_config=use_config,
|
|
552
564
|
config=config,
|
|
@@ -577,6 +589,7 @@ def index(
|
|
|
577
589
|
base_url: str | None = None,
|
|
578
590
|
api_key: str | None = None,
|
|
579
591
|
local_cuda: bool | None = None,
|
|
592
|
+
embedding_dimensions: int | None = None,
|
|
580
593
|
use_config: bool = True,
|
|
581
594
|
config: Config | Mapping[str, object] | str | None = None,
|
|
582
595
|
data_dir: Path | str | None = None,
|
|
@@ -601,6 +614,7 @@ def index(
|
|
|
601
614
|
base_url=base_url,
|
|
602
615
|
api_key=api_key,
|
|
603
616
|
local_cuda=local_cuda,
|
|
617
|
+
embedding_dimensions=embedding_dimensions,
|
|
604
618
|
use_config=use_config,
|
|
605
619
|
config=config,
|
|
606
620
|
runtime_config=_RUNTIME_CONFIG,
|
|
@@ -628,6 +642,7 @@ def index_in_memory(
|
|
|
628
642
|
base_url: str | None = None,
|
|
629
643
|
api_key: str | None = None,
|
|
630
644
|
local_cuda: bool | None = None,
|
|
645
|
+
embedding_dimensions: int | None = None,
|
|
631
646
|
use_config: bool = True,
|
|
632
647
|
config: Config | Mapping[str, object] | str | None = None,
|
|
633
648
|
no_cache: bool = True,
|
|
@@ -653,6 +668,7 @@ def index_in_memory(
|
|
|
653
668
|
base_url=base_url,
|
|
654
669
|
api_key=api_key,
|
|
655
670
|
local_cuda=local_cuda,
|
|
671
|
+
embedding_dimensions=embedding_dimensions,
|
|
656
672
|
use_config=use_config,
|
|
657
673
|
config=config,
|
|
658
674
|
no_cache=no_cache,
|
|
@@ -711,6 +727,7 @@ def _search_with_settings(
|
|
|
711
727
|
base_url: str | None,
|
|
712
728
|
api_key: str | None,
|
|
713
729
|
local_cuda: bool | None,
|
|
730
|
+
embedding_dimensions: int | None,
|
|
714
731
|
auto_index: bool | None,
|
|
715
732
|
use_config: bool,
|
|
716
733
|
config: Config | Mapping[str, object] | str | None,
|
|
@@ -747,6 +764,7 @@ def _search_with_settings(
|
|
|
747
764
|
base_url=base_url,
|
|
748
765
|
api_key=api_key,
|
|
749
766
|
local_cuda=local_cuda,
|
|
767
|
+
embedding_dimensions=embedding_dimensions,
|
|
750
768
|
auto_index=auto_index,
|
|
751
769
|
use_config=use_config,
|
|
752
770
|
runtime_config=runtime_config,
|
|
@@ -776,6 +794,7 @@ def _search_with_settings(
|
|
|
776
794
|
temporary_index=temporary_index,
|
|
777
795
|
no_cache=no_cache,
|
|
778
796
|
rerank=settings.rerank,
|
|
797
|
+
embedding_dimensions=settings.embedding_dimensions,
|
|
779
798
|
flashrank_model=settings.flashrank_model,
|
|
780
799
|
remote_rerank=settings.remote_rerank,
|
|
781
800
|
)
|
|
@@ -800,6 +819,7 @@ def _index_with_settings(
|
|
|
800
819
|
base_url: str | None,
|
|
801
820
|
api_key: str | None,
|
|
802
821
|
local_cuda: bool | None,
|
|
822
|
+
embedding_dimensions: int | None,
|
|
803
823
|
use_config: bool,
|
|
804
824
|
config: Config | Mapping[str, object] | str | None,
|
|
805
825
|
runtime_config: Config | None,
|
|
@@ -825,6 +845,7 @@ def _index_with_settings(
|
|
|
825
845
|
base_url=base_url,
|
|
826
846
|
api_key=api_key,
|
|
827
847
|
local_cuda=local_cuda,
|
|
848
|
+
embedding_dimensions=embedding_dimensions,
|
|
828
849
|
auto_index=None,
|
|
829
850
|
use_config=use_config,
|
|
830
851
|
runtime_config=runtime_config,
|
|
@@ -846,6 +867,7 @@ def _index_with_settings(
|
|
|
846
867
|
base_url=settings.base_url,
|
|
847
868
|
api_key=settings.api_key,
|
|
848
869
|
local_cuda=settings.local_cuda,
|
|
870
|
+
embedding_dimensions=settings.embedding_dimensions,
|
|
849
871
|
exclude_patterns=normalized_excludes,
|
|
850
872
|
extensions=normalized_exts,
|
|
851
873
|
)
|
|
@@ -869,6 +891,7 @@ def _index_in_memory_with_settings(
|
|
|
869
891
|
base_url: str | None,
|
|
870
892
|
api_key: str | None,
|
|
871
893
|
local_cuda: bool | None,
|
|
894
|
+
embedding_dimensions: int | None,
|
|
872
895
|
use_config: bool,
|
|
873
896
|
config: Config | Mapping[str, object] | str | None,
|
|
874
897
|
no_cache: bool,
|
|
@@ -895,6 +918,7 @@ def _index_in_memory_with_settings(
|
|
|
895
918
|
base_url=base_url,
|
|
896
919
|
api_key=api_key,
|
|
897
920
|
local_cuda=local_cuda,
|
|
921
|
+
embedding_dimensions=embedding_dimensions,
|
|
898
922
|
auto_index=None,
|
|
899
923
|
use_config=use_config,
|
|
900
924
|
runtime_config=runtime_config,
|
|
@@ -916,6 +940,7 @@ def _index_in_memory_with_settings(
|
|
|
916
940
|
base_url=settings.base_url,
|
|
917
941
|
api_key=settings.api_key,
|
|
918
942
|
local_cuda=settings.local_cuda,
|
|
943
|
+
embedding_dimensions=settings.embedding_dimensions,
|
|
919
944
|
exclude_patterns=normalized_excludes,
|
|
920
945
|
extensions=normalized_exts,
|
|
921
946
|
no_cache=no_cache,
|
|
@@ -933,6 +958,7 @@ def _index_in_memory_with_settings(
|
|
|
933
958
|
base_url=settings.base_url,
|
|
934
959
|
api_key=settings.api_key,
|
|
935
960
|
local_cuda=settings.local_cuda,
|
|
961
|
+
embedding_dimensions=settings.embedding_dimensions,
|
|
936
962
|
rerank=settings.rerank,
|
|
937
963
|
flashrank_model=settings.flashrank_model,
|
|
938
964
|
remote_rerank=settings.remote_rerank,
|
|
@@ -1011,6 +1037,7 @@ def _resolve_settings(
|
|
|
1011
1037
|
base_url: str | None,
|
|
1012
1038
|
api_key: str | None,
|
|
1013
1039
|
local_cuda: bool | None,
|
|
1040
|
+
embedding_dimensions: int | None,
|
|
1014
1041
|
auto_index: bool | None,
|
|
1015
1042
|
use_config: bool,
|
|
1016
1043
|
runtime_config: Config | None = None,
|
|
@@ -1047,6 +1074,19 @@ def _resolve_settings(
|
|
|
1047
1074
|
extract_backend_value = (
|
|
1048
1075
|
extract_backend if extract_backend is not None else config.extract_backend
|
|
1049
1076
|
)
|
|
1077
|
+
resolved_embedding_dimensions = _coerce_embedding_dimensions(
|
|
1078
|
+
embedding_dimensions
|
|
1079
|
+
if embedding_dimensions is not None
|
|
1080
|
+
else config.embedding_dimensions
|
|
1081
|
+
)
|
|
1082
|
+
try:
|
|
1083
|
+
validate_embedding_dimensions_for_model(
|
|
1084
|
+
resolved_embedding_dimensions,
|
|
1085
|
+
model_name,
|
|
1086
|
+
)
|
|
1087
|
+
except ValueError as exc:
|
|
1088
|
+
raise VexorError(str(exc)) from exc
|
|
1089
|
+
|
|
1050
1090
|
return RuntimeSettings(
|
|
1051
1091
|
provider=provider_value,
|
|
1052
1092
|
model_name=model_name,
|
|
@@ -1061,6 +1101,7 @@ def _resolve_settings(
|
|
|
1061
1101
|
rerank=rerank_value,
|
|
1062
1102
|
flashrank_model=config.flashrank_model,
|
|
1063
1103
|
remote_rerank=config.remote_rerank,
|
|
1104
|
+
embedding_dimensions=resolved_embedding_dimensions,
|
|
1064
1105
|
)
|
|
1065
1106
|
|
|
1066
1107
|
|
|
@@ -1074,3 +1115,17 @@ def _apply_config_override(
|
|
|
1074
1115
|
return config_from_json(override, base=base)
|
|
1075
1116
|
except ValueError as exc:
|
|
1076
1117
|
raise VexorError(str(exc)) from exc
|
|
1118
|
+
|
|
1119
|
+
|
|
1120
|
+
def _coerce_embedding_dimensions(value: int | None) -> int | None:
|
|
1121
|
+
if value is None:
|
|
1122
|
+
return None
|
|
1123
|
+
if isinstance(value, bool):
|
|
1124
|
+
raise VexorError(Messages.ERROR_EMBEDDING_DIMENSIONS_INVALID)
|
|
1125
|
+
if not isinstance(value, int):
|
|
1126
|
+
raise VexorError(Messages.ERROR_EMBEDDING_DIMENSIONS_INVALID)
|
|
1127
|
+
if value == 0:
|
|
1128
|
+
return None
|
|
1129
|
+
if value < 0:
|
|
1130
|
+
raise VexorError(Messages.ERROR_EMBEDDING_DIMENSIONS_INVALID)
|
|
1131
|
+
return value
|
|
@@ -30,7 +30,7 @@ EMBED_CACHE_TTL_DAYS = 30
|
|
|
30
30
|
EMBED_CACHE_MAX_ENTRIES = 50_000
|
|
31
31
|
EMBED_MEMORY_CACHE_MAX_ENTRIES = 2_048
|
|
32
32
|
|
|
33
|
-
_EMBED_MEMORY_CACHE: "OrderedDict[tuple[str, str], np.ndarray]" = OrderedDict()
|
|
33
|
+
_EMBED_MEMORY_CACHE: "OrderedDict[tuple[str, int | None, str], np.ndarray]" = OrderedDict()
|
|
34
34
|
_EMBED_MEMORY_LOCK = Lock()
|
|
35
35
|
|
|
36
36
|
|
|
@@ -89,11 +89,20 @@ def query_cache_key(query: str, model: str) -> str:
|
|
|
89
89
|
return hashlib.sha1(base.encode("utf-8")).hexdigest()
|
|
90
90
|
|
|
91
91
|
|
|
92
|
-
def embedding_cache_key(text: str) -> str:
|
|
93
|
-
"""Return a stable hash for embedding cache lookups.
|
|
92
|
+
def embedding_cache_key(text: str, dimension: int | None = None) -> str:
|
|
93
|
+
"""Return a stable hash for embedding cache lookups.
|
|
94
94
|
|
|
95
|
+
Args:
|
|
96
|
+
text: The text to hash
|
|
97
|
+
dimension: Optional embedding dimension (included in hash for dimension-aware caching)
|
|
98
|
+
"""
|
|
95
99
|
clean_text = text or ""
|
|
96
|
-
|
|
100
|
+
# Include dimension in hash to prevent cross-dimension cache pollution
|
|
101
|
+
if dimension is not None:
|
|
102
|
+
base = f"{clean_text}|dim={dimension}"
|
|
103
|
+
else:
|
|
104
|
+
base = clean_text
|
|
105
|
+
return hashlib.sha1(base.encode("utf-8")).hexdigest()
|
|
97
106
|
|
|
98
107
|
|
|
99
108
|
def _clear_embedding_memory_cache() -> None:
|
|
@@ -106,6 +115,7 @@ def _clear_embedding_memory_cache() -> None:
|
|
|
106
115
|
def _load_embedding_memory_cache(
|
|
107
116
|
model: str,
|
|
108
117
|
text_hashes: Sequence[str],
|
|
118
|
+
dimension: int | None = None,
|
|
109
119
|
) -> dict[str, np.ndarray]:
|
|
110
120
|
if EMBED_MEMORY_CACHE_MAX_ENTRIES <= 0:
|
|
111
121
|
return {}
|
|
@@ -114,7 +124,8 @@ def _load_embedding_memory_cache(
|
|
|
114
124
|
for text_hash in text_hashes:
|
|
115
125
|
if not text_hash:
|
|
116
126
|
continue
|
|
117
|
-
key
|
|
127
|
+
# Include dimension in cache key to prevent cross-dimension pollution
|
|
128
|
+
key = (model, dimension, text_hash)
|
|
118
129
|
vector = _EMBED_MEMORY_CACHE.pop(key, None)
|
|
119
130
|
if vector is None:
|
|
120
131
|
continue
|
|
@@ -127,6 +138,7 @@ def _store_embedding_memory_cache(
|
|
|
127
138
|
*,
|
|
128
139
|
model: str,
|
|
129
140
|
embeddings: Mapping[str, np.ndarray],
|
|
141
|
+
dimension: int | None = None,
|
|
130
142
|
) -> None:
|
|
131
143
|
if EMBED_MEMORY_CACHE_MAX_ENTRIES <= 0 or not embeddings:
|
|
132
144
|
return
|
|
@@ -137,7 +149,8 @@ def _store_embedding_memory_cache(
|
|
|
137
149
|
array = np.asarray(vector, dtype=np.float32)
|
|
138
150
|
if array.size == 0:
|
|
139
151
|
continue
|
|
140
|
-
key
|
|
152
|
+
# Include dimension in cache key to prevent cross-dimension pollution
|
|
153
|
+
key = (model, dimension, text_hash)
|
|
141
154
|
if key in _EMBED_MEMORY_CACHE:
|
|
142
155
|
_EMBED_MEMORY_CACHE.pop(key, None)
|
|
143
156
|
_EMBED_MEMORY_CACHE[key] = array
|
|
@@ -1388,13 +1401,22 @@ def load_embedding_cache(
|
|
|
1388
1401
|
model: str,
|
|
1389
1402
|
text_hashes: Sequence[str],
|
|
1390
1403
|
conn: sqlite3.Connection | None = None,
|
|
1404
|
+
*,
|
|
1405
|
+
dimension: int | None = None,
|
|
1391
1406
|
) -> dict[str, np.ndarray]:
|
|
1392
|
-
"""Load cached embeddings keyed by (model, text_hash).
|
|
1393
|
-
|
|
1407
|
+
"""Load cached embeddings keyed by (model, text_hash).
|
|
1408
|
+
|
|
1409
|
+
Args:
|
|
1410
|
+
model: The embedding model name
|
|
1411
|
+
text_hashes: Sequence of text hashes to look up (should be generated with
|
|
1412
|
+
embedding_cache_key() using the same dimension parameter)
|
|
1413
|
+
conn: Optional database connection
|
|
1414
|
+
dimension: Embedding dimension (used for memory cache segmentation)
|
|
1415
|
+
"""
|
|
1394
1416
|
unique_hashes = list(dict.fromkeys([value for value in text_hashes if value]))
|
|
1395
1417
|
if not unique_hashes:
|
|
1396
1418
|
return {}
|
|
1397
|
-
results = _load_embedding_memory_cache(model, unique_hashes)
|
|
1419
|
+
results = _load_embedding_memory_cache(model, unique_hashes, dimension=dimension)
|
|
1398
1420
|
missing = [value for value in unique_hashes if value not in results]
|
|
1399
1421
|
if not missing:
|
|
1400
1422
|
return results
|
|
@@ -1429,7 +1451,9 @@ def load_embedding_cache(
|
|
|
1429
1451
|
continue
|
|
1430
1452
|
disk_results[row["text_hash"]] = vector
|
|
1431
1453
|
if disk_results:
|
|
1432
|
-
_store_embedding_memory_cache(
|
|
1454
|
+
_store_embedding_memory_cache(
|
|
1455
|
+
model=model, embeddings=disk_results, dimension=dimension
|
|
1456
|
+
)
|
|
1433
1457
|
results.update(disk_results)
|
|
1434
1458
|
return results
|
|
1435
1459
|
finally:
|
|
@@ -1442,12 +1466,20 @@ def store_embedding_cache(
|
|
|
1442
1466
|
model: str,
|
|
1443
1467
|
embeddings: Mapping[str, np.ndarray],
|
|
1444
1468
|
conn: sqlite3.Connection | None = None,
|
|
1469
|
+
dimension: int | None = None,
|
|
1445
1470
|
) -> None:
|
|
1446
|
-
"""Store embedding vectors keyed by (model, text_hash).
|
|
1447
|
-
|
|
1471
|
+
"""Store embedding vectors keyed by (model, text_hash).
|
|
1472
|
+
|
|
1473
|
+
Args:
|
|
1474
|
+
model: The embedding model name
|
|
1475
|
+
embeddings: Dict mapping text_hash -> vector (hashes should be generated with
|
|
1476
|
+
embedding_cache_key() using the same dimension parameter)
|
|
1477
|
+
conn: Optional database connection
|
|
1478
|
+
dimension: Embedding dimension (used for memory cache segmentation)
|
|
1479
|
+
"""
|
|
1448
1480
|
if not embeddings:
|
|
1449
1481
|
return
|
|
1450
|
-
_store_embedding_memory_cache(model=model, embeddings=embeddings)
|
|
1482
|
+
_store_embedding_memory_cache(model=model, embeddings=embeddings, dimension=dimension)
|
|
1451
1483
|
db_path = cache_db_path()
|
|
1452
1484
|
owns_connection = conn is None
|
|
1453
1485
|
connection = conn or _connect(db_path)
|
|
@@ -31,14 +31,18 @@ from .config import (
|
|
|
31
31
|
DEFAULT_MODEL,
|
|
32
32
|
DEFAULT_PROVIDER,
|
|
33
33
|
DEFAULT_RERANK,
|
|
34
|
+
DEFAULT_VOYAGE_MODEL,
|
|
35
|
+
DIMENSION_SUPPORTED_MODELS,
|
|
34
36
|
SUPPORTED_EXTRACT_BACKENDS,
|
|
35
37
|
SUPPORTED_PROVIDERS,
|
|
36
38
|
SUPPORTED_RERANKERS,
|
|
37
39
|
flashrank_cache_dir,
|
|
40
|
+
get_supported_dimensions,
|
|
38
41
|
load_config,
|
|
39
42
|
normalize_remote_rerank_url,
|
|
40
43
|
resolve_remote_rerank_api_key,
|
|
41
44
|
resolve_default_model,
|
|
45
|
+
supports_dimensions,
|
|
42
46
|
)
|
|
43
47
|
from .modes import available_modes, get_strategy
|
|
44
48
|
from .services.cache_service import is_cache_current, load_index_metadata_safe
|
|
@@ -454,6 +458,7 @@ def search(
|
|
|
454
458
|
rerank=rerank,
|
|
455
459
|
flashrank_model=flashrank_model,
|
|
456
460
|
remote_rerank=remote_rerank,
|
|
461
|
+
embedding_dimensions=config.embedding_dimensions,
|
|
457
462
|
)
|
|
458
463
|
if output_format == SearchOutputFormat.rich:
|
|
459
464
|
if no_cache:
|
|
@@ -488,7 +493,7 @@ def search(
|
|
|
488
493
|
else:
|
|
489
494
|
typer.echo(message, err=True)
|
|
490
495
|
raise typer.Exit(code=1)
|
|
491
|
-
except RuntimeError as exc:
|
|
496
|
+
except (RuntimeError, ValueError) as exc:
|
|
492
497
|
if output_format == SearchOutputFormat.rich:
|
|
493
498
|
console.print(_styled(str(exc), Styles.ERROR))
|
|
494
499
|
else:
|
|
@@ -688,8 +693,9 @@ def index(
|
|
|
688
693
|
local_cuda=bool(config.local_cuda),
|
|
689
694
|
exclude_patterns=normalized_excludes,
|
|
690
695
|
extensions=normalized_exts,
|
|
696
|
+
embedding_dimensions=config.embedding_dimensions,
|
|
691
697
|
)
|
|
692
|
-
except RuntimeError as exc:
|
|
698
|
+
except (RuntimeError, ValueError) as exc:
|
|
693
699
|
console.print(_styled(str(exc), Styles.ERROR))
|
|
694
700
|
raise typer.Exit(code=1)
|
|
695
701
|
if result.status == IndexStatus.EMPTY:
|
|
@@ -768,6 +774,16 @@ def config(
|
|
|
768
774
|
"--clear-base-url",
|
|
769
775
|
help=Messages.HELP_CLEAR_BASE_URL,
|
|
770
776
|
),
|
|
777
|
+
set_embedding_dimensions_option: int | None = typer.Option(
|
|
778
|
+
None,
|
|
779
|
+
"--set-embedding-dimensions",
|
|
780
|
+
help=Messages.HELP_SET_EMBEDDING_DIMENSIONS,
|
|
781
|
+
),
|
|
782
|
+
clear_embedding_dimensions: bool = typer.Option(
|
|
783
|
+
False,
|
|
784
|
+
"--clear-embedding-dimensions",
|
|
785
|
+
help=Messages.HELP_CLEAR_EMBEDDING_DIMENSIONS,
|
|
786
|
+
),
|
|
771
787
|
set_auto_index_option: str | None = typer.Option(
|
|
772
788
|
None,
|
|
773
789
|
"--set-auto-index",
|
|
@@ -989,6 +1005,33 @@ def config(
|
|
|
989
1005
|
except ValueError as exc:
|
|
990
1006
|
raise typer.BadParameter(str(exc)) from exc
|
|
991
1007
|
|
|
1008
|
+
effective_embedding_dimensions = set_embedding_dimensions_option
|
|
1009
|
+
effective_clear_embedding_dimensions = clear_embedding_dimensions
|
|
1010
|
+
if effective_embedding_dimensions == 0:
|
|
1011
|
+
effective_embedding_dimensions = None
|
|
1012
|
+
effective_clear_embedding_dimensions = True
|
|
1013
|
+
|
|
1014
|
+
# Validate embedding dimensions if set
|
|
1015
|
+
if effective_embedding_dimensions is not None:
|
|
1016
|
+
if effective_embedding_dimensions < 0:
|
|
1017
|
+
raise typer.BadParameter(
|
|
1018
|
+
f"--set-embedding-dimensions must be non-negative, got {effective_embedding_dimensions}"
|
|
1019
|
+
)
|
|
1020
|
+
if effective_embedding_dimensions > 0:
|
|
1021
|
+
# Resolve effective model from provider + model to account for provider defaults
|
|
1022
|
+
effective_model = resolve_default_model(pending_provider, pending_model)
|
|
1023
|
+
if not supports_dimensions(effective_model):
|
|
1024
|
+
raise typer.BadParameter(
|
|
1025
|
+
f"Model '{effective_model}' does not support custom dimensions. "
|
|
1026
|
+
f"Supported model names/prefixes: {', '.join(DIMENSION_SUPPORTED_MODELS.keys())}"
|
|
1027
|
+
)
|
|
1028
|
+
supported = get_supported_dimensions(effective_model)
|
|
1029
|
+
if supported and effective_embedding_dimensions not in supported:
|
|
1030
|
+
raise typer.BadParameter(
|
|
1031
|
+
f"Dimension {effective_embedding_dimensions} is not supported for model '{effective_model}'. "
|
|
1032
|
+
f"Supported dimensions: {supported}"
|
|
1033
|
+
)
|
|
1034
|
+
|
|
992
1035
|
updates = apply_config_updates(
|
|
993
1036
|
api_key=set_api_key_option,
|
|
994
1037
|
clear_api_key=clear_api_key,
|
|
@@ -1007,6 +1050,8 @@ def config(
|
|
|
1007
1050
|
remote_rerank_model=set_remote_rerank_model_option,
|
|
1008
1051
|
remote_rerank_api_key=set_remote_rerank_api_key_option,
|
|
1009
1052
|
clear_remote_rerank=clear_remote_rerank,
|
|
1053
|
+
embedding_dimensions=effective_embedding_dimensions,
|
|
1054
|
+
clear_embedding_dimensions=effective_clear_embedding_dimensions,
|
|
1010
1055
|
)
|
|
1011
1056
|
|
|
1012
1057
|
if updates.api_key_set:
|
|
@@ -1109,6 +1154,17 @@ def config(
|
|
|
1109
1154
|
console.print(_styled(Messages.INFO_REMOTE_RERANK_API_KEY_SET, Styles.SUCCESS))
|
|
1110
1155
|
if updates.remote_rerank_cleared and clear_remote_rerank:
|
|
1111
1156
|
console.print(_styled(Messages.INFO_REMOTE_RERANK_CLEARED, Styles.SUCCESS))
|
|
1157
|
+
if updates.embedding_dimensions_set and effective_embedding_dimensions is not None:
|
|
1158
|
+
console.print(
|
|
1159
|
+
_styled(
|
|
1160
|
+
Messages.INFO_EMBEDDING_DIMENSIONS_SET.format(
|
|
1161
|
+
value=effective_embedding_dimensions
|
|
1162
|
+
),
|
|
1163
|
+
Styles.SUCCESS,
|
|
1164
|
+
)
|
|
1165
|
+
)
|
|
1166
|
+
if updates.embedding_dimensions_cleared:
|
|
1167
|
+
console.print(_styled(Messages.INFO_EMBEDDING_DIMENSIONS_CLEARED, Styles.SUCCESS))
|
|
1112
1168
|
|
|
1113
1169
|
if clear_flashrank:
|
|
1114
1170
|
cache_dir = flashrank_cache_dir(create=False)
|
|
@@ -1188,6 +1244,7 @@ def config(
|
|
|
1188
1244
|
api="yes" if cfg.api_key else "no",
|
|
1189
1245
|
provider=provider,
|
|
1190
1246
|
model=resolve_default_model(provider, cfg.model),
|
|
1247
|
+
embedding_dimensions=cfg.embedding_dimensions if cfg.embedding_dimensions else "default",
|
|
1191
1248
|
batch=cfg.batch_size if cfg.batch_size is not None else DEFAULT_BATCH_SIZE,
|
|
1192
1249
|
concurrency=cfg.embed_concurrency,
|
|
1193
1250
|
extract_concurrency=cfg.extract_concurrency,
|