vexor 0.20.0__py3-none-any.whl → 0.21.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vexor/__init__.py +1 -1
- vexor/api.py +26 -0
- vexor/cache.py +470 -274
- vexor/cli.py +53 -0
- vexor/config.py +54 -1
- vexor/providers/gemini.py +79 -13
- vexor/providers/openai.py +79 -13
- vexor/services/config_service.py +14 -0
- vexor/services/index_service.py +132 -5
- vexor/services/search_service.py +94 -27
- vexor/text.py +10 -0
- {vexor-0.20.0.dist-info → vexor-0.21.0.dist-info}/METADATA +15 -13
- {vexor-0.20.0.dist-info → vexor-0.21.0.dist-info}/RECORD +16 -16
- {vexor-0.20.0.dist-info → vexor-0.21.0.dist-info}/WHEEL +0 -0
- {vexor-0.20.0.dist-info → vexor-0.21.0.dist-info}/entry_points.txt +0 -0
- {vexor-0.20.0.dist-info → vexor-0.21.0.dist-info}/licenses/LICENSE +0 -0
vexor/services/index_service.py
CHANGED
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import itertools
|
|
5
6
|
import os
|
|
7
|
+
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
|
6
8
|
from dataclasses import dataclass, field
|
|
7
9
|
from datetime import datetime, timezone
|
|
8
10
|
from enum import Enum
|
|
@@ -15,12 +17,18 @@ from .cache_service import load_index_metadata_safe
|
|
|
15
17
|
from .content_extract_service import TEXT_EXTENSIONS
|
|
16
18
|
from .js_parser import JSTS_EXTENSIONS
|
|
17
19
|
from ..cache import CACHE_VERSION, IndexedChunk, backfill_chunk_lines
|
|
18
|
-
from ..config import
|
|
20
|
+
from ..config import (
|
|
21
|
+
DEFAULT_EMBED_CONCURRENCY,
|
|
22
|
+
DEFAULT_EXTRACT_BACKEND,
|
|
23
|
+
DEFAULT_EXTRACT_CONCURRENCY,
|
|
24
|
+
)
|
|
19
25
|
from ..modes import get_strategy, ModePayload
|
|
20
26
|
|
|
21
27
|
INCREMENTAL_CHANGE_THRESHOLD = 0.5
|
|
22
28
|
MTIME_TOLERANCE = 5e-1
|
|
23
29
|
MARKDOWN_EXTENSIONS = {".md", ".markdown", ".mdx"}
|
|
30
|
+
_EXTRACT_PROCESS_MIN_FILES = 16
|
|
31
|
+
_CPU_HEAVY_MODES = {"auto", "code", "outline", "full"}
|
|
24
32
|
|
|
25
33
|
|
|
26
34
|
class IndexStatus(str, Enum):
|
|
@@ -36,6 +44,85 @@ class IndexResult:
|
|
|
36
44
|
files_indexed: int = 0
|
|
37
45
|
|
|
38
46
|
|
|
47
|
+
def _resolve_extract_concurrency(value: int) -> int:
|
|
48
|
+
return max(int(value or 1), 1)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _resolve_extract_backend(
|
|
52
|
+
value: str | None,
|
|
53
|
+
*,
|
|
54
|
+
mode: str,
|
|
55
|
+
file_count: int,
|
|
56
|
+
concurrency: int,
|
|
57
|
+
) -> str:
|
|
58
|
+
normalized = (value or DEFAULT_EXTRACT_BACKEND).strip().lower()
|
|
59
|
+
if normalized not in {"auto", "thread", "process"}:
|
|
60
|
+
normalized = DEFAULT_EXTRACT_BACKEND
|
|
61
|
+
if normalized == "auto":
|
|
62
|
+
if (
|
|
63
|
+
concurrency > 1
|
|
64
|
+
and file_count >= _EXTRACT_PROCESS_MIN_FILES
|
|
65
|
+
and mode in _CPU_HEAVY_MODES
|
|
66
|
+
):
|
|
67
|
+
return "process"
|
|
68
|
+
return "thread"
|
|
69
|
+
return normalized
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _extract_payloads_for_mode(path: Path, mode: str) -> list[ModePayload]:
|
|
73
|
+
strategy = get_strategy(mode)
|
|
74
|
+
return strategy.payloads_for_files([path])
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _payloads_for_files(
|
|
78
|
+
strategy,
|
|
79
|
+
files: Sequence[Path],
|
|
80
|
+
*,
|
|
81
|
+
mode: str,
|
|
82
|
+
extract_concurrency: int,
|
|
83
|
+
extract_backend: str,
|
|
84
|
+
) -> list[ModePayload]:
|
|
85
|
+
if not files:
|
|
86
|
+
return []
|
|
87
|
+
concurrency = _resolve_extract_concurrency(extract_concurrency)
|
|
88
|
+
if concurrency <= 1 or len(files) <= 1:
|
|
89
|
+
return strategy.payloads_for_files(files)
|
|
90
|
+
max_workers = min(concurrency, len(files))
|
|
91
|
+
|
|
92
|
+
def _extract_with_thread_pool() -> list[ModePayload]:
|
|
93
|
+
def _extract_one(path: Path) -> list[ModePayload]:
|
|
94
|
+
return strategy.payloads_for_files([path])
|
|
95
|
+
|
|
96
|
+
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
97
|
+
results = executor.map(_extract_one, files)
|
|
98
|
+
payloads: list[ModePayload] = []
|
|
99
|
+
for batch in results:
|
|
100
|
+
payloads.extend(batch)
|
|
101
|
+
return payloads
|
|
102
|
+
|
|
103
|
+
effective_backend = _resolve_extract_backend(
|
|
104
|
+
extract_backend,
|
|
105
|
+
mode=mode,
|
|
106
|
+
file_count=len(files),
|
|
107
|
+
concurrency=concurrency,
|
|
108
|
+
)
|
|
109
|
+
if effective_backend == "process":
|
|
110
|
+
try:
|
|
111
|
+
with ProcessPoolExecutor(max_workers=max_workers) as executor:
|
|
112
|
+
results = executor.map(
|
|
113
|
+
_extract_payloads_for_mode,
|
|
114
|
+
files,
|
|
115
|
+
itertools.repeat(mode),
|
|
116
|
+
)
|
|
117
|
+
payloads: list[ModePayload] = []
|
|
118
|
+
for batch in results:
|
|
119
|
+
payloads.extend(batch)
|
|
120
|
+
return payloads
|
|
121
|
+
except Exception:
|
|
122
|
+
return _extract_with_thread_pool()
|
|
123
|
+
return _extract_with_thread_pool()
|
|
124
|
+
|
|
125
|
+
|
|
39
126
|
def build_index(
|
|
40
127
|
directory: Path,
|
|
41
128
|
*,
|
|
@@ -46,6 +133,8 @@ def build_index(
|
|
|
46
133
|
model_name: str,
|
|
47
134
|
batch_size: int,
|
|
48
135
|
embed_concurrency: int = DEFAULT_EMBED_CONCURRENCY,
|
|
136
|
+
extract_concurrency: int = DEFAULT_EXTRACT_CONCURRENCY,
|
|
137
|
+
extract_backend: str = DEFAULT_EXTRACT_BACKEND,
|
|
49
138
|
provider: str,
|
|
50
139
|
base_url: str | None,
|
|
51
140
|
api_key: str | None,
|
|
@@ -71,6 +160,7 @@ def build_index(
|
|
|
71
160
|
if not files:
|
|
72
161
|
return IndexResult(status=IndexStatus.EMPTY)
|
|
73
162
|
stat_cache: dict[Path, os.stat_result] = {}
|
|
163
|
+
extract_concurrency = _resolve_extract_concurrency(extract_concurrency)
|
|
74
164
|
|
|
75
165
|
existing_meta = load_index_metadata_safe(
|
|
76
166
|
directory,
|
|
@@ -111,6 +201,9 @@ def build_index(
|
|
|
111
201
|
files=files,
|
|
112
202
|
missing_rel_paths=missing_line_files,
|
|
113
203
|
root=directory,
|
|
204
|
+
extract_concurrency=extract_concurrency,
|
|
205
|
+
extract_backend=extract_backend,
|
|
206
|
+
mode=mode,
|
|
114
207
|
)
|
|
115
208
|
cache_path = backfill_chunk_lines(
|
|
116
209
|
root=directory,
|
|
@@ -169,7 +262,15 @@ def build_index(
|
|
|
169
262
|
path for rel, path in files_with_rel if rel in changed_rel_paths
|
|
170
263
|
]
|
|
171
264
|
changed_payloads = (
|
|
172
|
-
|
|
265
|
+
_payloads_for_files(
|
|
266
|
+
strategy,
|
|
267
|
+
changed_files,
|
|
268
|
+
mode=mode,
|
|
269
|
+
extract_concurrency=extract_concurrency,
|
|
270
|
+
extract_backend=extract_backend,
|
|
271
|
+
)
|
|
272
|
+
if changed_files
|
|
273
|
+
else []
|
|
173
274
|
)
|
|
174
275
|
|
|
175
276
|
cache_path = _apply_incremental_update(
|
|
@@ -199,6 +300,9 @@ def build_index(
|
|
|
199
300
|
files=files,
|
|
200
301
|
missing_rel_paths=line_backfill_targets,
|
|
201
302
|
root=directory,
|
|
303
|
+
extract_concurrency=extract_concurrency,
|
|
304
|
+
extract_backend=extract_backend,
|
|
305
|
+
mode=mode,
|
|
202
306
|
)
|
|
203
307
|
cache_path = backfill_chunk_lines(
|
|
204
308
|
root=directory,
|
|
@@ -217,7 +321,13 @@ def build_index(
|
|
|
217
321
|
files_indexed=len(files),
|
|
218
322
|
)
|
|
219
323
|
|
|
220
|
-
payloads =
|
|
324
|
+
payloads = _payloads_for_files(
|
|
325
|
+
strategy,
|
|
326
|
+
files,
|
|
327
|
+
mode=mode,
|
|
328
|
+
extract_concurrency=extract_concurrency,
|
|
329
|
+
extract_backend=extract_backend,
|
|
330
|
+
)
|
|
221
331
|
file_labels = [payload.label for payload in payloads]
|
|
222
332
|
embeddings = _embed_labels_with_cache(
|
|
223
333
|
searcher=searcher,
|
|
@@ -255,6 +365,8 @@ def build_index_in_memory(
|
|
|
255
365
|
model_name: str,
|
|
256
366
|
batch_size: int,
|
|
257
367
|
embed_concurrency: int = DEFAULT_EMBED_CONCURRENCY,
|
|
368
|
+
extract_concurrency: int = DEFAULT_EXTRACT_CONCURRENCY,
|
|
369
|
+
extract_backend: str = DEFAULT_EXTRACT_BACKEND,
|
|
258
370
|
provider: str,
|
|
259
371
|
base_url: str | None,
|
|
260
372
|
api_key: str | None,
|
|
@@ -307,7 +419,13 @@ def build_index_in_memory(
|
|
|
307
419
|
api_key=api_key,
|
|
308
420
|
local_cuda=local_cuda,
|
|
309
421
|
)
|
|
310
|
-
payloads =
|
|
422
|
+
payloads = _payloads_for_files(
|
|
423
|
+
strategy,
|
|
424
|
+
files,
|
|
425
|
+
mode=mode,
|
|
426
|
+
extract_concurrency=extract_concurrency,
|
|
427
|
+
extract_backend=extract_backend,
|
|
428
|
+
)
|
|
311
429
|
if not payloads:
|
|
312
430
|
empty = np.empty((0, 0), dtype=np.float32)
|
|
313
431
|
metadata = {
|
|
@@ -809,6 +927,9 @@ def _build_line_backfill_updates(
|
|
|
809
927
|
files: Sequence[Path],
|
|
810
928
|
missing_rel_paths: set[str],
|
|
811
929
|
root: Path,
|
|
930
|
+
extract_concurrency: int,
|
|
931
|
+
extract_backend: str,
|
|
932
|
+
mode: str,
|
|
812
933
|
) -> list[tuple[str, int, int | None, int | None]]:
|
|
813
934
|
if not missing_rel_paths:
|
|
814
935
|
return []
|
|
@@ -816,7 +937,13 @@ def _build_line_backfill_updates(
|
|
|
816
937
|
targets = [files_by_rel[rel] for rel in missing_rel_paths if rel in files_by_rel]
|
|
817
938
|
if not targets:
|
|
818
939
|
return []
|
|
819
|
-
payloads =
|
|
940
|
+
payloads = _payloads_for_files(
|
|
941
|
+
strategy,
|
|
942
|
+
targets,
|
|
943
|
+
mode=mode,
|
|
944
|
+
extract_concurrency=extract_concurrency,
|
|
945
|
+
extract_backend=extract_backend,
|
|
946
|
+
)
|
|
820
947
|
return [
|
|
821
948
|
(
|
|
822
949
|
_relative_to_root(payload.file, root),
|
vexor/services/search_service.py
CHANGED
|
@@ -7,12 +7,15 @@ from functools import lru_cache
|
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
import json
|
|
9
9
|
import re
|
|
10
|
+
import numpy as np
|
|
10
11
|
from typing import Sequence, TYPE_CHECKING
|
|
11
12
|
from urllib import error as urlerror
|
|
12
13
|
from urllib import request as urlrequest
|
|
13
14
|
|
|
14
15
|
from ..config import (
|
|
15
16
|
DEFAULT_EMBED_CONCURRENCY,
|
|
17
|
+
DEFAULT_EXTRACT_BACKEND,
|
|
18
|
+
DEFAULT_EXTRACT_CONCURRENCY,
|
|
16
19
|
DEFAULT_FLASHRANK_MAX_LENGTH,
|
|
17
20
|
DEFAULT_FLASHRANK_MODEL,
|
|
18
21
|
DEFAULT_RERANK,
|
|
@@ -48,6 +51,8 @@ class SearchRequest:
|
|
|
48
51
|
temporary_index: bool = False
|
|
49
52
|
no_cache: bool = False
|
|
50
53
|
embed_concurrency: int = DEFAULT_EMBED_CONCURRENCY
|
|
54
|
+
extract_concurrency: int = DEFAULT_EXTRACT_CONCURRENCY
|
|
55
|
+
extract_backend: str = DEFAULT_EXTRACT_BACKEND
|
|
51
56
|
rerank: str = DEFAULT_RERANK
|
|
52
57
|
flashrank_model: str | None = None
|
|
53
58
|
remote_rerank: RemoteRerankConfig | None = None
|
|
@@ -112,6 +117,15 @@ def _resolve_rerank_candidates(top_k: int) -> int:
|
|
|
112
117
|
return max(20, min(candidate, 150))
|
|
113
118
|
|
|
114
119
|
|
|
120
|
+
def _top_indices(scores: np.ndarray, limit: int) -> list[int]:
|
|
121
|
+
if limit <= 0:
|
|
122
|
+
return []
|
|
123
|
+
if limit >= scores.size:
|
|
124
|
+
return sorted(range(scores.size), key=lambda idx: (-scores[idx], idx))
|
|
125
|
+
indices = np.argpartition(-scores, limit - 1)[:limit]
|
|
126
|
+
return sorted(indices.tolist(), key=lambda idx: (-scores[idx], idx))
|
|
127
|
+
|
|
128
|
+
|
|
115
129
|
def _bm25_scores(
|
|
116
130
|
query_tokens: Sequence[str],
|
|
117
131
|
documents: Sequence[Sequence[str]],
|
|
@@ -349,6 +363,7 @@ def perform_search(request: SearchRequest) -> SearchResponse:
|
|
|
349
363
|
from ..cache import ( # local import
|
|
350
364
|
embedding_cache_key,
|
|
351
365
|
list_cache_entries,
|
|
366
|
+
load_chunk_metadata,
|
|
352
367
|
load_embedding_cache,
|
|
353
368
|
load_index_vectors,
|
|
354
369
|
load_query_vector,
|
|
@@ -385,6 +400,8 @@ def perform_search(request: SearchRequest) -> SearchResponse:
|
|
|
385
400
|
model_name=request.model_name,
|
|
386
401
|
batch_size=request.batch_size,
|
|
387
402
|
embed_concurrency=request.embed_concurrency,
|
|
403
|
+
extract_concurrency=request.extract_concurrency,
|
|
404
|
+
extract_backend=request.extract_backend,
|
|
388
405
|
provider=request.provider,
|
|
389
406
|
base_url=request.base_url,
|
|
390
407
|
api_key=request.api_key,
|
|
@@ -446,6 +463,7 @@ def perform_search(request: SearchRequest) -> SearchResponse:
|
|
|
446
463
|
|
|
447
464
|
file_snapshot = metadata.get("files", [])
|
|
448
465
|
chunk_entries = metadata.get("chunks", [])
|
|
466
|
+
chunk_ids = metadata.get("chunk_ids", [])
|
|
449
467
|
stale = bool(file_snapshot) and not is_cache_current(
|
|
450
468
|
request.directory,
|
|
451
469
|
request.include_hidden,
|
|
@@ -466,6 +484,8 @@ def perform_search(request: SearchRequest) -> SearchResponse:
|
|
|
466
484
|
model_name=request.model_name,
|
|
467
485
|
batch_size=request.batch_size,
|
|
468
486
|
embed_concurrency=request.embed_concurrency,
|
|
487
|
+
extract_concurrency=request.extract_concurrency,
|
|
488
|
+
extract_backend=request.extract_backend,
|
|
469
489
|
provider=request.provider,
|
|
470
490
|
base_url=request.base_url,
|
|
471
491
|
api_key=request.api_key,
|
|
@@ -541,7 +561,6 @@ def perform_search(request: SearchRequest) -> SearchResponse:
|
|
|
541
561
|
index_empty=True,
|
|
542
562
|
)
|
|
543
563
|
|
|
544
|
-
from sklearn.metrics.pairwise import cosine_similarity # local import
|
|
545
564
|
from ..search import SearchResult, VexorSearcher # local import
|
|
546
565
|
searcher = VexorSearcher(
|
|
547
566
|
model_name=request.model_name,
|
|
@@ -595,13 +614,37 @@ def perform_search(request: SearchRequest) -> SearchResponse:
|
|
|
595
614
|
store_query_vector(int(index_id), query_hash, request.query, query_vector)
|
|
596
615
|
except Exception: # pragma: no cover - best-effort cache storage
|
|
597
616
|
pass
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
617
|
+
reranker = None
|
|
618
|
+
rerank = (request.rerank or DEFAULT_RERANK).strip().lower()
|
|
619
|
+
use_rerank = rerank in {"bm25", "flashrank", "remote"}
|
|
620
|
+
if use_rerank:
|
|
621
|
+
candidate_limit = _resolve_rerank_candidates(request.top_k)
|
|
622
|
+
else:
|
|
623
|
+
candidate_limit = request.top_k
|
|
624
|
+
candidate_count = min(len(paths), candidate_limit)
|
|
625
|
+
|
|
626
|
+
query_vector = np.asarray(query_vector, dtype=np.float32).ravel()
|
|
627
|
+
similarities = np.asarray(file_vectors @ query_vector, dtype=np.float32)
|
|
628
|
+
top_indices = _top_indices(similarities, candidate_count)
|
|
629
|
+
chunk_meta_by_id: dict[int, dict] = {}
|
|
630
|
+
if chunk_ids:
|
|
631
|
+
candidate_ids = [
|
|
632
|
+
chunk_ids[idx] for idx in top_indices if idx < len(chunk_ids)
|
|
633
|
+
]
|
|
634
|
+
if candidate_ids:
|
|
635
|
+
try:
|
|
636
|
+
chunk_meta_by_id = load_chunk_metadata(candidate_ids)
|
|
637
|
+
except Exception: # pragma: no cover - best-effort metadata lookup
|
|
638
|
+
chunk_meta_by_id = {}
|
|
639
|
+
scored: list[SearchResult] = []
|
|
640
|
+
for idx in top_indices:
|
|
641
|
+
path = paths[idx]
|
|
642
|
+
score = similarities[idx]
|
|
643
|
+
chunk_meta = {}
|
|
644
|
+
if chunk_ids and idx < len(chunk_ids):
|
|
645
|
+
chunk_meta = chunk_meta_by_id.get(chunk_ids[idx], {})
|
|
646
|
+
elif idx < len(chunk_entries):
|
|
647
|
+
chunk_meta = chunk_entries[idx]
|
|
605
648
|
start_line = chunk_meta.get("start_line")
|
|
606
649
|
end_line = chunk_meta.get("end_line")
|
|
607
650
|
scored.append(
|
|
@@ -614,12 +657,8 @@ def perform_search(request: SearchRequest) -> SearchResponse:
|
|
|
614
657
|
end_line=int(end_line) if end_line is not None else None,
|
|
615
658
|
)
|
|
616
659
|
)
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
rerank = (request.rerank or DEFAULT_RERANK).strip().lower()
|
|
620
|
-
if rerank in {"bm25", "flashrank", "remote"}:
|
|
621
|
-
candidate_count = min(len(scored), _resolve_rerank_candidates(request.top_k))
|
|
622
|
-
candidates = scored[:candidate_count]
|
|
660
|
+
if use_rerank:
|
|
661
|
+
candidates = scored
|
|
623
662
|
if rerank == "bm25":
|
|
624
663
|
candidates = _apply_bm25_rerank(request.query, candidates)
|
|
625
664
|
reranker = "bm25"
|
|
@@ -662,6 +701,8 @@ def _perform_search_with_temporary_index(request: SearchRequest) -> SearchRespon
|
|
|
662
701
|
model_name=request.model_name,
|
|
663
702
|
batch_size=request.batch_size,
|
|
664
703
|
embed_concurrency=request.embed_concurrency,
|
|
704
|
+
extract_concurrency=request.extract_concurrency,
|
|
705
|
+
extract_backend=request.extract_backend,
|
|
665
706
|
provider=request.provider,
|
|
666
707
|
base_url=request.base_url,
|
|
667
708
|
api_key=request.api_key,
|
|
@@ -680,7 +721,6 @@ def _perform_search_with_temporary_index(request: SearchRequest) -> SearchRespon
|
|
|
680
721
|
index_empty=True,
|
|
681
722
|
)
|
|
682
723
|
|
|
683
|
-
from sklearn.metrics.pairwise import cosine_similarity # local import
|
|
684
724
|
from ..search import SearchResult, VexorSearcher # local import
|
|
685
725
|
|
|
686
726
|
searcher = VexorSearcher(
|
|
@@ -717,13 +757,23 @@ def _perform_search_with_temporary_index(request: SearchRequest) -> SearchRespon
|
|
|
717
757
|
)
|
|
718
758
|
except Exception: # pragma: no cover - best-effort cache storage
|
|
719
759
|
pass
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
760
|
+
reranker = None
|
|
761
|
+
rerank = (request.rerank or DEFAULT_RERANK).strip().lower()
|
|
762
|
+
use_rerank = rerank in {"bm25", "flashrank", "remote"}
|
|
763
|
+
if use_rerank:
|
|
764
|
+
candidate_limit = _resolve_rerank_candidates(request.top_k)
|
|
765
|
+
else:
|
|
766
|
+
candidate_limit = request.top_k
|
|
767
|
+
candidate_count = min(len(paths), candidate_limit)
|
|
768
|
+
|
|
769
|
+
query_vector = np.asarray(query_vector, dtype=np.float32).ravel()
|
|
770
|
+
similarities = np.asarray(file_vectors @ query_vector, dtype=np.float32)
|
|
771
|
+
top_indices = _top_indices(similarities, candidate_count)
|
|
724
772
|
chunk_entries = metadata.get("chunks", [])
|
|
725
|
-
scored = []
|
|
726
|
-
for idx
|
|
773
|
+
scored: list[SearchResult] = []
|
|
774
|
+
for idx in top_indices:
|
|
775
|
+
path = paths[idx]
|
|
776
|
+
score = similarities[idx]
|
|
727
777
|
chunk_meta = chunk_entries[idx] if idx < len(chunk_entries) else {}
|
|
728
778
|
start_line = chunk_meta.get("start_line")
|
|
729
779
|
end_line = chunk_meta.get("end_line")
|
|
@@ -737,12 +787,8 @@ def _perform_search_with_temporary_index(request: SearchRequest) -> SearchRespon
|
|
|
737
787
|
end_line=int(end_line) if end_line is not None else None,
|
|
738
788
|
)
|
|
739
789
|
)
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
rerank = (request.rerank or DEFAULT_RERANK).strip().lower()
|
|
743
|
-
if rerank in {"bm25", "flashrank", "remote"}:
|
|
744
|
-
candidate_count = min(len(scored), _resolve_rerank_candidates(request.top_k))
|
|
745
|
-
candidates = scored[:candidate_count]
|
|
790
|
+
if use_rerank:
|
|
791
|
+
candidates = scored
|
|
746
792
|
if rerank == "bm25":
|
|
747
793
|
candidates = _apply_bm25_rerank(request.query, candidates)
|
|
748
794
|
reranker = "bm25"
|
|
@@ -908,6 +954,7 @@ def _filter_index_by_extensions(
|
|
|
908
954
|
ext_set = {ext.lower() for ext in extensions if ext}
|
|
909
955
|
if not ext_set:
|
|
910
956
|
return list(paths), file_vectors, metadata
|
|
957
|
+
chunk_ids = metadata.get("chunk_ids")
|
|
911
958
|
keep_indices: list[int] = []
|
|
912
959
|
filtered_paths: list[Path] = []
|
|
913
960
|
for idx, path in enumerate(paths):
|
|
@@ -922,6 +969,8 @@ def _filter_index_by_extensions(
|
|
|
922
969
|
ext_set,
|
|
923
970
|
)
|
|
924
971
|
filtered_metadata["chunks"] = []
|
|
972
|
+
if chunk_ids is not None:
|
|
973
|
+
filtered_metadata["chunk_ids"] = []
|
|
925
974
|
return [], filtered_vectors, filtered_metadata
|
|
926
975
|
filtered_vectors = file_vectors[keep_indices]
|
|
927
976
|
chunk_entries = metadata.get("chunks", [])
|
|
@@ -934,6 +983,10 @@ def _filter_index_by_extensions(
|
|
|
934
983
|
ext_set,
|
|
935
984
|
)
|
|
936
985
|
filtered_metadata["chunks"] = filtered_chunks
|
|
986
|
+
if chunk_ids is not None:
|
|
987
|
+
filtered_metadata["chunk_ids"] = [
|
|
988
|
+
chunk_ids[idx] for idx in keep_indices if idx < len(chunk_ids)
|
|
989
|
+
]
|
|
937
990
|
return filtered_paths, filtered_vectors, filtered_metadata
|
|
938
991
|
|
|
939
992
|
|
|
@@ -946,6 +999,7 @@ def _filter_index_by_exclude_patterns(
|
|
|
946
999
|
) -> tuple[list[Path], Sequence[Sequence[float]], dict]:
|
|
947
1000
|
if exclude_spec is None:
|
|
948
1001
|
return list(paths), file_vectors, metadata
|
|
1002
|
+
chunk_ids = metadata.get("chunk_ids")
|
|
949
1003
|
keep_indices: list[int] = []
|
|
950
1004
|
filtered_paths: list[Path] = []
|
|
951
1005
|
root_resolved = root.resolve()
|
|
@@ -966,6 +1020,8 @@ def _filter_index_by_exclude_patterns(
|
|
|
966
1020
|
exclude_spec,
|
|
967
1021
|
)
|
|
968
1022
|
filtered_metadata["chunks"] = []
|
|
1023
|
+
if chunk_ids is not None:
|
|
1024
|
+
filtered_metadata["chunk_ids"] = []
|
|
969
1025
|
return [], filtered_vectors, filtered_metadata
|
|
970
1026
|
filtered_vectors = file_vectors[keep_indices]
|
|
971
1027
|
chunk_entries = metadata.get("chunks", [])
|
|
@@ -978,6 +1034,10 @@ def _filter_index_by_exclude_patterns(
|
|
|
978
1034
|
exclude_spec,
|
|
979
1035
|
)
|
|
980
1036
|
filtered_metadata["chunks"] = filtered_chunks
|
|
1037
|
+
if chunk_ids is not None:
|
|
1038
|
+
filtered_metadata["chunk_ids"] = [
|
|
1039
|
+
chunk_ids[idx] for idx in keep_indices if idx < len(chunk_ids)
|
|
1040
|
+
]
|
|
981
1041
|
return filtered_paths, filtered_vectors, filtered_metadata
|
|
982
1042
|
|
|
983
1043
|
|
|
@@ -994,6 +1054,7 @@ def _filter_index_by_directory(
|
|
|
994
1054
|
relative_dir = directory.resolve().relative_to(index_root.resolve())
|
|
995
1055
|
except ValueError:
|
|
996
1056
|
return list(paths), file_vectors, metadata
|
|
1057
|
+
chunk_ids = metadata.get("chunk_ids")
|
|
997
1058
|
keep_indices: list[int] = []
|
|
998
1059
|
filtered_paths: list[Path] = []
|
|
999
1060
|
for idx, path in enumerate(paths):
|
|
@@ -1014,6 +1075,8 @@ def _filter_index_by_directory(
|
|
|
1014
1075
|
recursive=recursive,
|
|
1015
1076
|
)
|
|
1016
1077
|
filtered_metadata["chunks"] = []
|
|
1078
|
+
if chunk_ids is not None:
|
|
1079
|
+
filtered_metadata["chunk_ids"] = []
|
|
1017
1080
|
filtered_metadata["root"] = str(directory)
|
|
1018
1081
|
return [], filtered_vectors, filtered_metadata
|
|
1019
1082
|
filtered_vectors = file_vectors[keep_indices]
|
|
@@ -1028,6 +1091,10 @@ def _filter_index_by_directory(
|
|
|
1028
1091
|
recursive=recursive,
|
|
1029
1092
|
)
|
|
1030
1093
|
filtered_metadata["chunks"] = filtered_chunks
|
|
1094
|
+
if chunk_ids is not None:
|
|
1095
|
+
filtered_metadata["chunk_ids"] = [
|
|
1096
|
+
chunk_ids[idx] for idx in keep_indices if idx < len(chunk_ids)
|
|
1097
|
+
]
|
|
1031
1098
|
filtered_metadata["root"] = str(directory)
|
|
1032
1099
|
return filtered_paths, filtered_vectors, filtered_metadata
|
|
1033
1100
|
|
vexor/text.py
CHANGED
|
@@ -59,6 +59,8 @@ class Messages:
|
|
|
59
59
|
HELP_SET_MODEL = "Set the default embedding model."
|
|
60
60
|
HELP_SET_BATCH = "Set the default batch size (0 = single request)."
|
|
61
61
|
HELP_SET_EMBED_CONCURRENCY = "Set the number of concurrent embedding requests."
|
|
62
|
+
HELP_SET_EXTRACT_CONCURRENCY = "Set the number of concurrent file extraction workers."
|
|
63
|
+
HELP_SET_EXTRACT_BACKEND = "Set the extraction backend (auto, thread, process)."
|
|
62
64
|
HELP_SET_PROVIDER = "Set the default embedding provider (e.g., gemini, openai, custom, or local)."
|
|
63
65
|
HELP_SET_BASE_URL = "Override the provider's base URL (leave unset for official endpoints)."
|
|
64
66
|
HELP_CLEAR_BASE_URL = "Remove the custom base URL override."
|
|
@@ -117,6 +119,10 @@ class Messages:
|
|
|
117
119
|
ERROR_EMPTY_QUERY = "Query text must not be empty."
|
|
118
120
|
ERROR_BATCH_NEGATIVE = "Batch size must be >= 0"
|
|
119
121
|
ERROR_CONCURRENCY_INVALID = "Embedding concurrency must be >= 1"
|
|
122
|
+
ERROR_EXTRACT_CONCURRENCY_INVALID = "Extraction concurrency must be >= 1"
|
|
123
|
+
ERROR_EXTRACT_BACKEND_INVALID = (
|
|
124
|
+
"Unsupported extraction backend '{value}'. Allowed values: {allowed}."
|
|
125
|
+
)
|
|
120
126
|
ERROR_MODE_INVALID = "Unsupported mode '{value}'. Allowed values: {allowed}."
|
|
121
127
|
ERROR_PROVIDER_INVALID = "Unsupported provider '{value}'. Allowed values: {allowed}."
|
|
122
128
|
ERROR_RERANK_INVALID = "Unsupported rerank value '{value}'. Allowed values: {allowed}."
|
|
@@ -266,6 +272,8 @@ class Messages:
|
|
|
266
272
|
INFO_MODEL_SET = "Default model set to {value}."
|
|
267
273
|
INFO_BATCH_SET = "Default batch size set to {value}."
|
|
268
274
|
INFO_EMBED_CONCURRENCY_SET = "Embedding concurrency set to {value}."
|
|
275
|
+
INFO_EXTRACT_CONCURRENCY_SET = "Extraction concurrency set to {value}."
|
|
276
|
+
INFO_EXTRACT_BACKEND_SET = "Extraction backend set to {value}."
|
|
269
277
|
INFO_PROVIDER_SET = "Default provider set to {value}."
|
|
270
278
|
INFO_BASE_URL_SET = "Base URL override set to {value}."
|
|
271
279
|
INFO_BASE_URL_CLEARED = "Base URL override cleared."
|
|
@@ -308,6 +316,8 @@ class Messages:
|
|
|
308
316
|
"Default model: {model}\n"
|
|
309
317
|
"Default batch size: {batch}\n"
|
|
310
318
|
"Embedding concurrency: {concurrency}\n"
|
|
319
|
+
"Extract concurrency: {extract_concurrency}\n"
|
|
320
|
+
"Extract backend: {extract_backend}\n"
|
|
311
321
|
"Auto index: {auto_index}\n"
|
|
312
322
|
"Rerank: {rerank}\n"
|
|
313
323
|
"{flashrank_line}"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vexor
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.21.0
|
|
4
4
|
Summary: A vector-powered CLI for semantic search over files.
|
|
5
5
|
Project-URL: Repository, https://github.com/scarletkc/vexor
|
|
6
6
|
Author: scarletkc
|
|
@@ -150,13 +150,26 @@ for hit in response.results:
|
|
|
150
150
|
By default it reads `~/.vexor/config.json`. For runtime config overrides, cache
|
|
151
151
|
controls, and per-call options, see [`docs/api/python.md`](https://github.com/scarletkc/vexor/tree/main/docs/api/python.md).
|
|
152
152
|
|
|
153
|
+
## AI Agent Skill
|
|
154
|
+
|
|
155
|
+
This repo includes a skill for AI agents to use Vexor effectively:
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
vexor install --skills claude # Claude Code
|
|
159
|
+
vexor install --skills codex # Codex
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Skill source: [`plugins/vexor/skills/vexor-cli`](https://github.com/scarletkc/vexor/raw/refs/heads/main/plugins/vexor/skills/vexor-cli/SKILL.md)
|
|
163
|
+
|
|
153
164
|
## Configuration
|
|
154
165
|
|
|
155
166
|
```bash
|
|
156
167
|
vexor config --set-provider openai # default; also supports gemini/custom/local
|
|
157
168
|
vexor config --set-model text-embedding-3-small
|
|
158
169
|
vexor config --set-batch-size 0 # 0 = single request
|
|
159
|
-
vexor config --set-embed-concurrency
|
|
170
|
+
vexor config --set-embed-concurrency 4 # parallel embedding requests
|
|
171
|
+
vexor config --set-extract-concurrency 4 # parallel file extraction workers
|
|
172
|
+
vexor config --set-extract-backend auto # auto|thread|process (default: auto)
|
|
160
173
|
vexor config --set-auto-index true # auto-index before search (default)
|
|
161
174
|
vexor config --rerank bm25 # optional BM25 rerank for top-k results
|
|
162
175
|
vexor config --rerank flashrank # FlashRank rerank (requires optional extra)
|
|
@@ -298,17 +311,6 @@ Re-running `vexor index` only re-embeds changed files; >50% changes trigger full
|
|
|
298
311
|
|
|
299
312
|
Porcelain output fields: `rank`, `similarity`, `path`, `chunk_index`, `start_line`, `end_line`, `preview` (line fields are `-` when unavailable).
|
|
300
313
|
|
|
301
|
-
## AI Agent Skill
|
|
302
|
-
|
|
303
|
-
This repo includes a skill for AI agents to use Vexor effectively:
|
|
304
|
-
|
|
305
|
-
```bash
|
|
306
|
-
vexor install --skills claude # Claude Code
|
|
307
|
-
vexor install --skills codex # Codex
|
|
308
|
-
```
|
|
309
|
-
|
|
310
|
-
Skill source: [`plugins/vexor/skills/vexor-cli`](https://github.com/scarletkc/vexor/raw/refs/heads/main/plugins/vexor/skills/vexor-cli/SKILL.md)
|
|
311
|
-
|
|
312
314
|
## Documentation
|
|
313
315
|
|
|
314
316
|
See [docs](https://github.com/scarletkc/vexor/tree/main/docs) for more details.
|
|
@@ -1,33 +1,33 @@
|
|
|
1
|
-
vexor/__init__.py,sha256=
|
|
1
|
+
vexor/__init__.py,sha256=i0ly8cFA4N_PEQ_rhYgoLp2NPRQc3_ln8Gfi8QWjXSQ,441
|
|
2
2
|
vexor/__main__.py,sha256=ZFzom1wCfP6TPXe3aoDFpNcUgjbCZ7Quy_vfzNsH5Fw,426
|
|
3
|
-
vexor/api.py,sha256=
|
|
4
|
-
vexor/cache.py,sha256=
|
|
5
|
-
vexor/cli.py,sha256=
|
|
6
|
-
vexor/config.py,sha256=
|
|
3
|
+
vexor/api.py,sha256=YCHpiydbPbRJUqdQYrpwe1JrRI-w_7LRuyZDGBP1_d4,11506
|
|
4
|
+
vexor/cache.py,sha256=3i9FKFLSyZ1kx-w1apc12umPaQxWqMP-P8_lvo67hBw,52832
|
|
5
|
+
vexor/cli.py,sha256=M9GKdD_mJ068Zpm62znTp0KhhKp1dkh_WHmfJHR9hwU,68094
|
|
6
|
+
vexor/config.py,sha256=CiPfEH7Ilt6XepEx4p02qfW5HfkpNDBjhEMyckbSWaA,17413
|
|
7
7
|
vexor/modes.py,sha256=N_wAWoqbxmCfko-v520p59tpAYvUwraCSSQRtMaF4ac,11549
|
|
8
8
|
vexor/output.py,sha256=iooZgLlK8dh7ajJ4XMHUNNx0qyTVtD_OAAwrBx5MeqE,864
|
|
9
9
|
vexor/search.py,sha256=MSU4RmH6waFYOofkIdo8_ElTiz1oNaKuvr-3umif7Bs,6826
|
|
10
|
-
vexor/text.py,sha256=
|
|
10
|
+
vexor/text.py,sha256=2aK5nJHkosmbmyzp9o_Tzb3YlmVnju_IX8BcEPUdhTA,24794
|
|
11
11
|
vexor/utils.py,sha256=GzfYW2rz1-EuJjkevqZVe8flLRtrQ60OWMmFNbMh62k,12472
|
|
12
12
|
vexor/providers/__init__.py,sha256=kCEoV03TSLKcxDUYVNjXnrVoLU5NpfNXjp1w1Ak2imE,92
|
|
13
|
-
vexor/providers/gemini.py,sha256
|
|
13
|
+
vexor/providers/gemini.py,sha256=IWHHjCMJC0hUHQPhuaJ_L_97c_mnOXkPkCVdrIR6z-g,5705
|
|
14
14
|
vexor/providers/local.py,sha256=5X_WYCXgyBGIVvvVLgMnDjTkPR4GBF0ksNPyviBlB7w,4838
|
|
15
|
-
vexor/providers/openai.py,sha256=
|
|
15
|
+
vexor/providers/openai.py,sha256=YnJDY9gJW7RfGGdkgswVHvmOKNvgLRQUsbpA1MUuLPg,5356
|
|
16
16
|
vexor/services/__init__.py,sha256=dA_i2N03vlYmbZbEK2knzJLWviunkNWbzN2LWPNvMk0,160
|
|
17
17
|
vexor/services/cache_service.py,sha256=ywt6AgupCJ7_wC3je4znCMw5_VBouw3skbDTAt8xw6o,1639
|
|
18
|
-
vexor/services/config_service.py,sha256=
|
|
18
|
+
vexor/services/config_service.py,sha256=PojolfbSKh9pW8slF4qxCOs9hz5L6xvjf_nB7vfVlsU,5039
|
|
19
19
|
vexor/services/content_extract_service.py,sha256=zdhLxpNv70BU7irLf3Uc0ou9rKSvdjtrDcHkgRKlMn4,26421
|
|
20
|
-
vexor/services/index_service.py,sha256=
|
|
20
|
+
vexor/services/index_service.py,sha256=FXf1bBoqj4-K1l38ItxHf6Oh7QHVIdNAdVY2kg_Zoq8,32265
|
|
21
21
|
vexor/services/init_service.py,sha256=3D04hylGA9FRQhLHCfR95nMko3vb5MNBcRb9nWWaUE8,26863
|
|
22
22
|
vexor/services/js_parser.py,sha256=eRtW6KlK4JBYDGbyoecHVqLZ0hcx-Cc0kx6bOujHPAQ,16254
|
|
23
23
|
vexor/services/keyword_service.py,sha256=vmke8tII9kTwRDdBaLHBc6Hpy_B3p98L65iGkCQgtMU,2211
|
|
24
|
-
vexor/services/search_service.py,sha256=
|
|
24
|
+
vexor/services/search_service.py,sha256=K7SiAuMA7bGeyPWOHPMKpFFvzzkj5kHWwa3p94NakJs,38663
|
|
25
25
|
vexor/services/skill_service.py,sha256=Rrgt3OMsKPPiXOiRhSNAWjBM9UNz9qmSWQe3uYGzq4M,4863
|
|
26
26
|
vexor/services/system_service.py,sha256=KPlv83v3rTvBiNiH7vrp6tDmt_AqHxuUd-5RI0TfvWs,24638
|
|
27
27
|
vexor/_bundled_skills/vexor-cli/SKILL.md,sha256=m3FlyqgHBdRwyGPEp8PrUS21K0G2jEl88tRvhSPta08,2798
|
|
28
28
|
vexor/_bundled_skills/vexor-cli/references/install-vexor.md,sha256=IUBShLI1mAxugwUIMAJQ5_j6KcaPWfobe0gSd6MWU7w,1245
|
|
29
|
-
vexor-0.
|
|
30
|
-
vexor-0.
|
|
31
|
-
vexor-0.
|
|
32
|
-
vexor-0.
|
|
33
|
-
vexor-0.
|
|
29
|
+
vexor-0.21.0.dist-info/METADATA,sha256=Lc5PHY_Ir3F56ILYe6IBlkwhN6gMQGZvf48f7x_uVDg,13494
|
|
30
|
+
vexor-0.21.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
31
|
+
vexor-0.21.0.dist-info/entry_points.txt,sha256=dvxp6Q1R1d6bozR7TwmpdJ0X_v83MkzsLPagGY_lfr0,40
|
|
32
|
+
vexor-0.21.0.dist-info/licenses/LICENSE,sha256=wP7TAKRll1t9LoYGxWS9NikPM_0hCc00LmlLyvQBsL8,1066
|
|
33
|
+
vexor-0.21.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|