vexor 0.20.0__py3-none-any.whl → 0.21.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,9 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
+ import itertools
5
6
  import os
7
+ from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
6
8
  from dataclasses import dataclass, field
7
9
  from datetime import datetime, timezone
8
10
  from enum import Enum
@@ -15,12 +17,18 @@ from .cache_service import load_index_metadata_safe
15
17
  from .content_extract_service import TEXT_EXTENSIONS
16
18
  from .js_parser import JSTS_EXTENSIONS
17
19
  from ..cache import CACHE_VERSION, IndexedChunk, backfill_chunk_lines
18
- from ..config import DEFAULT_EMBED_CONCURRENCY
20
+ from ..config import (
21
+ DEFAULT_EMBED_CONCURRENCY,
22
+ DEFAULT_EXTRACT_BACKEND,
23
+ DEFAULT_EXTRACT_CONCURRENCY,
24
+ )
19
25
  from ..modes import get_strategy, ModePayload
20
26
 
21
27
  INCREMENTAL_CHANGE_THRESHOLD = 0.5
22
28
  MTIME_TOLERANCE = 5e-1
23
29
  MARKDOWN_EXTENSIONS = {".md", ".markdown", ".mdx"}
30
+ _EXTRACT_PROCESS_MIN_FILES = 16
31
+ _CPU_HEAVY_MODES = {"auto", "code", "outline", "full"}
24
32
 
25
33
 
26
34
  class IndexStatus(str, Enum):
@@ -36,6 +44,85 @@ class IndexResult:
36
44
  files_indexed: int = 0
37
45
 
38
46
 
47
+ def _resolve_extract_concurrency(value: int) -> int:
48
+ return max(int(value or 1), 1)
49
+
50
+
51
+ def _resolve_extract_backend(
52
+ value: str | None,
53
+ *,
54
+ mode: str,
55
+ file_count: int,
56
+ concurrency: int,
57
+ ) -> str:
58
+ normalized = (value or DEFAULT_EXTRACT_BACKEND).strip().lower()
59
+ if normalized not in {"auto", "thread", "process"}:
60
+ normalized = DEFAULT_EXTRACT_BACKEND
61
+ if normalized == "auto":
62
+ if (
63
+ concurrency > 1
64
+ and file_count >= _EXTRACT_PROCESS_MIN_FILES
65
+ and mode in _CPU_HEAVY_MODES
66
+ ):
67
+ return "process"
68
+ return "thread"
69
+ return normalized
70
+
71
+
72
+ def _extract_payloads_for_mode(path: Path, mode: str) -> list[ModePayload]:
73
+ strategy = get_strategy(mode)
74
+ return strategy.payloads_for_files([path])
75
+
76
+
77
+ def _payloads_for_files(
78
+ strategy,
79
+ files: Sequence[Path],
80
+ *,
81
+ mode: str,
82
+ extract_concurrency: int,
83
+ extract_backend: str,
84
+ ) -> list[ModePayload]:
85
+ if not files:
86
+ return []
87
+ concurrency = _resolve_extract_concurrency(extract_concurrency)
88
+ if concurrency <= 1 or len(files) <= 1:
89
+ return strategy.payloads_for_files(files)
90
+ max_workers = min(concurrency, len(files))
91
+
92
+ def _extract_with_thread_pool() -> list[ModePayload]:
93
+ def _extract_one(path: Path) -> list[ModePayload]:
94
+ return strategy.payloads_for_files([path])
95
+
96
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
97
+ results = executor.map(_extract_one, files)
98
+ payloads: list[ModePayload] = []
99
+ for batch in results:
100
+ payloads.extend(batch)
101
+ return payloads
102
+
103
+ effective_backend = _resolve_extract_backend(
104
+ extract_backend,
105
+ mode=mode,
106
+ file_count=len(files),
107
+ concurrency=concurrency,
108
+ )
109
+ if effective_backend == "process":
110
+ try:
111
+ with ProcessPoolExecutor(max_workers=max_workers) as executor:
112
+ results = executor.map(
113
+ _extract_payloads_for_mode,
114
+ files,
115
+ itertools.repeat(mode),
116
+ )
117
+ payloads: list[ModePayload] = []
118
+ for batch in results:
119
+ payloads.extend(batch)
120
+ return payloads
121
+ except Exception:
122
+ return _extract_with_thread_pool()
123
+ return _extract_with_thread_pool()
124
+
125
+
39
126
  def build_index(
40
127
  directory: Path,
41
128
  *,
@@ -46,6 +133,8 @@ def build_index(
46
133
  model_name: str,
47
134
  batch_size: int,
48
135
  embed_concurrency: int = DEFAULT_EMBED_CONCURRENCY,
136
+ extract_concurrency: int = DEFAULT_EXTRACT_CONCURRENCY,
137
+ extract_backend: str = DEFAULT_EXTRACT_BACKEND,
49
138
  provider: str,
50
139
  base_url: str | None,
51
140
  api_key: str | None,
@@ -71,6 +160,7 @@ def build_index(
71
160
  if not files:
72
161
  return IndexResult(status=IndexStatus.EMPTY)
73
162
  stat_cache: dict[Path, os.stat_result] = {}
163
+ extract_concurrency = _resolve_extract_concurrency(extract_concurrency)
74
164
 
75
165
  existing_meta = load_index_metadata_safe(
76
166
  directory,
@@ -111,6 +201,9 @@ def build_index(
111
201
  files=files,
112
202
  missing_rel_paths=missing_line_files,
113
203
  root=directory,
204
+ extract_concurrency=extract_concurrency,
205
+ extract_backend=extract_backend,
206
+ mode=mode,
114
207
  )
115
208
  cache_path = backfill_chunk_lines(
116
209
  root=directory,
@@ -169,7 +262,15 @@ def build_index(
169
262
  path for rel, path in files_with_rel if rel in changed_rel_paths
170
263
  ]
171
264
  changed_payloads = (
172
- strategy.payloads_for_files(changed_files) if changed_files else []
265
+ _payloads_for_files(
266
+ strategy,
267
+ changed_files,
268
+ mode=mode,
269
+ extract_concurrency=extract_concurrency,
270
+ extract_backend=extract_backend,
271
+ )
272
+ if changed_files
273
+ else []
173
274
  )
174
275
 
175
276
  cache_path = _apply_incremental_update(
@@ -199,6 +300,9 @@ def build_index(
199
300
  files=files,
200
301
  missing_rel_paths=line_backfill_targets,
201
302
  root=directory,
303
+ extract_concurrency=extract_concurrency,
304
+ extract_backend=extract_backend,
305
+ mode=mode,
202
306
  )
203
307
  cache_path = backfill_chunk_lines(
204
308
  root=directory,
@@ -217,7 +321,13 @@ def build_index(
217
321
  files_indexed=len(files),
218
322
  )
219
323
 
220
- payloads = strategy.payloads_for_files(files)
324
+ payloads = _payloads_for_files(
325
+ strategy,
326
+ files,
327
+ mode=mode,
328
+ extract_concurrency=extract_concurrency,
329
+ extract_backend=extract_backend,
330
+ )
221
331
  file_labels = [payload.label for payload in payloads]
222
332
  embeddings = _embed_labels_with_cache(
223
333
  searcher=searcher,
@@ -255,6 +365,8 @@ def build_index_in_memory(
255
365
  model_name: str,
256
366
  batch_size: int,
257
367
  embed_concurrency: int = DEFAULT_EMBED_CONCURRENCY,
368
+ extract_concurrency: int = DEFAULT_EXTRACT_CONCURRENCY,
369
+ extract_backend: str = DEFAULT_EXTRACT_BACKEND,
258
370
  provider: str,
259
371
  base_url: str | None,
260
372
  api_key: str | None,
@@ -307,7 +419,13 @@ def build_index_in_memory(
307
419
  api_key=api_key,
308
420
  local_cuda=local_cuda,
309
421
  )
310
- payloads = strategy.payloads_for_files(files)
422
+ payloads = _payloads_for_files(
423
+ strategy,
424
+ files,
425
+ mode=mode,
426
+ extract_concurrency=extract_concurrency,
427
+ extract_backend=extract_backend,
428
+ )
311
429
  if not payloads:
312
430
  empty = np.empty((0, 0), dtype=np.float32)
313
431
  metadata = {
@@ -809,6 +927,9 @@ def _build_line_backfill_updates(
809
927
  files: Sequence[Path],
810
928
  missing_rel_paths: set[str],
811
929
  root: Path,
930
+ extract_concurrency: int,
931
+ extract_backend: str,
932
+ mode: str,
812
933
  ) -> list[tuple[str, int, int | None, int | None]]:
813
934
  if not missing_rel_paths:
814
935
  return []
@@ -816,7 +937,13 @@ def _build_line_backfill_updates(
816
937
  targets = [files_by_rel[rel] for rel in missing_rel_paths if rel in files_by_rel]
817
938
  if not targets:
818
939
  return []
819
- payloads = strategy.payloads_for_files(targets)
940
+ payloads = _payloads_for_files(
941
+ strategy,
942
+ targets,
943
+ mode=mode,
944
+ extract_concurrency=extract_concurrency,
945
+ extract_backend=extract_backend,
946
+ )
820
947
  return [
821
948
  (
822
949
  _relative_to_root(payload.file, root),
@@ -7,12 +7,15 @@ from functools import lru_cache
7
7
  from pathlib import Path
8
8
  import json
9
9
  import re
10
+ import numpy as np
10
11
  from typing import Sequence, TYPE_CHECKING
11
12
  from urllib import error as urlerror
12
13
  from urllib import request as urlrequest
13
14
 
14
15
  from ..config import (
15
16
  DEFAULT_EMBED_CONCURRENCY,
17
+ DEFAULT_EXTRACT_BACKEND,
18
+ DEFAULT_EXTRACT_CONCURRENCY,
16
19
  DEFAULT_FLASHRANK_MAX_LENGTH,
17
20
  DEFAULT_FLASHRANK_MODEL,
18
21
  DEFAULT_RERANK,
@@ -48,6 +51,8 @@ class SearchRequest:
48
51
  temporary_index: bool = False
49
52
  no_cache: bool = False
50
53
  embed_concurrency: int = DEFAULT_EMBED_CONCURRENCY
54
+ extract_concurrency: int = DEFAULT_EXTRACT_CONCURRENCY
55
+ extract_backend: str = DEFAULT_EXTRACT_BACKEND
51
56
  rerank: str = DEFAULT_RERANK
52
57
  flashrank_model: str | None = None
53
58
  remote_rerank: RemoteRerankConfig | None = None
@@ -112,6 +117,15 @@ def _resolve_rerank_candidates(top_k: int) -> int:
112
117
  return max(20, min(candidate, 150))
113
118
 
114
119
 
120
+ def _top_indices(scores: np.ndarray, limit: int) -> list[int]:
121
+ if limit <= 0:
122
+ return []
123
+ if limit >= scores.size:
124
+ return sorted(range(scores.size), key=lambda idx: (-scores[idx], idx))
125
+ indices = np.argpartition(-scores, limit - 1)[:limit]
126
+ return sorted(indices.tolist(), key=lambda idx: (-scores[idx], idx))
127
+
128
+
115
129
  def _bm25_scores(
116
130
  query_tokens: Sequence[str],
117
131
  documents: Sequence[Sequence[str]],
@@ -349,6 +363,7 @@ def perform_search(request: SearchRequest) -> SearchResponse:
349
363
  from ..cache import ( # local import
350
364
  embedding_cache_key,
351
365
  list_cache_entries,
366
+ load_chunk_metadata,
352
367
  load_embedding_cache,
353
368
  load_index_vectors,
354
369
  load_query_vector,
@@ -385,6 +400,8 @@ def perform_search(request: SearchRequest) -> SearchResponse:
385
400
  model_name=request.model_name,
386
401
  batch_size=request.batch_size,
387
402
  embed_concurrency=request.embed_concurrency,
403
+ extract_concurrency=request.extract_concurrency,
404
+ extract_backend=request.extract_backend,
388
405
  provider=request.provider,
389
406
  base_url=request.base_url,
390
407
  api_key=request.api_key,
@@ -446,6 +463,7 @@ def perform_search(request: SearchRequest) -> SearchResponse:
446
463
 
447
464
  file_snapshot = metadata.get("files", [])
448
465
  chunk_entries = metadata.get("chunks", [])
466
+ chunk_ids = metadata.get("chunk_ids", [])
449
467
  stale = bool(file_snapshot) and not is_cache_current(
450
468
  request.directory,
451
469
  request.include_hidden,
@@ -466,6 +484,8 @@ def perform_search(request: SearchRequest) -> SearchResponse:
466
484
  model_name=request.model_name,
467
485
  batch_size=request.batch_size,
468
486
  embed_concurrency=request.embed_concurrency,
487
+ extract_concurrency=request.extract_concurrency,
488
+ extract_backend=request.extract_backend,
469
489
  provider=request.provider,
470
490
  base_url=request.base_url,
471
491
  api_key=request.api_key,
@@ -541,7 +561,6 @@ def perform_search(request: SearchRequest) -> SearchResponse:
541
561
  index_empty=True,
542
562
  )
543
563
 
544
- from sklearn.metrics.pairwise import cosine_similarity # local import
545
564
  from ..search import SearchResult, VexorSearcher # local import
546
565
  searcher = VexorSearcher(
547
566
  model_name=request.model_name,
@@ -595,13 +614,37 @@ def perform_search(request: SearchRequest) -> SearchResponse:
595
614
  store_query_vector(int(index_id), query_hash, request.query, query_vector)
596
615
  except Exception: # pragma: no cover - best-effort cache storage
597
616
  pass
598
- similarities = cosine_similarity(
599
- query_vector.reshape(1, -1),
600
- file_vectors,
601
- )[0]
602
- scored = []
603
- for idx, (path, score) in enumerate(zip(paths, similarities)):
604
- chunk_meta = chunk_entries[idx] if idx < len(chunk_entries) else {}
617
+ reranker = None
618
+ rerank = (request.rerank or DEFAULT_RERANK).strip().lower()
619
+ use_rerank = rerank in {"bm25", "flashrank", "remote"}
620
+ if use_rerank:
621
+ candidate_limit = _resolve_rerank_candidates(request.top_k)
622
+ else:
623
+ candidate_limit = request.top_k
624
+ candidate_count = min(len(paths), candidate_limit)
625
+
626
+ query_vector = np.asarray(query_vector, dtype=np.float32).ravel()
627
+ similarities = np.asarray(file_vectors @ query_vector, dtype=np.float32)
628
+ top_indices = _top_indices(similarities, candidate_count)
629
+ chunk_meta_by_id: dict[int, dict] = {}
630
+ if chunk_ids:
631
+ candidate_ids = [
632
+ chunk_ids[idx] for idx in top_indices if idx < len(chunk_ids)
633
+ ]
634
+ if candidate_ids:
635
+ try:
636
+ chunk_meta_by_id = load_chunk_metadata(candidate_ids)
637
+ except Exception: # pragma: no cover - best-effort metadata lookup
638
+ chunk_meta_by_id = {}
639
+ scored: list[SearchResult] = []
640
+ for idx in top_indices:
641
+ path = paths[idx]
642
+ score = similarities[idx]
643
+ chunk_meta = {}
644
+ if chunk_ids and idx < len(chunk_ids):
645
+ chunk_meta = chunk_meta_by_id.get(chunk_ids[idx], {})
646
+ elif idx < len(chunk_entries):
647
+ chunk_meta = chunk_entries[idx]
605
648
  start_line = chunk_meta.get("start_line")
606
649
  end_line = chunk_meta.get("end_line")
607
650
  scored.append(
@@ -614,12 +657,8 @@ def perform_search(request: SearchRequest) -> SearchResponse:
614
657
  end_line=int(end_line) if end_line is not None else None,
615
658
  )
616
659
  )
617
- scored.sort(key=lambda item: item.score, reverse=True)
618
- reranker = None
619
- rerank = (request.rerank or DEFAULT_RERANK).strip().lower()
620
- if rerank in {"bm25", "flashrank", "remote"}:
621
- candidate_count = min(len(scored), _resolve_rerank_candidates(request.top_k))
622
- candidates = scored[:candidate_count]
660
+ if use_rerank:
661
+ candidates = scored
623
662
  if rerank == "bm25":
624
663
  candidates = _apply_bm25_rerank(request.query, candidates)
625
664
  reranker = "bm25"
@@ -662,6 +701,8 @@ def _perform_search_with_temporary_index(request: SearchRequest) -> SearchRespon
662
701
  model_name=request.model_name,
663
702
  batch_size=request.batch_size,
664
703
  embed_concurrency=request.embed_concurrency,
704
+ extract_concurrency=request.extract_concurrency,
705
+ extract_backend=request.extract_backend,
665
706
  provider=request.provider,
666
707
  base_url=request.base_url,
667
708
  api_key=request.api_key,
@@ -680,7 +721,6 @@ def _perform_search_with_temporary_index(request: SearchRequest) -> SearchRespon
680
721
  index_empty=True,
681
722
  )
682
723
 
683
- from sklearn.metrics.pairwise import cosine_similarity # local import
684
724
  from ..search import SearchResult, VexorSearcher # local import
685
725
 
686
726
  searcher = VexorSearcher(
@@ -717,13 +757,23 @@ def _perform_search_with_temporary_index(request: SearchRequest) -> SearchRespon
717
757
  )
718
758
  except Exception: # pragma: no cover - best-effort cache storage
719
759
  pass
720
- similarities = cosine_similarity(
721
- query_vector.reshape(1, -1),
722
- file_vectors,
723
- )[0]
760
+ reranker = None
761
+ rerank = (request.rerank or DEFAULT_RERANK).strip().lower()
762
+ use_rerank = rerank in {"bm25", "flashrank", "remote"}
763
+ if use_rerank:
764
+ candidate_limit = _resolve_rerank_candidates(request.top_k)
765
+ else:
766
+ candidate_limit = request.top_k
767
+ candidate_count = min(len(paths), candidate_limit)
768
+
769
+ query_vector = np.asarray(query_vector, dtype=np.float32).ravel()
770
+ similarities = np.asarray(file_vectors @ query_vector, dtype=np.float32)
771
+ top_indices = _top_indices(similarities, candidate_count)
724
772
  chunk_entries = metadata.get("chunks", [])
725
- scored = []
726
- for idx, (path, score) in enumerate(zip(paths, similarities)):
773
+ scored: list[SearchResult] = []
774
+ for idx in top_indices:
775
+ path = paths[idx]
776
+ score = similarities[idx]
727
777
  chunk_meta = chunk_entries[idx] if idx < len(chunk_entries) else {}
728
778
  start_line = chunk_meta.get("start_line")
729
779
  end_line = chunk_meta.get("end_line")
@@ -737,12 +787,8 @@ def _perform_search_with_temporary_index(request: SearchRequest) -> SearchRespon
737
787
  end_line=int(end_line) if end_line is not None else None,
738
788
  )
739
789
  )
740
- scored.sort(key=lambda item: item.score, reverse=True)
741
- reranker = None
742
- rerank = (request.rerank or DEFAULT_RERANK).strip().lower()
743
- if rerank in {"bm25", "flashrank", "remote"}:
744
- candidate_count = min(len(scored), _resolve_rerank_candidates(request.top_k))
745
- candidates = scored[:candidate_count]
790
+ if use_rerank:
791
+ candidates = scored
746
792
  if rerank == "bm25":
747
793
  candidates = _apply_bm25_rerank(request.query, candidates)
748
794
  reranker = "bm25"
@@ -908,6 +954,7 @@ def _filter_index_by_extensions(
908
954
  ext_set = {ext.lower() for ext in extensions if ext}
909
955
  if not ext_set:
910
956
  return list(paths), file_vectors, metadata
957
+ chunk_ids = metadata.get("chunk_ids")
911
958
  keep_indices: list[int] = []
912
959
  filtered_paths: list[Path] = []
913
960
  for idx, path in enumerate(paths):
@@ -922,6 +969,8 @@ def _filter_index_by_extensions(
922
969
  ext_set,
923
970
  )
924
971
  filtered_metadata["chunks"] = []
972
+ if chunk_ids is not None:
973
+ filtered_metadata["chunk_ids"] = []
925
974
  return [], filtered_vectors, filtered_metadata
926
975
  filtered_vectors = file_vectors[keep_indices]
927
976
  chunk_entries = metadata.get("chunks", [])
@@ -934,6 +983,10 @@ def _filter_index_by_extensions(
934
983
  ext_set,
935
984
  )
936
985
  filtered_metadata["chunks"] = filtered_chunks
986
+ if chunk_ids is not None:
987
+ filtered_metadata["chunk_ids"] = [
988
+ chunk_ids[idx] for idx in keep_indices if idx < len(chunk_ids)
989
+ ]
937
990
  return filtered_paths, filtered_vectors, filtered_metadata
938
991
 
939
992
 
@@ -946,6 +999,7 @@ def _filter_index_by_exclude_patterns(
946
999
  ) -> tuple[list[Path], Sequence[Sequence[float]], dict]:
947
1000
  if exclude_spec is None:
948
1001
  return list(paths), file_vectors, metadata
1002
+ chunk_ids = metadata.get("chunk_ids")
949
1003
  keep_indices: list[int] = []
950
1004
  filtered_paths: list[Path] = []
951
1005
  root_resolved = root.resolve()
@@ -966,6 +1020,8 @@ def _filter_index_by_exclude_patterns(
966
1020
  exclude_spec,
967
1021
  )
968
1022
  filtered_metadata["chunks"] = []
1023
+ if chunk_ids is not None:
1024
+ filtered_metadata["chunk_ids"] = []
969
1025
  return [], filtered_vectors, filtered_metadata
970
1026
  filtered_vectors = file_vectors[keep_indices]
971
1027
  chunk_entries = metadata.get("chunks", [])
@@ -978,6 +1034,10 @@ def _filter_index_by_exclude_patterns(
978
1034
  exclude_spec,
979
1035
  )
980
1036
  filtered_metadata["chunks"] = filtered_chunks
1037
+ if chunk_ids is not None:
1038
+ filtered_metadata["chunk_ids"] = [
1039
+ chunk_ids[idx] for idx in keep_indices if idx < len(chunk_ids)
1040
+ ]
981
1041
  return filtered_paths, filtered_vectors, filtered_metadata
982
1042
 
983
1043
 
@@ -994,6 +1054,7 @@ def _filter_index_by_directory(
994
1054
  relative_dir = directory.resolve().relative_to(index_root.resolve())
995
1055
  except ValueError:
996
1056
  return list(paths), file_vectors, metadata
1057
+ chunk_ids = metadata.get("chunk_ids")
997
1058
  keep_indices: list[int] = []
998
1059
  filtered_paths: list[Path] = []
999
1060
  for idx, path in enumerate(paths):
@@ -1014,6 +1075,8 @@ def _filter_index_by_directory(
1014
1075
  recursive=recursive,
1015
1076
  )
1016
1077
  filtered_metadata["chunks"] = []
1078
+ if chunk_ids is not None:
1079
+ filtered_metadata["chunk_ids"] = []
1017
1080
  filtered_metadata["root"] = str(directory)
1018
1081
  return [], filtered_vectors, filtered_metadata
1019
1082
  filtered_vectors = file_vectors[keep_indices]
@@ -1028,6 +1091,10 @@ def _filter_index_by_directory(
1028
1091
  recursive=recursive,
1029
1092
  )
1030
1093
  filtered_metadata["chunks"] = filtered_chunks
1094
+ if chunk_ids is not None:
1095
+ filtered_metadata["chunk_ids"] = [
1096
+ chunk_ids[idx] for idx in keep_indices if idx < len(chunk_ids)
1097
+ ]
1031
1098
  filtered_metadata["root"] = str(directory)
1032
1099
  return filtered_paths, filtered_vectors, filtered_metadata
1033
1100
 
vexor/text.py CHANGED
@@ -59,6 +59,8 @@ class Messages:
59
59
  HELP_SET_MODEL = "Set the default embedding model."
60
60
  HELP_SET_BATCH = "Set the default batch size (0 = single request)."
61
61
  HELP_SET_EMBED_CONCURRENCY = "Set the number of concurrent embedding requests."
62
+ HELP_SET_EXTRACT_CONCURRENCY = "Set the number of concurrent file extraction workers."
63
+ HELP_SET_EXTRACT_BACKEND = "Set the extraction backend (auto, thread, process)."
62
64
  HELP_SET_PROVIDER = "Set the default embedding provider (e.g., gemini, openai, custom, or local)."
63
65
  HELP_SET_BASE_URL = "Override the provider's base URL (leave unset for official endpoints)."
64
66
  HELP_CLEAR_BASE_URL = "Remove the custom base URL override."
@@ -117,6 +119,10 @@ class Messages:
117
119
  ERROR_EMPTY_QUERY = "Query text must not be empty."
118
120
  ERROR_BATCH_NEGATIVE = "Batch size must be >= 0"
119
121
  ERROR_CONCURRENCY_INVALID = "Embedding concurrency must be >= 1"
122
+ ERROR_EXTRACT_CONCURRENCY_INVALID = "Extraction concurrency must be >= 1"
123
+ ERROR_EXTRACT_BACKEND_INVALID = (
124
+ "Unsupported extraction backend '{value}'. Allowed values: {allowed}."
125
+ )
120
126
  ERROR_MODE_INVALID = "Unsupported mode '{value}'. Allowed values: {allowed}."
121
127
  ERROR_PROVIDER_INVALID = "Unsupported provider '{value}'. Allowed values: {allowed}."
122
128
  ERROR_RERANK_INVALID = "Unsupported rerank value '{value}'. Allowed values: {allowed}."
@@ -266,6 +272,8 @@ class Messages:
266
272
  INFO_MODEL_SET = "Default model set to {value}."
267
273
  INFO_BATCH_SET = "Default batch size set to {value}."
268
274
  INFO_EMBED_CONCURRENCY_SET = "Embedding concurrency set to {value}."
275
+ INFO_EXTRACT_CONCURRENCY_SET = "Extraction concurrency set to {value}."
276
+ INFO_EXTRACT_BACKEND_SET = "Extraction backend set to {value}."
269
277
  INFO_PROVIDER_SET = "Default provider set to {value}."
270
278
  INFO_BASE_URL_SET = "Base URL override set to {value}."
271
279
  INFO_BASE_URL_CLEARED = "Base URL override cleared."
@@ -308,6 +316,8 @@ class Messages:
308
316
  "Default model: {model}\n"
309
317
  "Default batch size: {batch}\n"
310
318
  "Embedding concurrency: {concurrency}\n"
319
+ "Extract concurrency: {extract_concurrency}\n"
320
+ "Extract backend: {extract_backend}\n"
311
321
  "Auto index: {auto_index}\n"
312
322
  "Rerank: {rerank}\n"
313
323
  "{flashrank_line}"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: vexor
3
- Version: 0.20.0
3
+ Version: 0.21.0
4
4
  Summary: A vector-powered CLI for semantic search over files.
5
5
  Project-URL: Repository, https://github.com/scarletkc/vexor
6
6
  Author: scarletkc
@@ -150,13 +150,26 @@ for hit in response.results:
150
150
  By default it reads `~/.vexor/config.json`. For runtime config overrides, cache
151
151
  controls, and per-call options, see [`docs/api/python.md`](https://github.com/scarletkc/vexor/tree/main/docs/api/python.md).
152
152
 
153
+ ## AI Agent Skill
154
+
155
+ This repo includes a skill for AI agents to use Vexor effectively:
156
+
157
+ ```bash
158
+ vexor install --skills claude # Claude Code
159
+ vexor install --skills codex # Codex
160
+ ```
161
+
162
+ Skill source: [`plugins/vexor/skills/vexor-cli`](https://github.com/scarletkc/vexor/raw/refs/heads/main/plugins/vexor/skills/vexor-cli/SKILL.md)
163
+
153
164
  ## Configuration
154
165
 
155
166
  ```bash
156
167
  vexor config --set-provider openai # default; also supports gemini/custom/local
157
168
  vexor config --set-model text-embedding-3-small
158
169
  vexor config --set-batch-size 0 # 0 = single request
159
- vexor config --set-embed-concurrency 2 # parallel embedding requests
170
+ vexor config --set-embed-concurrency 4 # parallel embedding requests
171
+ vexor config --set-extract-concurrency 4 # parallel file extraction workers
172
+ vexor config --set-extract-backend auto # auto|thread|process (default: auto)
160
173
  vexor config --set-auto-index true # auto-index before search (default)
161
174
  vexor config --rerank bm25 # optional BM25 rerank for top-k results
162
175
  vexor config --rerank flashrank # FlashRank rerank (requires optional extra)
@@ -298,17 +311,6 @@ Re-running `vexor index` only re-embeds changed files; >50% changes trigger full
298
311
 
299
312
  Porcelain output fields: `rank`, `similarity`, `path`, `chunk_index`, `start_line`, `end_line`, `preview` (line fields are `-` when unavailable).
300
313
 
301
- ## AI Agent Skill
302
-
303
- This repo includes a skill for AI agents to use Vexor effectively:
304
-
305
- ```bash
306
- vexor install --skills claude # Claude Code
307
- vexor install --skills codex # Codex
308
- ```
309
-
310
- Skill source: [`plugins/vexor/skills/vexor-cli`](https://github.com/scarletkc/vexor/raw/refs/heads/main/plugins/vexor/skills/vexor-cli/SKILL.md)
311
-
312
314
  ## Documentation
313
315
 
314
316
  See [docs](https://github.com/scarletkc/vexor/tree/main/docs) for more details.
@@ -1,33 +1,33 @@
1
- vexor/__init__.py,sha256=DAhiDVmFBHG2bu_3wkHBS2OubxR3yjbqa1nFLjQ0-Uw,441
1
+ vexor/__init__.py,sha256=i0ly8cFA4N_PEQ_rhYgoLp2NPRQc3_ln8Gfi8QWjXSQ,441
2
2
  vexor/__main__.py,sha256=ZFzom1wCfP6TPXe3aoDFpNcUgjbCZ7Quy_vfzNsH5Fw,426
3
- vexor/api.py,sha256=84GxMt4laq9bpfesPJSFUzkTTCrCpIDDRpVLXKGZ-rg,10470
4
- vexor/cache.py,sha256=B1seuKU0eYLNvFi7Lpy_X13cSvBfhsuQeH4rZ7Hc29Y,45106
5
- vexor/cli.py,sha256=hnANtRGO5ypEftMyuTmlZhttSuBZy9ivxymQK11gZ9c,65736
6
- vexor/config.py,sha256=f5Wom1yUzScp52xpdhLlCG6x7ZEgtFV7kQlarxvD9hU,15372
3
+ vexor/api.py,sha256=YCHpiydbPbRJUqdQYrpwe1JrRI-w_7LRuyZDGBP1_d4,11506
4
+ vexor/cache.py,sha256=3i9FKFLSyZ1kx-w1apc12umPaQxWqMP-P8_lvo67hBw,52832
5
+ vexor/cli.py,sha256=M9GKdD_mJ068Zpm62znTp0KhhKp1dkh_WHmfJHR9hwU,68094
6
+ vexor/config.py,sha256=CiPfEH7Ilt6XepEx4p02qfW5HfkpNDBjhEMyckbSWaA,17413
7
7
  vexor/modes.py,sha256=N_wAWoqbxmCfko-v520p59tpAYvUwraCSSQRtMaF4ac,11549
8
8
  vexor/output.py,sha256=iooZgLlK8dh7ajJ4XMHUNNx0qyTVtD_OAAwrBx5MeqE,864
9
9
  vexor/search.py,sha256=MSU4RmH6waFYOofkIdo8_ElTiz1oNaKuvr-3umif7Bs,6826
10
- vexor/text.py,sha256=ntqwx2hP5QtlUXnIvBh1NFSn8cxRVhvYtFb7aMt_Tus,24171
10
+ vexor/text.py,sha256=2aK5nJHkosmbmyzp9o_Tzb3YlmVnju_IX8BcEPUdhTA,24794
11
11
  vexor/utils.py,sha256=GzfYW2rz1-EuJjkevqZVe8flLRtrQ60OWMmFNbMh62k,12472
12
12
  vexor/providers/__init__.py,sha256=kCEoV03TSLKcxDUYVNjXnrVoLU5NpfNXjp1w1Ak2imE,92
13
- vexor/providers/gemini.py,sha256=-bKSubZRELefJmZzclepXNSWUPsXo94EAM9l0JtfbFM,3739
13
+ vexor/providers/gemini.py,sha256=IWHHjCMJC0hUHQPhuaJ_L_97c_mnOXkPkCVdrIR6z-g,5705
14
14
  vexor/providers/local.py,sha256=5X_WYCXgyBGIVvvVLgMnDjTkPR4GBF0ksNPyviBlB7w,4838
15
- vexor/providers/openai.py,sha256=KylfxVxoTRHrK7KHwMotgD7_fq-CLhx43MQGeGz2dfo,3388
15
+ vexor/providers/openai.py,sha256=YnJDY9gJW7RfGGdkgswVHvmOKNvgLRQUsbpA1MUuLPg,5356
16
16
  vexor/services/__init__.py,sha256=dA_i2N03vlYmbZbEK2knzJLWviunkNWbzN2LWPNvMk0,160
17
17
  vexor/services/cache_service.py,sha256=ywt6AgupCJ7_wC3je4znCMw5_VBouw3skbDTAt8xw6o,1639
18
- vexor/services/config_service.py,sha256=yJTBbOmxpbzskHPuLlxYXQ-COJC6-qKtvMsSfuneJoA,4471
18
+ vexor/services/config_service.py,sha256=PojolfbSKh9pW8slF4qxCOs9hz5L6xvjf_nB7vfVlsU,5039
19
19
  vexor/services/content_extract_service.py,sha256=zdhLxpNv70BU7irLf3Uc0ou9rKSvdjtrDcHkgRKlMn4,26421
20
- vexor/services/index_service.py,sha256=pteAG-eRA8FJmDc4GEwhHXGZE8Dm5L8uqzBB0Y8Rrgo,28312
20
+ vexor/services/index_service.py,sha256=FXf1bBoqj4-K1l38ItxHf6Oh7QHVIdNAdVY2kg_Zoq8,32265
21
21
  vexor/services/init_service.py,sha256=3D04hylGA9FRQhLHCfR95nMko3vb5MNBcRb9nWWaUE8,26863
22
22
  vexor/services/js_parser.py,sha256=eRtW6KlK4JBYDGbyoecHVqLZ0hcx-Cc0kx6bOujHPAQ,16254
23
23
  vexor/services/keyword_service.py,sha256=vmke8tII9kTwRDdBaLHBc6Hpy_B3p98L65iGkCQgtMU,2211
24
- vexor/services/search_service.py,sha256=_3WMzHNV0MCGWFXqwYCQ-XF08aJwF9L4mOGGnmXOATs,36076
24
+ vexor/services/search_service.py,sha256=K7SiAuMA7bGeyPWOHPMKpFFvzzkj5kHWwa3p94NakJs,38663
25
25
  vexor/services/skill_service.py,sha256=Rrgt3OMsKPPiXOiRhSNAWjBM9UNz9qmSWQe3uYGzq4M,4863
26
26
  vexor/services/system_service.py,sha256=KPlv83v3rTvBiNiH7vrp6tDmt_AqHxuUd-5RI0TfvWs,24638
27
27
  vexor/_bundled_skills/vexor-cli/SKILL.md,sha256=m3FlyqgHBdRwyGPEp8PrUS21K0G2jEl88tRvhSPta08,2798
28
28
  vexor/_bundled_skills/vexor-cli/references/install-vexor.md,sha256=IUBShLI1mAxugwUIMAJQ5_j6KcaPWfobe0gSd6MWU7w,1245
29
- vexor-0.20.0.dist-info/METADATA,sha256=RBTym4NL38S6OjijOg5fWPW9VmD7AuLEEqjZtQMkZqA,13331
30
- vexor-0.20.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
31
- vexor-0.20.0.dist-info/entry_points.txt,sha256=dvxp6Q1R1d6bozR7TwmpdJ0X_v83MkzsLPagGY_lfr0,40
32
- vexor-0.20.0.dist-info/licenses/LICENSE,sha256=wP7TAKRll1t9LoYGxWS9NikPM_0hCc00LmlLyvQBsL8,1066
33
- vexor-0.20.0.dist-info/RECORD,,
29
+ vexor-0.21.0.dist-info/METADATA,sha256=Lc5PHY_Ir3F56ILYe6IBlkwhN6gMQGZvf48f7x_uVDg,13494
30
+ vexor-0.21.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
31
+ vexor-0.21.0.dist-info/entry_points.txt,sha256=dvxp6Q1R1d6bozR7TwmpdJ0X_v83MkzsLPagGY_lfr0,40
32
+ vexor-0.21.0.dist-info/licenses/LICENSE,sha256=wP7TAKRll1t9LoYGxWS9NikPM_0hCc00LmlLyvQBsL8,1066
33
+ vexor-0.21.0.dist-info/RECORD,,
File without changes