vexor 0.22.0__py3-none-any.whl → 0.23.0rc1__py3-none-any.whl

vexor/services/search_service.py

@@ -56,6 +56,7 @@ class SearchRequest:
     rerank: str = DEFAULT_RERANK
     flashrank_model: str | None = None
     remote_rerank: RemoteRerankConfig | None = None
+    embedding_dimensions: int | None = None
 
 
 @dataclass(slots=True)
@@ -409,6 +410,7 @@ def perform_search(request: SearchRequest) -> SearchResponse:
             exclude_patterns=request.exclude_patterns,
             extensions=request.extensions,
             no_cache=request.no_cache,
+            embedding_dimensions=request.embedding_dimensions,
         )
         if result.status == IndexStatus.EMPTY:
             return SearchResponse(
@@ -493,6 +495,7 @@ def perform_search(request: SearchRequest) -> SearchResponse:
             exclude_patterns=index_excludes,
             extensions=index_extensions,
             no_cache=request.no_cache,
+            embedding_dimensions=request.embedding_dimensions,
         )
         if result.status == IndexStatus.EMPTY:
             return SearchResponse(
@@ -570,6 +573,7 @@ def perform_search(request: SearchRequest) -> SearchResponse:
         base_url=request.base_url,
         api_key=request.api_key,
         local_cuda=request.local_cuda,
+        embedding_dimensions=request.embedding_dimensions,
     )
     query_vector = None
     query_hash = None
@@ -586,8 +590,10 @@ def perform_search(request: SearchRequest) -> SearchResponse:
             query_vector = None
 
     if query_vector is None and not request.no_cache:
-        query_text_hash = embedding_cache_key(request.query)
-        cached = load_embedding_cache(request.model_name, [query_text_hash])
+        query_text_hash = embedding_cache_key(request.query, dimension=request.embedding_dimensions)
+        cached = load_embedding_cache(
+            request.model_name, [query_text_hash], dimension=request.embedding_dimensions
+        )
         query_vector = cached.get(query_text_hash)
         if query_vector is not None and query_vector.size != file_vectors.shape[1]:
             query_vector = None
@@ -596,11 +602,12 @@ def perform_search(request: SearchRequest) -> SearchResponse:
         query_vector = searcher.embed_texts([request.query])[0]
         if not request.no_cache:
             if query_text_hash is None:
-                query_text_hash = embedding_cache_key(request.query)
+                query_text_hash = embedding_cache_key(request.query, dimension=request.embedding_dimensions)
             try:
                 store_embedding_cache(
                     model=request.model_name,
                     embeddings={query_text_hash: query_vector},
+                    dimension=request.embedding_dimensions,
                 )
             except Exception:  # pragma: no cover - best-effort cache storage
                 pass
@@ -624,6 +631,18 @@ def perform_search(request: SearchRequest) -> SearchResponse:
     candidate_count = min(len(paths), candidate_limit)
 
     query_vector = np.asarray(query_vector, dtype=np.float32).ravel()
+
+    # Validate dimension compatibility between query and index
+    index_dimension = file_vectors.shape[1] if file_vectors.ndim == 2 else 0
+    query_dimension = query_vector.shape[0]
+    if index_dimension != query_dimension:
+        raise ValueError(
+            f"Embedding dimension mismatch: index has {index_dimension}-dim vectors, "
+            f"but query embedding is {query_dimension}-dim. "
+            f"This typically happens when embedding_dimensions was changed after building the index. "
+            f"Rebuild the index with: vexor index {request.directory}"
+        )
+
     similarities = np.asarray(file_vectors @ query_vector, dtype=np.float32)
     top_indices = _top_indices(similarities, candidate_count)
     chunk_meta_by_id: dict[int, dict] = {}
@@ -718,14 +737,17 @@ def search_from_vectors(
         base_url=request.base_url,
         api_key=request.api_key,
         local_cuda=request.local_cuda,
+        embedding_dimensions=request.embedding_dimensions,
     )
     query_vector = None
     query_text_hash = None
     if not request.no_cache:
         from ..cache import embedding_cache_key, load_embedding_cache, store_embedding_cache
 
-        query_text_hash = embedding_cache_key(request.query)
-        cached = load_embedding_cache(request.model_name, [query_text_hash])
+        query_text_hash = embedding_cache_key(request.query, dimension=request.embedding_dimensions)
+        cached = load_embedding_cache(
+            request.model_name, [query_text_hash], dimension=request.embedding_dimensions
+        )
         query_vector = cached.get(query_text_hash)
         if query_vector is not None and query_vector.size != file_vectors.shape[1]:
             query_vector = None
@@ -736,11 +758,12 @@ def search_from_vectors(
             if query_text_hash is None:
                 from ..cache import embedding_cache_key, store_embedding_cache
 
-                query_text_hash = embedding_cache_key(request.query)
+                query_text_hash = embedding_cache_key(request.query, dimension=request.embedding_dimensions)
             try:
                 store_embedding_cache(
                     model=request.model_name,
                     embeddings={query_text_hash: query_vector},
+                    dimension=request.embedding_dimensions,
                 )
             except Exception:  # pragma: no cover - best-effort cache storage
                 pass
@@ -754,6 +777,18 @@ def search_from_vectors(
     candidate_count = min(len(paths), candidate_limit)
 
     query_vector = np.asarray(query_vector, dtype=np.float32).ravel()
+
+    # Validate dimension compatibility between query and index
+    index_dimension = file_vectors.shape[1] if file_vectors.ndim == 2 else 0
+    query_dimension = query_vector.shape[0]
+    if index_dimension != query_dimension:
+        raise ValueError(
+            f"Embedding dimension mismatch: index has {index_dimension}-dim vectors, "
+            f"but query embedding is {query_dimension}-dim. "
+            f"This typically happens when embedding_dimensions was changed after building the index. "
+            f"Rebuild the index with: vexor index {request.directory}"
+        )
+
     similarities = np.asarray(file_vectors @ query_vector, dtype=np.float32)
     top_indices = _top_indices(similarities, candidate_count)
     chunk_entries = metadata.get("chunks", [])
@@ -827,6 +862,7 @@ def _perform_search_with_temporary_index(request: SearchRequest) -> SearchRespon
         exclude_patterns=request.exclude_patterns,
         extensions=request.extensions,
         no_cache=request.no_cache,
+        embedding_dimensions=request.embedding_dimensions,
     )
     return search_from_vectors(
         request,
@@ -863,6 +899,18 @@ def _load_index_vectors_for_request(
             request.extensions,
             respect_gitignore=request.respect_gitignore,
         )
+        # Check dimension compatibility when user explicitly requests a specific dimension
+        cached_dimension = metadata.get("dimension")
+        requested_dimension = request.embedding_dimensions
+        if (
+            cached_dimension is not None
+            and requested_dimension is not None
+            and cached_dimension != requested_dimension
+        ):
+            raise FileNotFoundError(
+                f"Cached index has dimension {cached_dimension}, "
+                f"but requested {requested_dimension}"
+            )
         return (
             paths,
             file_vectors,
@@ -933,6 +981,15 @@ def _select_cache_superset(
             continue
         if entry.get("mode") != request.mode:
             continue
+        # Check embedding dimension compatibility when user explicitly requests a specific dimension
+        cached_dimension = entry.get("dimension")
+        requested_dimension = request.embedding_dimensions
+        if (
+            cached_dimension is not None
+            and requested_dimension is not None
+            and cached_dimension != requested_dimension
+        ):
+            continue
         cached_excludes = tuple(entry.get("exclude_patterns") or ())
         cached_exclude_set = set(normalize_exclude_patterns(cached_excludes))
         if requested_exclude_set:

vexor/text.py

@@ -61,7 +61,9 @@ class Messages:
     HELP_SET_EMBED_CONCURRENCY = "Set the number of concurrent embedding requests."
     HELP_SET_EXTRACT_CONCURRENCY = "Set the number of concurrent file extraction workers."
     HELP_SET_EXTRACT_BACKEND = "Set the extraction backend (auto, thread, process)."
-    HELP_SET_PROVIDER = "Set the default embedding provider (e.g., gemini, openai, custom, or local)."
+    HELP_SET_PROVIDER = (
+        "Set the default embedding provider (e.g., openai, gemini, voyageai, custom, or local)."
+    )
     HELP_SET_BASE_URL = "Override the provider's base URL (leave unset for official endpoints)."
     HELP_CLEAR_BASE_URL = "Remove the custom base URL override."
     HELP_SET_AUTO_INDEX = "Enable/disable automatic indexing before search (default: enabled)."
@@ -90,6 +92,11 @@ class Messages:
     HELP_LOCAL_CLEANUP = "Delete the local model cache stored under ~/.vexor/models."
     HELP_LOCAL_CUDA = "Enable CUDA for local embedding (requires onnxruntime-gpu)."
     HELP_LOCAL_CPU = "Disable CUDA and use CPU for local embedding."
+    HELP_SET_EMBEDDING_DIMENSIONS = (
+        "Set the embedding dimensions for providers that support it "
+        "(e.g., Voyage AI: 256, 512, 1024, 2048; OpenAI text-embedding-3: 256-3072)."
+    )
+    HELP_CLEAR_EMBEDDING_DIMENSIONS = "Clear the embedding dimensions setting (use model default)."
 
     ERROR_API_KEY_MISSING = (
         "API key is missing or still set to the placeholder. "
@@ -176,13 +183,16 @@ class Messages:
     INIT_OPTION_PROVIDER_OPENAI_DESC = "default"
     INIT_OPTION_PROVIDER_GEMINI = "Gemini"
     INIT_OPTION_PROVIDER_GEMINI_DESC = "Google AI"
+    INIT_OPTION_PROVIDER_VOYAGEAI = "Voyage AI"
+    INIT_OPTION_PROVIDER_VOYAGEAI_DESC = "embedding API"
     INIT_OPTION_PROVIDER_CUSTOM = "Custom"
     INIT_OPTION_PROVIDER_CUSTOM_DESC = "OpenAI-compatible"
-    INIT_PROMPT_PROVIDER = "Choose A/B/C"
+    INIT_PROMPT_PROVIDER = "Choose A/B/C/D"
     INIT_PROMPT_CUSTOM_BASE_URL = "Custom base URL"
     INIT_PROMPT_CUSTOM_MODEL = "Custom model name"
     INIT_PROMPT_API_KEY_OPENAI = "OpenAI API key (leave blank to use environment variables)"
     INIT_PROMPT_API_KEY_GEMINI = "Gemini API key (leave blank to use environment variables)"
+    INIT_PROMPT_API_KEY_VOYAGE = "Voyage API key (leave blank to use environment variables)"
     INIT_PROMPT_API_KEY_CUSTOM = "API key (leave blank to use environment variables)"
     INIT_CONFIRM_SKIP_API_KEY = "No API key found. Continue without it?"
     INIT_USING_ENV_API_KEY = "Using API key from environment."
@@ -293,6 +303,9 @@ class Messages:
     INFO_LOCAL_SETUP_DONE = "Local model ready: {model}. Provider set to local."
     INFO_LOCAL_CUDA_ENABLED = "Local embeddings will use CUDA."
     INFO_LOCAL_CUDA_DISABLED = "Local embeddings will use CPU."
+    INFO_EMBEDDING_DIMENSIONS_SET = "Embedding dimensions set to {value}."
+    INFO_EMBEDDING_DIMENSIONS_CLEARED = "Embedding dimensions cleared (using model default)."
+    ERROR_EMBEDDING_DIMENSIONS_INVALID = "Embedding dimensions must be a positive integer."
     INFO_FLASHRANK_SETUP_START = "Preparing FlashRank model..."
     INFO_FLASHRANK_SETUP_DONE = "FlashRank model ready."
     DOCTOR_LOCAL_CUDA_MISSING = "CUDA provider not available for local embeddings"
@@ -314,6 +327,7 @@ class Messages:
         "API key set: {api}\n"
         "Default provider: {provider}\n"
         "Default model: {model}\n"
+        "Embedding dimensions: {embedding_dimensions}\n"
         "Default batch size: {batch}\n"
         "Embedding concurrency: {concurrency}\n"
         "Extract concurrency: {extract_concurrency}\n"
@@ -350,7 +364,7 @@ class Messages:
     DOCTOR_API_KEY_MISSING = "API key not configured"
     DOCTOR_API_KEY_MISSING_DETAIL = (
         "Run `vexor config --set-api-key <KEY>` or set VEXOR_API_KEY / "
-        "OPENAI_API_KEY / GOOGLE_GENAI_API_KEY environment variable."
+        "OPENAI_API_KEY / GOOGLE_GENAI_API_KEY / VOYAGE_API_KEY environment variable."
     )
     DOCTOR_API_KEY_NOT_REQUIRED = "Local provider selected (no API key required)"
     DOCTOR_API_SKIPPED = "Skipped (no API key)"

{vexor-0.22.0.dist-info → vexor-0.23.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vexor
-Version: 0.22.0
+Version: 0.23.0rc1
 Summary: A vector-powered CLI for semantic search over files.
 Project-URL: Repository, https://github.com/scarletkc/vexor
 Author: scarletkc
@@ -22,7 +22,7 @@ Classifier: Topic :: Text Processing :: Indexing
 Classifier: Topic :: Utilities
 Requires-Python: >=3.9
 Requires-Dist: charset-normalizer>=3.3.0
-Requires-Dist: google-genai>=0.5.0
+Requires-Dist: google-genai>=1.57.0
 Requires-Dist: numpy>=1.23.0
 Requires-Dist: openai>=1.0.0
 Requires-Dist: pathspec>=0.12.1
@@ -64,6 +64,7 @@ Description-Content-Type: text/markdown
 [![CI](https://img.shields.io/github/actions/workflow/status/scarletkc/vexor/publish.yml?branch=main)](https://github.com/scarletkc/vexor/actions/workflows/publish.yml)
 [![Codecov](https://img.shields.io/codecov/c/github/scarletkc/vexor/main)](https://codecov.io/github/scarletkc/vexor)
 [![License](https://img.shields.io/github/license/scarletkc/vexor.svg)](https://github.com/scarletkc/vexor/blob/main/LICENSE)
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/scarletkc/vexor)
 
 </div>
 
@@ -171,12 +172,15 @@ Skill source: [`plugins/vexor/skills/vexor-cli`](https://github.com/scarletkc/ve
 ## Configuration
 
 ```bash
-vexor config --set-provider openai          # default; also supports gemini/custom/local
+vexor config --set-provider openai          # default; also supports gemini/voyageai/custom/local
 vexor config --set-model text-embedding-3-small
+vexor config --set-provider voyageai        # uses voyage defaults when model/base_url are unset
 vexor config --set-batch-size 0             # 0 = single request
 vexor config --set-embed-concurrency 4       # parallel embedding requests
 vexor config --set-extract-concurrency 4     # parallel file extraction workers
 vexor config --set-extract-backend auto      # auto|thread|process (default: auto)
+vexor config --set-embedding-dimensions 1024 # optional, model/provider dependent
+vexor config --clear-embedding-dimensions    # reset to model default dimension
 vexor config --set-auto-index true          # auto-index before search (default)
 vexor config --rerank bm25                  # optional BM25 rerank for top-k results
 vexor config --rerank flashrank             # FlashRank rerank (requires optional extra)
@@ -202,7 +206,7 @@ Config stored in `~/.vexor/config.json`.
 ```bash
 vexor config --set-api-key "YOUR_KEY"
 ```
-Or via environment: `VEXOR_API_KEY`, `OPENAI_API_KEY`, or `GOOGLE_GENAI_API_KEY`.
+Or via environment: `VEXOR_API_KEY`, `OPENAI_API_KEY`, `GOOGLE_GENAI_API_KEY`, or `VOYAGE_API_KEY`.
 
 ### Rerank
 
@@ -222,11 +226,30 @@ Recommended defaults:
 
 ### Providers: Remote vs Local
 
-Vexor supports both remote API providers (`openai`, `gemini`, `custom`) and a local provider (`local`):
+Vexor supports both remote API providers (`openai`, `gemini`, `voyageai`, `custom`) and a local provider (`local`):
 - Remote providers use `api_key` and optional `base_url`.
+- `voyageai` defaults to `https://api.voyageai.com/v1` when `base_url` is not set.
 - `custom` is OpenAI-compatible and requires both `model` and `base_url`.
 - Local provider ignores `api_key/base_url` and only uses `model` plus `local_cuda` (CPU/GPU switch).
 
+### Embedding Dimensions
+
+Embedding dimensions are optional. If unset, the provider/model default is used.
+Custom dimensions are validated for:
+- OpenAI `text-embedding-3-*`
+- Voyage `voyage-3*` and `voyage-code-3*`
+
+```bash
+vexor config --set-embedding-dimensions 1024
+vexor config --clear-embedding-dimensions
+```
+
+If you change dimensions after an index is built, rebuild the index:
+
+```bash
+vexor index --path .
+```
+
 ### Local Model (Offline)
 
 Install the lightweight local backend:

{vexor-0.22.0.dist-info → vexor-0.23.0rc1.dist-info}/RECORD

@@ -1,33 +1,33 @@
-vexor/__init__.py,sha256=EQXPbwsXfHJAK3mNlhYfc4UwVqOMqL52cO46xg1GNCo,632
+vexor/__init__.py,sha256=B7BRcp5NDsgB_zUnP_wghH_Orw5RZ-VTRWam3R7It6Q,635
 vexor/__main__.py,sha256=ZFzom1wCfP6TPXe3aoDFpNcUgjbCZ7Quy_vfzNsH5Fw,426
-vexor/api.py,sha256=W6eJLNbg5uBC2gcE8kq34iUciJCYZjY9Zsytxunl-vs,35860
-vexor/cache.py,sha256=irCGy5XIcRKX5EFk7plKDVqGHrTjRpaWgXnFDiVChXk,57323
-vexor/cli.py,sha256=M9GKdD_mJ068Zpm62znTp0KhhKp1dkh_WHmfJHR9hwU,68094
-vexor/config.py,sha256=CEL5u7afZV81Y0i9FVsj8GVzZlb2C2gC17r5lNKu-aM,18570
+vexor/api.py,sha256=VKTTKbp1I82eZ3LT-J6PI9WOoaKSF1cgGIPwq5w2BYE,38137
+vexor/cache.py,sha256=cjnYUWp0sYnlviSRMTopmQlepDEk8_4tlwf7SnwX_ck,58766
+vexor/cli.py,sha256=PCUIY4LF-xiifuH9D82CuLLq9tNs9uRNfAgcSap2gLw,70895
+vexor/config.py,sha256=DiXZiUsZ1hidIQ5LI-2Fik6qpGRmK6Eh7h0DugT6oCM,24303
 vexor/modes.py,sha256=N_wAWoqbxmCfko-v520p59tpAYvUwraCSSQRtMaF4ac,11549
 vexor/output.py,sha256=iooZgLlK8dh7ajJ4XMHUNNx0qyTVtD_OAAwrBx5MeqE,864
-vexor/search.py,sha256=MSU4RmH6waFYOofkIdo8_ElTiz1oNaKuvr-3umif7Bs,6826
-vexor/text.py,sha256=2aK5nJHkosmbmyzp9o_Tzb3YlmVnju_IX8BcEPUdhTA,24794
+vexor/search.py,sha256=fWzbTcRhWGaMkOwjWMGUUSoJLTvMm6472Yhij-VWeZE,7593
+vexor/text.py,sha256=VM7CQH8f1tqn_N19l4ycjAekVMxi002Y5Qsfp3EcDD0,25655
 vexor/utils.py,sha256=GzfYW2rz1-EuJjkevqZVe8flLRtrQ60OWMmFNbMh62k,12472
 vexor/providers/__init__.py,sha256=kCEoV03TSLKcxDUYVNjXnrVoLU5NpfNXjp1w1Ak2imE,92
 vexor/providers/gemini.py,sha256=IWHHjCMJC0hUHQPhuaJ_L_97c_mnOXkPkCVdrIR6z-g,5705
 vexor/providers/local.py,sha256=5X_WYCXgyBGIVvvVLgMnDjTkPR4GBF0ksNPyviBlB7w,4838
-vexor/providers/openai.py,sha256=YnJDY9gJW7RfGGdkgswVHvmOKNvgLRQUsbpA1MUuLPg,5356
+vexor/providers/openai.py,sha256=uGI2qYc8BixxL58s6yQHr7OuocBhpLG4m93C4Y2JYYg,5994
 vexor/services/__init__.py,sha256=dA_i2N03vlYmbZbEK2knzJLWviunkNWbzN2LWPNvMk0,160
 vexor/services/cache_service.py,sha256=ywt6AgupCJ7_wC3je4znCMw5_VBouw3skbDTAt8xw6o,1639
-vexor/services/config_service.py,sha256=PojolfbSKh9pW8slF4qxCOs9hz5L6xvjf_nB7vfVlsU,5039
+vexor/services/config_service.py,sha256=uU-jd-H18GW6R0-AJSoCXFYJ1vRqf28YzwZRjn1-S9E,6159
 vexor/services/content_extract_service.py,sha256=oO7Hbadwp3uiyqCbr_4MRXQsUeMix2D98i-Yp94PwFk,26495
-vexor/services/index_service.py,sha256=FXf1bBoqj4-K1l38ItxHf6Oh7QHVIdNAdVY2kg_Zoq8,32265
-vexor/services/init_service.py,sha256=3D04hylGA9FRQhLHCfR95nMko3vb5MNBcRb9nWWaUE8,26863
+vexor/services/index_service.py,sha256=iyIP8ZcqjwjPbQOcGiJLYTFMIsX1CxswhHC9v6c-W00,34940
+vexor/services/init_service.py,sha256=9foGfQqLK-iomEjH820yMue3AQeveLJzQtaQA4nvYRo,27217
 vexor/services/js_parser.py,sha256=eRtW6KlK4JBYDGbyoecHVqLZ0hcx-Cc0kx6bOujHPAQ,16254
 vexor/services/keyword_service.py,sha256=vmke8tII9kTwRDdBaLHBc6Hpy_B3p98L65iGkCQgtMU,2211
-vexor/services/search_service.py,sha256=-7qHfebMOmXWiVBVYoBji6eaZr8OOY3q1vbtJaY3I6E,39076
+vexor/services/search_service.py,sha256=HsN0QpGc7yWIj-xbuQjwUZp8PbQKGgR4_0aJTI_tsR0,41948
 vexor/services/skill_service.py,sha256=Rrgt3OMsKPPiXOiRhSNAWjBM9UNz9qmSWQe3uYGzq4M,4863
 vexor/services/system_service.py,sha256=KPlv83v3rTvBiNiH7vrp6tDmt_AqHxuUd-5RI0TfvWs,24638
 vexor/_bundled_skills/vexor-cli/SKILL.md,sha256=m3FlyqgHBdRwyGPEp8PrUS21K0G2jEl88tRvhSPta08,2798
 vexor/_bundled_skills/vexor-cli/references/install-vexor.md,sha256=IUBShLI1mAxugwUIMAJQ5_j6KcaPWfobe0gSd6MWU7w,1245
-vexor-0.22.0.dist-info/METADATA,sha256=UAqD6ciQCaP3eBrOGA3unO_XLX0eArcYHfcXuRhjN8c,14154
-vexor-0.22.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-vexor-0.22.0.dist-info/entry_points.txt,sha256=dvxp6Q1R1d6bozR7TwmpdJ0X_v83MkzsLPagGY_lfr0,40
-vexor-0.22.0.dist-info/licenses/LICENSE,sha256=wP7TAKRll1t9LoYGxWS9NikPM_0hCc00LmlLyvQBsL8,1066
-vexor-0.22.0.dist-info/RECORD,,
+vexor-0.23.0rc1.dist-info/METADATA,sha256=HRGdeudEBPih9In7KEm-kWCJNDOUcq_cmZOZlXsnTGc,15048
+vexor-0.23.0rc1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+vexor-0.23.0rc1.dist-info/entry_points.txt,sha256=dvxp6Q1R1d6bozR7TwmpdJ0X_v83MkzsLPagGY_lfr0,40
+vexor-0.23.0rc1.dist-info/licenses/LICENSE,sha256=wP7TAKRll1t9LoYGxWS9NikPM_0hCc00LmlLyvQBsL8,1066
+vexor-0.23.0rc1.dist-info/RECORD,,