vexor 0.22.0__py3-none-any.whl → 0.23.0rc1__py3-none-any.whl

vexor/config.py

@@ -23,6 +23,7 @@ _CONFIG_DIR_OVERRIDE: ContextVar[Path | None] = ContextVar(
 )
 DEFAULT_MODEL = "text-embedding-3-small"
 DEFAULT_GEMINI_MODEL = "gemini-embedding-001"
+DEFAULT_VOYAGE_MODEL = "voyage-3-large"
 DEFAULT_LOCAL_MODEL = "intfloat/multilingual-e5-small"
 DEFAULT_BATCH_SIZE = 64
 DEFAULT_EMBED_CONCURRENCY = 4
@@ -32,13 +33,22 @@ DEFAULT_PROVIDER = "openai"
 DEFAULT_RERANK = "off"
 DEFAULT_FLASHRANK_MODEL = "ms-marco-TinyBERT-L-2-v2"
 DEFAULT_FLASHRANK_MAX_LENGTH = 256
-SUPPORTED_PROVIDERS: tuple[str, ...] = (DEFAULT_PROVIDER, "gemini", "custom", "local")
+VOYAGE_BASE_URL = "https://api.voyageai.com/v1"
+SUPPORTED_PROVIDERS: tuple[str, ...] = (DEFAULT_PROVIDER, "gemini", "voyageai", "custom", "local")
 SUPPORTED_RERANKERS: tuple[str, ...] = ("off", "bm25", "flashrank", "remote")
 SUPPORTED_EXTRACT_BACKENDS: tuple[str, ...] = ("auto", "thread", "process")
+# Models that support the dimensions parameter (model prefix/name -> supported dimensions)
+DIMENSION_SUPPORTED_MODELS: dict[str, tuple[int, ...]] = {
+    "text-embedding-3-small": (256, 512, 1024, 1536),
+    "text-embedding-3-large": (256, 512, 1024, 1536, 3072),
+    "voyage-3": (256, 512, 1024, 2048),
+    "voyage-code-3": (256, 512, 1024, 2048),
+}
 ENV_API_KEY = "VEXOR_API_KEY"
 REMOTE_RERANK_ENV = "VEXOR_REMOTE_RERANK_API_KEY"
 LEGACY_GEMINI_ENV = "GOOGLE_GENAI_API_KEY"
 OPENAI_ENV = "OPENAI_API_KEY"
+VOYAGE_ENV = "VOYAGE_API_KEY"
 
 
 @dataclass
@@ -63,6 +73,7 @@ class Config:
     rerank: str = DEFAULT_RERANK
     flashrank_model: str | None = None
     remote_rerank: RemoteRerankConfig | None = None
+    embedding_dimensions: int | None = None
 
 
 def _parse_remote_rerank(raw: object) -> RemoteRerankConfig | None:
@@ -133,6 +144,7 @@ def load_config() -> Config:
         rerank=rerank,
         flashrank_model=raw.get("flashrank_model") or None,
         remote_rerank=_parse_remote_rerank(raw.get("remote_rerank")),
+        embedding_dimensions=_coerce_optional_int(raw.get("embedding_dimensions")),
     )
 
 
@@ -157,6 +169,8 @@ def save_config(config: Config) -> None:
     data["rerank"] = config.rerank
     if config.flashrank_model:
         data["flashrank_model"] = config.flashrank_model
+    if config.embedding_dimensions is not None:
+        data["embedding_dimensions"] = config.embedding_dimensions
     if config.remote_rerank is not None:
         remote_data: Dict[str, Any] = {}
         if config.remote_rerank.base_url:
@@ -223,9 +237,11 @@ def set_api_key(value: str | None) -> None:
     save_config(config)
 
 
-def set_model(value: str) -> None:
+def set_model(value: str, *, validate_embedding_dimensions: bool = True) -> None:
     config = load_config()
     config.model = value
+    if validate_embedding_dimensions:
+        _validate_config_embedding_dimensions(config)
     save_config(config)
 
 
@@ -253,9 +269,11 @@ def set_extract_backend(value: str) -> None:
     save_config(config)
 
 
-def set_provider(value: str) -> None:
+def set_provider(value: str, *, validate_embedding_dimensions: bool = True) -> None:
     config = load_config()
     config.provider = value
+    if validate_embedding_dimensions:
+        _validate_config_embedding_dimensions(config)
     save_config(config)
 
 
@@ -293,6 +311,43 @@ def set_flashrank_model(value: str | None) -> None:
     save_config(config)
 
 
+def set_embedding_dimensions(
+    value: int | None,
+    model: str | None = None,
+    provider: str | None = None,
+) -> None:
+    """Set the embedding dimensions for providers that support it (e.g., Voyage AI).
+
+    Args:
+        value: The dimension to set, or None/0 to clear
+        model: Optional model to validate against. If not provided, uses config model.
+        provider: Optional provider to resolve effective model. If not provided, uses config provider.
+
+    Raises:
+        ValueError: If value is negative, model doesn't support dimensions,
+                   or dimension is not valid for the model.
+    """
+    config = load_config()
+
+    # Reject negative values explicitly
+    if value is not None and value < 0:
+        raise ValueError(f"embedding_dimensions must be non-negative, got {value}")
+
+    # Treat 0 and None as "clear"
+    if not value or value <= 0:
+        config.embedding_dimensions = None
+        save_config(config)
+        return
+
+    # Validate against effective model (resolved from provider + model)
+    effective_provider = provider if provider else config.provider
+    effective_model = resolve_default_model(effective_provider, model if model else config.model)
+    validate_embedding_dimensions_for_model(value, effective_model)
+
+    config.embedding_dimensions = value
+    save_config(config)
+
+
 def update_remote_rerank(
     *,
     base_url: str | None = None,
@@ -345,11 +400,72 @@ def resolve_default_model(provider: str | None, model: str | None) -> str:
     normalized = (provider or DEFAULT_PROVIDER).lower()
     if normalized == "gemini" and (not clean_model or clean_model == DEFAULT_MODEL):
         return DEFAULT_GEMINI_MODEL
+    if normalized == "voyageai" and (not clean_model or clean_model == DEFAULT_MODEL):
+        return DEFAULT_VOYAGE_MODEL
     if clean_model:
         return clean_model
     return DEFAULT_MODEL
 
 
+def resolve_base_url(provider: str | None, configured_url: str | None) -> str | None:
+    """Return the effective base URL for the selected provider."""
+    if configured_url:
+        return configured_url
+    normalized = (provider or DEFAULT_PROVIDER).lower()
+    if normalized == "voyageai":
+        return VOYAGE_BASE_URL
+    return None
+
+
+def supports_dimensions(model: str) -> bool:
+    """Check if a model supports the dimensions parameter."""
+    return get_supported_dimensions(model) is not None
+
+
+def get_supported_dimensions(model: str) -> tuple[int, ...] | None:
+    """Return the supported dimensions for a model, or None if not supported."""
+    model_lower = model.lower()
+    for prefix, dims in DIMENSION_SUPPORTED_MODELS.items():
+        if model_lower.startswith(prefix):
+            return dims
+    return None
+
+
+def validate_embedding_dimensions_for_model(value: int | None, model: str) -> None:
+    """Validate that `value` is supported by `model` when value is set."""
+    if value is None:
+        return
+    supported = get_supported_dimensions(model)
+    if not supported:
+        raise ValueError(
+            f"Model '{model}' does not support custom dimensions. "
+            f"Supported model names/prefixes: {', '.join(DIMENSION_SUPPORTED_MODELS.keys())}"
+        )
+    if value not in supported:
+        raise ValueError(
+            f"Dimension {value} is not supported for model '{model}'. "
+            f"Supported dimensions: {supported}"
+        )
+
+
+def _validate_config_embedding_dimensions(config: Config) -> None:
+    """Ensure stored embedding dimensions remain compatible with provider/model."""
+    if config.embedding_dimensions is None:
+        return
+    effective_model = resolve_default_model(config.provider, config.model)
+    try:
+        validate_embedding_dimensions_for_model(
+            config.embedding_dimensions,
+            effective_model,
+        )
+    except ValueError as exc:
+        raise ValueError(
+            f"Current embedding_dimensions ({config.embedding_dimensions}) is incompatible with "
+            f"model '{effective_model}'. Clear it with "
+            "`vexor config --clear-embedding-dimensions` or set a supported value."
+        ) from exc
+
+
 def resolve_api_key(configured: str | None, provider: str) -> str | None:
     """Return the first available API key from config or environment."""
 
@@ -365,6 +481,10 @@ def resolve_api_key(configured: str | None, provider: str) -> str | None:
         legacy = os.getenv(LEGACY_GEMINI_ENV)
         if legacy:
             return legacy
+    if normalized == "voyageai":
+        voyage_key = os.getenv(VOYAGE_ENV)
+        if voyage_key:
+            return voyage_key
     if normalized in {"openai", "custom"}:
         openai_key = os.getenv(OPENAI_ENV)
         if openai_key:
@@ -422,6 +542,7 @@ def _clone_config(config: Config) -> Config:
                 model=remote.model,
             )
         ),
+        embedding_dimensions=config.embedding_dimensions,
     )
 
 
@@ -466,6 +587,8 @@ def _apply_config_payload(config: Config, payload: Mapping[str, object]) -> None
         )
     if "remote_rerank" in payload:
         config.remote_rerank = _coerce_remote_rerank(payload["remote_rerank"])
+    if "embedding_dimensions" in payload:
+        config.embedding_dimensions = _coerce_optional_int(payload["embedding_dimensions"])
 
 
 def _coerce_optional_str(value: object, field: str) -> str | None:
@@ -522,6 +645,30 @@ def _coerce_bool(value: object, field: str) -> bool:
     raise ValueError(Messages.ERROR_CONFIG_VALUE_INVALID.format(field=field))
 
 
+def _coerce_optional_int(value: object) -> int | None:
+    """Coerce a value to an optional integer, returning None for empty/null values."""
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value if value > 0 else None
+    if isinstance(value, float):
+        if value.is_integer() and value > 0:
+            return int(value)
+        return None
+    if isinstance(value, str):
+        cleaned = value.strip()
+        if not cleaned:
+            return None
+        try:
+            parsed = int(cleaned)
+            return parsed if parsed > 0 else None
+        except ValueError:
+            return None
+    return None
+
+
 def _normalize_extract_backend(value: object) -> str:
     if value is None:
         return DEFAULT_EXTRACT_BACKEND

vexor/providers/openai.py

@@ -24,12 +24,14 @@ class OpenAIEmbeddingBackend:
         chunk_size: int | None = None,
         concurrency: int = 1,
         base_url: str | None = None,
+        dimensions: int | None = None,
     ) -> None:
         load_dotenv()
         self.model_name = model_name
         self.chunk_size = chunk_size if chunk_size and chunk_size > 0 else None
         self.concurrency = max(int(concurrency or 1), 1)
         self.api_key = api_key
+        self.dimensions = dimensions if dimensions and dimensions > 0 else None
         if not self.api_key:
             raise RuntimeError(Messages.ERROR_API_KEY_MISSING)
         client_kwargs: dict[str, object] = {"api_key": self.api_key}
@@ -73,10 +75,18 @@ class OpenAIEmbeddingBackend:
         attempt = 0
         while True:
             try:
-                response = self._client.embeddings.create(
-                    model=self.model_name,
-                    input=list(batch),
-                )
+                create_kwargs: dict[str, object] = {
+                    "model": self.model_name,
+                    "input": list(batch),
+                }
+                if self.dimensions is not None:
+                    # Voyage AI uses output_dimension, OpenAI uses dimensions
+                    if self.model_name.startswith("voyage"):
+                        # Pass Voyage-specific params via extra_body
+                        create_kwargs["extra_body"] = {"output_dimension": self.dimensions}
+                    else:
+                        create_kwargs["dimensions"] = self.dimensions
+                response = self._client.embeddings.create(**create_kwargs)
                 break
             except Exception as exc:  # pragma: no cover - API client variations
                 if _should_retry_openai_error(exc) and attempt < _MAX_RETRIES:

vexor/search.py

@@ -15,6 +15,7 @@ from .config import (
     DEFAULT_PROVIDER,
     SUPPORTED_PROVIDERS,
     resolve_api_key,
+    resolve_base_url,
 )
 from .providers.gemini import GeminiEmbeddingBackend
 from .providers.local import LocalEmbeddingBackend
@@ -56,14 +57,16 @@ class VexorSearcher:
         base_url: str | None = None,
         api_key: str | None = None,
         local_cuda: bool = False,
+        embedding_dimensions: int | None = None,
     ) -> None:
         self.model_name = model_name
         self.batch_size = max(batch_size, 0)
         self.embed_concurrency = max(int(embed_concurrency or 1), 1)
         self.provider = (provider or DEFAULT_PROVIDER).lower()
-        self.base_url = base_url
+        self.base_url = resolve_base_url(self.provider, base_url)
         self.api_key = resolve_api_key(api_key, self.provider)
         self.local_cuda = bool(local_cuda)
+        self.embedding_dimensions = embedding_dimensions if embedding_dimensions and embedding_dimensions > 0 else None
         if backend is not None:
             self._backend = backend
             self._device = getattr(backend, "device", "Custom embedding backend")
@@ -142,6 +145,16 @@ class VexorSearcher:
                 concurrency=self.embed_concurrency,
                 cuda=self.local_cuda,
             )
+        if self.provider == "voyageai":
+            self._device = f"{self.model_name} via Voyage AI API"
+            return OpenAIEmbeddingBackend(
+                model_name=self.model_name,
+                chunk_size=self.batch_size,
+                concurrency=self.embed_concurrency,
+                base_url=self.base_url,
+                api_key=self.api_key,
+                dimensions=self.embedding_dimensions,
+            )
         if self.provider == "custom":
             base_url = (self.base_url or "").strip()
             if not base_url:
@@ -155,6 +168,7 @@ class VexorSearcher:
                 concurrency=self.embed_concurrency,
                 base_url=base_url,
                 api_key=self.api_key,
+                dimensions=self.embedding_dimensions,
             )
         if self.provider == "openai":
             self._device = f"{self.model_name} via OpenAI API"
@@ -164,6 +178,7 @@ class VexorSearcher:
                 concurrency=self.embed_concurrency,
                 base_url=self.base_url,
                 api_key=self.api_key,
+                dimensions=self.embedding_dimensions,
             )
         allowed = ", ".join(SUPPORTED_PROVIDERS)
         raise RuntimeError(

vexor/services/config_service.py

@@ -11,6 +11,7 @@ from ..config import (
     set_base_url,
     set_batch_size,
     set_embed_concurrency,
+    set_embedding_dimensions,
     set_extract_concurrency,
     set_extract_backend,
     set_auto_index,
@@ -43,6 +44,8 @@ class ConfigUpdateResult:
     remote_rerank_model_set: bool = False
     remote_rerank_api_key_set: bool = False
     remote_rerank_cleared: bool = False
+    embedding_dimensions_set: bool = False
+    embedding_dimensions_cleared: bool = False
 
     @property
     def changed(self) -> bool:
@@ -66,6 +69,8 @@ class ConfigUpdateResult:
                 self.remote_rerank_model_set,
                 self.remote_rerank_api_key_set,
                 self.remote_rerank_cleared,
+                self.embedding_dimensions_set,
+                self.embedding_dimensions_cleared,
             )
         )
 
@@ -90,6 +95,8 @@ def apply_config_updates(
     remote_rerank_model: str | None = None,
     remote_rerank_api_key: str | None = None,
     clear_remote_rerank: bool = False,
+    embedding_dimensions: int | None = None,
+    clear_embedding_dimensions: bool = False,
 ) -> ConfigUpdateResult:
     """Apply config mutations and report which fields were updated."""
 
@@ -101,7 +108,12 @@ def apply_config_updates(
         set_api_key(None)
         result.api_key_cleared = True
     if model is not None:
-        set_model(model)
+        set_model(
+            model,
+            validate_embedding_dimensions=not (
+                embedding_dimensions is not None or clear_embedding_dimensions
+            ),
+        )
         result.model_set = True
     if batch_size is not None:
         set_batch_size(batch_size)
@@ -116,7 +128,12 @@ def apply_config_updates(
         set_extract_backend(extract_backend)
         result.extract_backend_set = True
     if provider is not None:
-        set_provider(provider)
+        set_provider(
+            provider,
+            validate_embedding_dimensions=not (
+                embedding_dimensions is not None or clear_embedding_dimensions
+            ),
+        )
         result.provider_set = True
     if base_url is not None:
         set_base_url(base_url)
@@ -152,6 +169,17 @@ def apply_config_updates(
         result.remote_rerank_model_set = remote_rerank_model is not None
         result.remote_rerank_api_key_set = remote_rerank_api_key is not None
         result.remote_rerank_cleared = clear_remote_rerank
+    if embedding_dimensions is not None:
+        if embedding_dimensions > 0:
+            set_embedding_dimensions(embedding_dimensions)
+            result.embedding_dimensions_set = True
+        else:
+            set_embedding_dimensions(None)
+            result.embedding_dimensions_cleared = True
+    if clear_embedding_dimensions:
+        if not result.embedding_dimensions_cleared:
+            set_embedding_dimensions(None)
+        result.embedding_dimensions_cleared = True
     return result
 
 

vexor/services/index_service.py

@@ -142,6 +142,7 @@ def build_index(
     exclude_patterns: Sequence[str] | None = None,
     extensions: Sequence[str] | None = None,
     no_cache: bool = False,
+    embedding_dimensions: int | None = None,
 ) -> IndexResult:
     """Create or refresh the cached index for *directory*."""
 
@@ -183,8 +184,24 @@ def build_index(
         base_url=base_url,
         api_key=api_key,
         local_cuda=local_cuda,
+        embedding_dimensions=embedding_dimensions,
     )
 
+    # Check if dimensions changed - if so, force full rebuild with no embedding cache
+    # Only detect mismatch when user explicitly requests a specific dimension that differs
+    force_no_cache = False
+    if cached_files:
+        cached_dimension = existing_meta.get("dimension") if existing_meta else None
+        dimension_changed = (
+            cached_dimension is not None
+            and embedding_dimensions is not None
+            and cached_dimension != embedding_dimensions
+        )
+        if dimension_changed:
+            # Dimensions changed, need full rebuild without embedding cache
+            # (cached embeddings have wrong dimensions)
+            cached_files = []
+            force_no_cache = True
     if cached_files:
         cached_version = int(existing_meta.get("version", 0) or 0) if existing_meta else 0
         full_max_bytes = (
@@ -291,6 +308,8 @@ def build_index(
                 extensions=extensions,
                 stat_cache=stat_cache,
                 no_cache=no_cache,
+                embedding_dimensions=embedding_dimensions,
+                cached_index_dimension=existing_meta.get("dimension") if existing_meta else None,
             )
 
             line_backfill_targets = missing_line_files - changed_rel_paths - removed_rel_paths
@@ -333,7 +352,8 @@ def build_index(
         searcher=searcher,
         model_name=model_name,
         labels=file_labels,
-        no_cache=no_cache,
+        no_cache=no_cache or force_no_cache,
+        embedding_dimension=embedding_dimensions,
     )
     entries = _build_index_entries(payloads, embeddings, directory, stat_cache=stat_cache)
 
@@ -374,6 +394,7 @@ def build_index_in_memory(
     exclude_patterns: Sequence[str] | None = None,
     extensions: Sequence[str] | None = None,
     no_cache: bool = False,
+    embedding_dimensions: int | None = None,
 ) -> tuple[list[Path], np.ndarray, dict]:
     """Build an index in memory without writing to disk."""
 
@@ -418,6 +439,7 @@ def build_index_in_memory(
         base_url=base_url,
         api_key=api_key,
         local_cuda=local_cuda,
+        embedding_dimensions=embedding_dimensions,
     )
     payloads = _payloads_for_files(
         strategy,
@@ -455,6 +477,7 @@ def build_index_in_memory(
             searcher=searcher,
             model_name=model_name,
             labels=labels,
+            embedding_dimension=embedding_dimensions,
         )
     entries = _build_index_entries(
         payloads,
@@ -634,6 +657,8 @@ def _apply_incremental_update(
     extensions: Sequence[str] | None,
     stat_cache: MutableMapping[Path, os.stat_result] | None = None,
     no_cache: bool = False,
+    embedding_dimensions: int | None = None,
+    cached_index_dimension: int | None = None,
 ) -> Path:
     payloads_to_embed, payloads_to_touch = _split_payloads_by_label(
         changed_payloads,
@@ -655,7 +680,20 @@ def _apply_incremental_update(
             model_name=model_name,
             labels=labels,
             no_cache=no_cache,
+            embedding_dimension=embedding_dimensions,
         )
+
+        # Validate dimension compatibility with existing index
+        if cached_index_dimension is not None and embeddings.size > 0:
+            new_dimension = embeddings.shape[1] if embeddings.ndim == 2 else 0
+            if new_dimension != cached_index_dimension:
+                raise ValueError(
+                    f"Embedding dimension mismatch: existing index has {cached_index_dimension}-dim vectors, "
+                    f"but new embeddings are {new_dimension}-dim. "
+                    f"This typically happens when embedding_dimensions config was changed. "
+                    f"Clear the index and rebuild: vexor index --clear {directory}"
+                )
+
         changed_entries = _build_index_entries(
             payloads_to_embed,
             embeddings,
@@ -693,7 +731,18 @@ def _embed_labels_with_cache(
     model_name: str,
     labels: Sequence[str],
     no_cache: bool = False,
+    embedding_dimension: int | None = None,
 ) -> np.ndarray:
+    """Embed labels with caching support.
+
+    Args:
+        searcher: The embedding searcher instance
+        model_name: Name of the embedding model
+        labels: Sequence of label strings to embed
+        no_cache: If True, bypass cache entirely
+        embedding_dimension: Embedding dimension for cache segmentation (prevents
+            cross-dimension cache pollution when dimension settings change)
+    """
     if not labels:
         return np.empty((0, 0), dtype=np.float32)
     if no_cache:
@@ -701,8 +750,9 @@ def _embed_labels_with_cache(
         return np.asarray(vectors, dtype=np.float32)
     from ..cache import embedding_cache_key, load_embedding_cache, store_embedding_cache
 
-    hashes = [embedding_cache_key(label) for label in labels]
-    cached = load_embedding_cache(model_name, hashes)
+    # Include dimension in cache key to prevent cross-dimension cache pollution
+    hashes = [embedding_cache_key(label, dimension=embedding_dimension) for label in labels]
+    cached = load_embedding_cache(model_name, hashes, dimension=embedding_dimension)
     missing: dict[str, str] = {}
     for label, text_hash in zip(labels, hashes):
         vector = cached.get(text_hash)
@@ -719,7 +769,9 @@ def _embed_labels_with_cache(
             vector = np.asarray(new_vectors[idx], dtype=np.float32)
             cached[text_hash] = vector
             stored[text_hash] = vector
-        store_embedding_cache(model=model_name, embeddings=stored)
+        store_embedding_cache(
+            model=model_name, embeddings=stored, dimension=embedding_dimension
+        )
 
     vectors = [cached[text_hash] for text_hash in hashes]
     return np.vstack([np.asarray(vector, dtype=np.float32) for vector in vectors])

vexor/services/init_service.py

@@ -231,6 +231,11 @@ def _collect_remote_settings() -> dict[str, object]:
     )
     _print_option(
         "C",
+        Messages.INIT_OPTION_PROVIDER_VOYAGEAI,
+        Messages.INIT_OPTION_PROVIDER_VOYAGEAI_DESC,
+    )
+    _print_option(
+        "D",
         Messages.INIT_OPTION_PROVIDER_CUSTOM,
         Messages.INIT_OPTION_PROVIDER_CUSTOM_DESC,
     )
@@ -242,11 +247,14 @@ def _collect_remote_settings() -> dict[str, object]:
             "openai": "openai",
             "b": "gemini",
             "gemini": "gemini",
-            "c": "custom",
+            "c": "voyageai",
+            "voyageai": "voyageai",
+            "voyage": "voyageai",
+            "d": "custom",
             "custom": "custom",
         },
         default="A",
-        allowed="A/B/C",
+        allowed="A/B/C/D",
     )
     console.print()
 
@@ -266,6 +274,8 @@ def _collect_remote_settings() -> dict[str, object]:
 
     if provider == "gemini":
         api_key = _prompt_api_key(Messages.INIT_PROMPT_API_KEY_GEMINI, provider)
+    elif provider == "voyageai":
+        api_key = _prompt_api_key(Messages.INIT_PROMPT_API_KEY_VOYAGE, provider)
     else:
         api_key = _prompt_api_key(Messages.INIT_PROMPT_API_KEY_OPENAI, provider)
     updates["api_key"] = api_key