vortexa 0.3.1__tar.gz → 0.3.2__tar.gz

{vortexa-0.3.1 → vortexa-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vortexa
-Version: 0.3.1
+Version: 0.3.2
 Summary: Codebase indexing and semantic search engine
 Author-email: VortexAI <koulabhay25@gmail.com>
 License-Expression: Apache-2.0
@@ -29,7 +29,6 @@ Requires-Dist: huggingface-hub>=0.20.0
 Requires-Dist: tokenizers>=0.19.0
 Requires-Dist: safetensors>=0.4.0
 Requires-Dist: fastmcp>=2.0.0
-Requires-Dist: model2vec>=0.3.0
 Requires-Dist: sentence-transformers>=2.2.0
 Requires-Dist: tree-sitter-language-pack>=0.1.0
 Dynamic: license-file
@@ -375,11 +374,8 @@ indexer = CodebaseIndexer(root="/path/to/project", model_id="VTXAI/vtx-embed-7M"
 
 ### Alternative Embedders
 
-The `embedding.py` module also provides alternative embedders that can be passed directly:
-
-- `Model2VecEmbedder` — static embeddings via Model2Vec (`AI4free/JARVIS-tool-search-v1`)
-- `SentenceTransformerEmbedder` — Transformer-based dense embeddings (`all-MiniLM-L6-v2`)
-- `LF4Embedder` — 4-bit quantized static embeddings (`VTXAI/Vortex-Embed-4.7M`)
+The `embedding.py` module also provides `SentenceTransformerEmbedder`
+for transformer-based models (requires the `sentence-transformers` package).
 
 ---
 

{vortexa-0.3.1 → vortexa-0.3.2}/README.md

@@ -339,11 +339,8 @@ indexer = CodebaseIndexer(root="/path/to/project", model_id="VTXAI/vtx-embed-7M"
 
 ### Alternative Embedders
 
-The `embedding.py` module also provides alternative embedders that can be passed directly:
-
-- `Model2VecEmbedder` — static embeddings via Model2Vec (`AI4free/JARVIS-tool-search-v1`)
-- `SentenceTransformerEmbedder` — Transformer-based dense embeddings (`all-MiniLM-L6-v2`)
-- `LF4Embedder` — 4-bit quantized static embeddings (`VTXAI/Vortex-Embed-4.7M`)
+The `embedding.py` module also provides `SentenceTransformerEmbedder`
+for transformer-based models (requires the `sentence-transformers` package).
 
 ---
 

{vortexa-0.3.1 → vortexa-0.3.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vortexa"
-version = "0.3.1"
+version = "0.3.2"
 description = "Codebase indexing and semantic search engine"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -35,7 +35,6 @@ dependencies = [
     "tokenizers>=0.19.0",
     "safetensors>=0.4.0",
     "fastmcp>=2.0.0",
-    "model2vec>=0.3.0",
     "sentence-transformers>=2.2.0",
     "tree-sitter-language-pack>=0.1.0",
 ]

vortexa-0.3.2/src/vortexa/core/embedding.py

@@ -0,0 +1,88 @@
+"""Embedding model abstraction for the codebase indexer.
+
+Provides lazy-loading, thread-safe embedders with memoization support.
+Inspired by cocoindex's SentenceTransformerEmbedder pattern.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from typing import Protocol, runtime_checkable
+
+import numpy as np
+import numpy.typing as npt
+
+logger = logging.getLogger(__name__)
+
+
+@runtime_checkable
+class Embedder(Protocol):
+    """Protocol for embedding models used by the indexer."""
+
+    @property
+    def dim(self) -> int:
+        """Embedding dimensionality."""
+        ...
+
+    def embed(self, text: str) -> npt.NDArray[np.float32]:
+        """Embed a single text string."""
+        ...
+
+    def embed_batch(self, texts: list[str]) -> npt.NDArray[np.float32]:
+        """Embed a batch of text strings."""
+        ...
+
+    @property
+    def memo_key(self) -> tuple:
+        """Identity key for memoization cache invalidation."""
+        ...
+
+
+class SentenceTransformerEmbedder:
+    """Thread-safe embedder wrapping sentence-transformers.
+
+    Supports any sentence-transformers model with lazy loading.
+    Memo key includes model name and device for cache invalidation.
+    """
+
+    def __init__(self, model_name: str = "all-MiniLM-L6-v2", device: str | None = None) -> None:
+        self._model_name = model_name
+        self._device = device
+        self._model = None
+        self._lock = threading.Lock()
+
+    @property
+    def dim(self) -> int:
+        self._ensure_loaded()
+        assert self._model is not None
+        dim = self._model.get_embedding_dimension()
+        assert dim is not None
+        return dim
+
+    def _ensure_loaded(self) -> None:
+        if self._model is None:
+            with self._lock:
+                if self._model is None:
+                    from sentence_transformers import SentenceTransformer
+                    logger.info("Loading sentence-transformers model: %s", self._model_name)
+                    self._model = SentenceTransformer(self._model_name, device=self._device)
+
+    def embed(self, text: str) -> npt.NDArray[np.float32]:
+        """Embed a single text string."""
+        self._ensure_loaded()
+        assert self._model is not None
+        return self._model.encode(text, convert_to_numpy=True, normalize_embeddings=True)
+
+    def embed_batch(self, texts: list[str]) -> npt.NDArray[np.float32]:
+        """Embed a batch of text strings."""
+        if not texts:
+            return np.empty((0, 0), dtype=np.float32)
+        self._ensure_loaded()
+        assert self._model is not None
+        return self._model.encode(texts, convert_to_numpy=True, normalize_embeddings=True, show_progress_bar=False)
+
+    @property
+    def memo_key(self) -> tuple:
+        """Identity key: (class, model_name, device)."""
+        return ("SentenceTransformerEmbedder", self._model_name, self._device)

{vortexa-0.3.1 → vortexa-0.3.2}/src/vortexa/core/indexer.py

@@ -21,7 +21,7 @@ from typing import cast
 import lmdb
 import numpy as np
 from vortexa.core.chunking import chunk_source
-from vortexa.core.embedding import Embedder, LF4Embedder
+from vortexa.core.embedding import Embedder
 from vortexa.core.language import detect_language, get_extensions
 from vortexa.core.types import (
     Chunk,

{vortexa-0.3.1 → vortexa-0.3.2}/src/vortexa.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vortexa
-Version: 0.3.1
+Version: 0.3.2
 Summary: Codebase indexing and semantic search engine
 Author-email: VortexAI <koulabhay25@gmail.com>
 License-Expression: Apache-2.0
@@ -29,7 +29,6 @@ Requires-Dist: huggingface-hub>=0.20.0
 Requires-Dist: tokenizers>=0.19.0
 Requires-Dist: safetensors>=0.4.0
 Requires-Dist: fastmcp>=2.0.0
-Requires-Dist: model2vec>=0.3.0
 Requires-Dist: sentence-transformers>=2.2.0
 Requires-Dist: tree-sitter-language-pack>=0.1.0
 Dynamic: license-file
@@ -375,11 +374,8 @@ indexer = CodebaseIndexer(root="/path/to/project", model_id="VTXAI/vtx-embed-7M"
 
 ### Alternative Embedders
 
-The `embedding.py` module also provides alternative embedders that can be passed directly:
-
-- `Model2VecEmbedder` — static embeddings via Model2Vec (`AI4free/JARVIS-tool-search-v1`)
-- `SentenceTransformerEmbedder` — Transformer-based dense embeddings (`all-MiniLM-L6-v2`)
-- `LF4Embedder` — 4-bit quantized static embeddings (`VTXAI/Vortex-Embed-4.7M`)
+The `embedding.py` module also provides `SentenceTransformerEmbedder`
+for transformer-based models (requires the `sentence-transformers` package).
 
 ---
 

{vortexa-0.3.1 → vortexa-0.3.2}/src/vortexa.egg-info/SOURCES.txt

@@ -16,7 +16,6 @@ src/vortexa/core/graph.py
 src/vortexa/core/indexer.py
 src/vortexa/core/inference.py
 src/vortexa/core/language.py
-src/vortexa/core/lf4_model.py
 src/vortexa/core/lf4_v4_model.py
 src/vortexa/core/types.py
 src/vortexa/core/v4_embedder.py

{vortexa-0.3.1 → vortexa-0.3.2}/src/vortexa.egg-info/requires.txt

@@ -6,6 +6,5 @@ huggingface-hub>=0.20.0
 tokenizers>=0.19.0
 safetensors>=0.4.0
 fastmcp>=2.0.0
-model2vec>=0.3.0
 sentence-transformers>=2.2.0
 tree-sitter-language-pack>=0.1.0

vortexa-0.3.1/src/vortexa/core/embedding.py

@@ -1,180 +0,0 @@
-"""Embedding model abstraction for the codebase indexer.
-
-Provides lazy-loading, thread-safe embedders with memoization support.
-Inspired by cocoindex's SentenceTransformerEmbedder pattern.
-"""
-
-from __future__ import annotations
-
-import logging
-import threading
-from typing import Protocol, runtime_checkable
-
-import numpy as np
-import numpy.typing as npt
-
-logger = logging.getLogger(__name__)
-
-
-@runtime_checkable
-class Embedder(Protocol):
-    """Protocol for embedding models used by the indexer."""
-
-    @property
-    def dim(self) -> int:
-        """Embedding dimensionality."""
-        ...
-
-    def embed(self, text: str) -> npt.NDArray[np.float32]:
-        """Embed a single text string."""
-        ...
-
-    def embed_batch(self, texts: list[str]) -> npt.NDArray[np.float32]:
-        """Embed a batch of text strings."""
-        ...
-
-    @property
-    def memo_key(self) -> tuple:
-        """Identity key for memoization cache invalidation."""
-        ...
-
-
-class Model2VecEmbedder:
-    """Thread-safe, lazy-loading embedder wrapping model2vec.StaticModel.
-
-    The model is loaded on first use and cached. Thread-safe via a lock.
-    Memo key includes the model ID for cache invalidation.
-    """
-
-    def __init__(self, model_id: str = "AI4free/JARVIS-tool-search-v1") -> None:
-        self._model_id = model_id
-        self._model = None
-        self._lock = threading.Lock()
-
-    @property
-    def dim(self) -> int:
-        self._ensure_loaded()
-        assert self._model is not None
-        return self._model.dim
-
-    def _ensure_loaded(self) -> None:
-        if self._model is None:
-            with self._lock:
-                if self._model is None:  # Double-checked locking
-                    from model2vec import StaticModel
-                    logger.info("Loading embedding model: %s", self._model_id)
-                    self._model = StaticModel.from_pretrained(self._model_id)
-
-    def embed(self, text: str) -> npt.NDArray[np.float32]:
-        """Embed a single text string."""
-        self._ensure_loaded()
-        assert self._model is not None
-        return self._model.encode([text])[0]
-
-    def embed_batch(self, texts: list[str]) -> npt.NDArray[np.float32]:
-        """Embed a batch of text strings."""
-        if not texts:
-            return np.empty((0, 0), dtype=np.float32)
-        self._ensure_loaded()
-        assert self._model is not None
-        result = self._model.encode(texts)
-        return np.array(result, dtype=np.float32)
-
-    @property
-    def memo_key(self) -> tuple:
-        """Identity key: (class, model_id)."""
-        return ("Model2VecEmbedder", self._model_id)
-
-
-class SentenceTransformerEmbedder:
-    """Thread-safe embedder wrapping sentence-transformers.
-
-    Supports any sentence-transformers model with lazy loading.
-    Memo key includes model name and device for cache invalidation.
-    """
-
-    def __init__(self, model_name: str = "all-MiniLM-L6-v2", device: str | None = None) -> None:
-        self._model_name = model_name
-        self._device = device
-        self._model = None
-        self._lock = threading.Lock()
-
-    @property
-    def dim(self) -> int:
-        self._ensure_loaded()
-        assert self._model is not None
-        dim = self._model.get_embedding_dimension()
-        assert dim is not None
-        return dim
-
-    def _ensure_loaded(self) -> None:
-        if self._model is None:
-            with self._lock:
-                if self._model is None:
-                    from sentence_transformers import SentenceTransformer
-                    logger.info("Loading sentence-transformers model: %s", self._model_name)
-                    self._model = SentenceTransformer(self._model_name, device=self._device)
-
-    def embed(self, text: str) -> npt.NDArray[np.float32]:
-        """Embed a single text string."""
-        self._ensure_loaded()
-        assert self._model is not None
-        return self._model.encode(text, convert_to_numpy=True, normalize_embeddings=True)
-
-    def embed_batch(self, texts: list[str]) -> npt.NDArray[np.float32]:
-        """Embed a batch of text strings."""
-        if not texts:
-            return np.empty((0, 0), dtype=np.float32)
-        self._ensure_loaded()
-        assert self._model is not None
-        return self._model.encode(texts, convert_to_numpy=True, normalize_embeddings=True, show_progress_bar=False)
-
-    @property
-    def memo_key(self) -> tuple:
-        """Identity key: (class, model_name, device)."""
-        return ("SentenceTransformerEmbedder", self._model_name, self._device)
-
-
-class LF4Embedder:
-    """Thread-safe, lazy-loading embedder wrapping LF4StaticEmbedding (4-bit quantized).
-
-    Uses the VTXAI/Vortex-Embed-4.7M model by default — a 4-bit static embedding
-    model with ~3.5 MB footprint. Loads on first use, cached thereafter.
-    """
-
-    def __init__(self, model_id: str = "VTXAI/Vortex-Embed-4.7M") -> None:
-        self._model_id = model_id
-        self._model = None
-        self._lock = threading.Lock()
-
-    @property
-    def dim(self) -> int:
-        self._ensure_loaded()
-        assert self._model is not None
-        return self._model.dim
-
-    def _ensure_loaded(self) -> None:
-        if self._model is None:
-            with self._lock:
-                if self._model is None:
-                    logger.info("Loading LF4 embedding model: %s", self._model_id)
-                    from vortexa.core.lf4_model import LF4StaticEmbedding
-                    self._model = LF4StaticEmbedding.from_pretrained(self._model_id)
-
-    def embed(self, text: str) -> npt.NDArray[np.float32]:
-        """Embed a single text string."""
-        self._ensure_loaded()
-        assert self._model is not None
-        return self._model.encode([text])[0]
-
-    def embed_batch(self, texts: list[str]) -> npt.NDArray[np.float32]:
-        """Embed a batch of text strings."""
-        if not texts:
-            return np.empty((0, 0), dtype=np.float32)
-        self._ensure_loaded()
-        assert self._model is not None
-        return self._model.encode(texts)
-
-    @property
-    def memo_key(self) -> tuple:
-        return ("LF4Embedder", self._model_id)

vortexa-0.3.1/src/vortexa/core/lf4_model.py

@@ -1,168 +0,0 @@
-"""LF4 Static Embedding Model - Native 4-bit quantized sentence embeddings.
-
-Usage:
-    from lf4_model import LF4StaticEmbedding
-    model = LF4StaticEmbedding.from_pretrained("VTXAI/Vortex-Embed-4.7M")
-    embeddings = model.encode(["find python json parser", "weather API tool"])
-
-    # Search
-    scores, indices = model.search(query_emb, index_emb, top_k=10)
-"""
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-import numpy as np
-
-
-class LF4StaticEmbedding:
-    """Native LF4 4-bit static embedding model.
-
-    Weights are stored as packed 4-bit integers with per-block FP16 scales/zeros.
-    Total model size: ~3.5 MB (vs 29 MB FP32).
-    """
-
-    def __init__(self, packed, scales, zeros, tokenizer_data, config):
-        self.packed = packed  # uint8 (vocab, dim/2)
-        self.scales = scales  # float16 (vocab, num_blocks)
-        self.zeros = zeros  # float16 (vocab, num_blocks)
-        self.config = config
-        self.vocab_size = config["vocab_size"]
-        self.dim = config["embedding_dim"]
-        self.block_size = config["block_size"]
-        self._tokenizer_data = tokenizer_data
-        self._tokenizer = None
-
-        self._embedding_table = self._dequantize_all()
-
-    def _dequantize_all(self) -> np.ndarray:
-        """Dequantize full embedding table to FP32 for fast token lookup."""
-        N = self.packed.shape[0]
-        D = self.dim
-        B = self.block_size
-
-        low = (self.packed & 0x0F).astype(np.float32)
-        high = ((self.packed >> 4) & 0x0F).astype(np.float32)
-        D_padded = self.packed.shape[1] * 2
-
-        unpacked = np.empty((N, D_padded), dtype=np.float32)
-        unpacked[:, 0::2] = low
-        unpacked[:, 1::2] = high
-
-        num_blocks = D_padded // B
-        blocked = unpacked.reshape(N, num_blocks, B)
-        s = self.scales.astype(np.float32)[:, :, None]
-        z = self.zeros.astype(np.float32)[:, :, None]
-
-        return (blocked * s + z).reshape(N, D_padded)[:, :D]
-
-    @property
-    def tokenizer(self):
-        if self._tokenizer is None:
-            from tokenizers import Tokenizer
-            self._tokenizer = Tokenizer.from_str(self._tokenizer_data)
-        return self._tokenizer
-
-    @classmethod
-    def from_pretrained(cls, path_or_id: str) -> LF4StaticEmbedding:
-        """Load model from local path or HuggingFace Hub."""
-        p = Path(path_or_id)
-        if p.is_dir():
-            model_path = str(p / "model.safetensors")
-            config_path = p / "config.json"
-            tok_path = str(p / "tokenizer.json")
-        else:
-            from huggingface_hub import hf_hub_download
-            model_path = hf_hub_download(path_or_id, "model.safetensors")
-            config_path = Path(hf_hub_download(path_or_id, "config.json"))
-            tok_path = hf_hub_download(path_or_id, "tokenizer.json")
-
-        from safetensors.numpy import load_file
-        tensors = load_file(model_path)
-        config = json.loads(config_path.read_text())
-
-        return cls(
-            packed=tensors["embedding_packed"],
-            scales=tensors["embedding_scales"],
-            zeros=tensors["embedding_zeros"],
-            tokenizer_data=Path(tok_path).read_text(encoding="utf-8"),
-            config=config,
-        )
-
-    def encode(self, texts: str | list[str], normalize: bool = True) -> np.ndarray:
-        """Encode texts to embeddings.
-
-        Args:
-            texts: single string or list of strings
-            normalize: L2-normalize output embeddings (default True for cosine sim)
-
-        Returns:
-            np.ndarray of shape (N, dim)
-        """
-        if isinstance(texts, str):
-            texts = [texts]
-
-        embeddings = np.zeros((len(texts), self.dim), dtype=np.float32)
-
-        for i, text in enumerate(texts):
-            encoded = self.tokenizer.encode(text)
-            token_ids = encoded.ids
-
-            valid_ids = [tid for tid in token_ids if 0 <= tid < self.vocab_size]
-            if valid_ids:
-                token_embs = self._embedding_table[valid_ids]
-                embeddings[i] = token_embs.mean(axis=0)
-
-        if normalize:
-            norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
-            norms = np.where(norms == 0, 1.0, norms)
-            embeddings = embeddings / norms
-
-        return embeddings
-
-    def search(
-        self,
-        queries: np.ndarray,
-        index: np.ndarray,
-        top_k: int = 10,
-    ) -> tuple[np.ndarray, np.ndarray]:
-        """Cosine similarity search.
-
-        Args:
-            queries: (Q, D) query embeddings
-            index: (N, D) document embeddings
-            top_k: number of results
-
-        Returns:
-            (scores, indices) arrays
-        """
-        queries = np.asarray(queries, dtype=np.float32)
-        index = np.asarray(index, dtype=np.float32)
-        if queries.ndim == 1:
-            queries = queries[None, :]
-
-        qn = queries / (np.linalg.norm(queries, axis=1, keepdims=True) + 1e-8)
-        dn = index / (np.linalg.norm(index, axis=1, keepdims=True) + 1e-8)
-
-        scores = qn @ dn.T
-
-        if top_k >= scores.shape[1]:
-            idx = np.argsort(-scores, axis=1)
-            return np.take_along_axis(scores, idx, 1), idx
-
-        idx = np.argpartition(-scores, top_k, axis=1)[:, :top_k]
-        s = np.take_along_axis(scores, idx, 1)
-        order = np.argsort(-s, axis=1)
-        return np.take_along_axis(s, order, 1), np.take_along_axis(idx, order, 1)
-
-    @property
-    def model_size_mb(self) -> float:
-        return (self.packed.nbytes + self.scales.nbytes + self.zeros.nbytes) / 1e6
-
-    def __repr__(self):
-        return (
-            f"LF4StaticEmbedding(vocab={self.vocab_size}, dim={self.dim}, "
-            f"bits=4, size={self.model_size_mb:.1f}MB, "
-            f"block_size={self.block_size})"
-        )