vecforge 0.2.0__py3-none-any.whl

vecforge/__init__.py

@@ -0,0 +1,59 @@
+# VecForge — Universal Local-First Vector Database
+# Copyright (c) 2026 Suneel Bose K · ArcGX TechLabs Private Limited
+# Built by Suneel Bose K (Founder & CEO, ArcGX TechLabs)
+#
+# Licensed under the Business Source License 1.1 (BSL 1.1)
+# Free for personal, research, open-source, and non-commercial use.
+# Commercial use requires a separate license from ArcGX TechLabs.
+# See LICENSE file in the project root or contact: suneelbose@arcgx.in
+
+"""
+VecForge — Forge your vector database. Own it forever.
+
+A universal, local-first Python vector database with enterprise security,
+multimodal ingestion, and optional quantum-inspired acceleration.
+
+Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+
+Quick Start::
+
+    from vecforge import VecForge
+
+    db = VecForge("my_vault")
+    db.add("Patient admitted with type 2 diabetes", metadata={"ward": "7"})
+    results = db.search("diabetic patient")
+    print(results[0].text)
+"""
+
+from __future__ import annotations
+
+from vecforge.core.vault import SearchResult, VecForge
+from vecforge.exceptions import (
+    DeletionProtectedError,
+    EncryptionKeyError,
+    IngestError,
+    InvalidAlphaError,
+    NamespaceNotFoundError,
+    VaultEmptyError,
+    VecForgeError,
+    VecForgePermissionError,
+)
+
+__all__ = [
+    "VecForge",
+    "SearchResult",
+    "VecForgeError",
+    "VaultEmptyError",
+    "NamespaceNotFoundError",
+    "VecForgePermissionError",
+    "InvalidAlphaError",
+    "EncryptionKeyError",
+    "DeletionProtectedError",
+    "IngestError",
+]
+
+__version__ = "0.2.0"
+__author__ = "Suneel Bose K"
+__company__ = "ArcGX TechLabs Private Limited"
+__license__ = "BSL-1.1"
+__copyright__ = "Copyright (c) 2026 Suneel Bose K · ArcGX TechLabs"

vecforge/cli/__init__.py

@@ -0,0 +1,3 @@
+# VecForge — Universal Local-First Vector Database
+# Copyright (c) 2026 Suneel Bose K · ArcGX TechLabs Private Limited
+# Licensed under BSL 1.1 — see LICENSE for details.

vecforge/cli/main.py

@@ -0,0 +1,197 @@
+# VecForge — Universal Local-First Vector Database
+# Copyright (c) 2026 Suneel Bose K · ArcGX TechLabs Private Limited
+# Built by Suneel Bose K (Founder & CEO, ArcGX TechLabs)
+#
+# Licensed under the Business Source License 1.1 (BSL 1.1)
+# Free for personal, research, open-source, and non-commercial use.
+# Commercial use requires a separate license from ArcGX TechLabs.
+# See LICENSE file in the project root or contact: suneelbose@arcgx.in
+
+"""
+VecForge CLI — command-line interface.
+
+Provides commands for ingestion, search, statistics, and serving.
+
+Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+"""
+
+from __future__ import annotations
+
+import json
+
+import click
+
+
+@click.group()
+@click.version_option(version="0.2.0", prog_name="VecForge")
+def cli() -> None:
+    """VecForge — Forge your vector database. Own it forever.
+
+    Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+    """
+
+
+@cli.command()
+@click.argument("path")
+@click.option("--vault", required=True, help="Path to vault database")
+@click.option("--namespace", default="default", help="Target namespace")
+@click.option("--chunk-size", default=1000, help="Chunk size in characters")
+@click.option("--chunk-overlap", default=200, help="Chunk overlap in characters")
+def ingest(
+    path: str,
+    vault: str,
+    namespace: str,
+    chunk_size: int,
+    chunk_overlap: int,
+) -> None:
+    """Ingest documents from PATH into the vault.
+
+    Supports: .txt, .md, .pdf, .docx, .html
+
+    Example: vecforge ingest my_docs/ --vault my.db
+    """
+    from vecforge import VecForge
+
+    click.echo(f"VecForge — Ingesting from {path}...")
+    with VecForge(vault) as db:
+        count = db.ingest(
+            path,
+            namespace=namespace,
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+        )
+    click.echo(f"✅ Ingested {count} chunks into vault '{vault}'")
+
+
+@cli.command()
+@click.argument("query")
+@click.option("--vault", required=True, help="Path to vault database")
+@click.option("--top-k", default=5, help="Number of results")
+@click.option("--namespace", default=None, help="Restrict to namespace")
+@click.option("--alpha", default=0.5, help="Semantic weight (0.0-1.0)")
+@click.option("--rerank", is_flag=True, help="Enable cross-encoder reranking")
+def search(
+    query: str,
+    vault: str,
+    top_k: int,
+    namespace: str | None,
+    alpha: float,
+    rerank: bool,
+) -> None:
+    """Search the vault with a natural language query.
+
+    Example: vecforge search "diabetes treatment" --vault my.db
+    """
+    from vecforge import VecForge
+
+    with VecForge(vault) as db:
+        results = db.search(
+            query,
+            top_k=top_k,
+            namespace=namespace,
+            alpha=alpha,
+            rerank=rerank,
+        )
+
+    if not results:
+        click.echo("No results found.")
+        return
+
+    for i, r in enumerate(results, 1):
+        click.echo(f"\n{'─' * 60}")
+        click.echo(f"Result {i} | Score: {r.score:.4f} | ID: {r.doc_id[:8]}...")
+        click.echo(f"Namespace: {r.namespace} | Modality: {r.modality}")
+        if r.metadata:
+            click.echo(f"Metadata: {json.dumps(r.metadata, default=str)}")
+        click.echo(f"\n{r.text[:500]}")
+
+
+@cli.command()
+@click.argument("vault")
+def stats(vault: str) -> None:
+    """Show vault statistics.
+
+    Example: vecforge stats my.db
+    """
+    from vecforge import VecForge
+
+    with VecForge(vault) as db:
+        info = db.stats()
+
+    click.echo(f"\n{'═' * 50}")
+    click.echo("VecForge Vault Statistics")
+    click.echo(f"{'═' * 50}")
+    click.echo(f"Path:           {info['path']}")
+    click.echo(f"Documents:      {info['documents']}")
+    click.echo(f"Encrypted:      {info['encrypted']}")
+    click.echo(f"Quantum:        {info['quantum']}")
+    click.echo(f"Protection:     {info['deletion_protection']}")
+    click.echo(f"Namespaces:     {', '.join(info['namespaces'])}")
+    click.echo(f"Index vectors:  {info['index_vectors']}")
+    click.echo(f"BM25 docs:      {info['bm25_documents']}")
+    click.echo(f"\nBuilt by {info['built_by']}")
+
+
+@cli.command()
+@click.argument("vault")
+@click.option("--format", "fmt", default="json", help="Export format (json)")
+@click.option("--output", "-o", default=None, help="Output file path")
+@click.option("--namespace", default=None, help="Export specific namespace")
+def export(vault: str, fmt: str, output: str | None, namespace: str | None) -> None:
+    """Export vault data to JSON.
+
+    Example: vecforge export my.db -o data.json
+    """
+    from vecforge.core.storage import StorageBackend
+
+    docs = []
+    storage = StorageBackend(path=vault)
+    all_docs = storage.get_all_docs(namespace=namespace)
+
+    for doc in all_docs:
+        docs.append(
+            {
+                "doc_id": doc.doc_id,
+                "text": doc.text,
+                "metadata": doc.metadata,
+                "namespace": doc.namespace,
+                "modality": doc.modality,
+                "created_at": doc.created_at,
+            }
+        )
+    storage.close()
+
+    data = {"vault": vault, "documents": docs, "count": len(docs)}
+    json_str = json.dumps(data, indent=2, default=str)
+
+    if output:
+        with open(output, "w", encoding="utf-8") as f:
+            f.write(json_str)
+        click.echo(f"✅ Exported {len(docs)} documents to {output}")
+    else:
+        click.echo(json_str)
+
+
+@cli.command()
+@click.option("--vault", required=True, help="Path to vault database")
+@click.option("--port", default=8080, help="Server port")
+@click.option("--host", default="0.0.0.0", help="Server host")
+def serve(vault: str, port: int, host: str) -> None:
+    """Start VecForge REST API server.
+
+    Example: vecforge serve --vault my.db --port 8080
+    """
+    click.echo(f"VecForge REST Server — {vault}")
+    click.echo(f"Listening on {host}:{port}")
+    click.echo("Built by Suneel Bose K · ArcGX TechLabs\n")
+
+    import uvicorn
+
+    from vecforge.server.app import create_app
+
+    app = create_app(vault)
+    uvicorn.run(app, host=host, port=port)
+
+
+if __name__ == "__main__":
+    cli()

vecforge/core/__init__.py

@@ -0,0 +1,3 @@
+# VecForge — Universal Local-First Vector Database
+# Copyright (c) 2026 Suneel Bose K · ArcGX TechLabs Private Limited
+# Licensed under BSL 1.1 — see LICENSE for details.

vecforge/core/bm25.py

@@ -0,0 +1,187 @@
+# VecForge — Universal Local-First Vector Database
+# Copyright (c) 2026 Suneel Bose K · ArcGX TechLabs Private Limited
+# Built by Suneel Bose K (Founder & CEO, ArcGX TechLabs)
+#
+# Licensed under the Business Source License 1.1 (BSL 1.1)
+# Free for personal, research, open-source, and non-commercial use.
+# Commercial use requires a separate license from ArcGX TechLabs.
+# See LICENSE file in the project root or contact: suneelbose@arcgx.in
+
+"""
+BM25 keyword search engine for VecForge.
+
+Provides sparse keyword-based retrieval using BM25Okapi. Used alongside
+FAISS dense retrieval for hybrid search.
+
+Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass
+
+import numpy as np
+from rank_bm25 import BM25Okapi
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class BM25Result:
+    """A single BM25 search result.
+
+    Attributes:
+        doc_index: Index of the document in the corpus.
+        score: BM25 relevance score (higher = more relevant).
+    """
+
+    doc_index: int
+    score: float
+
+
+class BM25Engine:
+    """BM25 keyword search engine using Okapi BM25.
+
+    Maintains an in-memory inverted index for fast keyword retrieval.
+    Rebuilt on each add operation (efficient for batch ingestion).
+
+    Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+
+    Performance:
+        Build: O(N * L) where N = docs, L = avg doc length
+        Search: O(V * N) where V = query terms
+        Typical: <2ms search at 100k docs
+
+    Example:
+        >>> engine = BM25Engine()
+        >>> engine.add_documents(["patient with diabetes", "hip fracture case"])
+        >>> results = engine.search("diabetes", top_k=1)
+        >>> results[0].doc_index
+        0
+    """
+
+    def __init__(self) -> None:
+        self._corpus: list[list[str]] = []
+        self._bm25: BM25Okapi | None = None
+
+    @property
+    def count(self) -> int:
+        """Return number of documents in the corpus.
+
+        Performance:
+            Time: O(1)
+        """
+        return len(self._corpus)
+
+    @staticmethod
+    def _tokenize(text: str) -> list[str]:
+        """Tokenize text into lowercase words.
+
+        Simple whitespace + punctuation tokenizer. Adequate for BM25
+        where exact matching matters more than linguistic analysis.
+
+        Args:
+            text: Raw text string to tokenize.
+
+        Returns:
+            List of lowercase word tokens.
+
+        Performance:
+            Time: O(L) where L = length of text
+        """
+        # why: Simple regex tokenizer — BM25 doesn't need stemming for v0.1
+        text = text.lower()
+        tokens = re.findall(r"\b\w+\b", text)
+        return tokens
+
+    def add_documents(self, texts: list[str]) -> None:
+        """Add documents to the BM25 index.
+
+        Rebuilds the internal BM25 index after adding. For best
+        performance, batch all documents into a single call.
+
+        Args:
+            texts: List of document texts to add.
+
+        Performance:
+            Time: O(N * L) where N = total docs, L = avg doc length
+
+        Example:
+            >>> engine = BM25Engine()
+            >>> engine.add_documents(["doc one", "doc two", "doc three"])
+            >>> engine.count
+            3
+        """
+        for text in texts:
+            self._corpus.append(self._tokenize(text))
+
+        # why: Rebuild entire index — BM25Okapi doesn't support incremental add
+        self._bm25 = BM25Okapi(self._corpus)
+        logger.debug("BM25 index rebuilt with %d documents", len(self._corpus))
+
+    def add_document(self, text: str) -> None:
+        """Add a single document to the BM25 index.
+
+        Args:
+            text: Document text to add.
+
+        Performance:
+            Time: O(N * L) — rebuilds entire index
+        """
+        self.add_documents([text])
+
+    def search(self, query: str, top_k: int = 10) -> list[BM25Result]:
+        """Search for documents matching the query keywords.
+
+        Args:
+            query: Search query string.
+            top_k: Number of top results to return.
+
+        Returns:
+            List of BM25Result sorted by descending score.
+            Empty list if no documents in corpus.
+
+        Performance:
+            Time: O(V * N) where V = query terms, N = corpus size
+            Typical: <2ms at 100k docs
+
+        Example:
+            >>> results = engine.search("diabetes treatment", top_k=5)
+            >>> for r in results:
+            ...     print(f"Doc {r.doc_index}: score={r.score:.4f}")
+        """
+        if self._bm25 is None or len(self._corpus) == 0:
+            return []
+
+        query_tokens = self._tokenize(query)
+        if not query_tokens:
+            return []
+
+        # perf: BM25Okapi.get_scores returns all scores in one pass
+        scores = self._bm25.get_scores(query_tokens)
+
+        # perf: Use argpartition for O(N) top-k instead of O(N log N) sort
+        effective_k = min(top_k, len(scores))
+        if effective_k == 0:
+            return []
+
+        top_indices = np.argpartition(scores, -effective_k)[-effective_k:]
+        # why: Sort the top-k by score descending
+        top_indices = top_indices[np.argsort(scores[top_indices])[::-1]]
+
+        return [
+            BM25Result(doc_index=int(idx), score=float(scores[idx]))
+            for idx in top_indices
+            if scores[idx] > 0.0  # why: Filter zero-score matches
+        ]
+
+    def reset(self) -> None:
+        """Reset the BM25 index, removing all documents.
+
+        Performance:
+            Time: O(1)
+        """
+        self._corpus = []
+        self._bm25 = None

vecforge/core/embedder.py

@@ -0,0 +1,152 @@
+# VecForge — Universal Local-First Vector Database
+# Copyright (c) 2026 Suneel Bose K · ArcGX TechLabs Private Limited
+# Built by Suneel Bose K (Founder & CEO, ArcGX TechLabs)
+#
+# Licensed under the Business Source License 1.1 (BSL 1.1)
+# Free for personal, research, open-source, and non-commercial use.
+# Commercial use requires a separate license from ArcGX TechLabs.
+# See LICENSE file in the project root or contact: suneelbose@arcgx.in
+
+"""
+Embedding engine for VecForge.
+
+Wraps sentence-transformers for local text embedding. No internet
+required — models are downloaded once and cached locally.
+
+Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import numpy as np
+from numpy.typing import NDArray
+
+logger = logging.getLogger(__name__)
+
+# perf: Default model balances quality and speed for most use cases
+_DEFAULT_MODEL = "all-MiniLM-L6-v2"
+
+
+class Embedder:
+    """Local text embedding engine using sentence-transformers.
+
+    Lazily loads the model on first use to keep VecForge init fast.
+    All processing runs locally — zero cloud dependency.
+
+    Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+
+    Args:
+        model_name: Name of the sentence-transformers model.
+            Defaults to 'all-MiniLM-L6-v2' (384-dim, fast, good quality).
+        device: Device to run on ('cpu', 'cuda'). Auto-detected if None.
+
+    Performance:
+        Time: O(n * d) where n = number of texts, d = model dimension
+        Typical: ~5ms per text on CPU, ~0.5ms on GPU
+
+    Example:
+        >>> embedder = Embedder()
+        >>> vectors = embedder.encode(["hello world", "vector search"])
+        >>> vectors.shape
+        (2, 384)
+    """
+
+    def __init__(
+        self,
+        model_name: str = _DEFAULT_MODEL,
+        device: str | None = None,
+    ) -> None:
+        self._model_name = model_name
+        self._device = device
+        self._model: Any = None  # Lazy-loaded SentenceTransformer
+        self._dimension: int | None = None
+
+    @property
+    def dimension(self) -> int:
+        """Return embedding dimension, loading model if needed.
+
+        Returns:
+            Integer dimension of the embedding vectors.
+
+        Performance:
+            Time: O(1) after first call
+        """
+        if self._dimension is None:
+            self._load_model()
+        assert self._dimension is not None  # guaranteed after _load_model
+        return self._dimension
+
+    def _load_model(self) -> None:
+        """Lazily load the sentence-transformer model.
+
+        Performance:
+            Time: O(1) — one-time cost of ~1-3 seconds for model loading
+        """
+        if self._model is not None:
+            return
+
+        try:
+            from sentence_transformers import SentenceTransformer
+        except ImportError as e:
+            raise ImportError(
+                "sentence-transformers is required for VecForge embeddings.\n"
+                "Install with: pip install sentence-transformers\n"
+                "VecForge by Suneel Bose K · ArcGX TechLabs"
+            ) from e
+
+        logger.info("Loading embedding model: %s", self._model_name)
+        self._model = SentenceTransformer(self._model_name, device=self._device)
+        self._dimension = self._model.get_sentence_embedding_dimension()
+        logger.info(
+            "Embedding model loaded: %s (dim=%d)",
+            self._model_name,
+            self._dimension,
+        )
+
+    def encode(
+        self,
+        texts: list[str] | str,
+        batch_size: int = 64,
+        normalize: bool = True,
+        show_progress: bool = False,
+    ) -> NDArray[np.float32]:
+        """Encode texts into dense embedding vectors.
+
+        Args:
+            texts: Single string or list of strings to embed.
+            batch_size: Batch size for encoding. Defaults to 64.
+            normalize: If True, L2-normalize vectors for cosine similarity.
+                Defaults to True.
+            show_progress: Show progress bar for large batches.
+
+        Returns:
+            NumPy array of shape (n_texts, dimension) with float32 vectors.
+
+        Performance:
+            Time: O(n * d) where n = len(texts), d = model dimension
+            Typical: ~5ms per text on CPU with default model
+
+        Example:
+            >>> embedder = Embedder()
+            >>> vec = embedder.encode("patient with diabetes")
+            >>> vec.shape
+            (1, 384)
+        """
+        self._load_model()
+
+        if isinstance(texts, str):
+            texts = [texts]
+
+        # perf: sentence-transformers handles batching internally
+        vectors: NDArray[np.float32] = self._model.encode(
+            texts,
+            batch_size=batch_size,
+            normalize_embeddings=normalize,
+            show_progress_bar=show_progress,
+            convert_to_numpy=True,
+        )
+
+        return vectors.astype(np.float32)