vecforge 0.2.0__py3-none-any.whl

vecforge/search/cascade.py

@@ -0,0 +1,186 @@
+# VecForge — Universal Local-First Vector Database
+# Copyright (c) 2026 Suneel Bose K · ArcGX TechLabs Private Limited
+# Built by Suneel Bose K (Founder & CEO, ArcGX TechLabs)
+#
+# Licensed under the Business Source License 1.1 (BSL 1.1)
+# Free for personal, research, open-source, and non-commercial use.
+# Commercial use requires a separate license from ArcGX TechLabs.
+# See LICENSE file in the project root or contact: suneelbose@arcgx.in
+
+"""
+4-stage cascading retrieval pipeline for VecForge.
+
+Pipeline stages:
+    1. FAISS dense retrieval (broad recall)
+    2. BM25 keyword merge via hybrid fusion (precision boost)
+    3. Metadata + namespace filtering
+    4. Optional cross-encoder reranking
+
+Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+from numpy.typing import NDArray
+
+from vecforge.core.bm25 import BM25Engine
+from vecforge.core.indexer import FaissIndexer
+from vecforge.core.reranker import Reranker
+from vecforge.search.filters import MetadataFilter
+from vecforge.search.hybrid import reciprocal_rank_fusion
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class CascadeCandidate:
+    """Intermediate candidate during cascade search.
+
+    Attributes:
+        doc_index: Index in the document store.
+        score: Current relevance score.
+        text: Document text (loaded during cascade).
+        metadata: Document metadata.
+        namespace: Document namespace.
+        doc_id: Unique document identifier.
+        modality: Content modality.
+        created_at: Document creation timestamp.
+    """
+
+    doc_index: int
+    score: float
+    text: str = ""
+    metadata: dict[str, Any] | None = None
+    namespace: str = "default"
+    doc_id: str = ""
+    modality: str = "text"
+    created_at: float = 0.0
+
+
+class CascadeSearcher:
+    """4-stage cascading search pipeline.
+
+    Processes search through increasingly precise stages:
+    1. Dense: FAISS retrieves broad candidate set
+    2. Sparse: BM25 scores merged via RRF or linear fusion
+    3. Filter: Metadata and namespace filtering applied
+    4. Rerank: Optional cross-encoder reranking for precision
+
+    Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+
+    Args:
+        indexer: FAISS index for dense retrieval.
+        bm25: BM25 engine for keyword search.
+        reranker: Optional cross-encoder reranker.
+
+    Performance:
+        Time: O(log N) FAISS + O(k) rerank where k << N
+        Typical: <15ms at 100k docs, <50ms at 1M docs
+
+    Example:
+        >>> searcher = CascadeSearcher(indexer, bm25_engine)
+        >>> results = searcher.search(query_vec, "diabetes", top_k=10)
+    """
+
+    def __init__(
+        self,
+        indexer: FaissIndexer,
+        bm25: BM25Engine,
+        reranker: Reranker | None = None,
+    ) -> None:
+        self._indexer = indexer
+        self._bm25 = bm25
+        self._reranker = reranker
+
+    def search(
+        self,
+        query_vector: NDArray[np.float32],
+        query_text: str,
+        top_k: int = 10,
+        alpha: float = 0.5,
+        rerank: bool = False,
+        filters: dict[str, Any] | None = None,
+        recency_weight: float = 0.0,
+    ) -> list[CascadeCandidate]:
+        """Execute 4-stage cascading search.
+
+        Args:
+            query_vector: Dense query embedding from embedder.
+            query_text: Original query string for BM25 and reranking.
+            top_k: Number of final results to return.
+            alpha: Semantic vs keyword weight (0.0-1.0).
+            rerank: Enable cross-encoder reranking (Stage 4).
+            filters: Metadata filter conditions.
+            recency_weight: Weight for document recency (0.0-1.0).
+
+        Returns:
+            List of CascadeCandidate sorted by descending relevance.
+
+        Performance:
+            Time: O(log N) + O(k) + O(k*F) + O(k*d_rerank)
+            Typical: <15ms without rerank, <50ms with rerank
+        """
+        # why: Retrieve more candidates than top_k to allow filtering
+        retrieval_k = min(top_k * 4, self._indexer.count)
+        if retrieval_k == 0:
+            return []
+
+        # ─── Stage 1: Dense retrieval via FAISS ───
+        dense_scores, dense_ids = self._indexer.search(query_vector, top_k=retrieval_k)
+        logger.debug("Stage 1 (Dense): retrieved %d candidates", len(dense_ids))
+
+        # ─── Stage 2: Sparse keyword merge via RRF ───
+        bm25_results = self._bm25.search(query_text, top_k=retrieval_k)
+        sparse_ids = [r.doc_index for r in bm25_results]
+        sparse_scores = [r.score for r in bm25_results]
+
+        fused = reciprocal_rank_fusion(
+            dense_ids=dense_ids,
+            dense_scores=dense_scores,
+            sparse_ids=sparse_ids,
+            sparse_scores=sparse_scores,
+            alpha=alpha,
+        )
+        logger.debug("Stage 2 (Hybrid): fused %d candidates", len(fused))
+
+        # why: Convert to CascadeCandidate
+        candidates = [
+            CascadeCandidate(doc_index=doc_idx, score=score) for doc_idx, score in fused
+        ]
+
+        # ─── Stage 3: Metadata filtering ───
+        if filters:
+            meta_filter = MetadataFilter(filters)
+            candidates = [
+                c
+                for c in candidates
+                if c.metadata is not None and meta_filter.matches(c.metadata)
+            ]
+            logger.debug(
+                "Stage 3 (Filter): %d candidates after filtering",
+                len(candidates),
+            )
+
+        # ─── Stage 4: Cross-encoder reranking (optional) ───
+        if rerank and self._reranker is not None and candidates:
+            texts = [c.text for c in candidates if c.text]
+            if texts:
+                reranked = self._reranker.rerank(query_text, texts, top_k=top_k)
+                reranked_candidates = []
+                for orig_idx, rerank_score in reranked:
+                    if orig_idx < len(candidates):
+                        c = candidates[orig_idx]
+                        c.score = rerank_score
+                        reranked_candidates.append(c)
+                candidates = reranked_candidates
+                logger.debug(
+                    "Stage 4 (Rerank): reranked to %d results",
+                    len(candidates),
+                )
+
+        return candidates[:top_k]

vecforge/search/filters.py

@@ -0,0 +1,146 @@
+# VecForge — Universal Local-First Vector Database
+# Copyright (c) 2026 Suneel Bose K · ArcGX TechLabs Private Limited
+# Built by Suneel Bose K (Founder & CEO, ArcGX TechLabs)
+#
+# Licensed under the Business Source License 1.1 (BSL 1.1)
+# Free for personal, research, open-source, and non-commercial use.
+# Commercial use requires a separate license from ArcGX TechLabs.
+# See LICENSE file in the project root or contact: suneelbose@arcgx.in
+
+"""
+Metadata filtering logic for VecForge.
+
+Supports equality, range (gte, lte, gt, lt), in/not_in operators
+for flexible metadata-based result filtering.
+
+Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class MetadataFilter:
+    """Filter search results based on metadata conditions.
+
+    Supports operators: equality, gte, lte, gt, lt, in, not_in, ne.
+    Filters are specified as nested dictionaries.
+
+    Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+
+    Performance:
+        Time: O(N * F) where N = results, F = number of filter keys
+
+    Example:
+        >>> f = MetadataFilter({"type": "NDA", "year": {"gte": 2023}})
+        >>> f.matches({"type": "NDA", "year": 2024})
+        True
+        >>> f.matches({"type": "NDA", "year": 2020})
+        False
+    """
+
+    _OPERATORS = {"gte", "lte", "gt", "lt", "in", "not_in", "ne"}
+
+    def __init__(self, filters: dict[str, Any]) -> None:
+        self._filters = filters
+
+    def matches(self, metadata: dict[str, Any]) -> bool:
+        """Check if metadata satisfies all filter conditions.
+
+        Args:
+            metadata: Document metadata dictionary.
+
+        Returns:
+            True if all filter conditions are satisfied.
+
+        Performance:
+            Time: O(F) where F = number of filter keys
+        """
+        for key, condition in self._filters.items():
+            if key not in metadata:
+                return False
+
+            value = metadata[key]
+
+            if isinstance(condition, dict):
+                # why: Operator-based filtering
+                if not self._check_operators(value, condition):
+                    return False
+            else:
+                # why: Simple equality check
+                if value != condition:
+                    return False
+
+        return True
+
+    def _check_operators(self, value: Any, operators: dict[str, Any]) -> bool:
+        """Check operator-based conditions on a value.
+
+        Args:
+            value: Metadata field value.
+            operators: Dictionary of operator → threshold pairs.
+
+        Returns:
+            True if all operator conditions pass.
+
+        Performance:
+            Time: O(number of operators)
+        """
+        for op, threshold in operators.items():
+            if op not in self._OPERATORS:
+                # why: Treat unknown keys as nested equality
+                if value != threshold:
+                    return False
+                continue
+
+            if op == "gte" and not (value >= threshold):
+                return False
+            if op == "lte" and not (value <= threshold):
+                return False
+            if op == "gt" and not (value > threshold):
+                return False
+            if op == "lt" and not (value < threshold):
+                return False
+            if op == "in" and value not in threshold:
+                return False
+            if op == "not_in" and value in threshold:
+                return False
+            if op == "ne" and value == threshold:
+                return False
+
+        return True
+
+    def filter_results(
+        self, results: list[Any], metadata_getter: Any = None
+    ) -> list[Any]:
+        """Filter a list of results by metadata conditions.
+
+        Args:
+            results: List of result objects.
+            metadata_getter: Callable or attribute name to extract metadata.
+                If None, expects results to have a 'metadata' attribute.
+
+        Returns:
+            Filtered list of results.
+
+        Performance:
+            Time: O(N * F) where N = results, F = filter keys
+        """
+        if not self._filters:
+            return results
+
+        filtered = []
+        for result in results:
+            if metadata_getter is not None:
+                if callable(metadata_getter):
+                    meta = metadata_getter(result)
+                else:
+                    meta = getattr(result, metadata_getter)
+            else:
+                meta = getattr(result, "metadata", {})
+
+            if self.matches(meta):
+                filtered.append(result)
+
+        return filtered

vecforge/search/hybrid.py

@@ -0,0 +1,146 @@
+# VecForge — Universal Local-First Vector Database
+# Copyright (c) 2026 Suneel Bose K · ArcGX TechLabs Private Limited
+# Built by Suneel Bose K (Founder & CEO, ArcGX TechLabs)
+#
+# Licensed under the Business Source License 1.1 (BSL 1.1)
+# Free for personal, research, open-source, and non-commercial use.
+# Commercial use requires a separate license from ArcGX TechLabs.
+# See LICENSE file in the project root or contact: suneelbose@arcgx.in
+
+"""
+Hybrid dense + sparse search fusion for VecForge.
+
+Combines FAISS dense retrieval scores with BM25 sparse keyword scores
+using configurable weighted fusion. Alpha controls the balance between
+semantic understanding and keyword matching.
+
+Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from numpy.typing import NDArray
+
+
+def reciprocal_rank_fusion(
+    dense_ids: NDArray[np.int64],
+    dense_scores: NDArray[np.float32],
+    sparse_ids: list[int],
+    sparse_scores: list[float],
+    alpha: float = 0.5,
+    k: int = 60,
+) -> list[tuple[int, float]]:
+    """Fuse dense and sparse search results using weighted RRF.
+
+    Reciprocal Rank Fusion (RRF) combines rankings from multiple
+    retrieval systems. Each document's score is computed as:
+
+        score = alpha * 1/(k + dense_rank) + (1 - alpha) * 1/(k + sparse_rank)
+
+    Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+
+    Args:
+        dense_ids: Document indices from FAISS dense retrieval.
+        dense_scores: Corresponding FAISS inner-product scores.
+        sparse_ids: Document indices from BM25 sparse retrieval.
+        sparse_scores: Corresponding BM25 scores.
+        alpha: Weight for dense vs sparse (0.0 = keyword only,
+            1.0 = semantic only). Defaults to 0.5 (balanced).
+        k: RRF constant. Higher values reduce the impact of ranking
+            position. Defaults to 60 (standard RRF constant).
+
+    Returns:
+        List of (doc_index, fused_score) tuples sorted by descending
+        fused score.
+
+    Performance:
+        Time: O(D + S) where D = dense results, S = sparse results
+        Typical: <1ms for top-100 results
+
+    Example:
+        >>> fused = reciprocal_rank_fusion(
+        ...     dense_ids=np.array([0, 2, 5]),
+        ...     dense_scores=np.array([0.9, 0.7, 0.5]),
+        ...     sparse_ids=[2, 0, 3],
+        ...     sparse_scores=[5.0, 3.0, 1.0],
+        ...     alpha=0.5,
+        ... )
+        >>> fused[0]  # (doc_id, combined_score)
+    """
+    fused_scores: dict[int, float] = {}
+
+    # perf: Process dense results — already sorted by score descending
+    for rank, doc_id in enumerate(dense_ids):
+        doc_id_int = int(doc_id)
+        if doc_id_int < 0:  # why: FAISS returns -1 for padded slots
+            continue
+        rrf_score = alpha * (1.0 / (k + rank + 1))
+        fused_scores[doc_id_int] = fused_scores.get(doc_id_int, 0.0) + rrf_score
+
+    # perf: Process sparse results — already sorted by score descending
+    for rank, doc_id in enumerate(sparse_ids):
+        rrf_score = (1.0 - alpha) * (1.0 / (k + rank + 1))
+        fused_scores[doc_id] = fused_scores.get(doc_id, 0.0) + rrf_score
+
+    # why: Sort by fused score descending
+    results = sorted(fused_scores.items(), key=lambda x: x[1], reverse=True)
+    return results
+
+
+def weighted_linear_fusion(
+    dense_ids: NDArray[np.int64],
+    dense_scores: NDArray[np.float32],
+    sparse_ids: list[int],
+    sparse_scores: list[float],
+    alpha: float = 0.5,
+) -> list[tuple[int, float]]:
+    """Fuse dense and sparse scores using weighted linear combination.
+
+    Normalizes both score distributions to [0, 1] and combines:
+
+        score = alpha * norm_dense + (1 - alpha) * norm_sparse
+
+    Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+
+    Args:
+        dense_ids: Document indices from FAISS.
+        dense_scores: FAISS scores.
+        sparse_ids: Document indices from BM25.
+        sparse_scores: BM25 scores.
+        alpha: Semantic weight. Defaults to 0.5.
+
+    Returns:
+        Sorted list of (doc_index, fused_score) tuples.
+
+    Performance:
+        Time: O(D + S + U*log(U)) where U = unique docs
+    """
+    fused_scores: dict[int, float] = {}
+
+    # perf: Normalize dense scores to [0, 1]
+    if len(dense_scores) > 0:
+        d_min, d_max = float(dense_scores.min()), float(dense_scores.max())
+        d_range = d_max - d_min if d_max > d_min else 1.0
+
+        for doc_id, score in zip(dense_ids, dense_scores, strict=False):
+            doc_id_int = int(doc_id)
+            if doc_id_int < 0:
+                continue
+            norm_score = (float(score) - d_min) / d_range
+            fused_scores[doc_id_int] = alpha * norm_score
+
+    # perf: Normalize sparse scores to [0, 1]
+    if len(sparse_scores) > 0:
+        s_min = min(sparse_scores)
+        s_max = max(sparse_scores)
+        s_range = s_max - s_min if s_max > s_min else 1.0
+
+        for doc_id, score in zip(sparse_ids, sparse_scores, strict=False):
+            norm_score = (score - s_min) / s_range
+            fused_scores[doc_id] = fused_scores.get(doc_id, 0.0) + (
+                (1.0 - alpha) * norm_score
+            )
+
+    results = sorted(fused_scores.items(), key=lambda x: x[1], reverse=True)
+    return results

vecforge/security/__init__.py

@@ -0,0 +1,3 @@
+# VecForge — Universal Local-First Vector Database
+# Copyright (c) 2026 Suneel Bose K · ArcGX TechLabs Private Limited
+# Licensed under BSL 1.1 — see LICENSE for details.

vecforge/security/audit.py

@@ -0,0 +1,169 @@
+# VecForge — Universal Local-First Vector Database
+# Copyright (c) 2026 Suneel Bose K · ArcGX TechLabs Private Limited
+# Built by Suneel Bose K (Founder & CEO, ArcGX TechLabs)
+#
+# Licensed under the Business Source License 1.1 (BSL 1.1)
+# Free for personal, research, open-source, and non-commercial use.
+# Commercial use requires a separate license from ArcGX TechLabs.
+# See LICENSE file in the project root or contact: suneelbose@arcgx.in
+
+"""
+Audit log writer + reader for VecForge.
+
+Records all mutating operations as append-only JSONL audit events.
+Every add, delete, update, and admin action is logged for compliance.
+
+Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+class AuditLogger:
+    """Append-only JSONL audit log for compliance and security.
+
+    Every mutating operation emits an audit event with actor, operation,
+    target, timestamp, and metadata. Logs are append-only and tamper-evident.
+
+    Built by Suneel Bose K · ArcGX TechLabs Private Limited.
+
+    Args:
+        log_path: Path to the JSONL audit log file. If None, audit
+            logging is disabled (but a warning is emitted).
+
+    Performance:
+        Write: O(1) per event (append-only)
+        Read: O(N) where N = total events
+
+    Example:
+        >>> audit = AuditLogger("audit.jsonl")
+        >>> audit.log("admin", "add", doc_id="d123", namespace="default")
+        >>> events = audit.read_log()
+        >>> print(events[0]["operation"])
+        'add'
+    """
+
+    def __init__(self, log_path: str | None = None) -> None:
+        self._path = Path(log_path) if log_path else None
+        self._enabled = log_path is not None
+
+        if self._enabled and self._path is not None:
+            # why: Ensure parent directory exists
+            self._path.parent.mkdir(parents=True, exist_ok=True)
+            logger.info("Audit logging enabled: %s", self._path)
+        else:
+            logger.debug("Audit logging disabled")
+
+    @property
+    def enabled(self) -> bool:
+        """Return whether audit logging is active.
+
+        Performance:
+            Time: O(1)
+        """
+        return self._enabled
+
+    def log(
+        self,
+        actor: str,
+        operation: str,
+        doc_id: str | None = None,
+        namespace: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> None:
+        """Write an audit event to the log.
+
+        Args:
+            actor: Identifier of the user/key performing the action.
+            operation: Operation name (add, delete, update, search, etc.).
+            doc_id: Target document ID, if applicable.
+            namespace: Target namespace, if applicable.
+            metadata: Additional event metadata.
+
+        Performance:
+            Time: O(1) — single file append
+
+        Example:
+            >>> audit.log(
+            ...     actor="key-abc123",
+            ...     operation="add",
+            ...     doc_id="doc-xyz",
+            ...     namespace="ward_7",
+            ...     metadata={"chars": 1500}
+            ... )
+        """
+        if not self._enabled or self._path is None:
+            return
+
+        event = {
+            "timestamp": time.time(),
+            "actor": actor,
+            "operation": operation,
+            "doc_id": doc_id,
+            "namespace": namespace,
+            "metadata": metadata or {},
+        }
+
+        # security: Append-only write — no modification of existing entries
+        with open(self._path, "a", encoding="utf-8") as f:
+            f.write(json.dumps(event, default=str) + "\n")
+
+    def read_log(
+        self,
+        since: float | None = None,
+        until: float | None = None,
+        actor: str | None = None,
+        operation: str | None = None,
+    ) -> list[dict[str, Any]]:
+        """Read audit events with optional filters.
+
+        Args:
+            since: Unix timestamp — events after this time.
+            until: Unix timestamp — events before this time.
+            actor: Filter by actor identity.
+            operation: Filter by operation type.
+
+        Returns:
+            List of audit event dictionaries.
+
+        Performance:
+            Time: O(N) where N = total events in log
+        """
+        if not self._enabled or self._path is None or not self._path.exists():
+            return []
+
+        events = []
+        with open(self._path, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+
+                try:
+                    event = json.loads(line)
+                except json.JSONDecodeError:
+                    logger.warning("Skipping malformed audit log entry")
+                    continue
+
+                # why: Apply filters
+                ts = event.get("timestamp", 0)
+                if since is not None and ts < since:
+                    continue
+                if until is not None and ts > until:
+                    continue
+                if actor is not None and event.get("actor") != actor:
+                    continue
+                if operation is not None and event.get("operation") != operation:
+                    continue
+
+                events.append(event)
+
+        return events