simlar 1.0.0__py3-none-any.whl

simlar/__init__.py

@@ -0,0 +1,24 @@
+from simlar.contracts import SearchResult, TextIndex, VectorIndex
+from simlar.fusion import ReciprocalRankFusion
+from simlar.indexes.helix_index import HelixIndex
+from simlar.indexes.registry import load_from_directory, register
+from simlar.indexes.relevance_index import RelevanceIndex
+from simlar.indexes.simlar_engine import SimlarEngine
+from simlar.indexes.streaming_index import StreamingHelixIndex as StreamingHybridIndex
+
+__all__ = [
+    # Contracts — extension points
+    "SearchResult",
+    "TextIndex",
+    "VectorIndex",
+    # Fusion
+    "ReciprocalRankFusion",
+    # Indexes
+    "RelevanceIndex",
+    "SimlarEngine",
+    "HelixIndex",
+    "StreamingHybridIndex",
+    # Registry
+    "register",
+    "load_from_directory",
+]

simlar/contracts.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Protocol, runtime_checkable
+
+import numpy as np
+
+# SearchResult and _Parameters are defined in simlar_engine and re-exported here
+# so user code can import them from either package.
+from simlar_engine._types import SearchResult, _Parameters
+
+__all__ = [
+    "SearchResult",
+    "_Parameters",
+    "Index",
+    "TextIndex",
+    "VectorIndex",
+    "CompositeIndex",
+    "FusionStrategy",
+]
+
+# ── Base ──────────────────────────────────────────────────────────────────────
+
+
+class Index(ABC):
+    """Minimal contract: every index can persist and report its type and size."""
+
+    @abstractmethod
+    def save(self, path: str) -> None: ...
+
+    @classmethod
+    @abstractmethod
+    def load(cls, path: str) -> Index: ...
+
+    @property
+    @abstractmethod
+    def index_type(self) -> str: ...
+
+    @property
+    @abstractmethod
+    def size(self) -> int: ...
+
+    @property
+    @abstractmethod
+    def is_trained(self) -> bool: ...
+
+
+# ── Text index ────────────────────────────────────────────────────────────────
+
+
+class TextIndex(Index):
+    """Index over a text corpus."""
+
+    # ── Public API ─────────────────────────────────────────────────────────────
+
+    @abstractmethod
+    def add(self, ids: list[str], texts: list[str]) -> None:
+        """Append new documents. Raises ValueError on duplicate IDs; use update() to replace."""
+
+    @abstractmethod
+    def update(self, ids: list[str], texts: list[str]) -> None:
+        """Replace texts for existing document IDs, then rebuild."""
+
+    @abstractmethod
+    def delete(self, ids: list[str]) -> None:
+        """Remove documents by ID, rebuilding internal structures."""
+
+    @abstractmethod
+    def search(self, query: str, k: int) -> list[SearchResult]: ...
+
+    # ── Internal ───────────────────────────────────────────────────────────────
+
+    @abstractmethod
+    def fit(self, corpus: list[str], parallel: bool = False, **kwargs) -> None:
+        """Build the model from a raw corpus list. Called by add() and HelixIndex."""
+
+    @abstractmethod
+    def search_raw(
+        self,
+        queries: str | list[str],
+        k: int,
+        parallel: bool = False,
+    ) -> tuple[np.ndarray, np.ndarray]:
+        """Return (ids, scores) shaped (n_queries, k), int64/float64.
+        Used internally by HelixIndex._search_raw().
+        """
+
+
+# ── Vector index ──────────────────────────────────────────────────────────────
+
+
+class VectorIndex(Index):
+    """Index over dense or binary float embeddings."""
+
+    # ── Public API ─────────────────────────────────────────────────────────────
+
+    @abstractmethod
+    def add(self, ids: list[str], vectors: np.ndarray) -> None: ...
+
+    @abstractmethod
+    def search(self, query: np.ndarray, k: int) -> list[SearchResult]: ...
+
+    @abstractmethod
+    def update(self, ids: list[str], vectors: np.ndarray) -> None:
+        """Replace vectors for existing IDs in-place using frozen quantization params."""
+
+    # ── Internal ───────────────────────────────────────────────────────────────
+
+    @abstractmethod
+    def fit(
+        self,
+        embeddings: np.ndarray,
+        parallel: bool = False,
+        params: _Parameters | None = None,
+        **kwargs,
+    ) -> None:
+        """Build from embeddings. Pass params to reuse frozen quantization across shards."""
+
+    @abstractmethod
+    def search_raw(
+        self,
+        vectors: np.ndarray,
+        k: int,
+        candidates: np.ndarray | None = None,
+        parallel: bool = False,
+        n_candidates: int | None = None,
+    ) -> tuple[np.ndarray, np.ndarray]:
+        """Return (ids, distances) shaped (n_queries, k), int64/float64.
+        Used internally by HelixIndex._search_raw().
+        """
+
+    @abstractmethod
+    def delete(self, ids: list[str]) -> None:
+        """Remove documents by ID, physically rebuilding internal structures."""
+
+
+# ── Composite index ───────────────────────────────────────────────────────────
+
+
+class CompositeIndex(Index):
+    """Fuses N sub-indexes with a FusionStrategy."""
+
+    @abstractmethod
+    def add(
+        self,
+        ids: list[str],
+        texts: list[str] | None = None,
+        vectors: np.ndarray | None = None,
+    ) -> None: ...
+
+    @abstractmethod
+    def search(
+        self,
+        query_text: str | None = None,
+        query_vector: np.ndarray | None = None,
+        k: int = 10,
+    ) -> list[SearchResult]: ...
+
+    @abstractmethod
+    def fit(
+        self,
+        corpus: list,
+        vectors: np.ndarray,
+        parallel: bool = False,
+        **kwargs,
+    ) -> None:
+        """Internal: build all sub-indexes. Used by StreamingHybridIndex shards."""
+
+
+# ── Fusion ────────────────────────────────────────────────────────────────────
+
+
+@runtime_checkable
+class FusionStrategy(Protocol):
+    """Combines N result lists into a single ranked list."""
+
+    def __call__(
+        self,
+        results: list[list[SearchResult]],
+        k: int,
+    ) -> list[SearchResult]: ...

simlar/fusion/__init__.py

@@ -0,0 +1,3 @@
+from .rrf import ReciprocalRankFusion
+
+__all__ = ["ReciprocalRankFusion"]

simlar/fusion/_rrf.pyi

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from simlar.contracts import SearchResult
+
+class ReciprocalRankFusion:
+    def __init__(self, k: int = 2, weights: list[float] | None = None) -> None: ...
+    def __call__(self, results: list[list[SearchResult]], k: int) -> list[SearchResult]: ...

simlar/fusion/rrf.py

@@ -0,0 +1,3 @@
+from simlar_engine.fusion.rrf import ReciprocalRankFusion
+
+__all__ = ["ReciprocalRankFusion"]

simlar/indexes/__init__.py

@@ -0,0 +1,7 @@
+from .helix_index import HelixIndex
+from .lookup_index import LookupIndex
+from .relevance_index import RelevanceIndex
+from .simlar_engine import SimlarEngine
+from .streaming_index import StreamingHelixIndex
+
+__all__ = ["LookupIndex", "RelevanceIndex", "SimlarEngine", "StreamingHelixIndex", "HelixIndex"]

simlar/indexes/_helix.pyi

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import numpy as np
+
+from simlar.contracts import (
+    CompositeIndex,
+    FusionStrategy,
+    SearchResult,
+    TextIndex,
+    VectorIndex,
+    _Parameters,
+)
+
+class HelixIndex(CompositeIndex):
+    """
+    Example::
+
+        from simlar import HelixIndex, RelevanceIndex, SimlarEngine
+        from simlar.fusion import ReciprocalRankFusion
+
+        idx = HelixIndex(
+            text_index=RelevanceIndex(),
+            vector_index=SimlarEngine(),
+            fusion=ReciprocalRankFusion(),
+        )
+        idx.fit(corpus=texts, vectors=embeddings)
+        idx.add(ids=ids, texts=texts, vectors=embeddings)
+        results = idx.search(query_text="query", query_vector=q_vec, k=10)
+    """
+
+    def __init__(
+        self,
+        text_index: TextIndex | None = None,
+        vector_index: VectorIndex | None = None,
+        fusion: FusionStrategy | None = None,
+        text_k: int = 5000,
+        vector_k: int = 1000,
+        top_k: int = 100,
+        alpha_text: float = 0.10,
+        alpha_vector: float = 1.0,
+        rrf_k: int = 2,
+    ) -> None: ...
+    def fit(
+        self,
+        corpus: list[str],
+        vectors: np.ndarray,
+        parallel: bool = False,
+        params: _Parameters | None = None,
+        **kwargs: object,
+    ) -> None: ...
+    def add(
+        self,
+        ids: list[str],
+        texts: list[str] | None = None,
+        vectors: np.ndarray | None = None,
+    ) -> None: ...
+    def search(
+        self,
+        query_text: str | list[str] | None = None,
+        query_vector: np.ndarray | None = None,
+        k: int | None = None,
+        parallel: bool = False,
+    ) -> list[SearchResult]: ...
+    def save(self, directory: str) -> None: ...
+    @classmethod
+    def load(cls, directory: str) -> HelixIndex: ...
+    @property
+    def size(self) -> int: ...
+    @property
+    def is_trained(self) -> bool: ...
+    @property
+    def index_type(self) -> str: ...
+    @property
+    def text_index(self) -> TextIndex: ...
+    @property
+    def vector_index(self) -> VectorIndex: ...
+    @property
+    def boundaries(self) -> np.ndarray | None: ...
+    @property
+    def fit_values(self) -> np.ndarray | None: ...
+    @property
+    def quantization_params(self) -> _Parameters | None: ...

simlar/indexes/_relevance.pyi

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import numpy as np
+
+from simlar.contracts import SearchResult, TextIndex
+
+class RelevanceIndex(TextIndex):
+    """
+
+    Example::
+
+        idx = RelevanceIndex(k1=1.5, b=0.75)
+        idx.add(ids=["a", "b"], texts=["first doc", "second doc"])
+        results = idx.search("first", k=10)
+        idx.save("bm25.idx")
+        idx = RelevanceIndex.load("idx")
+    """
+
+    def __init__(
+        self,
+        method: str = "robertson",
+        k1: float = 1.5,
+        b: float = 0.75,
+        stopwords_lang: str = "english",
+        stemmer_lang: str = "english",
+    ) -> None: ...
+    def fit(self, corpus: list[str], parallel: bool = False, **kwargs: object) -> None: ...
+    def add(self, ids: list[str], texts: list[str]) -> None: ...
+    def update(self, ids: list[str], texts: list[str]) -> None: ...
+    def delete(self, ids: list[str]) -> None: ...
+    def search(self, query: str, k: int) -> list[SearchResult]: ...
+    def search_raw(
+        self,
+        queries: str | list[str],
+        k: int,
+        parallel: bool = False,
+    ) -> tuple[np.ndarray, np.ndarray]: ...
+    def save(self, directory: str) -> None: ...
+    @classmethod
+    def load(cls, directory: str) -> RelevanceIndex: ...
+    @property
+    def size(self) -> int: ...
+    @property
+    def is_trained(self) -> bool: ...
+    @property
+    def index_type(self) -> str: ...

simlar/indexes/_simlar.pyi

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import numpy as np
+
+from simlar.contracts import SearchResult, VectorIndex, _Parameters
+
+class SimlarEngine(VectorIndex):
+    """
+    Example::
+
+        idx = SimlarEngine()
+        idx.fit(embeddings=vecs)
+        idx.add(ids=["a", "b"], vectors=vecs)
+        results = idx.search(query=q_vec, k=10)
+    """
+
+    def __init__(self) -> None: ...
+    def fit(
+        self,
+        embeddings: np.ndarray,
+        parallel: bool = False,
+        params: _Parameters | None = None,
+        **kwargs: object,
+    ) -> None: ...
+    def add(self, ids: list[str], vectors: np.ndarray) -> None: ...
+    def update(self, ids: list[str], vectors: np.ndarray) -> None: ...
+    def delete(self, ids: list[str]) -> None: ...
+    def search(self, query: np.ndarray, k: int) -> list[SearchResult]: ...
+    def search_raw(
+        self,
+        vectors: np.ndarray,
+        k: int,
+        candidates: np.ndarray | None = None,
+        parallel: bool = False,
+        n_candidates: int | None = None,
+    ) -> tuple[np.ndarray, np.ndarray]: ...
+    def update_vector(self, doc_id: int, vector: np.ndarray) -> None: ...
+    def save(self, directory: str) -> None: ...
+    @classmethod
+    def load(cls, directory: str) -> SimlarEngine: ...
+    @property
+    def size(self) -> int: ...
+    @property
+    def is_trained(self) -> bool: ...
+    @property
+    def index_type(self) -> str: ...
+    @property
+    def boundaries(self) -> np.ndarray | None: ...
+    @property
+    def fit_values(self) -> np.ndarray | None: ...
+    @property
+    def _params(self) -> _Parameters | None: ...
+    @property
+    def _matrix(self) -> np.ndarray | None: ...

simlar/indexes/_streaming.pyi

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import numpy as np
+
+from simlar.contracts import TextIndex, VectorIndex, _Parameters
+
+class StreamingHybridIndex:
+    """
+    Example::
+
+        idx = StreamingHybridIndex()
+        idx.add_batch(corpus=texts[:1000], vectors=vecs[:1000])
+        idx.add_batch(corpus=texts[1000:], vectors=vecs[1000:])
+        ids, distances = idx.search(query_text="query", query_vector=q_vec, k=10)
+        idx.save("streaming.idx")
+        idx = StreamingHybridIndex.load("streaming.idx")
+    """
+
+    def __init__(
+        self,
+        text_index_cls: type[TextIndex] | None = None,
+        vector_index_cls: type[VectorIndex] | None = None,
+        text_k: int = 5000,
+        vector_k: int = 1000,
+        top_k: int = 100,
+        alpha_text: float = 0.10,
+        alpha_vector: float = 1.0,
+        rrf_k: int = 2,
+        n_candidates: int = 5000,
+    ) -> None: ...
+    def add_batch(self, corpus: list[str], vectors: np.ndarray, parallel: bool = False) -> None: ...
+    async def add_batch_async(
+        self, corpus: list[str], vectors: np.ndarray, parallel: bool = False
+    ) -> None: ...
+    def search(
+        self,
+        query_text: str | list[str],
+        query_vector: np.ndarray,
+        k: int | None = None,
+        parallel: bool = False,
+    ) -> tuple[np.ndarray, np.ndarray]: ...
+    def fit(self, corpus: list, vectors: np.ndarray, **kwargs: object) -> None: ...
+    def save(self, directory: str) -> None: ...
+    @classmethod
+    def load(cls, directory: str) -> StreamingHybridIndex: ...
+    @property
+    def size(self) -> int: ...
+    @property
+    def n_shards(self) -> int: ...
+    @property
+    def is_trained(self) -> bool: ...
+    @property
+    def index_type(self) -> str: ...
+    @property
+    def boundaries(self) -> np.ndarray | None: ...
+    @property
+    def fit_values(self) -> np.ndarray | None: ...
+    @property
+    def _params(self) -> _Parameters | None: ...

simlar/indexes/helix_index.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import numpy as np
+from simlar_engine.indexes._helix_impl import _HelixCore
+
+from simlar.contracts import (
+    CompositeIndex,
+    FusionStrategy,
+    SearchResult,
+    TextIndex,
+    VectorIndex,
+    _Parameters,
+)
+from simlar.indexes.registry import register
+
+
+@register("helix")
+class HelixIndex(CompositeIndex):
+    """Fuses N indexes with a FusionStrategy.
+
+    Example::
+
+        HelixIndex(indexes=[RelevanceIndex(), SimilarityIndex()], fusion=ReciprocalRankFusion())
+    """
+
+    def __init__(
+        self,
+        *,
+        text_index: TextIndex | None = None,
+        vector_index: VectorIndex | None = None,
+        fusion: FusionStrategy | None = None,
+        text_k: int | None = None,
+        vector_k: int | None = None,
+        top_k: int = 100,
+        alpha_text: float = 0.10,
+        alpha_vector: float = 1.0,
+    ) -> None:
+        self._core = _HelixCore(
+            text_index=text_index,
+            vector_index=vector_index,
+            fusion=fusion,
+            text_k=text_k,
+            vector_k=vector_k,
+            top_k=top_k,
+            alpha_text=alpha_text,
+            alpha_vector=alpha_vector,
+        )
+
+    # ── Public API ────────────────────────────────────────────────────────────
+
+    def add(
+        self,
+        ids: list[str],
+        texts: list[str] | None = None,
+        vectors: np.ndarray | None = None,
+    ) -> None:
+        self._core.add(ids, texts, vectors)
+
+    def search(
+        self,
+        query_text: str | list[str] | None = None,
+        query_vector: np.ndarray | None = None,
+        k: int | None = None,
+        parallel: bool = False,
+    ) -> list[SearchResult] | list[list[SearchResult]]:
+        return self._core.search(query_text, query_vector, k, parallel)
+
+    def fit(
+        self,
+        corpus: list,
+        vectors: np.ndarray,
+        parallel: bool = False,
+        **kwargs,
+    ) -> None:
+        params = kwargs.pop("params", None)
+        self._core.fit(corpus, vectors, parallel, params)
+
+    def save(self, directory: str) -> None:
+        self._core.save(directory)
+
+    @classmethod
+    def load(cls, directory: str) -> HelixIndex:
+        obj = cls.__new__(cls)
+        obj._core = _HelixCore.load(directory)
+        return obj
+
+    # ── Metadata ──────────────────────────────────────────────────────────────
+
+    @property
+    def size(self) -> int:
+        return self._core.size
+
+    @property
+    def is_trained(self) -> bool:
+        return self._core.is_trained
+
+    @property
+    def index_type(self) -> str:
+        return "helix"
+
+    @property
+    def boundaries(self) -> np.ndarray | None:
+        return self._core.boundaries
+
+    @property
+    def fit_values(self) -> np.ndarray | None:
+        return self._core.fit_values
+
+    @property
+    def _params(self) -> _Parameters | None:
+        return self._core._params
+
+    @property
+    def text_index(self) -> TextIndex:
+        return self._core.text_index
+
+    @property
+    def vector_index(self) -> VectorIndex:
+        return self._core.vector_index
+
+    def __repr__(self) -> str:
+        return (
+            f"HelixIndex(text={self._core.text_index.index_type!r}, "
+            f"vector={self._core.vector_index.index_type!r}, "
+            f"text_k={self._core._text_k}, vector_k={self._core._vector_k})"
+        )

simlar/indexes/lookup_index.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import numpy as np
+from simlar_engine.indexes._lookup_impl import _TextCore
+
+from simlar.contracts import SearchResult, TextIndex
+from simlar.indexes.registry import register
+
+
+@register("lookup")
+class LookupIndex(TextIndex):
+    """
+    Example:
+        >>> idx = LookupIndex()
+        >>> idx.add(["a", "b"], ["hello world", "foo bar"])
+        >>> results = idx.search("hello", k=5)
+        >>> results[0].id
+        'a'
+    """
+
+    def __init__(
+        self,
+        stopwords_lang: str = "english",
+        stemmer_lang: str = "english",
+    ) -> None:
+        self._core = _TextCore(stopwords_lang, stemmer_lang)
+
+    # ── Public contract ───────────────────────────────────────────────────────
+
+    def fit(self, corpus: list[str], parallel: bool = False, **kwargs: object) -> None:
+        self._core.fit(corpus, parallel)
+
+    def add(self, ids: list[str], texts: list[str]) -> None:
+        self._core.add(ids, texts)
+
+    def update(self, ids: list[str], texts: list[str]) -> None:
+        self._core.update(ids, texts)
+
+    def delete(self, ids: list[str]) -> None:
+        self._core.delete(ids)
+
+    def search(self, query: str, k: int = 10) -> list[SearchResult]:
+        return self._core.search(query, k)
+
+    def search_raw(
+        self,
+        queries: str | list[str],
+        k: int,
+        parallel: bool = False,
+    ) -> tuple[np.ndarray, np.ndarray]:
+        return self._core.search_raw(queries, k, parallel)
+
+    def save(self, directory: str) -> None:
+        self._core.save(directory)
+
+    @classmethod
+    def load(cls, directory: str) -> LookupIndex:
+        obj = cls.__new__(cls)
+        obj._core = _TextCore.load(directory)
+        return obj
+
+    # ── Properties ────────────────────────────────────────────────────────────
+
+    @property
+    def size(self) -> int:
+        return self._core.size
+
+    @property
+    def is_trained(self) -> bool:
+        return self._core.is_trained
+
+    @property
+    def index_type(self) -> str:
+        return self._core.index_type
+
+    @property
+    def ids(self) -> list[str]:
+        return self._core.ids