longcache 0.1.0__py3-none-any.whl

longcache/core/embedder.py

@@ -0,0 +1,155 @@
+"""
+Local embedding engine for LongCache.
+
+Uses sentence-transformers to generate embeddings locally with no API calls.
+The default model (all-MiniLM-L6-v2) produces 384-dimensional vectors
+in under 5ms on modern hardware.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from typing import Optional
+
+import numpy as np
+
+from longcache.core.base_embedder import BaseEmbedder
+
+_log = logging.getLogger(__name__)
+
+
+class LocalEmbedder(BaseEmbedder):
+    """
+    Local embedding engine using sentence-transformers.
+
+    Loads the model lazily on first use to avoid import penalties at module level.
+    Thread-safe — model loading is protected by a lock.
+
+    Usage:
+        embedder = LocalEmbedder()
+        vector = embedder.embed("What is your refund policy?")
+        # vector.shape = (384,) for MiniLM-L6-v2
+    """
+
+    # Class-level cache for model instances (singleton per model name)
+    _instances: dict[str, LocalEmbedder] = {}
+    _lock = threading.Lock()
+
+    def __new__(cls, model_name: str = "BAAI/bge-small-en-v1.5"):
+        """
+        Singleton pattern: one embedder instance per model name.
+
+        This avoids loading the same model multiple times in memory.
+
+        Args:
+            model_name: The sentence-transformers model identifier.
+
+        Returns:
+            The singleton LocalEmbedder instance for the given model.
+        """
+        with cls._lock:
+            if model_name not in cls._instances:
+                instance = super().__new__(cls)
+                instance._model_name = model_name
+                instance._model = None
+                instance._dimension = None
+                instance._model_lock = threading.Lock()
+                cls._instances[model_name] = instance
+            return cls._instances[model_name]
+
+    def __init__(self, model_name: str = "BAAI/bge-small-en-v1.5"):
+        """
+        Initialize the embedder. Model is loaded lazily on first embed() call.
+
+        Args:
+            model_name: The sentence-transformers model identifier.
+                Defaults to all-MiniLM-L6-v2 (384-dim, sub-5ms).
+        """
+        # __new__ handles actual initialization; __init__ just ensures
+        # the model name is set (idempotent for singleton)
+        self._model_name = model_name
+
+    def _load_model(self):
+        """
+        Lazily load the sentence-transformers model.
+
+        This is called on the first embed() invocation, not at import time,
+        so importing longcache does not trigger a heavy model download.
+        """
+        if self._model is not None:
+            return
+
+        with self._model_lock:
+            if self._model is not None:
+                return
+
+            try:
+                from sentence_transformers import SentenceTransformer
+            except ImportError:
+                raise ImportError(
+                    "sentence-transformers is required for LocalEmbedder. "
+                    "Install with: uv pip install longcache"
+                )
+
+            _log.info("Loading embedding model: %s", self._model_name)
+            self._model = SentenceTransformer(self._model_name)
+            self._dimension = self._model.get_embedding_dimension()
+            _log.info("Embedding model loaded: dim=%d", self._dimension)
+
+    @property
+    def dimension(self) -> int:
+        """
+        Get the embedding dimension for this model.
+
+        Loads the model if not already loaded.
+
+        Returns:
+            The dimensionality of the embedding vectors.
+        """
+        self._load_model()
+        return self._dimension
+
+    def embed(self, text: str) -> np.ndarray:
+        """
+        Generate an embedding vector for the input text.
+
+        The model is loaded lazily on the first call. Subsequent calls
+        use the cached model instance.
+
+        Args:
+            text: The input text to embed.
+
+        Returns:
+            L2-normalized numpy float32 array of shape (dimension,).
+            Normalization ensures cosine similarity equals inner product
+            when using FAISS IndexFlatIP.
+        """
+        self._load_model()
+        vector = self._model.encode(text, normalize_embeddings=True)
+        return vector.astype(np.float32)
+
+    def embed_batch(self, texts: list[str]) -> np.ndarray:
+        """
+        Generate embedding vectors for multiple texts in a single batch.
+
+        More efficient than calling embed() in a loop for multiple texts
+        because the model can parallelize computation across the batch.
+
+        Args:
+            texts: List of input texts to embed.
+
+        Returns:
+            numpy float32 array of shape (len(texts), dimension).
+            Each row is L2-normalized.
+        """
+        if not texts:
+            return np.array([], dtype=np.float32)
+
+        self._load_model()
+        vectors = self._model.encode(texts, normalize_embeddings=True, batch_size=32)
+        return vectors.astype(np.float32)
+
+    def __repr__(self) -> str:
+        model_status = "loaded" if self._model is not None else "not loaded"
+        return f"LocalEmbedder(model={self._model_name!r}, status={model_status})"

longcache/core/entry.py

@@ -0,0 +1,122 @@
+"""
+Cache entry data model.
+
+Represents a single cached query-response pair with metadata
+for TTL, namespace isolation, and hit tracking.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+import numpy as np
+
+
+@dataclass
+class CacheEntry:
+    """
+    A single cache entry storing a query, its embedding vector, and the cached response.
+
+    Attributes:
+        id: Unique identifier for the cache entry.
+        namespace: Namespace this entry belongs to (for isolation).
+        query: The original query text.
+        vector: The embedding vector for the query (numpy float32 array).
+        response: The cached LLM response. Can be any serializable Python object.
+        created_at: Unix timestamp when the entry was created.
+        expires_at: Unix timestamp when the entry expires. None means no expiry.
+        hit_count: Number of times this entry has been served from cache.
+    """
+
+    id: Optional[int] = None
+    namespace: str = "default"
+    query: str = ""
+    vector: Optional[np.ndarray] = None
+    response: Any = None
+    created_at: float = field(default_factory=time.time)
+    expires_at: Optional[float] = None
+    hit_count: int = 0
+    model: str = ""
+    system_prompt: str = ""
+    temperature: float = -1.0
+    tags: list[str] = field(default_factory=list)
+
+    def is_expired(self) -> bool:
+        """
+        Check if this cache entry has expired.
+
+        Returns:
+            True if the entry has expired, False otherwise.
+            Entries with expires_at=None never expire.
+        """
+        if self.expires_at is None:
+            return False
+        return time.time() > self.expires_at
+
+    def increment_hit(self) -> None:
+        """Increment the hit count for this cache entry."""
+        self.hit_count += 1
+
+    def to_dict(self) -> dict:
+        """
+        Serialize the cache entry to a dictionary.
+
+        The vector is converted to bytes for storage compatibility.
+        The response is stored as-is (should be JSON-serializable by backends).
+
+        Returns:
+            Dictionary representation of the cache entry.
+        """
+        return {
+            "id": self.id,
+            "namespace": self.namespace,
+            "query": self.query,
+            "vector": self.vector.tobytes() if self.vector is not None else None,
+            "response": self.response,
+            "created_at": self.created_at,
+            "expires_at": self.expires_at,
+            "hit_count": self.hit_count,
+            "model": self.model,
+            "system_prompt": self.system_prompt,
+            "temperature": self.temperature,
+            "tags": list(self.tags or []),
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict, vector_dim: Optional[int] = None) -> CacheEntry:
+        """
+        Deserialize a cache entry from a dictionary.
+
+        The vector field is converted from bytes back to a numpy array
+        if vector_dim is provided.
+
+        Args:
+            data: Dictionary with cache entry fields.
+            vector_dim: Expected dimension of the vector array.
+                Required if the vector is stored as raw bytes.
+
+        Returns:
+            CacheEntry instance.
+        """
+        vector = data.get("vector")
+        if isinstance(vector, bytes) and vector_dim is not None:
+            vector = np.frombuffer(vector, dtype=np.float32).copy()
+            if vector_dim and len(vector) != vector_dim:
+                vector = vector.reshape(vector_dim) if vector.size == vector_dim else vector
+
+        return cls(
+            id=data.get("id"),
+            namespace=data.get("namespace", "default"),
+            query=data.get("query", ""),
+            vector=vector,
+            response=data.get("response"),
+            created_at=data.get("created_at", time.time()),
+            expires_at=data.get("expires_at"),
+            hit_count=data.get("hit_count", 0),
+            model=data.get("model", ""),
+            system_prompt=data.get("system_prompt", ""),
+            temperature=float(data.get("temperature", -1.0)),
+            tags=list(data.get("tags") or []),
+        )

longcache/core/index.py

@@ -0,0 +1,272 @@
+"""
+Similarity index for LongCache using FAISS.
+
+Provides fast approximate nearest neighbor search over embedding vectors.
+Uses IndexFlatIP (inner product) for exact search, which works well
+for caches with up to ~100K entries. For larger caches, consider
+switching to an IVF index.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import threading
+from typing import Optional
+
+import numpy as np
+
+from longcache.core.entry import CacheEntry
+
+
+class SimilarityIndex:
+    """
+    FAISS-based similarity index for cache lookups.
+
+    Maintains an in-memory FAISS index alongside a mapping from
+    internal FAISS IDs to CacheEntry objects. All operations are
+    thread-safe via a reentrant lock.
+
+    Usage:
+        index = SimilarityIndex(backend, config)
+        index.add(vector, entry_id)
+        result = index.find_similar(query_vector, threshold=0.92)
+    """
+
+    def __init__(self, backend, config):
+        """
+        Initialize the similarity index.
+
+        Args:
+            backend: The cache backend (used for retrieving entries by ID).
+            config: CacheConfig instance with similarity_threshold and other settings.
+        """
+        self._backend = backend
+        self._config = config
+        self._lock = threading.RLock()
+        self._async_lock: Optional[asyncio.Lock] = None
+        self._faiss_index = None
+        self._id_map: dict[int, int] = {}  # faiss_internal_id -> entry_id
+        self._next_faiss_id = 0
+        self._dimension = None
+        self._orphan_count = 0  # vectors in FAISS with no id_map entry
+        self._REBUILD_THRESHOLD = 0.20  # rebuild when 20% of index is orphaned
+        self._removed_entry_ids: set[int] = set()  # entry ids explicitly removed via remove()
+
+    def _get_async_lock(self) -> asyncio.Lock:
+        """Lazily create asyncio.Lock on first async use."""
+        if self._async_lock is None:
+            self._async_lock = asyncio.Lock()
+        return self._async_lock
+
+    def _ensure_index(self, dimension: int):
+        """
+        Lazily create the FAISS index with the given dimension.
+
+        Called automatically on first add() with a vector of the
+        appropriate dimension.
+
+        Args:
+            dimension: The dimensionality of the embedding vectors.
+        """
+        if self._faiss_index is not None:
+            return
+
+        import faiss
+
+        index_type = getattr(self._config, "faiss_index_type", "flat")
+        if index_type == "hnsw":
+            m = getattr(self._config, "hnsw_m", 32)
+            ef = getattr(self._config, "hnsw_ef_search", 64)
+            # HNSW with inner product metric (cosine for normalized vectors)
+            self._faiss_index = faiss.IndexHNSWFlat(dimension, m, faiss.METRIC_INNER_PRODUCT)
+            self._faiss_index.hnsw.efSearch = ef
+        else:
+            # Default: exact flat index
+            self._faiss_index = faiss.IndexFlatIP(dimension)
+
+        self._dimension = dimension
+
+    def add(self, vector: np.ndarray, entry_id: int) -> None:
+        """
+        Add an embedding vector to the index.
+
+        Creates the FAISS index on first call, inferring dimension
+        from the input vector.
+
+        Args:
+            vector: L2-normalized embedding vector of shape (dimension,).
+            entry_id: The database ID of the corresponding CacheEntry.
+        """
+        with self._lock:
+            if self._faiss_index is None:
+                self._ensure_index(vector.shape[0])
+
+            # FAISS expects 2D arrays
+            vector_2d = vector.reshape(1, -1).astype(np.float32)
+            faiss_id = self._next_faiss_id
+            # Assign id_map BEFORE calling FAISS to keep mapping consistent
+            self._id_map[faiss_id] = entry_id
+            self._faiss_index.add(vector_2d)
+            self._next_faiss_id += 1
+
+    async def async_add(self, vector: np.ndarray, entry_id: int) -> None:
+        """Async-safe version of add() for use in async contexts."""
+        async with self._get_async_lock():
+            self.add(vector, entry_id)
+
+    def find_similar(
+        self,
+        vector: np.ndarray,
+        threshold: float = 0.92,
+        namespace: str = "default",
+    ) -> Optional[CacheEntry]:
+        """
+        Find the most similar cached entry for the given query vector.
+
+        Searches the FAISS index for the nearest neighbor, then verifies
+        the entry exists in the backend, belongs to the correct namespace,
+        and meets the similarity threshold.
+
+        Args:
+            vector: L2-normalized query embedding vector.
+            threshold: Minimum cosine similarity for a cache hit.
+            namespace: Only return entries from this namespace.
+
+        Returns:
+            CacheEntry if a similar entry is found above the threshold,
+            None otherwise.
+        """
+        with self._lock:
+            if self._faiss_index is None or self._faiss_index.ntotal == 0:
+                return None
+
+            # Search for top-k (k=1) nearest neighbors
+            vector_2d = vector.reshape(1, -1).astype(np.float32)
+            similarities, indices = self._faiss_index.search(vector_2d, 1)
+
+            if len(indices) == 0 or indices[0][0] < 0:
+                return None
+
+            best_similarity = float(similarities[0][0])
+            best_faiss_id = int(indices[0][0])
+
+            if best_similarity < threshold:
+                return None
+
+            # Look up the entry ID from our mapping
+            entry_id = self._id_map.get(best_faiss_id)
+            if entry_id is None:
+                return None
+
+            # Retrieve the full entry from the backend
+            entry = self._backend.retrieve(entry_id)
+            if entry is None:
+                return None
+
+            # Verify namespace
+            if entry.namespace != namespace:
+                return None
+
+            # Verify not expired
+            if entry.is_expired():
+                return None
+
+            # Update hit count
+            entry.increment_hit()
+            self._backend.update_hit_count(entry_id)
+
+            return entry
+
+    def remove(self, faiss_id: int) -> None:
+        """
+        Remove an entry from the ID map by its FAISS internal ID.
+
+        Note: FAISS IndexFlatIP does not support efficient removal.
+        The entry is removed from the ID map, so it won't be returned
+        in future searches even though the vector remains in the index.
+        For production use with high churn, consider rebuilding the index
+        periodically.
+
+        Args:
+            faiss_id: The internal FAISS index ID to remove.
+        """
+        with self._lock:
+            if faiss_id in self._id_map:
+                entry_id = self._id_map.pop(faiss_id)
+                self._removed_entry_ids.add(entry_id)
+                self._orphan_count += 1
+                self._auto_rebuild_if_needed()
+
+    def _auto_rebuild_if_needed(self) -> None:
+        """Rebuild FAISS index if orphan ratio exceeds threshold."""
+        total = self._faiss_index.ntotal if self._faiss_index else 0
+        if total > 0 and self._orphan_count / total >= self._REBUILD_THRESHOLD:
+            import logging
+
+            logging.getLogger(__name__).info(
+                "Auto-rebuilding FAISS index: %d orphaned vectors (%.0f%%)",
+                self._orphan_count,
+                (self._orphan_count / total) * 100,
+            )
+            entries = self._backend.list_entries(self._config.namespace)
+            entries = [e for e in entries if e.id not in self._removed_entry_ids]
+            self.rebuild(entries)
+            self._orphan_count = 0
+
+    def rebuild(self, entries: list[CacheEntry]) -> None:
+        """
+        Rebuild the FAISS index from a list of cache entries.
+
+        Useful for compacting the index after many removals, or for
+        loading the index from persistent storage on startup.
+
+        Args:
+            entries: List of CacheEntry objects with their vectors populated.
+        """
+        with self._lock:
+            import faiss
+
+            valid_entries = [
+                e for e in (entries or []) if e.vector is not None and e.id is not None
+            ]
+
+            if not valid_entries:
+                self._faiss_index = None
+                self._id_map = {}
+                self._next_faiss_id = 0
+                self._orphan_count = 0
+                self._removed_entry_ids.clear()
+                return
+
+            dimension = valid_entries[0].vector.shape[0]
+            self._ensure_index(dimension)
+
+            # Reset index
+            self._faiss_index = faiss.IndexFlatIP(dimension)
+            self._id_map = {}
+            self._next_faiss_id = 0
+
+            # Add all valid entries
+            vectors = []
+            for entry in valid_entries:
+                vectors.append(entry.vector)
+                self._id_map[self._next_faiss_id] = entry.id
+                self._next_faiss_id += 1
+
+            if vectors:
+                vectors_array = np.stack(vectors).astype(np.float32)
+                self._faiss_index.add(vectors_array)
+
+            self._orphan_count = 0
+            self._removed_entry_ids.clear()
+
+    @property
+    def size(self) -> int:
+        """Return the number of entries currently in the index."""
+        with self._lock:
+            return self._faiss_index.ntotal if self._faiss_index is not None else 0
+
+    def __repr__(self) -> str:
+        size = self.size
+        dim = self._dimension or "uninitialized"
+        return f"SimilarityIndex(size={size}, dimension={dim})"

longcache/integrations/__init__.py

@@ -0,0 +1,32 @@
+"""Integrations with LLM frameworks."""
+
+from longcache.integrations.langchain import CachedLLM, CachedChain
+
+__all__ = ["CachedLLM", "CachedChain"]
+
+
+# Lazy imports for optional integrations
+def get_llamaindex_wrapper():
+    """Import and return CachedQueryEngine. Raises ImportError if llama-index is not installed."""
+    try:
+        from longcache.integrations.llamaindex import CachedQueryEngine
+
+        return CachedQueryEngine
+    except ImportError:
+        raise ImportError(
+            "LlamaIndex integration requires the 'llamaindex' extra. "
+            "Install with: uv pip install longcache[llamaindex]"
+        )
+
+
+def get_openai_wrapper():
+    """Import and return CachedOpenAI. Raises ImportError if openai is not installed."""
+    try:
+        from longcache.integrations.openai import CachedOpenAI
+
+        return CachedOpenAI
+    except ImportError:
+        raise ImportError(
+            "OpenAI integration requires the 'openai' extra. "
+            "Install with: uv pip install longcache[openai]"
+        )