synaptoroute 0.3.0__py3-none-any.whl

synaptoroute/__init__.py

@@ -0,0 +1,18 @@
+"""
+SynaptoRoute
+A high-throughput, local semantic routing engine.
+"""
+
+import importlib.metadata
+
+try:
+    __version__ = importlib.metadata.version("synaptoroute")
+except importlib.metadata.PackageNotFoundError:
+    __version__ = "unknown"
+
+from synaptoroute.router import AdaptiveRouter
+from synaptoroute.encoder import Encoder
+from synaptoroute.storage import BaseStorage, SQLiteStorage
+from synaptoroute.models import Route
+
+__all__ = ["AdaptiveRouter", "Encoder", "BaseStorage", "SQLiteStorage", "Route", "__version__"]

synaptoroute/encoder.py

@@ -0,0 +1,136 @@
+import abc
+import numpy as np
+import numpy.typing as npt
+from typing import List, Optional
+
+class BaseEncoder(abc.ABC):
+    """
+    Abstract base class for all SynaptoRoute encoders.
+    """
+    @property
+    @abc.abstractmethod
+    def requires_lock(self) -> bool:
+        pass
+
+    @property
+    @abc.abstractmethod
+    def dim(self) -> int:
+        pass
+
+    @abc.abstractmethod
+    def encode(self, text: str) -> npt.NDArray[np.float32]:
+        pass
+
+    @abc.abstractmethod
+    def encode_batch(self, texts: List[str]) -> npt.NDArray[np.float32]:
+        pass
+
+class FastEmbedEncoder(BaseEncoder):
+    """
+    Handles local intent embeddings using fastembed ONNX models.
+    """
+    def __init__(self, model_name: str = "BAAI/bge-small-en-v1.5", providers: List[str] = None, threads: Optional[int] = None):
+        from fastembed import TextEmbedding
+        if providers is None:
+            providers = ["CPUExecutionProvider"]
+        self.model = TextEmbedding(model_name=model_name, providers=providers, threads=threads)
+        # Probe dimensionality directly from the model using a dummy token
+        self._dim = len(list(self.model.embed(["test"]))[0])
+    
+    @property
+    def requires_lock(self) -> bool:
+        return True
+
+    @property
+    def dim(self) -> int:
+        return self._dim
+
+    def encode(self, text: str) -> npt.NDArray[np.float32]:
+        embeddings = list(self.model.embed([text]))
+        return embeddings[0]
+        
+    def encode_batch(self, texts: List[str]) -> npt.NDArray[np.float32]:
+        if not texts:
+            # Dynamically use the model's true dimensionality
+            return np.empty((0, self.dim), dtype=np.float32)
+            
+        chunk_size = 500
+        all_embeddings = []
+        for i in range(0, len(texts), chunk_size):
+            chunk = texts[i:i + chunk_size]
+            embeddings = list(self.model.embed(chunk))
+            all_embeddings.extend(embeddings)
+            
+        return np.array(all_embeddings, dtype=np.float32)
+
+class OpenAIEncoder(BaseEncoder):
+    """
+    Handles remote intent embeddings using OpenAI models.
+    """
+    def __init__(self, model_name: str = "text-embedding-3-small", dim: Optional[int] = None, dimensions: Optional[int] = None, client=None):
+        try:
+            import openai
+        except ImportError as e:
+            raise RuntimeError("Please install synaptoroute[openai] to use the OpenAIEncoder. Run `pip install synaptoroute[openai]`.") from e
+        self.model_name = model_name
+        self.client = client or openai.OpenAI()
+        
+        self.dimensions = dimensions
+        if dimensions is not None:
+            self._dim = dimensions
+        elif dim is not None:
+            self._dim = dim
+        else:
+            # Hardcode based on known models to save an API call
+            if model_name == "text-embedding-3-small":
+                self._dim = 1536
+            elif model_name == "text-embedding-3-large":
+                self._dim = 3072
+            elif model_name == "text-embedding-ada-002":
+                self._dim = 1536
+            else:
+                raise ValueError("dim must be provided explicitly for unknown models.")
+                
+    @property
+    def requires_lock(self) -> bool:
+        return False
+
+    @property
+    def dim(self) -> int:
+        return self._dim
+
+    def encode(self, text: str) -> npt.NDArray[np.float32]:
+        import openai
+        from synaptoroute.exceptions import SynaptoRouteError
+        try:
+            kwargs = {"input": [text], "model": self.model_name}
+            if self.dimensions is not None:
+                kwargs["dimensions"] = self.dimensions
+            response = self.client.embeddings.create(**kwargs)
+            return np.array(response.data[0].embedding, dtype=np.float32)
+        except openai.OpenAIError as e:
+            raise SynaptoRouteError(f"OpenAI API Error: {e}") from e
+        
+    def encode_batch(self, texts: List[str]) -> npt.NDArray[np.float32]:
+        if not texts:
+            return np.empty((0, self.dim), dtype=np.float32)
+        
+        import openai
+        from synaptoroute.exceptions import SynaptoRouteError
+        try:
+            chunk_size = 2048
+            all_embeddings = []
+            for i in range(0, len(texts), chunk_size):
+                chunk = texts[i:i + chunk_size]
+                kwargs = {"input": chunk, "model": self.model_name}
+                if self.dimensions is not None:
+                    kwargs["dimensions"] = self.dimensions
+                response = self.client.embeddings.create(**kwargs)
+                embeddings = [data.embedding for data in response.data]
+                all_embeddings.extend(embeddings)
+            return np.array(all_embeddings, dtype=np.float32)
+        except openai.OpenAIError as e:
+            raise SynaptoRouteError(f"OpenAI API Error: {e}") from e
+
+# Preserve backwards compatibility
+Encoder = FastEmbedEncoder

synaptoroute/exceptions.py

@@ -0,0 +1,19 @@
+class SynaptoRouteError(Exception):
+    """Base exception for SynaptoRoute."""
+    pass
+
+class RouteNotFoundError(SynaptoRouteError):
+    """Raised when a specified route cannot be found."""
+    pass
+
+class ModelLoadError(SynaptoRouteError):
+    """Raised when an embedding model fails to load."""
+    pass
+
+class RouterOverloadedError(SynaptoRouteError):
+    """Raised when the dynamic batching queue is full (DDoS protection)."""
+    pass
+
+class RouterCapacityError(SynaptoRouteError):
+    """Raised when the router's maximum capacity is exceeded."""
+    pass

synaptoroute/index.py

@@ -0,0 +1,296 @@
+import numpy as np
+import threading
+from typing import List, Tuple
+
+try:
+    import faiss
+    HAS_FAISS = True
+except ImportError:
+    HAS_FAISS = False
+
+class NumpyIndex:
+    """
+    O(1) Lazy Memory Slicing dense numpy index.
+    Used as the default engine if FAISS is not installed.
+    """
+    def __init__(self, dim: int, max_capacity: int = 50000):
+        self.dim = dim
+        self.lock = threading.Lock()
+        self.embeddings = np.zeros((max_capacity, dim), dtype=np.float32)
+        self.tombstones = set()
+        self._tombstone_array = np.array([], dtype=int)
+        self._id_to_route = {}
+        self._route_to_ids = {}
+        self._next_id = 0
+        self.max_capacity = max_capacity
+        self.ntotal = 0
+        
+    def _add_unlocked(self, embeddings: np.ndarray, route_name: str):
+        num_embs = embeddings.shape[0]
+        if self._next_id + num_embs > self.max_capacity:
+            raise ValueError("ID_OVERFLOW")
+        
+        if embeddings.dtype != np.float32:
+            embeddings = embeddings.astype(np.float32)
+            
+        norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+        norms[norms == 0] = 1
+        embeddings = embeddings / norms
+        
+        self.embeddings[self._next_id:self._next_id + num_embs] = embeddings
+        ids = list(range(self._next_id, self._next_id + num_embs))
+        
+        if route_name not in self._route_to_ids:
+            self._route_to_ids[route_name] = []
+        self._route_to_ids[route_name].extend(ids)
+        for i in ids:
+            self._id_to_route[i] = route_name
+            
+        self._next_id += num_embs
+        self.ntotal += num_embs
+
+    def add(self, embeddings: np.ndarray, route_name: str):
+        with self.lock:
+            self._add_unlocked(embeddings, route_name)
+            num_embs = embeddings.shape[0]
+            if self._next_id + num_embs > self.max_capacity:
+                raise ValueError("Capacity exceeded")
+            
+            if embeddings.dtype != np.float32:
+                embeddings = embeddings.astype(np.float32)
+                
+            norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+            norms[norms == 0] = 1
+            embeddings = embeddings / norms
+            
+            self.embeddings[self._next_id:self._next_id + num_embs] = embeddings
+            ids = list(range(self._next_id, self._next_id + num_embs))
+            
+            if route_name not in self._route_to_ids:
+                self._route_to_ids[route_name] = []
+            self._route_to_ids[route_name].extend(ids)
+            for i in ids:
+                self._id_to_route[i] = route_name
+                
+            self._next_id += num_embs
+            self.ntotal += num_embs
+
+    def delete(self, route_name: str):
+        with self.lock:
+            if route_name in self._route_to_ids:
+                ids = self._route_to_ids[route_name]
+                self.tombstones.update(ids)
+                self._tombstone_array = np.array(list(self.tombstones), dtype=int)
+                for i in ids:
+                    self._id_to_route.pop(i, None)
+                del self._route_to_ids[route_name]
+
+    def search(self, query_embeddings: np.ndarray, top_k: int = 1) -> List[List[Tuple[float, str]]]:
+        with self.lock:
+            if self.ntotal == 0 or self._next_id == 0:
+                return [[] for _ in range(query_embeddings.shape[0])]
+                
+            if query_embeddings.dtype != np.float32:
+                query_embeddings = query_embeddings.astype(np.float32)
+                
+            norms = np.linalg.norm(query_embeddings, axis=1, keepdims=True)
+            norms[norms == 0] = 1
+            query_embeddings = query_embeddings / norms
+            
+            valid_mask = np.ones(self._next_id, dtype=bool)
+            if self._tombstone_array.size > 0:
+                valid_mask[self._tombstone_array] = False
+                
+            if not np.any(valid_mask):
+                return [[] for _ in range(query_embeddings.shape[0])]
+                
+            scores = np.dot(query_embeddings, self.embeddings[:self._next_id].T)
+            
+            results = []
+            for i in range(scores.shape[0]):
+                valid_scores = scores[i][valid_mask]
+                valid_indices = np.arange(self._next_id)[valid_mask]
+                
+                # Sort descending
+                num_results = min(top_k, len(valid_scores))
+                if num_results == 0:
+                    results.append([])
+                    continue
+                    
+                if len(valid_scores) > num_results:
+                    sort_idx = np.argpartition(valid_scores, -num_results)[-num_results:]
+                    sort_idx = sort_idx[np.argsort(valid_scores[sort_idx])[::-1]]
+                else:
+                    sort_idx = np.argsort(valid_scores)[::-1]
+                
+                query_results = []
+                for idx in sort_idx:
+                    real_idx = valid_indices[idx]
+                    route_name = self._id_to_route[real_idx]
+                    query_results.append((float(valid_scores[idx]), route_name))
+                results.append(query_results)
+            return results
+
+    @property
+    def total_vectors(self) -> int:
+        return self.ntotal - len(self.tombstones)
+
+    def rebuild(self, route_map: dict, embeddings_map: dict):
+        with self.lock:
+            self.embeddings = np.zeros((self.max_capacity, self.dim), dtype=np.float32)
+            self._route_to_ids = {}
+            self._id_to_route = {}
+            self._next_id = 0
+            self.tombstones.clear()
+            self._tombstone_array = np.array([], dtype=int)
+            self.ntotal = 0
+            
+            for route_name, route in route_map.items():
+                if route_name not in embeddings_map:
+                    continue
+                embs_data = embeddings_map[route_name]
+                if not embs_data:
+                    continue
+                valid_embs = []
+                for e_bytes in embs_data:
+                    if e_bytes is not None:
+                        valid_embs.append(np.frombuffer(e_bytes, dtype=np.float32))
+                if not valid_embs:
+                    continue
+                embeddings = np.stack(valid_embs)
+                self._add_unlocked(embeddings, route_name)
+
+def get_index(dim: int, max_capacity: int = 50000):
+    if HAS_FAISS:
+        return FaissIndex(dim)
+    else:
+        return NumpyIndex(dim, max_capacity)
+
+class FaissIndex:
+    """
+    A FAISS-based vector index utilizing HNSW for sub-linear search latency.
+    Employs a Tombstone architecture for O(1) instantaneous deletions.
+    """
+    def __init__(self, dim: int):
+        self.dim = dim
+        self.lock = threading.Lock()
+        
+        # Inner Product (Cosine Similarity for normalized embeddings)
+        base_index = faiss.IndexHNSWFlat(self.dim, 32, faiss.METRIC_INNER_PRODUCT)
+        self.index = faiss.IndexIDMap(base_index)
+        
+        self.tombstones = set()
+        
+        # Bidirectional mapping
+        self._id_to_route = {}
+        self._route_to_ids = {}
+        self._next_id = 0
+
+    def add(self, embeddings: np.ndarray, route_name: str):
+        with self.lock:
+            num_embs = embeddings.shape[0]
+            ids = np.arange(self._next_id, self._next_id + num_embs, dtype=np.int64)
+            
+            if embeddings.dtype != np.float32:
+                embeddings = embeddings.astype(np.float32)
+                
+            faiss.normalize_L2(embeddings)
+            self.index.add_with_ids(embeddings, ids)
+            
+            if route_name not in self._route_to_ids:
+                self._route_to_ids[route_name] = []
+                
+            self._route_to_ids[route_name].extend(ids.tolist())
+            for i in ids:
+                self._id_to_route[int(i)] = route_name
+                
+            self._next_id += num_embs
+
+    def delete(self, route_name: str):
+        with self.lock:
+            if route_name in self._route_to_ids:
+                ids = self._route_to_ids[route_name]
+                self.tombstones.update(ids)
+                for i in ids:
+                    self._id_to_route.pop(i, None)
+                del self._route_to_ids[route_name]
+
+    def search(self, query_embeddings: np.ndarray, top_k: int = 1) -> List[List[Tuple[float, str]]]:
+        with self.lock:
+            # Overfetch to account for tombstones 
+            search_k = min(self.index.ntotal, max(top_k + len(self.tombstones) * 2, 2048))
+            
+            if search_k == 0:
+                return [[] for _ in range(query_embeddings.shape[0])]
+                
+            if query_embeddings.dtype != np.float32:
+                query_embeddings = query_embeddings.astype(np.float32)
+                
+            faiss.normalize_L2(query_embeddings)
+            distances, indices = self.index.search(query_embeddings, search_k)
+            
+            results = []
+            for i in range(query_embeddings.shape[0]):
+                query_results = []
+                for j in range(search_k):
+                    idx = int(indices[i][j])
+                    if idx != -1 and idx not in self.tombstones:
+                        route_name = self._id_to_route[idx]
+                        query_results.append((float(distances[i][j]), route_name))
+                        if len(query_results) == top_k:
+                            break
+                results.append(query_results)
+                
+            return results
+
+    @property
+    def total_vectors(self) -> int:
+        return self.index.ntotal - len(self.tombstones)
+
+    def rebuild(self, route_map: dict, embeddings_map: dict):
+        """Garbage Collection: Completely reconstructs the HNSW index to flush dead vectors."""
+        with self.lock:
+            # Create a brand new index
+            base_index = faiss.IndexHNSWFlat(self.dim, 32, faiss.METRIC_INNER_PRODUCT)
+            new_index = faiss.IndexIDMap(base_index)
+            
+            new_route_to_ids = {}
+            new_id_to_route = {}
+            next_id = 0
+            
+            for route_name, route in route_map.items():
+                if route_name not in embeddings_map:
+                    continue
+                
+                embs_data = embeddings_map[route_name]
+                if not embs_data:
+                    continue
+                    
+                # Collect embeddings for this route
+                valid_embs = []
+                for e_bytes in embs_data:
+                    if e_bytes is not None:
+                        valid_embs.append(np.frombuffer(e_bytes, dtype=np.float32))
+                        
+                if not valid_embs:
+                    continue
+                    
+                embeddings = np.stack(valid_embs)
+                num_embs = embeddings.shape[0]
+                ids = np.arange(next_id, next_id + num_embs, dtype=np.int64)
+                
+                faiss.normalize_L2(embeddings)
+                new_index.add_with_ids(embeddings, ids)
+                
+                new_route_to_ids[route_name] = ids.tolist()
+                for i in ids:
+                    new_id_to_route[int(i)] = route_name
+                    
+                next_id += num_embs
+                
+            # Atomic swap
+            self.index = new_index
+            self._route_to_ids = new_route_to_ids
+            self._id_to_route = new_id_to_route
+            self._next_id = next_id
+            self.tombstones.clear()

synaptoroute/integrations/__init__.py

@@ -0,0 +1,3 @@
+"""
+SynaptoRoute integrations for external frameworks like LangChain and LlamaIndex.
+"""

synaptoroute/integrations/langchain.py

@@ -0,0 +1,25 @@
+from typing import Any, Optional
+from langchain_core.runnables import Runnable, RunnableConfig
+
+from synaptoroute.router import AdaptiveRouter
+
+class SynaptoRouteChain(Runnable):
+    """
+    A LangChain Runnable that routes string inputs using SynaptoRoute.
+    Returns the route name if matched, else 'default'.
+    """
+    
+    def __init__(self, router: AdaptiveRouter):
+        self.router = router
+
+    def invoke(self, input: str, config: Optional[RunnableConfig] = None, **kwargs: Any) -> str:
+        route = self.router(input)
+        if route and hasattr(route, 'name'):
+            return route.name
+        return "default"
+        
+    async def ainvoke(self, input: str, config: Optional[RunnableConfig] = None, **kwargs: Any) -> str:
+        route = await self.router.aquery(input)
+        if route and hasattr(route, 'name'):
+            return route.name
+        return "default"

synaptoroute/integrations/llamaindex.py

@@ -0,0 +1,47 @@
+from typing import Sequence, Any
+
+from llama_index.core.schema import QueryBundle
+from llama_index.core.selectors import BaseSelector, SelectorResult, SingleSelection
+from llama_index.core.tools.types import ToolMetadata
+
+from synaptoroute.router import AdaptiveRouter
+
+class SynaptoRouteSelector(BaseSelector):
+    """
+    A selector that uses SynaptoRoute's AdaptiveRouter to select a route.
+    """
+    router: Any
+
+    def __init__(self, router: AdaptiveRouter):
+        super().__init__()
+        self.router = router
+
+    def _get_prompts(self) -> dict:
+        return {}
+    
+    def _update_prompts(self, prompts: dict) -> None:
+        pass
+
+    def _select(self, choices: Sequence[ToolMetadata], query: QueryBundle) -> SelectorResult:
+        result = self.router(query.query_str)
+        route_name = result.name if result else "default"
+        
+        for i, choice in enumerate(choices):
+            if choice.name == route_name:
+                return SelectorResult(
+                    selections=[SingleSelection(index=i, reason=f"Matched route: {route_name}")]
+                )
+        
+        raise ValueError(f"No matching choice found for route: {route_name}")
+
+    async def _aselect(self, choices: Sequence[ToolMetadata], query: QueryBundle) -> SelectorResult:
+        result = await self.router.aquery(query.query_str)
+        route_name = result.name if result else "default"
+        
+        for i, choice in enumerate(choices):
+            if choice.name == route_name:
+                return SelectorResult(
+                    selections=[SingleSelection(index=i, reason=f"Matched route: {route_name}")]
+                )
+                
+        raise ValueError(f"No matching choice found for route: {route_name}")

synaptoroute/metrics.py

@@ -0,0 +1,72 @@
+try:
+    from prometheus_client import Histogram, Gauge, Counter, generate_latest, CollectorRegistry
+    HAS_PROMETHEUS = True
+except ImportError:
+    HAS_PROMETHEUS = False
+
+
+import collections
+
+class _MockHistogram:
+    def __init__(self, name):
+        self.name = name
+        self.observations = collections.deque(maxlen=1000)
+
+    def observe(self, amount):
+        self.observations.append(amount)
+
+
+class _MockGauge:
+    def __init__(self, name):
+        self.name = name
+        self.value = 0.0
+
+    def set(self, value):
+        self.value = float(value)
+
+    def inc(self, amount=1.0):
+        self.value += float(amount)
+
+    def dec(self, amount=1.0):
+        self.value -= float(amount)
+
+
+class _MockCounter:
+    def __init__(self, name):
+        self.name = name
+        self.value = 0.0
+
+    def inc(self, amount=1.0):
+        self.value += float(amount)
+
+
+class MetricsRegistry:
+    def __init__(self):
+        if HAS_PROMETHEUS:
+            self.registry = CollectorRegistry()
+            self.inference_latency_seconds = Histogram("inference_latency_seconds", "Latency of routing", registry=self.registry)
+            self.batch_size = Histogram("batch_size", "Batch size of queries processed", registry=self.registry)
+            self.queue_depth = Gauge("queue_depth", "Depth of the query queue", registry=self.registry)
+            self.capacity_usage = Gauge("capacity_usage", "Number of vectors stored in capacity", registry=self.registry)
+            self.gc_errors = Counter("gc_errors_total", "Number of failed background index rebuilds", registry=self.registry)
+        else:
+            self.registry = None
+            self.inference_latency_seconds = _MockHistogram("inference_latency_seconds")
+            self.batch_size = _MockHistogram("batch_size")
+            self.queue_depth = _MockGauge("queue_depth")
+            self.capacity_usage = _MockGauge("capacity_usage")
+            self.gc_errors = _MockCounter("gc_errors_total")
+
+    def export_metrics(self) -> str:
+        if HAS_PROMETHEUS:
+            return generate_latest(self.registry).decode("utf-8")
+        else:
+            lines = []
+            for name, metric in vars(self).items():
+                if isinstance(metric, _MockHistogram):
+                    obs = metric.observations
+                    avg = sum(obs) / len(obs) if obs else 0
+                    lines.append(f"{name}: count={len(obs)} avg={avg}")
+                elif isinstance(metric, (_MockGauge, _MockCounter)):
+                    lines.append(f"{name}: value={metric.value}")
+            return "\n".join(lines)

synaptoroute/models.py

@@ -0,0 +1,39 @@
+import json
+from typing import List, Optional, Dict, Any
+from pydantic import BaseModel, Field, field_validator, StringConstraints, ConfigDict
+from typing_extensions import Annotated
+
+NonEmptyString = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
+
+class Route(BaseModel):
+    """
+    Represents a single semantic route or intent.
+    """
+    model_config = ConfigDict(validate_assignment=True)
+    name: str = Field(..., min_length=1, pattern=r"^[a-zA-Z0-9_-]+$")
+    utterances: List[NonEmptyString] = Field(..., min_length=1)
+    threshold: float = Field(0.5, ge=-1.0, le=1.0)
+    metadata: Optional[Dict[str, Any]] = None
+
+    @field_validator('metadata')
+    @classmethod
+    def validate_metadata_serializable(cls, v: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+        if v is not None:
+            try:
+                json.dumps(v)
+            except (TypeError, ValueError) as e:
+                raise ValueError(f"Metadata must be JSON serializable: {e}")
+        return v
+
+    @field_validator('utterances')
+    @classmethod
+    def deduplicate_utterances(cls, v: List[str]) -> List[str]:
+        seen = set()
+        deduped = []
+        for utt in v:
+            if utt not in seen:
+                seen.add(utt)
+                deduped.append(utt)
+        if not deduped:
+            raise ValueError("Route must have at least one valid utterance.")
+        return deduped

synaptoroute/profile.py

@@ -0,0 +1,31 @@
+import os
+from enum import Enum
+from dataclasses import dataclass
+
+class ProfileType(Enum):
+    LATENCY = "latency"
+    THROUGHPUT = "throughput"
+
+@dataclass
+class OptimizationProfile:
+    type: ProfileType
+    threads: int
+    batch_size: int
+    batch_timeout: float
+
+def get_profile(profile_type: ProfileType = ProfileType.THROUGHPUT) -> OptimizationProfile:
+    cpu_count = os.cpu_count() or 4
+    if profile_type == ProfileType.LATENCY:
+        return OptimizationProfile(
+            type=ProfileType.LATENCY,
+            threads=max(1, cpu_count - 1),
+            batch_size=1,
+            batch_timeout=0.0
+        )
+    else:
+        return OptimizationProfile(
+            type=ProfileType.THROUGHPUT,
+            threads=1,
+            batch_size=32,
+            batch_timeout=0.005
+        )