semantic-router 0.1.0.dev0__tar.gz → 0.1.0.dev2__tar.gz

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: semantic-router
-Version: 0.1.0.dev0
+Version: 0.1.0.dev2
 Summary: Super fast semantic router for AI decision making
 License: MIT
 Author: Aurelio AI
@@ -21,10 +21,10 @@ Provides-Extra: local
 Provides-Extra: mistralai
 Provides-Extra: pinecone
 Provides-Extra: postgres
-Provides-Extra: processing
 Provides-Extra: qdrant
 Provides-Extra: vision
-Requires-Dist: aiohttp (>=3.9.5,<4.0.0)
+Requires-Dist: aiohttp (>=3.10.11,<4.0.0)
+Requires-Dist: aurelio-sdk (>=0.0.16,<0.0.17)
 Requires-Dist: boto3 (>=1.34.98,<2.0.0) ; extra == "bedrock"
 Requires-Dist: botocore (>=1.34.110,<2.0.0) ; extra == "bedrock"
 Requires-Dist: cohere (>=5.9.4,<6.00) ; extra == "cohere"
@@ -33,14 +33,13 @@ Requires-Dist: colorlog (>=6.8.0,<7.0.0)
 Requires-Dist: fastembed (>=0.3.0,<0.4.0) ; extra == "fastembed"
 Requires-Dist: google-cloud-aiplatform (>=1.45.0,<2.0.0) ; extra == "google"
 Requires-Dist: llama-cpp-python (>=0.2.28,<0.2.86) ; extra == "local"
-Requires-Dist: matplotlib (>=3.8.3,<4.0.0) ; extra == "processing"
 Requires-Dist: mistralai (>=0.0.12,<0.1.0) ; extra == "mistralai"
 Requires-Dist: numpy (>=1.25.2,<2.0.0)
 Requires-Dist: openai (>=1.10.0,<2.0.0)
 Requires-Dist: pillow (>=10.2.0,<11.0.0) ; extra == "vision"
 Requires-Dist: pinecone (>=5.0.0) ; extra == "pinecone"
 Requires-Dist: pinecone-text (>=0.7.1,<0.10.0) ; extra == "hybrid"
-Requires-Dist: psycopg2 (>=2.9.9,<3.0.0) ; extra == "postgres"
+Requires-Dist: psycopg2-binary (>=2.9.9,<3.0.0)
 Requires-Dist: pydantic (>=2.5.3,<3.0.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: qdrant-client (>=1.11.1,<2.0.0) ; extra == "qdrant"
@@ -52,6 +51,7 @@ Requires-Dist: tiktoken (>=0.6.0,<1.0.0)
 Requires-Dist: tokenizers (>=0.19) ; extra == "local"
 Requires-Dist: torch (>=2.1.0,<2.6.0) ; extra == "local" or extra == "vision"
 Requires-Dist: torchvision (>=0.17.0,<0.18.0) ; extra == "vision"
+Requires-Dist: tornado (>=6.4.2,<7.0.0)
 Requires-Dist: transformers (>=4.36.2) ; extra == "local" or extra == "vision"
 Description-Content-Type: text/markdown
 

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "semantic-router"
-version = "0.1.0.dev0"
+version = "0.1.0.dev2"
 description = "Super fast semantic router for AI decision making"
 authors = ["Aurelio AI <hello@aurelio.ai>"]
 readme = "README.md"
@@ -16,6 +16,7 @@ mistralai= {version = ">=0.0.12,<0.1.0", optional = true}
 numpy = "^1.25.2"
 colorlog = "^6.8.0"
 pyyaml = "^6.0.1"
+aurelio-sdk = {version = "^0.0.16"}
 pinecone-text = {version = ">=0.7.1,<0.10.0", optional = true}
 torch = {version = ">=2.1.0,<2.6.0", optional = true}
 transformers = {version = ">=4.36.2", optional = true}
@@ -27,24 +28,23 @@ regex = ">=2023.12.25"
 torchvision = { version = ">=0.17.0,<0.18.0", optional = true}
 pillow = { version = ">=10.2.0,<11.0.0", optional = true}
 tiktoken = ">=0.6.0,<1.0.0"
-matplotlib = { version = "^3.8.3", optional = true}
 qdrant-client = {version = "^1.11.1", optional = true}
 google-cloud-aiplatform = {version = "^1.45.0", optional = true}
 requests-mock = "^1.12.1"
 boto3 = { version = "^1.34.98", optional = true }
 botocore = {version = "^1.34.110", optional = true}
-aiohttp = "^3.9.5"
+aiohttp = "^3.10.11"
 fastembed = {version = "^0.3.0", optional = true}
-psycopg2 = {version = "^2.9.9", optional = true}
+psycopg2-binary = {version = "^2.9.9", optional = true}
 sphinx = {version = "^7.0.0", optional = true}
 sphinxawesome-theme = {version = "^5.2.0", optional = true}
+tornado = {version = "^6.4.2", optional = true}
 
 [tool.poetry.extras]
 hybrid = ["pinecone-text"]
 local = ["torch", "transformers", "tokenizers", "huggingface-hub", "llama-cpp-python"]
 pinecone = ["pinecone"]
 vision = ["torch", "torchvision", "transformers", "pillow"]
-processing = ["matplotlib"]
 mistralai = ["mistralai"]
 qdrant = ["qdrant-client"]
 google = ["google-cloud-aiplatform"]
@@ -57,10 +57,11 @@ cohere = ["cohere"]
 [tool.poetry.group.dev.dependencies]
 ipykernel = "^6.25.0"
 ruff = "^0.1.5"
-pytest = "^7.4.3"
+pytest = "^8.2"
 pytest-mock = "^3.12.0"
 pytest-cov = "^4.1.0"
 pytest-xdist = "^3.5.0"
+pytest-asyncio = "^0.24.0"
 mypy = "^1.7.1"
 types-pyyaml = "^6.0.12.12"
 types-requests = "^2.31.0"

semantic_router-0.1.0.dev2/semantic_router/__init__.py

@@ -0,0 +1,6 @@
+from semantic_router.routers import RouterConfig, SemanticRouter, HybridRouter
+from semantic_router.route import Route
+
+__all__ = ["SemanticRouter", "HybridRouter", "Route", "RouterConfig"]
+
+__version__ = "0.1.0.dev2"

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/encoders/__init__.py

@@ -1,6 +1,7 @@
 from typing import List, Optional
 
-from semantic_router.encoders.base import BaseEncoder
+from semantic_router.encoders.aurelio import AurelioSparseEncoder
+from semantic_router.encoders.base import DenseEncoder, SparseEncoder
 from semantic_router.encoders.bedrock import BedrockEncoder
 from semantic_router.encoders.bm25 import BM25Encoder
 from semantic_router.encoders.clip import CLIPEncoder
@@ -14,10 +15,12 @@ from semantic_router.encoders.openai import OpenAIEncoder
 from semantic_router.encoders.tfidf import TfidfEncoder
 from semantic_router.encoders.vit import VitEncoder
 from semantic_router.encoders.zure import AzureOpenAIEncoder
-from semantic_router.schema import EncoderType
+from semantic_router.schema import EncoderType, SparseEmbedding
 
 __all__ = [
-    "BaseEncoder",
+    "AurelioSparseEncoder",
+    "DenseEncoder",
+    "SparseEncoder",
     "AzureOpenAIEncoder",
     "CohereEncoder",
     "OpenAIEncoder",
@@ -37,7 +40,7 @@ __all__ = [
 class AutoEncoder:
     type: EncoderType
     name: Optional[str]
-    model: BaseEncoder
+    model: DenseEncoder | SparseEncoder
 
     def __init__(self, type: str, name: Optional[str]):
         self.type = EncoderType(type)
@@ -49,6 +52,8 @@ class AutoEncoder:
             self.model = CohereEncoder(name=name)
         elif self.type == EncoderType.OPENAI:
             self.model = OpenAIEncoder(name=name)
+        elif self.type == EncoderType.AURELIO:
+            self.model = AurelioSparseEncoder(name=name)
         elif self.type == EncoderType.BM25:
             if name is None:
                 name = "bm25"
@@ -74,5 +79,5 @@ class AutoEncoder:
         else:
             raise ValueError(f"Encoder type '{type}' not supported")
 
-    def __call__(self, texts: List[str]) -> List[List[float]]:
+    def __call__(self, texts: List[str]) -> List[List[float]] | List[SparseEmbedding]:
         return self.model(texts)

semantic_router-0.1.0.dev2/semantic_router/encoders/aurelio.py

@@ -0,0 +1,47 @@
+import os
+from typing import Any, List, Optional
+from pydantic.v1 import Field
+
+from aurelio_sdk import AurelioClient, AsyncAurelioClient, EmbeddingResponse
+
+from semantic_router.encoders.base import SparseEncoder
+from semantic_router.schema import SparseEmbedding
+
+
+class AurelioSparseEncoder(SparseEncoder):
+    model: Optional[Any] = None
+    client: AurelioClient = Field(default_factory=AurelioClient, exclude=True)
+    async_client: AsyncAurelioClient = Field(
+        default_factory=AsyncAurelioClient, exclude=True
+    )
+    type: str = "sparse"
+
+    def __init__(
+        self,
+        name: str | None = None,
+        api_key: Optional[str] = None,
+    ):
+        if name is None:
+            name = "bm25"
+        super().__init__(name=name)
+        if api_key is None:
+            api_key = os.getenv("AURELIO_API_KEY")
+        if api_key is None:
+            raise ValueError("AURELIO_API_KEY environment variable is not set.")
+        self.client = AurelioClient(api_key=api_key)
+        self.async_client = AsyncAurelioClient(api_key=api_key)
+
+    def __call__(self, docs: list[str]) -> list[SparseEmbedding]:
+        res: EmbeddingResponse = self.client.embedding(input=docs, model=self.name)
+        embeds = [SparseEmbedding.from_aurelio(r.embedding) for r in res.data]
+        return embeds
+
+    async def acall(self, docs: list[str]) -> list[SparseEmbedding]:
+        res: EmbeddingResponse = await self.async_client.embedding(
+            input=docs, model=self.name
+        )
+        embeds = [SparseEmbedding.from_aurelio(r.embedding) for r in res.data]
+        return embeds
+
+    def fit(self, docs: List[str]):
+        raise NotImplementedError("AurelioSparseEncoder does not support fit.")

semantic_router-0.1.0.dev2/semantic_router/encoders/base.py

@@ -0,0 +1,53 @@
+from typing import Any, Coroutine, List, Optional
+
+from pydantic.v1 import BaseModel, Field, validator
+import numpy as np
+
+from semantic_router.schema import SparseEmbedding
+
+
+class DenseEncoder(BaseModel):
+    name: str
+    score_threshold: Optional[float] = None
+    type: str = Field(default="base")
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    @validator("score_threshold", pre=True, always=True)
+    def set_score_threshold(cls, v):
+        return float(v) if v is not None else None
+
+    def __call__(self, docs: List[Any]) -> List[List[float]]:
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def acall(self, docs: List[Any]) -> Coroutine[Any, Any, List[List[float]]]:
+        raise NotImplementedError("Subclasses must implement this method")
+
+
+class SparseEncoder(BaseModel):
+    name: str
+    type: str = Field(default="base")
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def __call__(self, docs: List[str]) -> List[SparseEmbedding]:
+        raise NotImplementedError("Subclasses must implement this method")
+
+    async def acall(self, docs: List[str]) -> list[SparseEmbedding]:
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def _array_to_sparse_embeddings(
+        self, sparse_arrays: np.ndarray
+    ) -> List[SparseEmbedding]:
+        """Consumes several sparse vectors containing zero-values and returns a compact array."""
+        if sparse_arrays.ndim != 2:
+            raise ValueError(f"Expected a 2D array, got a {sparse_arrays.ndim}D array.")
+        # get coordinates of non-zero values
+        coords = np.nonzero(sparse_arrays)
+        # create compact array
+        compact_array = np.array([coords[0], coords[1], sparse_arrays[coords]]).T
+        arr_range = range(compact_array[:, 0].max().astype(int) + 1)
+        arrs = [compact_array[compact_array[:, 0] == i, :][:, 1:3] for i in arr_range]
+        return [SparseEmbedding.from_compact_array(arr) for arr in arrs]

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/encoders/bedrock.py

@@ -1,7 +1,7 @@
 """
 This module provides the BedrockEncoder class for generating embeddings using Amazon's Bedrock Platform.
 
-The BedrockEncoder class is a subclass of BaseEncoder and utilizes the TextEmbeddingModel from the
+The BedrockEncoder class is a subclass of DenseEncoder and utilizes the TextEmbeddingModel from the
 Amazon's Bedrock Platform to generate embeddings for given documents. It requires an AWS Access Key ID
 and AWS Secret Access Key and supports customization of the pre-trained model, score threshold, and region.
 
@@ -21,12 +21,12 @@ from typing import List, Optional, Any
 import os
 from time import sleep
 import tiktoken
-from semantic_router.encoders import BaseEncoder
+from semantic_router.encoders import DenseEncoder
 from semantic_router.utils.defaults import EncoderDefault
 from semantic_router.utils.logger import logger
 
 
-class BedrockEncoder(BaseEncoder):
+class BedrockEncoder(DenseEncoder):
     client: Any = None
     type: str = "bedrock"
     input_type: Optional[str] = "search_query"

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/encoders/bm25.py

@@ -1,21 +1,24 @@
 from typing import Any, Dict, List, Optional
 
-from semantic_router.encoders import BaseEncoder
+from semantic_router.encoders.tfidf import TfidfEncoder
 from semantic_router.utils.logger import logger
+from semantic_router.schema import SparseEmbedding
+from semantic_router.route import Route
 
 
-class BM25Encoder(BaseEncoder):
+class BM25Encoder(TfidfEncoder):
     model: Optional[Any] = None
     idx_mapping: Optional[Dict[int, int]] = None
     type: str = "sparse"
 
     def __init__(
         self,
-        name: str = "bm25",
-        score_threshold: float = 0.82,
+        name: str | None = None,
         use_default_params: bool = True,
     ):
-        super().__init__(name=name, score_threshold=score_threshold)
+        if name is None:
+            name = "bm25"
+        super().__init__(name=name)
         try:
             from pinecone_text.sparse import BM25Encoder as encoder
         except ImportError:
@@ -32,6 +35,7 @@ class BM25Encoder(BaseEncoder):
             self._set_idx_mapping()
 
     def _set_idx_mapping(self):
+        # TODO JB: this is training the model somehow - not sure how...
         params = self.model.get_params()
         doc_freq = params["doc_freq"]
         if isinstance(doc_freq, dict):
@@ -40,8 +44,20 @@ class BM25Encoder(BaseEncoder):
         else:
             raise TypeError("Expected a dictionary for 'doc_freq'")
 
-    def __call__(self, docs: List[str]) -> List[List[float]]:
-        if self.model is None or self.idx_mapping is None:
+    def fit(self, routes: List[Route]):
+        """Trains the encoder weights on the provided routes.
+
+        :param routes: List of routes to train the encoder on.
+        :type routes: List[Route]
+        """
+        self._fit_validate(routes=routes)
+        if self.model is None:
+            raise ValueError("Model is not initialized.")
+        utterances = [utterance for route in routes for utterance in route.utterances]
+        self.model.fit(corpus=utterances)
+
+    def __call__(self, docs: List[str]) -> list[SparseEmbedding]:
+        if self.model is None:
             raise ValueError("Model or index mapping is not initialized.")
         if len(docs) == 1:
             sparse_dicts = self.model.encode_queries(docs)
@@ -50,18 +66,8 @@ class BM25Encoder(BaseEncoder):
         else:
             raise ValueError("No documents to encode.")
 
-        embeds = [[0.0] * len(self.idx_mapping)] * len(docs)
+        embeds = []
         for i, output in enumerate(sparse_dicts):
-            indices = output["indices"]
-            values = output["values"]
-            for idx, val in zip(indices, values):
-                if idx in self.idx_mapping:
-                    position = self.idx_mapping[idx]
-                    embeds[i][position] = val
+            if isinstance(output, dict):
+                embeds.append(SparseEmbedding.from_pinecone_dict(output))
         return embeds
-
-    def fit(self, docs: List[str]):
-        if self.model is None:
-            raise ValueError("Model is not initialized.")
-        self.model.fit(docs)
-        self._set_idx_mapping()

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/encoders/clip.py

@@ -3,10 +3,10 @@ from typing import Any, List, Optional
 import numpy as np
 from pydantic.v1 import PrivateAttr
 from typing import Dict
-from semantic_router.encoders import BaseEncoder
+from semantic_router.encoders import DenseEncoder
 
 
-class CLIPEncoder(BaseEncoder):
+class CLIPEncoder(DenseEncoder):
     name: str = "openai/clip-vit-base-patch16"
     type: str = "huggingface"
     score_threshold: float = 0.2

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/encoders/cohere.py

@@ -3,11 +3,11 @@ from typing import Any, List, Optional
 
 from pydantic.v1 import PrivateAttr
 
-from semantic_router.encoders import BaseEncoder
+from semantic_router.encoders import DenseEncoder
 from semantic_router.utils.defaults import EncoderDefault
 
 
-class CohereEncoder(BaseEncoder):
+class CohereEncoder(DenseEncoder):
     _client: Any = PrivateAttr()
     _embed_type: Any = PrivateAttr()
     type: str = "cohere"

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/encoders/fastembed.py

@@ -3,10 +3,10 @@ from typing import Any, List, Optional
 import numpy as np
 from pydantic.v1 import PrivateAttr
 
-from semantic_router.encoders import BaseEncoder
+from semantic_router.encoders import DenseEncoder
 
 
-class FastEmbedEncoder(BaseEncoder):
+class FastEmbedEncoder(DenseEncoder):
     type: str = "fastembed"
     name: str = "BAAI/bge-small-en-v1.5"
     max_length: int = 512

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/encoders/google.py

@@ -1,7 +1,7 @@
 """
 This module provides the GoogleEncoder class for generating embeddings using Google's AI Platform.
 
-The GoogleEncoder class is a subclass of BaseEncoder and utilizes the TextEmbeddingModel from the
+The GoogleEncoder class is a subclass of DenseEncoder and utilizes the TextEmbeddingModel from the
 Google AI Platform to generate embeddings for given documents. It requires a Google Cloud project ID
 and supports customization of the pre-trained model, score threshold, location, and API endpoint.
 
@@ -19,11 +19,11 @@ Classes:
 import os
 from typing import Any, List, Optional
 
-from semantic_router.encoders import BaseEncoder
+from semantic_router.encoders import DenseEncoder
 from semantic_router.utils.defaults import EncoderDefault
 
 
-class GoogleEncoder(BaseEncoder):
+class GoogleEncoder(DenseEncoder):
     """GoogleEncoder class for generating embeddings using Google's AI Platform.
 
     Attributes:

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/encoders/huggingface.py

@@ -1,7 +1,7 @@
 """
 This module provides the HFEndpointEncoder class to embeddings models using Huggingface's endpoint.
 
-The HFEndpointEncoder class is a subclass of BaseEncoder and utilizes a specified Huggingface 
+The HFEndpointEncoder class is a subclass of DenseEncoder and utilizes a specified Huggingface 
 endpoint to generate embeddings for given documents. It requires the URL of the Huggingface 
 API endpoint and an API key for authentication. The class supports customization of the score 
 threshold for filtering or processing the embeddings.
@@ -27,11 +27,11 @@ from typing import Any, List, Optional, Dict
 
 from pydantic.v1 import PrivateAttr
 
-from semantic_router.encoders import BaseEncoder
+from semantic_router.encoders import DenseEncoder
 from semantic_router.utils.logger import logger
 
 
-class HuggingFaceEncoder(BaseEncoder):
+class HuggingFaceEncoder(DenseEncoder):
     name: str = "sentence-transformers/all-MiniLM-L6-v2"
     type: str = "huggingface"
     score_threshold: float = 0.5
@@ -140,7 +140,7 @@ class HuggingFaceEncoder(BaseEncoder):
         return self._torch.max(token_embeddings, 1)[0]
 
 
-class HFEndpointEncoder(BaseEncoder):
+class HFEndpointEncoder(DenseEncoder):
     """
     A class to encode documents using a Hugging Face transformer model endpoint.
 

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/encoders/mistral.py

@@ -6,11 +6,11 @@ from typing import Any, List, Optional
 
 from pydantic.v1 import PrivateAttr
 
-from semantic_router.encoders import BaseEncoder
+from semantic_router.encoders import DenseEncoder
 from semantic_router.utils.defaults import EncoderDefault
 
 
-class MistralEncoder(BaseEncoder):
+class MistralEncoder(DenseEncoder):
     """Class to encode text using MistralAI"""
 
     _client: Any = PrivateAttr()

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/encoders/openai.py

@@ -10,7 +10,7 @@ from openai._types import NotGiven
 from openai.types import CreateEmbeddingResponse
 import tiktoken
 
-from semantic_router.encoders import BaseEncoder
+from semantic_router.encoders import DenseEncoder
 from semantic_router.schema import EncoderInfo
 from semantic_router.utils.defaults import EncoderDefault
 from semantic_router.utils.logger import logger
@@ -35,7 +35,7 @@ model_configs = {
 }
 
 
-class OpenAIEncoder(BaseEncoder):
+class OpenAIEncoder(DenseEncoder):
     client: Optional[openai.Client]
     async_client: Optional[openai.AsyncClient]
     dimensions: Union[int, NotGiven] = NotGiven()

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/encoders/tfidf.py

@@ -3,24 +3,25 @@ from collections import Counter
 from typing import Dict, List
 
 import numpy as np
-from numpy import ndarray
-from numpy.linalg import norm
 
-from semantic_router.encoders import BaseEncoder
+from semantic_router.encoders import SparseEncoder
 from semantic_router.route import Route
+from semantic_router.schema import SparseEmbedding
 
 
-class TfidfEncoder(BaseEncoder):
-    idf: ndarray = np.array([])
+class TfidfEncoder(SparseEncoder):
+    idf: np.ndarray = np.array([])
+    # TODO: add option to use default params like with BM25Encoder
     word_index: Dict = {}
 
-    def __init__(self, name: str = "tfidf", score_threshold: float = 0.82):
-        # TODO default score_threshold not thoroughly tested, should optimize
-        super().__init__(name=name, score_threshold=score_threshold)
+    def __init__(self, name: str | None = None):
+        if name is None:
+            name = "tfidf"
+        super().__init__(name=name)
         self.word_index = {}
         self.idf = np.array([])
 
-    def __call__(self, docs: List[str]) -> List[List[float]]:
+    def __call__(self, docs: List[str]) -> list[SparseEmbedding]:
         if len(self.word_index) == 0 or self.idf.size == 0:
             raise ValueError("Vectorizer is not initialized.")
         if len(docs) == 0:
@@ -29,16 +30,28 @@ class TfidfEncoder(BaseEncoder):
         docs = [self._preprocess(doc) for doc in docs]
         tf = self._compute_tf(docs)
         tfidf = tf * self.idf
-        return tfidf.tolist()
+        return self._array_to_sparse_embeddings(tfidf)
 
     def fit(self, routes: List[Route]):
+        """Trains the encoder weights on the provided routes.
+
+        :param routes: List of routes to train the encoder on.
+        :type routes: List[Route]
+        """
+        self._fit_validate(routes=routes)
         docs = []
         for route in routes:
             for doc in route.utterances:
                 docs.append(self._preprocess(doc))  # type: ignore
         self.word_index = self._build_word_index(docs)
+        if len(self.word_index) == 0:
+            raise ValueError(f"Too little data to fit {self.__class__.__name__}.")
         self.idf = self._compute_idf(docs)
 
+    def _fit_validate(self, routes: List[Route]):
+        if not isinstance(routes, list) or not isinstance(routes[0], Route):
+            raise TypeError("`routes` parameter must be a list of Route objects.")
+
     def _build_word_index(self, docs: List[str]) -> Dict:
         words = set()
         for doc in docs:
@@ -57,7 +70,7 @@ class TfidfEncoder(BaseEncoder):
                 if word in self.word_index:
                     tf[i, self.word_index[word]] = count
         # L2 normalization
-        tf = tf / norm(tf, axis=1, keepdims=True)
+        tf = tf / np.linalg.norm(tf, axis=1, keepdims=True)
         return tf
 
     def _compute_idf(self, docs: List[str]) -> np.ndarray:

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/encoders/vit.py

@@ -2,10 +2,10 @@ from typing import Any, List, Optional, Dict
 
 from pydantic.v1 import PrivateAttr
 
-from semantic_router.encoders import BaseEncoder
+from semantic_router.encoders import DenseEncoder
 
 
-class VitEncoder(BaseEncoder):
+class VitEncoder(DenseEncoder):
     name: str = "google/vit-base-patch16-224"
     type: str = "huggingface"
     score_threshold: float = 0.5

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/encoders/zure.py

@@ -8,12 +8,12 @@ from openai._types import NotGiven
 from openai import OpenAIError
 from openai.types import CreateEmbeddingResponse
 
-from semantic_router.encoders import BaseEncoder
+from semantic_router.encoders import DenseEncoder
 from semantic_router.utils.defaults import EncoderDefault
 from semantic_router.utils.logger import logger
 
 
-class AzureOpenAIEncoder(BaseEncoder):
+class AzureOpenAIEncoder(DenseEncoder):
     client: Optional[openai.AzureOpenAI] = None
     async_client: Optional[openai.AsyncAzureOpenAI] = None
     dimensions: Union[int, NotGiven] = NotGiven()

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/index/__init__.py

@@ -1,11 +1,15 @@
 from semantic_router.index.base import BaseIndex
+from semantic_router.index.hybrid_local import HybridLocalIndex
 from semantic_router.index.local import LocalIndex
 from semantic_router.index.pinecone import PineconeIndex
+from semantic_router.index.postgres import PostgresIndex
 from semantic_router.index.qdrant import QdrantIndex
 
 __all__ = [
     "BaseIndex",
+    "HybridLocalIndex",
     "LocalIndex",
     "QdrantIndex",
     "PineconeIndex",
+    "PostgresIndex",
 ]

{semantic_router-0.1.0.dev0 → semantic_router-0.1.0.dev2}/semantic_router/index/base.py

@@ -4,7 +4,7 @@ import json
 import numpy as np
 from pydantic.v1 import BaseModel
 
-from semantic_router.schema import ConfigParameter, Utterance
+from semantic_router.schema import ConfigParameter, SparseEmbedding, Utterance
 from semantic_router.route import Route
 from semantic_router.utils.logger import logger
 
@@ -18,12 +18,12 @@ class BaseIndex(BaseModel):
 
     # You can define common attributes here if there are any.
     # For example, a placeholder for the index attribute:
-    index: Optional[Any] = None
     routes: Optional[np.ndarray] = None
     utterances: Optional[np.ndarray] = None
     dimensions: Union[int, None] = None
     type: str = "base"
     init_async_index: bool = False
+    index: Optional[Any] = None
 
     def add(
         self,
@@ -108,6 +108,7 @@ class BaseIndex(BaseModel):
         vector: np.ndarray,
         top_k: int = 5,
         route_filter: Optional[List[str]] = None,
+        sparse_vector: dict[int, float] | SparseEmbedding | None = None,
     ) -> Tuple[np.ndarray, List[str]]:
         """
         Search the index for the query_vector and return top_k results.
@@ -120,6 +121,7 @@ class BaseIndex(BaseModel):
         vector: np.ndarray,
         top_k: int = 5,
         route_filter: Optional[List[str]] = None,
+        sparse_vector: dict[int, float] | SparseEmbedding | None = None,
     ) -> Tuple[np.ndarray, List[str]]:
         """
         Search the index for the query_vector and return top_k results.