semantic-router 0.1.0.dev6__tar.gz → 0.1.0.dev8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/PKG-INFO +1 -1
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/pyproject.toml +1 -1
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/__init__.py +2 -2
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/__init__.py +1 -2
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/aurelio.py +2 -2
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/bedrock.py +3 -1
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/clip.py +2 -2
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/huggingface.py +3 -3
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/openai.py +3 -4
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/index/base.py +4 -5
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/index/hybrid_local.py +2 -3
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/index/local.py +2 -3
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/index/pinecone.py +109 -98
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/index/postgres.py +1 -1
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/llms/base.py +1 -1
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/llms/llamacpp.py +1 -1
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/llms/openai.py +11 -11
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/llms/openrouter.py +1 -2
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/routers/base.py +11 -7
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/routers/hybrid.py +15 -11
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/schema.py +5 -3
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/utils/function_call.py +1 -2
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/LICENSE +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/README.md +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/base.py +1 -1
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/bm25.py +2 -2
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/cohere.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/fastembed.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/google.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/mistral.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/tfidf.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/vit.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/zure.py +2 -2
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/index/__init__.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/index/qdrant.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/linear.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/llms/__init__.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/llms/cohere.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/llms/grammars/json.gbnf +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/llms/mistral.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/llms/ollama.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/llms/zure.py +1 -1
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/py.typed +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/route.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/routers/__init__.py +1 -1
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/routers/semantic.py +1 -1
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/utils/__init__.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/utils/defaults.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/utils/llm.py +0 -0
- {semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/utils/logger.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
from semantic_router.routers import RouterConfig, SemanticRouter, HybridRouter
|
|
2
1
|
from semantic_router.route import Route
|
|
2
|
+
from semantic_router.routers import HybridRouter, RouterConfig, SemanticRouter
|
|
3
3
|
|
|
4
4
|
__all__ = ["SemanticRouter", "HybridRouter", "Route", "RouterConfig"]
|
|
5
5
|
|
|
6
|
-
__version__ = "0.1.0.
|
|
6
|
+
__version__ = "0.1.0.dev8"
|
{semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/__init__.py
RENAMED
|
@@ -8,8 +8,7 @@ from semantic_router.encoders.clip import CLIPEncoder
|
|
|
8
8
|
from semantic_router.encoders.cohere import CohereEncoder
|
|
9
9
|
from semantic_router.encoders.fastembed import FastEmbedEncoder
|
|
10
10
|
from semantic_router.encoders.google import GoogleEncoder
|
|
11
|
-
from semantic_router.encoders.huggingface import HuggingFaceEncoder
|
|
12
|
-
from semantic_router.encoders.huggingface import HFEndpointEncoder
|
|
11
|
+
from semantic_router.encoders.huggingface import HFEndpointEncoder, HuggingFaceEncoder
|
|
13
12
|
from semantic_router.encoders.mistral import MistralEncoder
|
|
14
13
|
from semantic_router.encoders.openai import OpenAIEncoder
|
|
15
14
|
from semantic_router.encoders.tfidf import TfidfEncoder
|
{semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/aurelio.py
RENAMED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import os
|
|
2
2
|
from typing import Any, List, Optional
|
|
3
|
-
from pydantic import Field
|
|
4
3
|
|
|
5
|
-
from aurelio_sdk import
|
|
4
|
+
from aurelio_sdk import AsyncAurelioClient, AurelioClient, EmbeddingResponse
|
|
5
|
+
from pydantic import Field
|
|
6
6
|
|
|
7
7
|
from semantic_router.encoders.base import SparseEncoder
|
|
8
8
|
from semantic_router.schema import SparseEmbedding
|
{semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/bedrock.py
RENAMED
|
@@ -17,10 +17,12 @@ Classes:
|
|
|
17
17
|
"""
|
|
18
18
|
|
|
19
19
|
import json
|
|
20
|
-
from typing import Dict, List, Optional, Any, Union
|
|
21
20
|
import os
|
|
22
21
|
from time import sleep
|
|
22
|
+
from typing import Any, Dict, List, Optional, Union
|
|
23
|
+
|
|
23
24
|
import tiktoken
|
|
25
|
+
|
|
24
26
|
from semantic_router.encoders import DenseEncoder
|
|
25
27
|
from semantic_router.utils.defaults import EncoderDefault
|
|
26
28
|
from semantic_router.utils.logger import logger
|
{semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/huggingface.py
RENAMED
|
@@ -20,11 +20,11 @@ Classes:
|
|
|
20
20
|
HFEndpointEncoder: A class for generating embeddings using a Huggingface endpoint.
|
|
21
21
|
"""
|
|
22
22
|
|
|
23
|
-
import requests
|
|
24
|
-
import time
|
|
25
23
|
import os
|
|
26
|
-
|
|
24
|
+
import time
|
|
25
|
+
from typing import Any, Dict, List, Optional
|
|
27
26
|
|
|
27
|
+
import requests
|
|
28
28
|
from pydantic import PrivateAttr
|
|
29
29
|
|
|
30
30
|
from semantic_router.encoders import DenseEncoder
|
{semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/openai.py
RENAMED
|
@@ -1,21 +1,20 @@
|
|
|
1
|
-
from asyncio import sleep as asleep
|
|
2
1
|
import os
|
|
2
|
+
from asyncio import sleep as asleep
|
|
3
3
|
from time import sleep
|
|
4
4
|
from typing import Any, List, Optional, Union
|
|
5
|
-
from pydantic import PrivateAttr
|
|
6
5
|
|
|
7
6
|
import openai
|
|
7
|
+
import tiktoken
|
|
8
8
|
from openai import OpenAIError
|
|
9
9
|
from openai._types import NotGiven
|
|
10
10
|
from openai.types import CreateEmbeddingResponse
|
|
11
|
-
import
|
|
11
|
+
from pydantic import PrivateAttr
|
|
12
12
|
|
|
13
13
|
from semantic_router.encoders import DenseEncoder
|
|
14
14
|
from semantic_router.schema import EncoderInfo
|
|
15
15
|
from semantic_router.utils.defaults import EncoderDefault
|
|
16
16
|
from semantic_router.utils.logger import logger
|
|
17
17
|
|
|
18
|
-
|
|
19
18
|
model_configs = {
|
|
20
19
|
"text-embedding-ada-002": EncoderInfo(
|
|
21
20
|
name="text-embedding-ada-002",
|
|
@@ -1,17 +1,16 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
-
from datetime import datetime
|
|
3
|
-
import time
|
|
4
|
-
from typing import Any, List, Optional, Tuple, Union, Dict
|
|
5
2
|
import json
|
|
3
|
+
import time
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
6
6
|
|
|
7
7
|
import numpy as np
|
|
8
8
|
from pydantic import BaseModel
|
|
9
9
|
|
|
10
|
-
from semantic_router.schema import ConfigParameter, SparseEmbedding, Utterance
|
|
11
10
|
from semantic_router.route import Route
|
|
11
|
+
from semantic_router.schema import ConfigParameter, SparseEmbedding, Utterance
|
|
12
12
|
from semantic_router.utils.logger import logger
|
|
13
13
|
|
|
14
|
-
|
|
15
14
|
RETRY_WAIT_TIME = 2.5
|
|
16
15
|
|
|
17
16
|
|
{semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/index/hybrid_local.py
RENAMED
|
@@ -1,12 +1,11 @@
|
|
|
1
|
-
from typing import List, Optional, Tuple
|
|
1
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
2
2
|
|
|
3
3
|
import numpy as np
|
|
4
4
|
from numpy.linalg import norm
|
|
5
5
|
|
|
6
|
-
from semantic_router.schema import ConfigParameter, SparseEmbedding, Utterance
|
|
7
6
|
from semantic_router.index.local import LocalIndex
|
|
7
|
+
from semantic_router.schema import ConfigParameter, SparseEmbedding, Utterance
|
|
8
8
|
from semantic_router.utils.logger import logger
|
|
9
|
-
from typing import Any
|
|
10
9
|
|
|
11
10
|
|
|
12
11
|
class HybridLocalIndex(LocalIndex):
|
|
@@ -1,12 +1,11 @@
|
|
|
1
|
-
from typing import List, Optional, Tuple
|
|
1
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
2
2
|
|
|
3
3
|
import numpy as np
|
|
4
4
|
|
|
5
|
-
from semantic_router.schema import ConfigParameter, SparseEmbedding, Utterance
|
|
6
5
|
from semantic_router.index.base import BaseIndex, IndexConfig
|
|
7
6
|
from semantic_router.linear import similarity_matrix, top_scores
|
|
7
|
+
from semantic_router.schema import ConfigParameter, SparseEmbedding, Utterance
|
|
8
8
|
from semantic_router.utils.logger import logger
|
|
9
|
-
from typing import Any
|
|
10
9
|
|
|
11
10
|
|
|
12
11
|
class LocalIndex(BaseIndex):
|
|
@@ -1,12 +1,11 @@
|
|
|
1
|
-
import aiohttp
|
|
2
1
|
import asyncio
|
|
3
2
|
import hashlib
|
|
3
|
+
import json
|
|
4
4
|
import os
|
|
5
5
|
import time
|
|
6
|
-
import
|
|
7
|
-
|
|
8
|
-
from typing import Any, Dict, List, Optional, Union, Tuple
|
|
6
|
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
9
7
|
|
|
8
|
+
import aiohttp
|
|
10
9
|
import numpy as np
|
|
11
10
|
from pydantic import BaseModel, Field
|
|
12
11
|
|
|
@@ -112,11 +111,11 @@ class PineconeIndex(BaseIndex):
|
|
|
112
111
|
region: str = "us-east-1"
|
|
113
112
|
host: str = ""
|
|
114
113
|
client: Any = Field(default=None, exclude=True)
|
|
115
|
-
async_client: Any = Field(default=None, exclude=True)
|
|
116
114
|
index: Optional[Any] = Field(default=None, exclude=True)
|
|
117
115
|
ServerlessSpec: Any = Field(default=None, exclude=True)
|
|
118
116
|
namespace: Optional[str] = ""
|
|
119
117
|
base_url: Optional[str] = "https://api.pinecone.io"
|
|
118
|
+
headers: dict[str, str] = {}
|
|
120
119
|
|
|
121
120
|
def __init__(
|
|
122
121
|
self,
|
|
@@ -132,6 +131,15 @@ class PineconeIndex(BaseIndex):
|
|
|
132
131
|
init_async_index: bool = False,
|
|
133
132
|
):
|
|
134
133
|
super().__init__()
|
|
134
|
+
self.api_key = api_key or os.getenv("PINECONE_API_KEY")
|
|
135
|
+
if not self.api_key:
|
|
136
|
+
raise ValueError("Pinecone API key is required.")
|
|
137
|
+
self.headers = {
|
|
138
|
+
"Api-Key": self.api_key,
|
|
139
|
+
"Content-Type": "application/json",
|
|
140
|
+
"X-Pinecone-API-Version": "2024-07",
|
|
141
|
+
"User-Agent": "source_tag=semanticrouter",
|
|
142
|
+
}
|
|
135
143
|
self.index_name = index_name
|
|
136
144
|
self.dimensions = dimensions
|
|
137
145
|
self.metric = metric
|
|
@@ -153,10 +161,7 @@ class PineconeIndex(BaseIndex):
|
|
|
153
161
|
raise ValueError("Pinecone API key is required.")
|
|
154
162
|
|
|
155
163
|
self.client = self._initialize_client(api_key=self.api_key)
|
|
156
|
-
|
|
157
|
-
self.async_client = self._initialize_async_client(api_key=self.api_key)
|
|
158
|
-
else:
|
|
159
|
-
self.async_client = None
|
|
164
|
+
|
|
160
165
|
# try initializing index
|
|
161
166
|
self.index = self._init_index()
|
|
162
167
|
|
|
@@ -181,20 +186,6 @@ class PineconeIndex(BaseIndex):
|
|
|
181
186
|
|
|
182
187
|
return Pinecone(**pinecone_args)
|
|
183
188
|
|
|
184
|
-
def _initialize_async_client(self, api_key: Optional[str] = None):
|
|
185
|
-
api_key = api_key or self.api_key
|
|
186
|
-
if api_key is None:
|
|
187
|
-
raise ValueError("Pinecone API key is required.")
|
|
188
|
-
async_client = aiohttp.ClientSession(
|
|
189
|
-
headers={
|
|
190
|
-
"Api-Key": api_key,
|
|
191
|
-
"Content-Type": "application/json",
|
|
192
|
-
"X-Pinecone-API-Version": "2024-07",
|
|
193
|
-
"User-Agent": "source_tag=semanticrouter",
|
|
194
|
-
}
|
|
195
|
-
)
|
|
196
|
-
return async_client
|
|
197
|
-
|
|
198
189
|
def _init_index(self, force_create: bool = False) -> Union[Any, None]:
|
|
199
190
|
"""Initializing the index can be done after the object has been created
|
|
200
191
|
to allow for the user to set the dimensions and other parameters.
|
|
@@ -659,8 +650,8 @@ class PineconeIndex(BaseIndex):
|
|
|
659
650
|
:rtype: Tuple[np.ndarray, List[str]]
|
|
660
651
|
:raises ValueError: If the index is not populated.
|
|
661
652
|
"""
|
|
662
|
-
if self.
|
|
663
|
-
raise ValueError("
|
|
653
|
+
if self.host == "":
|
|
654
|
+
raise ValueError("Host is not initialized.")
|
|
664
655
|
query_vector_list = vector.tolist()
|
|
665
656
|
if route_filter is not None:
|
|
666
657
|
filter_query = {"sr_route": {"$in": route_filter}}
|
|
@@ -693,8 +684,8 @@ class PineconeIndex(BaseIndex):
|
|
|
693
684
|
:return: A list of (route_name, utterance) objects.
|
|
694
685
|
:rtype: List[Tuple]
|
|
695
686
|
"""
|
|
696
|
-
if self.
|
|
697
|
-
raise ValueError("
|
|
687
|
+
if self.host == "":
|
|
688
|
+
raise ValueError("Host is not initialized.")
|
|
698
689
|
|
|
699
690
|
return await self._async_get_routes()
|
|
700
691
|
|
|
@@ -722,15 +713,21 @@ class PineconeIndex(BaseIndex):
|
|
|
722
713
|
}
|
|
723
714
|
if self.host == "":
|
|
724
715
|
raise ValueError("self.host is not initialized.")
|
|
725
|
-
async with
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
716
|
+
async with aiohttp.ClientSession() as session:
|
|
717
|
+
async with session.post(
|
|
718
|
+
f"https://{self.host}/query",
|
|
719
|
+
json=params,
|
|
720
|
+
headers=self.headers,
|
|
721
|
+
) as response:
|
|
722
|
+
return await response.json(content_type=None)
|
|
730
723
|
|
|
731
724
|
async def _async_list_indexes(self):
|
|
732
|
-
async with
|
|
733
|
-
|
|
725
|
+
async with aiohttp.ClientSession() as session:
|
|
726
|
+
async with session.get(
|
|
727
|
+
f"{self.base_url}/indexes",
|
|
728
|
+
headers=self.headers,
|
|
729
|
+
) as response:
|
|
730
|
+
return await response.json(content_type=None)
|
|
734
731
|
|
|
735
732
|
async def _async_upsert(
|
|
736
733
|
self,
|
|
@@ -741,12 +738,14 @@ class PineconeIndex(BaseIndex):
|
|
|
741
738
|
"vectors": vectors,
|
|
742
739
|
"namespace": namespace,
|
|
743
740
|
}
|
|
744
|
-
async with
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
741
|
+
async with aiohttp.ClientSession() as session:
|
|
742
|
+
async with session.post(
|
|
743
|
+
f"https://{self.host}/vectors/upsert",
|
|
744
|
+
json=params,
|
|
745
|
+
headers=self.headers,
|
|
746
|
+
) as response:
|
|
747
|
+
res = await response.json(content_type=None)
|
|
748
|
+
return res
|
|
750
749
|
|
|
751
750
|
async def _async_create_index(
|
|
752
751
|
self,
|
|
@@ -762,26 +761,34 @@ class PineconeIndex(BaseIndex):
|
|
|
762
761
|
"metric": metric,
|
|
763
762
|
"spec": {"serverless": {"cloud": cloud, "region": region}},
|
|
764
763
|
}
|
|
765
|
-
async with
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
764
|
+
async with aiohttp.ClientSession() as session:
|
|
765
|
+
async with session.post(
|
|
766
|
+
f"{self.base_url}/indexes",
|
|
767
|
+
json=params,
|
|
768
|
+
headers=self.headers,
|
|
769
|
+
) as response:
|
|
770
|
+
return await response.json(content_type=None)
|
|
770
771
|
|
|
771
772
|
async def _async_delete(self, ids: list[str], namespace: str = ""):
|
|
772
773
|
params = {
|
|
773
774
|
"ids": ids,
|
|
774
775
|
"namespace": namespace,
|
|
775
776
|
}
|
|
776
|
-
async with
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
777
|
+
async with aiohttp.ClientSession() as session:
|
|
778
|
+
async with session.post(
|
|
779
|
+
f"https://{self.host}/vectors/delete",
|
|
780
|
+
json=params,
|
|
781
|
+
headers=self.headers,
|
|
782
|
+
) as response:
|
|
783
|
+
return await response.json(content_type=None)
|
|
781
784
|
|
|
782
785
|
async def _async_describe_index(self, name: str):
|
|
783
|
-
async with
|
|
784
|
-
|
|
786
|
+
async with aiohttp.ClientSession() as session:
|
|
787
|
+
async with session.get(
|
|
788
|
+
f"{self.base_url}/indexes/{name}",
|
|
789
|
+
headers=self.headers,
|
|
790
|
+
) as response:
|
|
791
|
+
return await response.json(content_type=None)
|
|
785
792
|
|
|
786
793
|
async def _async_get_all(
|
|
787
794
|
self, prefix: Optional[str] = None, include_metadata: bool = False
|
|
@@ -815,33 +822,38 @@ class PineconeIndex(BaseIndex):
|
|
|
815
822
|
params["namespace"] = self.namespace
|
|
816
823
|
metadata = []
|
|
817
824
|
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
825
|
+
async with aiohttp.ClientSession() as session:
|
|
826
|
+
while True:
|
|
827
|
+
if next_page_token:
|
|
828
|
+
params["paginationToken"] = next_page_token
|
|
829
|
+
|
|
830
|
+
async with session.get(
|
|
831
|
+
list_url,
|
|
832
|
+
params=params,
|
|
833
|
+
headers=self.headers,
|
|
834
|
+
) as response:
|
|
835
|
+
if response.status != 200:
|
|
836
|
+
error_text = await response.text()
|
|
837
|
+
logger.error(f"Error fetching vectors: {error_text}")
|
|
838
|
+
break
|
|
839
|
+
|
|
840
|
+
response_data = await response.json(content_type=None)
|
|
841
|
+
|
|
842
|
+
vector_ids = [vec["id"] for vec in response_data.get("vectors", [])]
|
|
843
|
+
if not vector_ids:
|
|
828
844
|
break
|
|
845
|
+
all_vector_ids.extend(vector_ids)
|
|
829
846
|
|
|
830
|
-
|
|
847
|
+
if include_metadata:
|
|
848
|
+
metadata_tasks = [
|
|
849
|
+
self._async_fetch_metadata(id) for id in vector_ids
|
|
850
|
+
]
|
|
851
|
+
metadata_results = await asyncio.gather(*metadata_tasks)
|
|
852
|
+
metadata.extend(metadata_results)
|
|
831
853
|
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
all_vector_ids.extend(vector_ids)
|
|
836
|
-
|
|
837
|
-
if include_metadata:
|
|
838
|
-
metadata_tasks = [self._async_fetch_metadata(id) for id in vector_ids]
|
|
839
|
-
metadata_results = await asyncio.gather(*metadata_tasks)
|
|
840
|
-
metadata.extend(metadata_results)
|
|
841
|
-
|
|
842
|
-
next_page_token = response_data.get("pagination", {}).get("next")
|
|
843
|
-
if not next_page_token:
|
|
844
|
-
break
|
|
854
|
+
next_page_token = response_data.get("pagination", {}).get("next")
|
|
855
|
+
if not next_page_token:
|
|
856
|
+
break
|
|
845
857
|
|
|
846
858
|
return all_vector_ids, metadata
|
|
847
859
|
|
|
@@ -851,7 +863,7 @@ class PineconeIndex(BaseIndex):
|
|
|
851
863
|
namespace: str | None = None,
|
|
852
864
|
) -> dict:
|
|
853
865
|
"""Fetch metadata for a single vector ID asynchronously using the
|
|
854
|
-
|
|
866
|
+
ClientSession.
|
|
855
867
|
|
|
856
868
|
:param vector_id: The ID of the vector to fetch metadata for.
|
|
857
869
|
:type vector_id: str
|
|
@@ -873,27 +885,26 @@ class PineconeIndex(BaseIndex):
|
|
|
873
885
|
elif self.namespace:
|
|
874
886
|
params["namespace"] = [self.namespace]
|
|
875
887
|
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
)
|
|
888
|
+
async with aiohttp.ClientSession() as session:
|
|
889
|
+
async with session.get(
|
|
890
|
+
url, params=params, headers=self.headers
|
|
891
|
+
) as response:
|
|
892
|
+
if response.status != 200:
|
|
893
|
+
error_text = await response.text()
|
|
894
|
+
logger.error(f"Error fetching metadata: {error_text}")
|
|
895
|
+
return {}
|
|
896
|
+
|
|
897
|
+
try:
|
|
898
|
+
response_data = await response.json(content_type=None)
|
|
899
|
+
except Exception as e:
|
|
900
|
+
logger.warning(f"No metadata found for vector {vector_id}: {e}")
|
|
901
|
+
return {}
|
|
902
|
+
|
|
903
|
+
return (
|
|
904
|
+
response_data.get("vectors", {})
|
|
905
|
+
.get(vector_id, {})
|
|
906
|
+
.get("metadata", {})
|
|
907
|
+
)
|
|
897
908
|
|
|
898
909
|
def __len__(self):
|
|
899
910
|
namespace_stats = self.index.describe_index_stats()["namespaces"].get(
|
|
@@ -1,24 +1,24 @@
|
|
|
1
|
+
import inspect
|
|
2
|
+
import json
|
|
1
3
|
import os
|
|
2
|
-
|
|
3
|
-
from
|
|
4
|
+
import re
|
|
5
|
+
from typing import Any, Callable, Dict, List, Optional, Union
|
|
4
6
|
|
|
5
7
|
import openai
|
|
6
|
-
from openai._types import
|
|
8
|
+
from openai._types import NOT_GIVEN, NotGiven
|
|
9
|
+
from openai.types.chat.chat_completion_message_tool_call import (
|
|
10
|
+
ChatCompletionMessageToolCall,
|
|
11
|
+
)
|
|
12
|
+
from pydantic import PrivateAttr
|
|
7
13
|
|
|
8
14
|
from semantic_router.llms import BaseLLM
|
|
9
15
|
from semantic_router.schema import Message
|
|
10
16
|
from semantic_router.utils.defaults import EncoderDefault
|
|
11
|
-
from semantic_router.utils.logger import logger
|
|
12
|
-
import json
|
|
13
17
|
from semantic_router.utils.function_call import (
|
|
14
|
-
get_schema,
|
|
15
18
|
convert_python_type_to_json_type,
|
|
19
|
+
get_schema,
|
|
16
20
|
)
|
|
17
|
-
import
|
|
18
|
-
import re
|
|
19
|
-
from openai.types.chat.chat_completion_message_tool_call import (
|
|
20
|
-
ChatCompletionMessageToolCall,
|
|
21
|
-
)
|
|
21
|
+
from semantic_router.utils.logger import logger
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
class OpenAILLM(BaseLLM):
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
+
import hashlib
|
|
1
2
|
import importlib
|
|
2
3
|
import json
|
|
3
4
|
import os
|
|
4
5
|
import random
|
|
5
|
-
import hashlib
|
|
6
6
|
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
|
7
|
-
from typing_extensions import deprecated
|
|
8
|
-
from pydantic import BaseModel, Field
|
|
9
7
|
|
|
10
8
|
import numpy as np
|
|
11
9
|
import yaml # type: ignore
|
|
10
|
+
from pydantic import BaseModel, Field
|
|
12
11
|
from tqdm.auto import tqdm
|
|
12
|
+
from typing_extensions import deprecated
|
|
13
13
|
|
|
14
14
|
from semantic_router.encoders import (
|
|
15
15
|
AutoEncoder,
|
|
@@ -1343,11 +1343,15 @@ class BaseRouter(BaseModel):
|
|
|
1343
1343
|
# Switch to a local index for fitting
|
|
1344
1344
|
from semantic_router.index.local import LocalIndex
|
|
1345
1345
|
|
|
1346
|
-
|
|
1346
|
+
remote_utterances = self.index.get_utterances(include_metadata=True)
|
|
1347
1347
|
# TODO Enhance by retrieving directly the vectors instead of embedding all utterances again
|
|
1348
|
-
routes
|
|
1349
|
-
|
|
1350
|
-
|
|
1348
|
+
routes = []
|
|
1349
|
+
utterances = []
|
|
1350
|
+
metadata = []
|
|
1351
|
+
for utterance in remote_utterances:
|
|
1352
|
+
routes.append(utterance.route)
|
|
1353
|
+
utterances.append(utterance.utterance)
|
|
1354
|
+
metadata.append(utterance.metadata)
|
|
1351
1355
|
embeddings = self.encoder(utterances)
|
|
1352
1356
|
self.index = LocalIndex()
|
|
1353
1357
|
self.index.add(
|
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
from typing import Any, Dict, List, Optional, Union
|
|
2
|
-
from tqdm.auto import tqdm
|
|
3
1
|
import asyncio
|
|
4
|
-
from
|
|
2
|
+
from typing import Any, Dict, List, Optional, Union
|
|
5
3
|
|
|
6
4
|
import numpy as np
|
|
5
|
+
from pydantic import Field
|
|
6
|
+
from tqdm.auto import tqdm
|
|
7
7
|
|
|
8
8
|
from semantic_router.encoders import (
|
|
9
|
+
BM25Encoder,
|
|
9
10
|
DenseEncoder,
|
|
10
11
|
SparseEncoder,
|
|
11
|
-
BM25Encoder,
|
|
12
12
|
TfidfEncoder,
|
|
13
13
|
)
|
|
14
|
-
from semantic_router.route import Route
|
|
15
14
|
from semantic_router.index import BaseIndex, HybridLocalIndex
|
|
15
|
+
from semantic_router.llms import BaseLLM
|
|
16
|
+
from semantic_router.route import Route
|
|
17
|
+
from semantic_router.routers.base import BaseRouter, threshold_random_search, xq_reshape
|
|
16
18
|
from semantic_router.schema import RouteChoice, SparseEmbedding, Utterance
|
|
17
19
|
from semantic_router.utils.logger import logger
|
|
18
|
-
from semantic_router.routers.base import BaseRouter, xq_reshape, threshold_random_search
|
|
19
|
-
from semantic_router.llms import BaseLLM
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
class HybridRouter(BaseRouter):
|
|
@@ -286,11 +286,15 @@ class HybridRouter(BaseRouter):
|
|
|
286
286
|
# Switch to a local index for fitting
|
|
287
287
|
from semantic_router.index.hybrid_local import HybridLocalIndex
|
|
288
288
|
|
|
289
|
-
|
|
289
|
+
remote_utterances = self.index.get_utterances(include_metadata=True)
|
|
290
290
|
# TODO Enhance by retrieving directly the vectors instead of embedding all utterances again
|
|
291
|
-
routes
|
|
292
|
-
|
|
293
|
-
|
|
291
|
+
routes = []
|
|
292
|
+
utterances = []
|
|
293
|
+
metadata = []
|
|
294
|
+
for utterance in remote_utterances:
|
|
295
|
+
routes.append(utterance.route)
|
|
296
|
+
utterances.append(utterance.utterance)
|
|
297
|
+
metadata.append(utterance.metadata)
|
|
294
298
|
embeddings = self.encoder(utterances)
|
|
295
299
|
sparse_embeddings = self.sparse_encoder(utterances)
|
|
296
300
|
self.index = HybridLocalIndex()
|
|
@@ -1,12 +1,14 @@
|
|
|
1
|
+
import json
|
|
1
2
|
from datetime import datetime, timezone
|
|
2
3
|
from difflib import Differ
|
|
3
4
|
from enum import Enum
|
|
4
|
-
import
|
|
5
|
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
6
|
+
|
|
5
7
|
import numpy as np
|
|
6
|
-
from
|
|
8
|
+
from aurelio_sdk.schema import BM25Embedding
|
|
7
9
|
from pydantic import BaseModel, Field
|
|
10
|
+
|
|
8
11
|
from semantic_router.utils.logger import logger
|
|
9
|
-
from aurelio_sdk.schema import BM25Embedding
|
|
10
12
|
|
|
11
13
|
|
|
12
14
|
class EncoderType(Enum):
|
{semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/utils/function_call.py
RENAMED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
import inspect
|
|
2
2
|
from typing import Any, Callable, Dict, List, Optional, Union
|
|
3
3
|
|
|
4
|
-
from pydantic import BaseModel
|
|
4
|
+
from pydantic import BaseModel, Field
|
|
5
5
|
|
|
6
6
|
from semantic_router.llms import BaseLLM
|
|
7
7
|
from semantic_router.schema import Message, RouteChoice
|
|
8
8
|
from semantic_router.utils.logger import logger
|
|
9
|
-
from pydantic import Field
|
|
10
9
|
|
|
11
10
|
|
|
12
11
|
class Parameter(BaseModel):
|
|
File without changes
|
|
File without changes
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
from typing import Any, Dict, List, Optional
|
|
2
2
|
|
|
3
3
|
from semantic_router.encoders.tfidf import TfidfEncoder
|
|
4
|
-
from semantic_router.utils.logger import logger
|
|
5
|
-
from semantic_router.schema import SparseEmbedding
|
|
6
4
|
from semantic_router.route import Route
|
|
5
|
+
from semantic_router.schema import SparseEmbedding
|
|
6
|
+
from semantic_router.utils.logger import logger
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
class BM25Encoder(TfidfEncoder):
|
{semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/cohere.py
RENAMED
|
File without changes
|
{semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/fastembed.py
RENAMED
|
File without changes
|
{semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/google.py
RENAMED
|
File without changes
|
{semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/encoders/mistral.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
from asyncio import sleep as asleep
|
|
2
1
|
import os
|
|
2
|
+
from asyncio import sleep as asleep
|
|
3
3
|
from time import sleep
|
|
4
4
|
from typing import List, Optional, Union
|
|
5
5
|
|
|
6
6
|
import openai
|
|
7
|
-
from openai._types import NotGiven
|
|
8
7
|
from openai import OpenAIError
|
|
8
|
+
from openai._types import NotGiven
|
|
9
9
|
from openai.types import CreateEmbeddingResponse
|
|
10
10
|
|
|
11
11
|
from semantic_router.encoders import DenseEncoder
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/llms/grammars/json.gbnf
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/routers/__init__.py
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from semantic_router.routers.base import BaseRouter, RouterConfig
|
|
2
|
-
from semantic_router.routers.semantic import SemanticRouter
|
|
3
2
|
from semantic_router.routers.hybrid import HybridRouter
|
|
3
|
+
from semantic_router.routers.semantic import SemanticRouter
|
|
4
4
|
|
|
5
5
|
__all__ = [
|
|
6
6
|
"BaseRouter",
|
{semantic_router-0.1.0.dev6 → semantic_router-0.1.0.dev8}/semantic_router/routers/semantic.py
RENAMED
|
@@ -5,9 +5,9 @@ import numpy as np
|
|
|
5
5
|
from semantic_router.encoders import DenseEncoder
|
|
6
6
|
from semantic_router.index.base import BaseIndex
|
|
7
7
|
from semantic_router.llms import BaseLLM
|
|
8
|
-
from semantic_router.utils.logger import logger
|
|
9
8
|
from semantic_router.route import Route
|
|
10
9
|
from semantic_router.routers.base import BaseRouter
|
|
10
|
+
from semantic_router.utils.logger import logger
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
class SemanticRouter(BaseRouter):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|