pinecone 6.0.0.dev1__tar.gz → 6.0.0.dev3__tar.gz

{pinecone-6.0.0.dev1 → pinecone-6.0.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pinecone
-Version: 6.0.0.dev1
+Version: 6.0.0.dev3
 Summary: Pinecone client and SDK
 Home-page: https://www.pinecone.io
 License: Apache-2.0

pinecone-6.0.0.dev3/pinecone/__version__

@@ -0,0 +1 @@
+6.0.0.dev3

{pinecone-6.0.0.dev1 → pinecone-6.0.0.dev3}/pinecone/core/openapi/shared/api_client.py

@@ -8,6 +8,24 @@ import re
 import typing
 from urllib.parse import quote
 from urllib3.fields import RequestField
+import time
+import random
+
+def retry_api_call(
+    func, args=(), kwargs={}, retries=3, backoff=1, jitter=0.5
+):
+    attempts = 0
+    while attempts < retries:
+        try:
+            return func(*args, **kwargs)  # Attempt to call __call_api
+        except Exception as e:
+            attempts += 1
+            if attempts >= retries:
+                print(f"API call failed after {attempts} attempts: {e}")
+                raise  # Re-raise exception if retries are exhausted
+            sleep_time = backoff * (2 ** (attempts - 1)) + random.uniform(0, jitter)
+            # print(f"Retrying ({attempts}/{retries}) in {sleep_time:.2f} seconds after error: {e}")
+            time.sleep(sleep_time)
 
 
 from pinecone.core.openapi.shared import rest
@@ -397,25 +415,32 @@ class ApiClient(object):
             )
 
         return self.pool.apply_async(
-            self.__call_api,
-            (
-                resource_path,
-                method,
-                path_params,
-                query_params,
-                header_params,
-                body,
-                post_params,
-                files,
-                response_type,
-                auth_settings,
-                _return_http_data_only,
-                collection_formats,
-                _preload_content,
-                _request_timeout,
-                _host,
-                _check_type,
-            ),
+            retry_api_call,
+            args=(
+                self.__call_api,  # Pass the API call function as the first argument
+                (
+                    resource_path,
+                    method,
+                    path_params,
+                    query_params,
+                    header_params,
+                    body,
+                    post_params,
+                    files,
+                    response_type,
+                    auth_settings,
+                    _return_http_data_only,
+                    collection_formats,
+                    _preload_content,
+                    _request_timeout,
+                    _host,
+                    _check_type,
+                ),
+                {},  # empty kwargs dictionary
+                3,    # retries
+                1,    # backoff time
+                0.5   # jitter
+            )
         )
 
     def request(

{pinecone-6.0.0.dev1 → pinecone-6.0.0.dev3}/pinecone/data/index.py

@@ -27,6 +27,8 @@ from .features.bulk_import import ImportFeatureMixin
 from pinecone.core.openapi.data.api.data_plane_api import DataPlaneApi
 from ..utils import setup_openapi_client, parse_non_empty_args
 from .vector_factory import VectorFactory
+from .query_results_aggregator import QueryResultsAggregator, QueryNamespacesResults
+from multiprocessing.pool import ApplyResult
 
 __all__ = [
     "Index",
@@ -361,7 +363,7 @@ class Index(ImportFeatureMixin):
             Union[SparseValues, Dict[str, Union[List[float], List[int]]]]
         ] = None,
         **kwargs,
-    ) -> QueryResponse:
+    ) -> Union[QueryResponse, ApplyResult]:
         """
         The Query operation searches a namespace, using a query vector.
         It retrieves the ids of the most similar items in a namespace, along with their similarity scores.
@@ -403,6 +405,39 @@ class Index(ImportFeatureMixin):
                  and namespace name.
         """
 
+        response = self._query(
+            *args,
+            top_k=top_k,
+            vector=vector,
+            id=id,
+            namespace=namespace,
+            filter=filter,
+            include_values=include_values,
+            include_metadata=include_metadata,
+            sparse_vector=sparse_vector,
+            **kwargs,
+        )
+
+        if kwargs.get("async_req", False):
+            return response
+        else:
+            return parse_query_response(response)
+
+    def _query(
+        self,
+        *args,
+        top_k: int,
+        vector: Optional[List[float]] = None,
+        id: Optional[str] = None,
+        namespace: Optional[str] = None,
+        filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None,
+        include_values: Optional[bool] = None,
+        include_metadata: Optional[bool] = None,
+        sparse_vector: Optional[
+            Union[SparseValues, Dict[str, Union[List[float], List[int]]]]
+        ] = None,
+        **kwargs,
+    ) -> QueryResponse:
         if len(args) > 0:
             raise ValueError(
                 "The argument order for `query()` has changed; please use keyword arguments instead of positional arguments. Example: index.query(vector=[0.1, 0.2, 0.3], top_k=10, namespace='my_namespace')"
@@ -435,7 +470,58 @@ class Index(ImportFeatureMixin):
             ),
             **{k: v for k, v in kwargs.items() if k in _OPENAPI_ENDPOINT_PARAMS},
         )
-        return parse_query_response(response)
+        return response
+
+    @validate_and_convert_errors
+    def query_namespaces(
+        self,
+        vector: List[float],
+        namespaces: List[str],
+        top_k: Optional[int] = None,
+        filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None,
+        include_values: Optional[bool] = None,
+        include_metadata: Optional[bool] = None,
+        sparse_vector: Optional[
+            Union[SparseValues, Dict[str, Union[List[float], List[int]]]]
+        ] = None,
+        **kwargs,
+    ) -> QueryNamespacesResults:
+        if len(namespaces) == 0:
+            raise ValueError("At least one namespace must be specified")
+        if len(vector) == 0:
+            raise ValueError("Query vector must not be empty")
+
+        # The caller may only want the top_k=1 result across all queries,
+        # but we need to get at least 2 results from each query in order to
+        # aggregate them correctly. So we'll temporarily set topK to 2 for the
+        # subqueries, and then we'll take the topK=1 results from the aggregated
+        # results.
+        overall_topk = top_k if top_k is not None else 10
+        aggregator = QueryResultsAggregator(top_k=overall_topk)
+        subquery_topk = overall_topk if overall_topk > 2 else 2
+
+        target_namespaces = set(namespaces)  # dedup namespaces
+        async_results = [
+            self.query(
+                vector=vector,
+                namespace=ns,
+                top_k=subquery_topk,
+                filter=filter,
+                include_values=include_values,
+                include_metadata=include_metadata,
+                sparse_vector=sparse_vector,
+                async_req=True,
+                **kwargs,
+            )
+            for ns in target_namespaces
+        ]
+
+        for result in async_results:
+            response = result.get()
+            aggregator.add_results(response)
+
+        final_results = aggregator.get_results()
+        return final_results
 
     @validate_and_convert_errors
     def update(

{pinecone-6.0.0.dev1/pinecone/grpc → pinecone-6.0.0.dev3/pinecone/data}/query_results_aggregator.py

@@ -61,7 +61,7 @@ class ScoredVectorWithNamespace:
 
 
 @dataclass
-class CompositeQueryResults:
+class QueryNamespacesResults:
     usage: Usage
     matches: List[ScoredVectorWithNamespace]
 
@@ -69,7 +69,7 @@ class CompositeQueryResults:
         if hasattr(self, key):
             return getattr(self, key)
         else:
-            raise KeyError(f"'{key}' not found in CompositeQueryResults")
+            raise KeyError(f"'{key}' not found in QueryNamespacesResults")
 
     def __repr__(self):
         return json.dumps(
@@ -81,28 +81,23 @@ class CompositeQueryResults:
         )
 
 
-class QueryResultsAggregationEmptyResultsError(Exception):
-    def __init__(self, namespace: str):
-        super().__init__(
-            f"Query results for namespace '{namespace}' were empty. Check that you have upserted vectors into this namespace (see describe_index_stats) and that the namespace name is spelled correctly."
-        )
-
-
 class QueryResultsAggregregatorNotEnoughResultsError(Exception):
-    def __init__(self, top_k: int, num_results: int):
+    def __init__(self):
         super().__init__(
-            f"Cannot interpret results without at least two matches. In order to aggregate results from multiple queries, top_k must be greater than 1 in order to correctly infer the similarity metric from scores. Expected at least {top_k} results but got {num_results}."
+            "Cannot interpret results without at least two matches. In order to aggregate results from multiple queries, top_k must be greater than 1 in order to correctly infer the similarity metric from scores."
         )
 
 
 class QueryResultsAggregatorInvalidTopKError(Exception):
     def __init__(self, top_k: int):
-        super().__init__(f"Invalid top_k value {top_k}. top_k must be a positive integer.")
+        super().__init__(
+            f"Invalid top_k value {top_k}. To aggregate results from multiple queries the top_k must be at least 2."
+        )
 
 
 class QueryResultsAggregator:
     def __init__(self, top_k: int):
-        if top_k < 1:
+        if top_k < 2:
             raise QueryResultsAggregatorInvalidTopKError(top_k)
         self.top_k = top_k
         self.usage_read_units = 0
@@ -110,7 +105,7 @@ class QueryResultsAggregator:
         self.insertion_counter = 0
         self.is_dotproduct = None
         self.read = False
-        self.final_results: Optional[CompositeQueryResults] = None
+        self.final_results: Optional[QueryNamespacesResults] = None
 
     def _is_dotproduct_index(self, matches):
         # The interpretation of the score depends on the similar metric used.
@@ -134,6 +129,13 @@ class QueryResultsAggregator:
             if len(self.heap) < self.top_k:
                 heapq.heappush(self.heap, heap_item_fn(match, ns))
             else:
+                # Assume we have dotproduct scores sorted in descending order
+                if self.is_dotproduct and match["score"] < self.heap[0][0]:
+                    # No further matches can improve the top-K heap
+                    break
+                elif not self.is_dotproduct and match["score"] > -self.heap[0][0]:
+                    # No further matches can improve the top-K heap
+                    break
                 heapq.heappushpop(self.heap, heap_item_fn(match, ns))
 
     def add_results(self, results: Dict[str, Any]):
@@ -148,11 +150,14 @@ class QueryResultsAggregator:
         self.usage_read_units += results.get("usage", {}).get("readUnits", 0)
 
         if len(matches) == 0:
-            raise QueryResultsAggregationEmptyResultsError(ns)
+            return
 
         if self.is_dotproduct is None:
             if len(matches) == 1:
-                raise QueryResultsAggregregatorNotEnoughResultsError(self.top_k, len(matches))
+                # This condition should match the second time we add results containing
+                # only one match. We need at least two matches in a single response in order
+                # to infer the similarity metric
+                raise QueryResultsAggregregatorNotEnoughResultsError()
             self.is_dotproduct = self._is_dotproduct_index(matches)
 
         if self.is_dotproduct:
@@ -160,7 +165,7 @@ class QueryResultsAggregator:
         else:
             self._process_matches(matches, ns, self._non_dotproduct_heap_item)
 
-    def get_results(self) -> CompositeQueryResults:
+    def get_results(self) -> QueryNamespacesResults:
         if self.read:
             if self.final_results is not None:
                 return self.final_results
@@ -169,7 +174,7 @@ class QueryResultsAggregator:
                 raise ValueError("Results have already been read. Cannot get results again.")
         self.read = True
 
-        self.final_results = CompositeQueryResults(
+        self.final_results = QueryNamespacesResults(
             usage=Usage(read_units=self.usage_read_units),
             matches=[
                 ScoredVectorWithNamespace(heapq.heappop(self.heap)) for _ in range(len(self.heap))

{pinecone-6.0.0.dev1 → pinecone-6.0.0.dev3}/pinecone/exceptions/__init__.py

@@ -12,6 +12,8 @@ from pinecone.core.openapi.shared.exceptions import (
 )
 from .exceptions import PineconeConfigurationError, PineconeProtocolError, ListConversionException
 
+PineconeNotFoundException = NotFoundException
+
 __all__ = [
     "PineconeConfigurationError",
     "PineconeProtocolError",
@@ -22,6 +24,7 @@ __all__ = [
     "PineconeApiKeyError",
     "PineconeApiException",
     "NotFoundException",
+    "PineconeNotFoundException",
     "UnauthorizedException",
     "ForbiddenException",
     "ServiceException",

{pinecone-6.0.0.dev1 → pinecone-6.0.0.dev3}/pinecone/grpc/grpc_runner.py

@@ -8,10 +8,19 @@ from pinecone import Config
 from .utils import _generate_request_id
 from .config import GRPCClientConfig
 from pinecone.utils.constants import REQUEST_ID, CLIENT_VERSION
-from pinecone.exceptions.exceptions import PineconeException
-from grpc import CallCredentials, Compression
+from grpc import CallCredentials, Compression, StatusCode
+from grpc.aio import AioRpcError
 from google.protobuf.message import Message
 
+from pinecone.exceptions import (
+    PineconeException,
+    PineconeApiValueError,
+    PineconeApiException,
+    UnauthorizedException,
+    PineconeNotFoundException,
+    ServiceException,
+)
+
 
 class GrpcRunner:
     def __init__(self, index_name: str, config: Config, grpc_config: GRPCClientConfig):
@@ -50,7 +59,7 @@ class GrpcRunner:
                     compression=compression,
                 )
             except _InactiveRpcError as e:
-                raise PineconeException(e._state.debug_error_string) from e
+                self._map_exception(e, e._state.code, e._state.details)
 
         return wrapped()
 
@@ -89,8 +98,8 @@ class GrpcRunner:
                         wait_for_ready=wait_for_ready,
                         compression=compression,
                     )
-            except _InactiveRpcError as e:
-                raise PineconeException(e._state.debug_error_string) from e
+            except AioRpcError as e:
+                self._map_exception(e, e.code(), e.details())
 
         return await wrapped()
 
@@ -108,3 +117,37 @@ class GrpcRunner:
 
     def _request_metadata(self) -> Dict[str, str]:
         return {REQUEST_ID: _generate_request_id()}
+
+    def _map_exception(self, e: Exception, code: Optional[StatusCode], details: Optional[str]):
+        # Client / connection issues
+        details = details or ""
+
+        if code in [StatusCode.DEADLINE_EXCEEDED]:
+            raise TimeoutError(details) from e
+
+        # Permissions stuff
+        if code in [StatusCode.PERMISSION_DENIED, StatusCode.UNAUTHENTICATED]:
+            raise UnauthorizedException(status=code, reason=details) from e
+
+        # 400ish stuff
+        if code in [StatusCode.NOT_FOUND]:
+            raise PineconeNotFoundException(status=code, reason=details) from e
+        if code in [StatusCode.INVALID_ARGUMENT, StatusCode.OUT_OF_RANGE]:
+            raise PineconeApiValueError(details) from e
+        if code in [
+            StatusCode.ALREADY_EXISTS,
+            StatusCode.FAILED_PRECONDITION,
+            StatusCode.UNIMPLEMENTED,
+            StatusCode.RESOURCE_EXHAUSTED,
+        ]:
+            raise PineconeApiException(status=code, reason=details) from e
+
+        # 500ish stuff
+        if code in [StatusCode.INTERNAL, StatusCode.UNAVAILABLE]:
+            raise ServiceException(status=code, reason=details) from e
+        if code in [StatusCode.UNKNOWN, StatusCode.DATA_LOSS, StatusCode.ABORTED]:
+            # abandon hope, all ye who enter here
+            raise PineconeException(code, details) from e
+
+        # If you get here, you're in a bad place
+        raise PineconeException(code, details) from e

{pinecone-6.0.0.dev1 → pinecone-6.0.0.dev3}/pinecone/grpc/index_grpc_asyncio.py

@@ -42,7 +42,7 @@ from .utils import (
     parse_sparse_values_arg,
 )
 from .vector_factory_grpc import VectorFactoryGRPC
-from .query_results_aggregator import QueryResultsAggregator, CompositeQueryResults
+from ..data.query_results_aggregator import QueryResultsAggregator, QueryNamespacesResults
 
 
 class GRPCIndexAsyncio(GRPCIndexBase):
@@ -151,7 +151,7 @@ class GRPCIndexAsyncio(GRPCIndexBase):
         sparse_vector: Optional[Union[GRPCSparseValues, SparseVectorTypedDict]] = None,
         semaphore: Optional[asyncio.Semaphore] = None,
         **kwargs,
-    ) -> dict[str, Any]:
+    ) -> Dict[str, Any]:
         if vector is not None and id is not None:
             raise ValueError("Cannot specify both `id` and `vector`")
 
@@ -256,7 +256,7 @@ class GRPCIndexAsyncio(GRPCIndexBase):
         )
         return parse_query_response(json_response, _check_type=False)
 
-    async def composite_query(
+    async def query_namespaces(
         self,
         vector: List[float],
         namespaces: List[str],
@@ -269,7 +269,7 @@ class GRPCIndexAsyncio(GRPCIndexBase):
         max_concurrent_requests: Optional[int] = None,
         semaphore: Optional[asyncio.Semaphore] = None,
         **kwargs,
-    ) -> CompositeQueryResults:
+    ) -> QueryNamespacesResults:
         aggregator_lock = asyncio.Lock()
         semaphore = self._get_semaphore(max_concurrent_requests, semaphore)
 

{pinecone-6.0.0.dev1 → pinecone-6.0.0.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pinecone"
-version = "6.0.0.dev1"
+version = "6.0.0.dev3"
 packages = [
     { include="pinecone", from="." },
 ]

pinecone-6.0.0.dev1/pinecone/__version__

@@ -1 +0,0 @@
-6.0.0.dev1