nucliadb-utils 5.0.1.post1044__py3-none-any.whl → 5.0.1.post1075__py3-none-any.whl

nucliadb_utils/aiopynecone/client.py

@@ -37,6 +37,7 @@ from nucliadb_utils.aiopynecone.exceptions import (
 from nucliadb_utils.aiopynecone.models import (
     CreateIndexRequest,
     CreateIndexResponse,
+    IndexStats,
     ListResponse,
     QueryResponse,
     UpsertRequest,
@@ -64,6 +65,7 @@ BASE_API_HEADERS = {
     # are coming from the Nuclia integration:
     # https://docs.pinecone.io/integrations/build-integration/attribute-usage-to-your-integration
     "User-Agent": "source_tag=nuclia",
+    "X-Pinecone-API-Version": "2024-10",
 }
 MEGA_BYTE = 1024 * 1024
 MAX_UPSERT_PAYLOAD_SIZE = 2 * MEGA_BYTE
@@ -75,6 +77,7 @@ RETRIABLE_EXCEPTIONS = (
     PineconeRateLimitError,
     httpx.ConnectError,
     httpx.NetworkError,
+    httpx.WriteTimeout,
 )
 
 
@@ -159,6 +162,25 @@ class DataPlane:
     def _get_request_timeout(self, timeout: Optional[float] = None) -> Optional[float]:
         return timeout or self.client_timeout
 
+    @pinecone_observer.wrap({"type": "stats"})
+    async def stats(self, filter: Optional[dict[str, Any]] = None) -> IndexStats:
+        """
+        Get the index stats.
+        Params:
+        - `filter`: to filter the stats by their metadata. See:
+        https://docs.pinecone.io/reference/api/2024-07/data-plane/describeindexstats
+        """
+        post_kwargs: dict[str, Any] = {
+            "headers": {"Api-Key": self.api_key},
+        }
+        if filter is not None:
+            post_kwargs["json"] = {
+                "filter": filter,
+            }
+        response = await self.http_session.post("/describe_index_stats", **post_kwargs)
+        raise_for_status("stats", response)
+        return IndexStats.model_validate(response.json())
+
     @backoff.on_exception(
         backoff.expo,
         RETRIABLE_EXCEPTIONS,

nucliadb_utils/aiopynecone/models.py

@@ -112,3 +112,13 @@ class VectorMatch(BaseModel):
 
 class QueryResponse(BaseModel):
     matches: list[VectorMatch]
+
+
+class IndexNamespaceStats(BaseModel):
+    vectorCount: int
+
+
+class IndexStats(BaseModel):
+    dimension: int
+    namespaces: dict[str, IndexNamespaceStats] = {}
+    totalVectorCount: int

nucliadb_utils/audit/audit.py

@@ -107,10 +107,3 @@ class AuditStorage:
 
     def delete_kb(self, kbid: str):
         raise NotImplementedError
-
-    def suggest(
-        self,
-        kbid: str,
-        client_type: int,
-    ):
-        raise NotImplementedError

nucliadb_utils/audit/basic.py

@@ -108,10 +108,3 @@ class BasicAuditStorage(AuditStorage):
 
     def delete_kb(self, kbid: str):
         logger.debug(f"DELETE_KB {kbid}")
-
-    def suggest(
-        self,
-        kbid: str,
-        client_type: int,
-    ):
-        logger.debug(f"SUGGEST {kbid} {client_type}")

nucliadb_utils/audit/stream.py

@@ -372,27 +372,6 @@ class StreamAuditStorage(AuditStorage):
             ],
         )
 
-    def suggest(
-        self,
-        kbid: str,
-        client_type: int,
-    ):
-        self.kb_usage_utility.send_kb_usage(
-            service=Service.NUCLIA_DB,
-            account_id=None,
-            kb_id=kbid,
-            kb_source=KBSource.HOSTED,
-            # TODO unify AuditRequest client type and Nuclia Usage client type
-            searches=[
-                Search(
-                    client=ClientTypeKbUsage.Value(ClientType.Name(client_type)),  # type: ignore
-                    type=SearchType.SUGGEST,
-                    tokens=0,
-                    num_searches=1,
-                )
-            ],
-        )
-
     def chat(
         self,
         kbid: str,

nucliadb_utils/const.py

@@ -81,3 +81,5 @@ class Features:
     FIND_MERGE_ORDER_FIX = "nucliadb_find_merge_order_fix"
     PG_CATALOG_READ = "nucliadb_pg_catalog_read"
     PG_CATALOG_WRITE = "nucliadb_pg_catalog_write"
+    VECTORSETS_V0 = "vectorsets_v0_new_kbs_with_multiple_vectorsets"
+    SKIP_EXTERNAL_INDEX = "nucliadb_skip_external_index"

nucliadb_utils/featureflagging.py

@@ -73,6 +73,14 @@ DEFAULT_FLAG_DATA: dict[str, Any] = {
         "rollout": 0,
         "variants": {"environment": ["local"]},
     },
+    const.Features.VECTORSETS_V0: {
+        "rollout": 0,
+        "variants": {"environment": ["local"]},
+    },
+    const.Features.SKIP_EXTERNAL_INDEX: {
+        "rollout": 0,
+        "variants": {"environment": ["none"]},
+    },
 }
 
 

nucliadb_utils/storages/gcs.py

@@ -652,7 +652,7 @@ class GCSStorage(Storage):
         return deleted
 
     @storage_ops_observer.wrap({"type": "delete"})
-    async def delete_kb(self, kbid: str):
+    async def delete_kb(self, kbid: str) -> tuple[bool, bool]:
         if self.session is None:
             raise AttributeError()
         bucket_name = self.get_bucket_name(kbid)

nucliadb_utils/storages/s3.py

@@ -441,7 +441,7 @@ class S3Storage(Storage):
             deleted = True
         return deleted
 
-    async def delete_kb(self, kbid: str):
+    async def delete_kb(self, kbid: str) -> tuple[bool, bool]:
         bucket_name = self.get_bucket_name(kbid)
 
         missing = False

nucliadb_utils/storages/storage.py

@@ -29,7 +29,6 @@ from typing import (
     AsyncIterator,
     List,
     Optional,
-    Tuple,
     Type,
     Union,
     cast,
@@ -495,7 +494,7 @@ class Storage(abc.ABC, metaclass=abc.ABCMeta):
     async def create_kb(self, kbid: str) -> bool: ...
 
     @abc.abstractmethod
-    async def delete_kb(self, kbid: str) -> Tuple[bool, bool]: ...
+    async def delete_kb(self, kbid: str) -> tuple[bool, bool]: ...
 
     @abc.abstractmethod
     async def schedule_delete_kb(self, kbid: str) -> bool: ...

nucliadb_utils/tests/azure.py

@@ -31,7 +31,7 @@ from nucliadb_utils.storages.azure import AzureStorage
 
 images.settings["azurite"] = {
     "image": "mcr.microsoft.com/azure-storage/azurite",
-    "version": "3.30.0",
+    "version": "3.31.0",
     "options": {
         "ports": {"10000": None},
         "command": " ".join(
@@ -51,6 +51,8 @@ class Azurite(BaseImage):
     name = "azurite"
     port = 10000
 
+    _errors = 0
+
     def check(self):
         try:
             from azure.storage.blob import BlobServiceClient  # type: ignore
@@ -58,13 +60,15 @@ class Azurite(BaseImage):
             container_port = self.port
             host_port = self.get_port(port=container_port)
             conn_string = get_connection_string(self.host, host_port)
-
             client = BlobServiceClient.from_connection_string(conn_string)
             container_client = client.get_container_client("foo")
             container_client.create_container()
             container_client.delete_container()
             return True
         except Exception as ex:
+            self._errors += 1
+            if self._errors > 10:
+                raise
             print(ex)
             return False
 

nucliadb_utils-5.0.1.post1075.dist-info/METADATA

@@ -0,0 +1,60 @@
+Metadata-Version: 2.1
+Name: nucliadb_utils
+Version: 5.0.1.post1075
+Home-page: https://nuclia.com
+License: BSD
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9, <4
+Requires-Dist: pydantic>=2.6
+Requires-Dist: pydantic-settings>=2.2
+Requires-Dist: aiohttp>=3.9.4
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: prometheus-client>=0.12.0
+Requires-Dist: types-requests>=2.27.7
+Requires-Dist: mmh3>=3.0.0
+Requires-Dist: nats-py[nkeys]>=2.6.0
+Requires-Dist: PyNaCl
+Requires-Dist: pyjwt>=2.4.0
+Requires-Dist: memorylru>=1.1.2
+Requires-Dist: mrflagly>=0.2.9
+Requires-Dist: nucliadb-protos>=5.0.1.post1075
+Requires-Dist: nucliadb-telemetry>=5.0.1.post1075
+Provides-Extra: cache
+Requires-Dist: redis>=4.3.4; extra == "cache"
+Requires-Dist: orjson>=3.6.7; extra == "cache"
+Requires-Dist: lru-dict>=1.1.7; extra == "cache"
+Provides-Extra: fastapi
+Requires-Dist: fastapi>=0.95.2; extra == "fastapi"
+Requires-Dist: uvicorn<0.19.0,>=0.16.0; extra == "fastapi"
+Requires-Dist: starlette>=0.21.0; extra == "fastapi"
+Provides-Extra: storages
+Requires-Dist: oauth2client>=4.1.3; extra == "storages"
+Requires-Dist: aiobotocore>=2.9.0; extra == "storages"
+Requires-Dist: google-api-python-client>=2.37.0; extra == "storages"
+Requires-Dist: types-aiofiles>=0.8.3; extra == "storages"
+Requires-Dist: aiofiles>=0.8.0; extra == "storages"
+Requires-Dist: backoff>=1.11.1; extra == "storages"
+Requires-Dist: google-auth>=2.4.1; extra == "storages"
+Requires-Dist: azure-storage-blob==12.20.0; extra == "storages"
+Requires-Dist: azure-identity>=1.16.1; extra == "storages"
+
+# nucliadb util python library
+
+- Nats driver
+- FastAPI fixes
+- S3/GCS drivers
+
+
+# Install and run tests
+
+```bash
+pdm sync -d
+make test
+```

{nucliadb_utils-5.0.1.post1044.dist-info → nucliadb_utils-5.0.1.post1075.dist-info}/RECORD

@@ -1,10 +1,10 @@
 nucliadb_utils/__init__.py,sha256=EvBCH1iTODe-AgXm48aj4kVUt_Std3PeL8QnwimR5wI,895
 nucliadb_utils/asyncio_utils.py,sha256=h8Y-xpcFFRgNzaiIW0eidz7griAQa7ggbNk34-tAt2c,2888
 nucliadb_utils/authentication.py,sha256=N__d2Ez3JHJv5asYK5TgUcIkKqcAC8ZTLlnfLhfSneM,5837
-nucliadb_utils/const.py,sha256=x6QNG0dAgn87fxwlH6vB0unKl_UDsUjJOgQBINIokko,2550
+nucliadb_utils/const.py,sha256=ERzAa78DCPnE7ln9ChjP7A3ILdA_qGd9HStosBZkqwQ,2676
 nucliadb_utils/debug.py,sha256=Q56Nx9Dp7V2ae3CU2H0ztaZcHTJXdlflPLKLeOPZ170,2436
 nucliadb_utils/exceptions.py,sha256=y_3wk77WLVUtdo-5FtbBsdSkCtK_DsJkdWb5BoPn3qo,1094
-nucliadb_utils/featureflagging.py,sha256=-Ts3syVIXKCdCu2UdzkhdsSIlw0IKd32bjuDyXNwz6k,3077
+nucliadb_utils/featureflagging.py,sha256=0nN3vmtaEs6rB2uFQn0kI88It9568gcZ0GWWeWI1AOk,3308
 nucliadb_utils/grpc.py,sha256=apu0uePnkGHCAT7GRQ9YZfRYyFj26kJ440i8jitbM3U,3314
 nucliadb_utils/helpers.py,sha256=nPw8yod3hP-pxq80VF8QC36s7ygSg0dBUdfI-LatvCs,1600
 nucliadb_utils/indexing.py,sha256=Luaqcar3CySpdYOFp6Q9Fyr8ZYwhYhaKRHQ_VGL78f8,2318
@@ -18,13 +18,13 @@ nucliadb_utils/store.py,sha256=kQ35HemE0v4_Qg6xVqNIJi8vSFAYQtwI3rDtMsNy62Y,890
 nucliadb_utils/transaction.py,sha256=mwcI3aIHAvU5KOGqd_Uz_d1XQzXhk_-NWY8NqU1lfb0,7307
 nucliadb_utils/utilities.py,sha256=idajCm_4Sojh7b3HTkP0fTfG2Mb6PIB9xtMmcfB7Nl0,15758
 nucliadb_utils/aiopynecone/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb_utils/aiopynecone/client.py,sha256=N4aVWS0zUaAfKyQKxJ_0M-8p_j5vKxD0r-GL4pgxFKc,20319
+nucliadb_utils/aiopynecone/client.py,sha256=EFAQbNp-OHZ55vyXWzdzzp_59FGacGs_nfM2sMBIG8A,21153
 nucliadb_utils/aiopynecone/exceptions.py,sha256=EEE0XoGs1zIB5yOJ_fy6yoG4uIb4cWIawYdJeNe4eDo,3012
-nucliadb_utils/aiopynecone/models.py,sha256=tUugQ-ACQAyT-lhsfLwKAOgb6ilLchLwZfnRV7xwFck,3047
+nucliadb_utils/aiopynecone/models.py,sha256=ketK2IYLWiwFZ76rnJmwfcuopFJrCAtCUszdTSurm_Q,3236
 nucliadb_utils/audit/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb_utils/audit/audit.py,sha256=ictuErJy3X36S7s_f_sdNQQrQ4ackAYCT6M2-4nyenE,3086
-nucliadb_utils/audit/basic.py,sha256=-Yztp0I745J6dz_zHPKdXv60OH1m9_d69NlpbhsGFZI,3717
-nucliadb_utils/audit/stream.py,sha256=yPWe9iCJmTKLtB1iktSOKR8HDB0I3KF7vsWOwloljGM,14299
+nucliadb_utils/audit/audit.py,sha256=2OyAQ3sUfFfVqBgaA3xiAby3eSIwpey2wg7Ctf0wDxc,2968
+nucliadb_utils/audit/basic.py,sha256=NC3bEuEKS1EV2v7HuH0zt8YobK_xdpM45L9SBsR-GFU,3579
+nucliadb_utils/audit/stream.py,sha256=Zysj70rwxCziJtptDCc4fHoACHyC5FVjkG47NaKx214,13661
 nucliadb_utils/cache/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
 nucliadb_utils/cache/exceptions.py,sha256=Zu-O_-0-yctOEgoDGI92gPzWfBMRrpiAyESA62ld6MA,975
 nucliadb_utils/cache/nats.py,sha256=-AjCfkFgKVdJUlGR0hT9JDSNkPVFg4S6w9eW-ZIcXPM,7037
@@ -47,25 +47,25 @@ nucliadb_utils/nuclia_usage/utils/kb_usage_report.py,sha256=P1fKvrcYdSPNJ1ycd0Ar
 nucliadb_utils/storages/__init__.py,sha256=5Qc8AUWiJv9_JbGCBpAn88AIJhwDlm0OPQpg2ZdRL4U,872
 nucliadb_utils/storages/azure.py,sha256=egMDwLNIGSQyVevuySt2AswzFdNAcih05BbRg3-p8IU,16015
 nucliadb_utils/storages/exceptions.py,sha256=mm_wX4YRtp7u7enkk_4pMSlX5AQQuFbq4xLmupVDt3Y,2502
-nucliadb_utils/storages/gcs.py,sha256=WblkxWoa1brevsJV3ebiE6s7Wb_eXFScw41202f5uP4,26999
+nucliadb_utils/storages/gcs.py,sha256=k1WXZDBQXSizO_3kEJef7EmyInnASJsAy5VTtEw8ly4,27020
 nucliadb_utils/storages/local.py,sha256=NxC_nMBd38NDsR266DSgoBLdQlvUwf0_sd50r-BLI0E,10288
 nucliadb_utils/storages/nuclia.py,sha256=vEv94xAT7QM2g80S25QyrOw2pzvP2BAX-ADgZLtuCVc,2097
 nucliadb_utils/storages/object_store.py,sha256=Tw10GmpYfM5TMqJ3Tk9pLQ9wLMBk1-snL_m6uasiZDQ,4257
-nucliadb_utils/storages/s3.py,sha256=8KV-V7EiqRYhXYlGN0UjzM-v1Pj2Zh7NtXDikG96knU,19272
+nucliadb_utils/storages/s3.py,sha256=8c5yFcoYWOnC_xWU3HveS6bMIH2_ZwAoNpdkFu9Fl8A,19293
 nucliadb_utils/storages/settings.py,sha256=ugCPy1zxBOmA2KosT-4tsjpvP002kg5iQyi42yCGCJA,1285
-nucliadb_utils/storages/storage.py,sha256=gknIjRiHZ_f0EhTVYKVRgT4VqMlgDy9Ia6Av_1zz1tk,19706
+nucliadb_utils/storages/storage.py,sha256=4O2vUEAQz5SA93m9tVgNxOVuuy96gDNNvMrmutXpnjM,19695
 nucliadb_utils/storages/utils.py,sha256=8g2rIwJeYIumQLOB47Yw1rx3twlhRB_cJxer65QfZmk,1479
 nucliadb_utils/tests/__init__.py,sha256=Oo9CAE7B0eW5VHn8sHd6o30SQzOWUhktLPRXdlDOleA,1456
 nucliadb_utils/tests/asyncbenchmark.py,sha256=x4be2IwCawle9zWgMOJkmwoUwk5p1tv7cLQGmybkEOg,10587
-nucliadb_utils/tests/azure.py,sha256=uf1Ltt1sHyoN0Yt5QwkAm7GZxx-SXST1JdB4x3P_dEY,4372
+nucliadb_utils/tests/azure.py,sha256=Dg-Eb4KVScG-O6P9y-bVQZTAKTNUMQ0i-CKEd9IdrWw,4474
 nucliadb_utils/tests/fixtures.py,sha256=b-e6Ciewh7GSh9rriG01bN9GiMW6xbGgrp3KiX7HNiY,2089
 nucliadb_utils/tests/gcs.py,sha256=Bx-kxf2NoYYeyNFPpBFxbRN8kIiYFpELEvqG0V1_9Ks,4183
 nucliadb_utils/tests/indexing.py,sha256=YW2QhkhO9Q_8A4kKWJaWSvXvyQ_AiAwY1VylcfVQFxk,1513
 nucliadb_utils/tests/local.py,sha256=7nuP8EFUAiA8ZH50R1iPV9EUXBySQxOanVm3Zht_e0g,1835
 nucliadb_utils/tests/nats.py,sha256=xqpww4jZjTKY9oPGlJdDJG67L3FIBQsa9qDHxILR8r8,7687
 nucliadb_utils/tests/s3.py,sha256=IdMxK_cNdSHLvO1u8BwsKFzD87Hk1MVPDZ57zx6h-rA,3656
-nucliadb_utils-5.0.1.post1044.dist-info/METADATA,sha256=tb6nwHLxxxB6EkIz5H3tFkHtSskd3L8aOVkiFYCurNs,2084
-nucliadb_utils-5.0.1.post1044.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-nucliadb_utils-5.0.1.post1044.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
-nucliadb_utils-5.0.1.post1044.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-nucliadb_utils-5.0.1.post1044.dist-info/RECORD,,
+nucliadb_utils-5.0.1.post1075.dist-info/METADATA,sha256=5p7gO6sOPQ7sB8B_KiftcxYaq36LbZux-OgdslnKF8Q,2071
+nucliadb_utils-5.0.1.post1075.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+nucliadb_utils-5.0.1.post1075.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
+nucliadb_utils-5.0.1.post1075.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+nucliadb_utils-5.0.1.post1075.dist-info/RECORD,,

nucliadb_utils-5.0.1.post1044.dist-info/METADATA

@@ -1,59 +0,0 @@
-Metadata-Version: 2.1
-Name: nucliadb_utils
-Version: 5.0.1.post1044
-Home-page: https://nuclia.com
-License: BSD
-Classifier: Development Status :: 4 - Beta
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Python: >=3.9, <4
-Requires-Dist: pydantic >=2.6
-Requires-Dist: pydantic-settings >=2.2
-Requires-Dist: aiohttp >=3.9.4
-Requires-Dist: prometheus-client >=0.12.0
-Requires-Dist: types-requests >=2.27.7
-Requires-Dist: mmh3 >=3.0.0
-Requires-Dist: nats-py[nkeys] >=2.6.0
-Requires-Dist: PyNaCl
-Requires-Dist: pyjwt >=2.4.0
-Requires-Dist: memorylru >=1.1.2
-Requires-Dist: mrflagly >=0.2.9
-Requires-Dist: nucliadb-protos >=5.0.1.post1044
-Requires-Dist: nucliadb-telemetry >=5.0.1.post1044
-Provides-Extra: cache
-Requires-Dist: redis >=4.3.4 ; extra == 'cache'
-Requires-Dist: orjson >=3.6.7 ; extra == 'cache'
-Requires-Dist: lru-dict >=1.1.7 ; extra == 'cache'
-Provides-Extra: fastapi
-Requires-Dist: fastapi >=0.95.2 ; extra == 'fastapi'
-Requires-Dist: uvicorn <0.19.0,>=0.16.0 ; extra == 'fastapi'
-Requires-Dist: starlette >=0.21.0 ; extra == 'fastapi'
-Provides-Extra: storages
-Requires-Dist: oauth2client >=4.1.3 ; extra == 'storages'
-Requires-Dist: aiobotocore >=2.9.0 ; extra == 'storages'
-Requires-Dist: google-api-python-client >=2.37.0 ; extra == 'storages'
-Requires-Dist: types-aiofiles >=0.8.3 ; extra == 'storages'
-Requires-Dist: aiofiles >=0.8.0 ; extra == 'storages'
-Requires-Dist: backoff >=1.11.1 ; extra == 'storages'
-Requires-Dist: google-auth >=2.4.1 ; extra == 'storages'
-Requires-Dist: azure-storage-blob >=12.20.0 ; extra == 'storages'
-Requires-Dist: azure-identity >=1.16.1 ; extra == 'storages'
-
-# nucliadb util python library
-
-- Nats driver
-- FastAPI fixes
-- S3/GCS drivers
-
-
-# Install and run tests
-
-```bash
-pdm sync -d
-make test
-```