nucliadb-utils 6.3.1.post3526__py3-none-any.whl → 6.3.1.post3531__py3-none-any.whl

nucliadb_utils/storages/azure.py

@@ -211,8 +211,10 @@ class AzureStorage(Storage):
         bucket_name = self.get_bucket_name(kbid)
         return await self.object_store.bucket_delete(bucket_name)
 
-    async def iterate_objects(self, bucket: str, prefix: str) -> AsyncGenerator[ObjectInfo, None]:
-        async for obj in self.object_store.iterate(bucket, prefix):
+    async def iterate_objects(
+        self, bucket: str, prefix: str, start: Optional[str] = None
+    ) -> AsyncGenerator[ObjectInfo, None]:
+        async for obj in self.object_store.iterate(bucket, prefix, start):
             yield obj
 
     async def insert_object(self, bucket_name: str, key: str, data: bytes) -> None:
@@ -373,9 +375,13 @@ class AzureObjectStore(ObjectStore):
         async for chunk in downloader.chunks():
             yield chunk
 
-    async def iterate(self, bucket: str, prefix: str) -> AsyncGenerator[ObjectInfo, None]:
+    async def iterate(
+        self, bucket: str, prefix: str, start: Optional[str] = None
+    ) -> AsyncGenerator[ObjectInfo, None]:
         container_client = self.service_client.get_container_client(bucket)
         async for blob in container_client.list_blobs(name_starts_with=prefix):
+            if start and blob.name <= start:
+                continue
             yield ObjectInfo(name=blob.name)
 
     async def get_metadata(self, bucket: str, key: str) -> ObjectMetadata:

nucliadb_utils/storages/gcs.py

@@ -695,20 +695,29 @@ class GCSStorage(Storage):
                     errors.capture_message(msg, "error", scope)
         return deleted, conflict
 
-    async def iterate_objects(self, bucket: str, prefix: str) -> AsyncGenerator[ObjectInfo, None]:
+    async def iterate_objects(
+        self, bucket: str, prefix: str, start: Optional[str] = None
+    ) -> AsyncGenerator[ObjectInfo, None]:
         if self.session is None:
             raise AttributeError()
         url = "{}/{}/o".format(self.object_base_url, bucket)
         headers = await self.get_access_headers()
+        params = {"prefix": prefix}
+        if start:
+            params["startOffset"] = start
         async with self.session.get(
             url,
             headers=headers,
-            params={"prefix": prefix},
+            params=params,
         ) as resp:
             assert resp.status == 200
             data = await resp.json()
             if "items" in data:
                 for item in data["items"]:
+                    if start is not None and item["name"] == start:
+                        # Skip the start item to be compatible with all
+                        # storage implementations
+                        continue
                     yield ObjectInfo(name=item["name"])
 
         page_token = data.get("nextPageToken")

nucliadb_utils/storages/local.py

@@ -241,14 +241,17 @@ class LocalStorage(Storage):
 
     async def create_kb(self, kbid: str):
         bucket = self.get_bucket_name(kbid)
-        path = self.get_bucket_path(bucket)
         try:
-            os.makedirs(path, exist_ok=True)
+            await self.create_bucket(bucket)
             created = True
         except FileExistsError:
             created = False
         return created
 
+    async def create_bucket(self, bucket_name: str):
+        path = self.get_bucket_path(bucket_name)
+        os.makedirs(path, exist_ok=True)
+
     async def delete_kb(self, kbid: str) -> tuple[bool, bool]:
         bucket = self.get_bucket_name(kbid)
         path = self.get_bucket_path(bucket)
@@ -277,20 +280,27 @@ class LocalStorage(Storage):
             deleted = False
         return deleted
 
-    async def iterate_objects(self, bucket: str, prefix: str) -> AsyncGenerator[ObjectInfo, None]:
-        pathname = f"{self.get_file_path(bucket, prefix)}*"
-        for key in glob.glob(pathname):
+    async def iterate_objects(
+        self, bucket: str, prefix: str, start: Optional[str] = None
+    ) -> AsyncGenerator[ObjectInfo, None]:
+        bucket_path = self.get_bucket_path(bucket)
+        pathname = f"{self.get_file_path(bucket, prefix)}**/*"
+        for key in sorted(glob.glob(pathname, recursive=True)):
+            if not os.path.isfile(key):
+                continue
             if key.endswith(".metadata"):
                 # Skip metadata files -- they are internal to the local-storage implementation.
                 continue
-            name = key.split("/")[-1]
+            name = key.split(bucket_path)[-1].lstrip("/")
+            if start is not None and name <= start:
+                continue
             yield ObjectInfo(name=name)
 
-    async def download(self, bucket_name: str, key: str, range: Optional[Range] = None):
-        key_path = self.get_file_path(bucket_name, key)
+    async def download(self, bucket: str, key: str, range: Optional[Range] = None):
+        key_path = self.get_file_path(bucket, key)
         if not os.path.exists(key_path):
             return
-        async for chunk in super().download(bucket_name, key, range=range):
+        async for chunk in super().download(bucket, key, range=range):
             yield chunk
 
     async def insert_object(self, bucket: str, key: str, data: bytes) -> None:

nucliadb_utils/storages/object_store.py

@@ -120,7 +120,9 @@ class ObjectStore(abc.ABC, metaclass=abc.ABCMeta):
         yield b""
 
     @abc.abstractmethod
-    async def iterate(self, bucket: str, prefix: str) -> AsyncGenerator[ObjectInfo, None]:
+    async def iterate(
+        self, bucket: str, prefix: str, start: Optional[str] = None
+    ) -> AsyncGenerator[ObjectInfo, None]:
         raise NotImplementedError()
         yield ObjectInfo(name="")
 

nucliadb_utils/storages/s3.py

@@ -422,10 +422,14 @@ class S3Storage(Storage):
         else:
             raise AttributeError("No valid uri")
 
-    async def iterate_objects(self, bucket: str, prefix: str = "/") -> AsyncGenerator[ObjectInfo, None]:
+    async def iterate_objects(
+        self, bucket: str, prefix: str = "/", start: Optional[str] = None
+    ) -> AsyncGenerator[ObjectInfo, None]:
         paginator = self._s3aioclient.get_paginator("list_objects")
-        async for result in paginator.paginate(Bucket=bucket, Prefix=prefix):
-            for item in result.get("Contents", []):
+        async for result in paginator.paginate(
+            Bucket=bucket, Prefix=prefix, PaginationConfig={"StartingToken": start}
+        ):
+            for item in result.get("Contents") or []:
                 yield ObjectInfo(name=item["Key"])
 
     async def create_kb(self, kbid: str):

nucliadb_utils/storages/storage.py

@@ -502,7 +502,9 @@ class Storage(abc.ABC, metaclass=abc.ABCMeta):
     async def finalize(self) -> None: ...
 
     @abc.abstractmethod
-    async def iterate_objects(self, bucket: str, prefix: str) -> AsyncGenerator[ObjectInfo, None]:
+    async def iterate_objects(
+        self, bucket: str, prefix: str, start: Optional[str] = None
+    ) -> AsyncGenerator[ObjectInfo, None]:
         raise NotImplementedError()
         yield ObjectInfo(name="")
 
@@ -553,6 +555,13 @@ class Storage(abc.ABC, metaclass=abc.ABCMeta):
         else:
             await self.insert_object(bucket, key, data)
 
+    @abc.abstractmethod
+    async def create_bucket(self, bucket_name: str) -> None:
+        """
+        Create a new bucket in the storage.
+        """
+        ...
+
 
 async def iter_and_add_size(
     stream: AsyncGenerator[bytes, None], cf: CloudFile

nucliadb_utils/transaction.py

@@ -51,6 +51,10 @@ class WaitFor:
         self.seq = seq
 
 
+class TransactionError(Exception):
+    pass
+
+
 class TransactionCommitTimeoutError(Exception):
     pass
 
@@ -80,7 +84,7 @@ class LocalTransactionUtility:
 
         resp = await ingest.ProcessMessage(iterator(writer))  # type: ignore
         if resp.status != OpStatusWriter.Status.OK:
-            logger.error(f"Local transaction failed processing {writer}")
+            raise TransactionError(f"Local transaction failed processing {writer}")
         return 0
 
     async def finalize(self):

{nucliadb_utils-6.3.1.post3526.dist-info → nucliadb_utils-6.3.1.post3531.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb_utils
-Version: 6.3.1.post3526
+Version: 6.3.1.post3531
 Summary: NucliaDB util library
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -27,8 +27,8 @@ Requires-Dist: nats-py[nkeys]>=2.6.0
 Requires-Dist: PyNaCl
 Requires-Dist: pyjwt>=2.4.0
 Requires-Dist: mrflagly>=0.2.9
-Requires-Dist: nucliadb-protos>=6.3.1.post3526
-Requires-Dist: nucliadb-telemetry>=6.3.1.post3526
+Requires-Dist: nucliadb-protos>=6.3.1.post3531
+Requires-Dist: nucliadb-telemetry>=6.3.1.post3531
 Provides-Extra: cache
 Requires-Dist: redis>=4.3.4; extra == "cache"
 Requires-Dist: orjson>=3.6.7; extra == "cache"

{nucliadb_utils-6.3.1.post3526.dist-info → nucliadb_utils-6.3.1.post3531.dist-info}/RECORD

@@ -14,7 +14,7 @@ nucliadb_utils/run.py,sha256=Es0_Bu5Yc-LWczvwL6gzWqSwC85RjDCk-0oFQAJi9g4,1827
 nucliadb_utils/settings.py,sha256=RnGhEUvwv6faNqALiqDCivvzNOyyXVBflYh_37uNkow,8193
 nucliadb_utils/signals.py,sha256=lo_Mk12NIX5Au--3H3WObvDOXq_OMurql2qiC2TnAao,2676
 nucliadb_utils/store.py,sha256=kQ35HemE0v4_Qg6xVqNIJi8vSFAYQtwI3rDtMsNy62Y,890
-nucliadb_utils/transaction.py,sha256=YYnTpxCDs56lo0tS6ErABjk9WjDuieUc4f7r63Q_OP8,7864
+nucliadb_utils/transaction.py,sha256=z_VeiTIta48rosS2SXMqx86XaavprSMRWf6s6zWIeEs,7920
 nucliadb_utils/utilities.py,sha256=Vc4zLpDf-FQh9bs0ZyAfGAjzlbbMTMWf3VWt2Ao5V3k,15379
 nucliadb_utils/aiopynecone/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb_utils/aiopynecone/client.py,sha256=MPyHnDXwhukJr7U3CJh7BpsekfSuOkyM4g5b9LLtzc8,22941
@@ -39,15 +39,15 @@ nucliadb_utils/nuclia_usage/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn
 nucliadb_utils/nuclia_usage/utils/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb_utils/nuclia_usage/utils/kb_usage_report.py,sha256=6lLuxCCPQVn3dOuZNL5ThPjl2yws-1TJ_7duhQSWkPU,3934
 nucliadb_utils/storages/__init__.py,sha256=5Qc8AUWiJv9_JbGCBpAn88AIJhwDlm0OPQpg2ZdRL4U,872
-nucliadb_utils/storages/azure.py,sha256=FK4c_v9AUAwagScm_F1uDmJeQQq7P4jZswiD2trwb4A,16394
+nucliadb_utils/storages/azure.py,sha256=b5WzTjwsyrFxkuD5GzOUSBw6Szm0OkNt8t2NGoMiTJI,16557
 nucliadb_utils/storages/exceptions.py,sha256=GOPKH-F3dPTfHEkwGNfVkSfF70eWJJXjI83yccw9WpA,2501
-nucliadb_utils/storages/gcs.py,sha256=5QODikIj26tfjz4KqhNgezPqqmqrG1yvUbarKAQ6sS0,28899
-nucliadb_utils/storages/local.py,sha256=JxlWNtu49JJ04dq6o7bBAqbpbeYpVyvvBM5jq1sGJ-4,11003
+nucliadb_utils/storages/gcs.py,sha256=KR267zRvBBjT0ksxcfCCz0CmqPD8W-Kj3_lWZXWFg0Q,29249
+nucliadb_utils/storages/local.py,sha256=USEYXavJfTK5bU71iLozVM9XjDdtPNte3uWJBuiWIOo,11371
 nucliadb_utils/storages/nuclia.py,sha256=vEv94xAT7QM2g80S25QyrOw2pzvP2BAX-ADgZLtuCVc,2097
-nucliadb_utils/storages/object_store.py,sha256=HtKjIKhErSBvuqx1SuCOnL0SkiHqgfyekNMP8o2piZU,4492
-nucliadb_utils/storages/s3.py,sha256=isw4v9L4tlwLklNotdF-l7mU4coshPzO5wj_tbuSNOc,20916
+nucliadb_utils/storages/object_store.py,sha256=2PueRP5Q3XOuWgKhj6B9Kp2fyBql5np0T400YRUbqn4,4535
+nucliadb_utils/storages/s3.py,sha256=4-cs_uFC4ZoLHLTZqvGi8v3nQcbrPBFJZuIeWltkFHI,21026
 nucliadb_utils/storages/settings.py,sha256=ugCPy1zxBOmA2KosT-4tsjpvP002kg5iQyi42yCGCJA,1285
-nucliadb_utils/storages/storage.py,sha256=1LSaZKQ4xSoBP85OVATh7zBZdBQ369Wl1uYBymZPfPw,21135
+nucliadb_utils/storages/storage.py,sha256=MGmwWmGuZFff5lOvTY6ZgKrYJUvJojISwAM3K4qzFKM,21344
 nucliadb_utils/storages/utils.py,sha256=8g2rIwJeYIumQLOB47Yw1rx3twlhRB_cJxer65QfZmk,1479
 nucliadb_utils/tests/__init__.py,sha256=Oo9CAE7B0eW5VHn8sHd6o30SQzOWUhktLPRXdlDOleA,1456
 nucliadb_utils/tests/asyncbenchmark.py,sha256=vrX_x9ifCXi18PfNShc23w9x_VUiB_Ph-2nuolh9z3Q,10707
@@ -57,7 +57,7 @@ nucliadb_utils/tests/gcs.py,sha256=MBMzn_UHU5SU6iILuCsB5zU4umhNcaCw_MKrxZhwvOc,4
 nucliadb_utils/tests/local.py,sha256=cxIfPrKuqs5Ef0nbrVYQQAH2mwc4E0iD9bC2sWegS-c,1934
 nucliadb_utils/tests/nats.py,sha256=RWHjwqq5esuO7OFbP24yYX1cXnpPLcWJwDUdmwCpH28,1897
 nucliadb_utils/tests/s3.py,sha256=DACUh3HvgH3BchKFZ9R7RFUzsrg3v9A-cxTcXx4nmvA,3734
-nucliadb_utils-6.3.1.post3526.dist-info/METADATA,sha256=aZS3f5jsY6xY837mfcDJxkqUsUksTbzTzpld0e5IRdg,2209
-nucliadb_utils-6.3.1.post3526.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-nucliadb_utils-6.3.1.post3526.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
-nucliadb_utils-6.3.1.post3526.dist-info/RECORD,,
+nucliadb_utils-6.3.1.post3531.dist-info/METADATA,sha256=42hW8sy26Suv1ZQdpxtvzqRARgqMWm-pfADzOY9-Bvk,2209
+nucliadb_utils-6.3.1.post3531.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+nucliadb_utils-6.3.1.post3531.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
+nucliadb_utils-6.3.1.post3531.dist-info/RECORD,,