nucliadb-utils 6.8.1.post4957__py3-none-any.whl → 6.8.1.post4961__py3-none-any.whl

nucliadb_utils/storages/azure.py

@@ -31,7 +31,6 @@ from azure.storage.blob.aio import BlobServiceClient
 
 from nucliadb_protos.resources_pb2 import CloudFile
 from nucliadb_telemetry import metrics
-from nucliadb_utils.storages.exceptions import ObjectNotFoundError
 from nucliadb_utils.storages.object_store import ObjectStore
 from nucliadb_utils.storages.storage import Storage, StorageField
 from nucliadb_utils.storages.utils import ObjectInfo, ObjectMetadata, Range
@@ -146,7 +145,7 @@ class AzureStorageField(StorageField):
             return None
         try:
             return await self.storage.object_store.get_metadata(bucket, key)
-        except ObjectNotFoundError:
+        except KeyError:
             return None
 
     async def upload(self, iterator: AsyncIterator, origin: CloudFile) -> CloudFile:
@@ -169,8 +168,8 @@ class AzureStorage(Storage):
     def __init__(
         self,
         account_url: str,
-        deadletter_bucket: str = "deadletter",
-        indexing_bucket: str = "indexing",
+        deadletter_bucket: Optional[str] = "deadletter",
+        indexing_bucket: Optional[str] = "indexing",
         connection_string: Optional[str] = None,
     ):
         self.object_store = AzureObjectStore(account_url, connection_string=connection_string)
@@ -196,7 +195,7 @@ class AzureStorage(Storage):
     async def delete_upload(self, uri: str, bucket_name: str):
         try:
             await self.object_store.delete(bucket_name, uri)
-        except ObjectNotFoundError:
+        except KeyError:
             pass
 
     async def create_bucket(self, bucket_name: str, kbid: Optional[str] = None):
@@ -328,7 +327,7 @@ class AzureObjectStore(ObjectStore):
         try:
             await container_client.delete_blob(key, delete_snapshots="include")
         except ResourceNotFoundError:
-            raise ObjectNotFoundError()
+            raise KeyError(f"Not found: {bucket}/{key}")
 
     @ops_observer.wrap({"type": "upload"})
     async def upload(
@@ -350,6 +349,7 @@ class AzureObjectStore(ObjectStore):
             name=key,
             data=data,
             length=length,
+            overwrite=True,
             blob_type=BlobType.BLOCKBLOB,
             metadata=custom_metadata,
             content_settings=ContentSettings(
@@ -361,7 +361,7 @@ class AzureObjectStore(ObjectStore):
     @ops_observer.wrap({"type": "insert"})
     async def insert(self, bucket: str, key: str, data: bytes) -> None:
         container_client = self.service_client.get_container_client(bucket)
-        await container_client.upload_blob(name=key, data=data, length=len(data))
+        await container_client.upload_blob(name=key, data=data, length=len(data), overwrite=True)
 
     @ops_observer.wrap({"type": "download"})
     async def download(self, bucket: str, key: str) -> bytes:
@@ -370,7 +370,7 @@ class AzureObjectStore(ObjectStore):
         try:
             downloader = await blob_client.download_blob()
         except ResourceNotFoundError:
-            raise ObjectNotFoundError()
+            raise KeyError(f"Not found: {bucket}/{key}")
         return await downloader.readall()
 
     async def download_stream(
@@ -383,14 +383,14 @@ class AzureObjectStore(ObjectStore):
         length = None
         if range.any():
             offset = range.start or 0
-            length = range.end - offset + 1 if range.end else None
+            length = range.end - offset + 1 if range.end is not None else None
         try:
             downloader = await blob_client.download_blob(
                 offset=offset,  # type: ignore
                 length=length,  # type: ignore
             )
         except ResourceNotFoundError:
-            raise ObjectNotFoundError()
+            raise KeyError(f"Not found: {bucket}/{key}")
         async for chunk in downloader.chunks():
             yield chunk
 
@@ -411,7 +411,7 @@ class AzureObjectStore(ObjectStore):
             properties: BlobProperties = await blob_client.get_blob_properties()
             return parse_object_metadata(properties, key)
         except ResourceNotFoundError:
-            raise ObjectNotFoundError()
+            raise KeyError(f"Not found: {bucket}/{key}")
 
     @ops_observer.wrap({"type": "multipart_start"})
     async def upload_multipart_start(self, bucket: str, key: str, metadata: ObjectMetadata) -> None:

nucliadb_utils/storages/exceptions.py

@@ -71,9 +71,3 @@ class UnparsableResponse(Exception):
     Raised when trying to parse a response from a storage API and it's not
     possible
     """
-
-
-class ObjectNotFoundError(Exception):
-    """
-    Raised when the object is not found in storage
-    """

nucliadb_utils/storages/settings.py

@@ -33,6 +33,9 @@ class Settings(BaseSettings):
     s3_deadletter_bucket: Optional[str] = None
     s3_indexing_bucket: Optional[str] = None
 
+    azure_deadletter_bucket: Optional[str] = None
+    azure_indexing_bucket: Optional[str] = None
+
     local_testing_files: str = os.path.dirname(__file__)
 
 

nucliadb_utils/tests/azure.py

@@ -28,6 +28,7 @@ from pytest_docker_fixtures.containers._base import BaseImage  # type: ignore  #
 
 from nucliadb_utils.settings import FileBackendConfig, storage_settings
 from nucliadb_utils.storages.azure import AzureStorage
+from nucliadb_utils.storages.settings import settings as extended_storage_settings
 
 images.settings["azurite"] = {
     "image": "mcr.microsoft.com/azure-storage/azurite",
@@ -105,7 +106,7 @@ def azurite() -> Generator[AzuriteFixture, None, None]:
             port=port,
             container=container.container_obj,
             connection_string=get_connection_string(host, port),
-            account_url=f"http://{host}:{port}/devstoreaccount1",
+            account_url=f"https://devstoreaccount1.blob.core.windows.net",
         )
     finally:
         container.stop()
@@ -118,12 +119,18 @@ def azure_storage_settings(azurite: AzuriteFixture) -> Iterator[dict[str, Any]]:
         "azure_account_url": azurite.account_url,
         "azure_connection_string": azurite.connection_string,
     }
+    extended_settings = {
+        "azure_deadletter_bucket": "deadletter",
+        "azure_indexing_bucket": "indexing",
+    }
     with ExitStack() as stack:
         for key, value in settings.items():
             context = patch.object(storage_settings, key, value)
             stack.enter_context(context)
-
-        yield settings
+        for key, value in extended_settings.items():
+            context = patch.object(extended_storage_settings, key, value)
+            stack.enter_context(context)
+        yield settings | extended_settings
 
 
 @pytest.fixture(scope="function")
@@ -135,5 +142,6 @@ async def azure_storage(azurite, azure_storage_settings: dict[str, Any]):
         connection_string=storage_settings.azure_connection_string,
     )
     await storage.initialize()
+    await storage.create_bucket("nidx")
     yield storage
     await storage.finalize()

nucliadb_utils/utilities.py

@@ -125,6 +125,8 @@ async def _create_storage(gcs_scopes: Optional[List[str]] = None) -> Storage:
         azureutil = AzureStorage(
             account_url=storage_settings.azure_account_url,
             connection_string=storage_settings.azure_connection_string,
+            deadletter_bucket=extended_storage_settings.azure_deadletter_bucket,
+            indexing_bucket=extended_storage_settings.azure_indexing_bucket,
         )
 
         logger.info("Configuring Azure Storage")

{nucliadb_utils-6.8.1.post4957.dist-info → nucliadb_utils-6.8.1.post4961.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb_utils
-Version: 6.8.1.post4957
+Version: 6.8.1.post4961
 Summary: NucliaDB util library
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -27,8 +27,8 @@ Requires-Dist: nats-py[nkeys]>=2.6.0
 Requires-Dist: PyNaCl
 Requires-Dist: pyjwt>=2.4.0
 Requires-Dist: mrflagly>=0.2.9
-Requires-Dist: nucliadb-protos>=6.8.1.post4957
-Requires-Dist: nucliadb-telemetry>=6.8.1.post4957
+Requires-Dist: nucliadb-protos>=6.8.1.post4961
+Requires-Dist: nucliadb-telemetry>=6.8.1.post4961
 Provides-Extra: cache
 Requires-Dist: redis>=4.3.4; extra == "cache"
 Requires-Dist: orjson>=3.6.7; extra == "cache"

{nucliadb_utils-6.8.1.post4957.dist-info → nucliadb_utils-6.8.1.post4961.dist-info}/RECORD

@@ -15,7 +15,7 @@ nucliadb_utils/settings.py,sha256=lZUCliwNKYfk_Tt0KiYeHsT4jRBG0gLAompuHWu9fBI,82
 nucliadb_utils/signals.py,sha256=lo_Mk12NIX5Au--3H3WObvDOXq_OMurql2qiC2TnAao,2676
 nucliadb_utils/store.py,sha256=kQ35HemE0v4_Qg6xVqNIJi8vSFAYQtwI3rDtMsNy62Y,890
 nucliadb_utils/transaction.py,sha256=l3ZvrITYMnAs_fv1OOC-1nDZxWPG5qmbBhzvuC3DUzQ,8039
-nucliadb_utils/utilities.py,sha256=D07dg5Ycm1sYkO65OqxHG9TqDfvBoy91143iln8pOhA,15782
+nucliadb_utils/utilities.py,sha256=C_qSKlCeCpi8wIWttjDp0TiQjSUUhd_yS08X5NuPbAs,15940
 nucliadb_utils/aiopynecone/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb_utils/aiopynecone/client.py,sha256=MPyHnDXwhukJr7U3CJh7BpsekfSuOkyM4g5b9LLtzc8,22941
 nucliadb_utils/aiopynecone/exceptions.py,sha256=fUErx3ceKQK1MUbOnYcZhIzpNe8UVAptZE9JIRDLXDE,4000
@@ -39,25 +39,25 @@ nucliadb_utils/nuclia_usage/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn
 nucliadb_utils/nuclia_usage/utils/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb_utils/nuclia_usage/utils/kb_usage_report.py,sha256=6lLuxCCPQVn3dOuZNL5ThPjl2yws-1TJ_7duhQSWkPU,3934
 nucliadb_utils/storages/__init__.py,sha256=5Qc8AUWiJv9_JbGCBpAn88AIJhwDlm0OPQpg2ZdRL4U,872
-nucliadb_utils/storages/azure.py,sha256=EEUyi-2c69FQz8iPhKixkZDp8xVMgMFGEPaZDVuillc,17429
-nucliadb_utils/storages/exceptions.py,sha256=07Isip18qxkEynGz28AkO2BBC34b_zjL5dEUzeSC2OU,2451
+nucliadb_utils/storages/azure.py,sha256=SdZnz4GEFyBftsdXk36iiUvj-EZiXQ6PeDbZadSVkTk,17484
+nucliadb_utils/storages/exceptions.py,sha256=6YhFLf8k0ABy5AVfxIJUo7w6AK0SJjktiyQTwF3gCdg,2344
 nucliadb_utils/storages/gcs.py,sha256=VyT72My34N4pEMmrQc5wdAMNLiuqpYl8OW3d50cJfSA,28222
 nucliadb_utils/storages/local.py,sha256=2aCHpZymORG_dUc1FDq0VFcgQulu0w2pZiUaj9dphFs,11686
 nucliadb_utils/storages/nuclia.py,sha256=vEv94xAT7QM2g80S25QyrOw2pzvP2BAX-ADgZLtuCVc,2097
 nucliadb_utils/storages/object_store.py,sha256=2PueRP5Q3XOuWgKhj6B9Kp2fyBql5np0T400YRUbqn4,4535
 nucliadb_utils/storages/s3.py,sha256=eFFVRgNTIxTz1Hpmd6ofRz9KQhPJAmiyetW4EmWN8EM,21835
-nucliadb_utils/storages/settings.py,sha256=ugCPy1zxBOmA2KosT-4tsjpvP002kg5iQyi42yCGCJA,1285
+nucliadb_utils/storages/settings.py,sha256=mepN3wbLGL0Pv5yI6D-sNjSAFinEWT7aRi6N3eClNDg,1384
 nucliadb_utils/storages/storage.py,sha256=2EIgnaCN5XzKpienounOjQ2AX3ANtQA2Xgl6hnMpHr4,21951
 nucliadb_utils/storages/utils.py,sha256=F4Iboa_0_bhDQr-JOKD9sGPld_-hKwJW5ptyZdn9Oag,1505
 nucliadb_utils/tests/__init__.py,sha256=Oo9CAE7B0eW5VHn8sHd6o30SQzOWUhktLPRXdlDOleA,1456
 nucliadb_utils/tests/asyncbenchmark.py,sha256=vrX_x9ifCXi18PfNShc23w9x_VUiB_Ph-2nuolh9z3Q,10707
-nucliadb_utils/tests/azure.py,sha256=NvMrPG6gfbpDE0m_aZgaa7eorbmA1r9rhAsAANhMlJk,4494
+nucliadb_utils/tests/azure.py,sha256=rBRnS0XWqoh7Qa31MmHIl-XM20FxuQGJDxLazqa64-8,4939
 nucliadb_utils/tests/fixtures.py,sha256=4lzz-khYvbGzdbT18IG6KKg40f7CVex2q3ho88I-jL8,3799
 nucliadb_utils/tests/gcs.py,sha256=JNqp5ymeNNU9Ci8rNYTh7-VqP4fjybElhyB3ap7EV1c,4721
 nucliadb_utils/tests/local.py,sha256=z9E11_ol1mu7N8Y6PkjKl-WMPPMl7JqQbDj3uhVa1A0,1933
 nucliadb_utils/tests/nats.py,sha256=RWHjwqq5esuO7OFbP24yYX1cXnpPLcWJwDUdmwCpH28,1897
 nucliadb_utils/tests/s3.py,sha256=kz9ULxrAYLVslZ59I8dtweZ9DJz5R8Ioy2XYrveZzHw,3829
-nucliadb_utils-6.8.1.post4957.dist-info/METADATA,sha256=NAdKMGS6yS8IeT3XEJkW2wuUk8gE1oqR_-_Gwm3Xv1Y,2180
-nucliadb_utils-6.8.1.post4957.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb_utils-6.8.1.post4957.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
-nucliadb_utils-6.8.1.post4957.dist-info/RECORD,,
+nucliadb_utils-6.8.1.post4961.dist-info/METADATA,sha256=oHPiOnIVpPX3nNpiU0ajcpquGso9H5LqJqJSsdoblkA,2180
+nucliadb_utils-6.8.1.post4961.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb_utils-6.8.1.post4961.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
+nucliadb_utils-6.8.1.post4961.dist-info/RECORD,,