nucliadb-utils 6.0.0.post2298__py3-none-any.whl → 6.0.0.post2310__py3-none-any.whl

nucliadb_utils/storages/s3.py

@@ -19,6 +19,7 @@
 #
 from __future__ import annotations
 
+import base64
 from contextlib import AsyncExitStack
 from datetime import datetime
 from typing import AsyncGenerator, AsyncIterator, Optional
@@ -31,12 +32,15 @@ from aiobotocore.client import AioBaseClient  # type: ignore
 from aiobotocore.session import AioSession, get_session  # type: ignore
 
 from nucliadb_protos.resources_pb2 import CloudFile
-from nucliadb_telemetry import errors
+from nucliadb_telemetry import errors, metrics
 from nucliadb_utils import logger
 from nucliadb_utils.storages.exceptions import UnparsableResponse
 from nucliadb_utils.storages.storage import Storage, StorageField
 from nucliadb_utils.storages.utils import ObjectInfo, ObjectMetadata, Range
 
+s3_ops_observer = metrics.Observer("s3_ops", labels={"type": ""})
+
+
 MB = 1024 * 1024
 MIN_UPLOAD_SIZE = 5 * MB
 CHUNK_SIZE = MIN_UPLOAD_SIZE
@@ -77,6 +81,7 @@ class S3StorageField(StorageField):
         jitter=backoff.random_jitter,
         max_tries=MAX_TRIES,
     )
+    @s3_ops_observer.wrap({"type": "download"})
     async def _download(
         self,
         uri,
@@ -97,6 +102,7 @@ class S3StorageField(StorageField):
             else:
                 raise
 
+    @s3_ops_observer.wrap({"type": "iter_data"})
     async def iter_data(self, range: Optional[Range] = None) -> AsyncGenerator[bytes, None]:
         # Suports field and key based iter
         uri = self.field.uri if self.field else self.key
@@ -113,6 +119,7 @@ class S3StorageField(StorageField):
             yield data
             data = await stream.read(CHUNK_SIZE)
 
+    @s3_ops_observer.wrap({"type": "abort_multipart"})
     async def _abort_multipart(self):
         try:
             mpu = self.field.resumable_uri
@@ -124,6 +131,7 @@ class S3StorageField(StorageField):
         except Exception:
             logger.warning("Could not abort multipart upload", exc_info=True)
 
+    @s3_ops_observer.wrap({"type": "start_upload"})
     async def start(self, cf: CloudFile) -> CloudFile:
         if self.field is not None and self.field.upload_uri != "":
             # Field has already a file beeing uploaded, cancel
@@ -165,6 +173,7 @@ class S3StorageField(StorageField):
         jitter=backoff.random_jitter,
         max_tries=MAX_TRIES,
     )
+    @s3_ops_observer.wrap({"type": "create_multipart"})
     async def _create_multipart(self, bucket_name: str, upload_id: str, cf: CloudFile):
         return await self.storage._s3aioclient.create_multipart_upload(
             Bucket=bucket_name,
@@ -176,6 +185,7 @@ class S3StorageField(StorageField):
             },
         )
 
+    @s3_ops_observer.wrap({"type": "append_data"})
     async def append(self, cf: CloudFile, iterable: AsyncIterator) -> int:
         size = 0
         if self.field is None:
@@ -203,6 +213,7 @@ class S3StorageField(StorageField):
         jitter=backoff.random_jitter,
         max_tries=MAX_TRIES,
     )
+    @s3_ops_observer.wrap({"type": "upload_part"})
     async def _upload_part(self, cf: CloudFile, data: bytes):
         if self.field is None:
             raise AttributeError("No field configured")
@@ -214,6 +225,7 @@ class S3StorageField(StorageField):
             Body=data,
         )
 
+    @s3_ops_observer.wrap({"type": "finish_upload"})
     async def finish(self):
         if self.field is None:
             raise AttributeError("No field configured")
@@ -244,6 +256,7 @@ class S3StorageField(StorageField):
         jitter=backoff.random_jitter,
         max_tries=MAX_TRIES,
     )
+    @s3_ops_observer.wrap({"type": "complete_multipart"})
     async def _complete_multipart_upload(self):
         # if blocks is 0, it means the file is of zero length so we need to
         # trick it to finish a multiple part with no data.
@@ -263,6 +276,7 @@ class S3StorageField(StorageField):
             MultipartUpload=part_info,
         )
 
+    @s3_ops_observer.wrap({"type": "exists"})
     async def exists(self) -> Optional[ObjectMetadata]:
         """
         Existence can be checked either with a CloudFile data in the field attribute
@@ -291,6 +305,7 @@ class S3StorageField(StorageField):
                 return None
             raise
 
+    @s3_ops_observer.wrap({"type": "copy"})
     async def copy(
         self,
         origin_uri: str,
@@ -304,6 +319,7 @@ class S3StorageField(StorageField):
             Key=destination_uri,
         )
 
+    @s3_ops_observer.wrap({"type": "move"})
     async def move(
         self,
         origin_uri: str,
@@ -395,6 +411,7 @@ class S3Storage(Storage):
     async def finalize(self):
         await self._exit_stack.__aexit__(None, None, None)
 
+    @s3_ops_observer.wrap({"type": "delete"})
     async def delete_upload(self, uri: str, bucket_name: str):
         if uri:
             try:
@@ -547,5 +564,10 @@ def parse_object_metadata(obj: dict, key: str) -> ObjectMetadata:
     # Content type
     content_type = custom_metadata.get("content_type") or obj.get("ContentType") or ""
     # Filename
-    filename = custom_metadata.get("filename") or key.split("/")[-1]
+    base64_filename = custom_metadata.get("base64_filename")
+    if base64_filename:
+        filename = base64.b64decode(base64_filename).decode()
+    else:
+        filename = custom_metadata.get("filename") or key.split("/")[-1]
+
     return ObjectMetadata(size=size, content_type=content_type, filename=filename)

{nucliadb_utils-6.0.0.post2298.dist-info → nucliadb_utils-6.0.0.post2310.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nucliadb_utils
-Version: 6.0.0.post2298
+Version: 6.0.0.post2310
 Home-page: https://nuclia.com
 License: BSD
 Classifier: Development Status :: 4 - Beta
@@ -24,8 +24,8 @@ Requires-Dist: PyNaCl
 Requires-Dist: pyjwt>=2.4.0
 Requires-Dist: memorylru>=1.1.2
 Requires-Dist: mrflagly>=0.2.9
-Requires-Dist: nucliadb-protos>=6.0.0.post2298
-Requires-Dist: nucliadb-telemetry>=6.0.0.post2298
+Requires-Dist: nucliadb-protos>=6.0.0.post2310
+Requires-Dist: nucliadb-telemetry>=6.0.0.post2310
 Provides-Extra: cache
 Requires-Dist: redis>=4.3.4; extra == "cache"
 Requires-Dist: orjson>=3.6.7; extra == "cache"

{nucliadb_utils-6.0.0.post2298.dist-info → nucliadb_utils-6.0.0.post2310.dist-info}/RECORD

@@ -46,7 +46,7 @@ nucliadb_utils/storages/gcs.py,sha256=MUPPTmPo5AfCWPykNvlIO35-C_8r94yWJgBxcDsRIg
 nucliadb_utils/storages/local.py,sha256=kxofHdNy4QBtdM797AeksNjIUP2kw9vIC0X7AHABJBI,10374
 nucliadb_utils/storages/nuclia.py,sha256=vEv94xAT7QM2g80S25QyrOw2pzvP2BAX-ADgZLtuCVc,2097
 nucliadb_utils/storages/object_store.py,sha256=Tw10GmpYfM5TMqJ3Tk9pLQ9wLMBk1-snL_m6uasiZDQ,4257
-nucliadb_utils/storages/s3.py,sha256=tCJxSXwSq2hKaFole71ctpEXjcjBElUuTFWlmhLkytM,19471
+nucliadb_utils/storages/s3.py,sha256=pbuukqpce_kqkmI_3eUTo390KbM5rmI7h8wsYAXtTAo,20377
 nucliadb_utils/storages/settings.py,sha256=ugCPy1zxBOmA2KosT-4tsjpvP002kg5iQyi42yCGCJA,1285
 nucliadb_utils/storages/storage.py,sha256=onwQJ4at-XewEG7dxcWdOqobfCw4w0PyPC7olvFJgjI,20295
 nucliadb_utils/storages/utils.py,sha256=8g2rIwJeYIumQLOB47Yw1rx3twlhRB_cJxer65QfZmk,1479
@@ -59,8 +59,8 @@ nucliadb_utils/tests/indexing.py,sha256=YW2QhkhO9Q_8A4kKWJaWSvXvyQ_AiAwY1VylcfVQ
 nucliadb_utils/tests/local.py,sha256=fXIBasrvdaFJM-sw2wk1_oiFzBcm9O10iCyC-OiXwY8,1914
 nucliadb_utils/tests/nats.py,sha256=xqpww4jZjTKY9oPGlJdDJG67L3FIBQsa9qDHxILR8r8,7687
 nucliadb_utils/tests/s3.py,sha256=pl-RJFjA4MH6iXkqhsh5g8gDuEhrYu1nPZ-laxlrMlE,3704
-nucliadb_utils-6.0.0.post2298.dist-info/METADATA,sha256=A18ZcppRhsDLeZXxhOqwCNDyr7y14t8YFZl3aASxsmw,2055
-nucliadb_utils-6.0.0.post2298.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-nucliadb_utils-6.0.0.post2298.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
-nucliadb_utils-6.0.0.post2298.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-nucliadb_utils-6.0.0.post2298.dist-info/RECORD,,
+nucliadb_utils-6.0.0.post2310.dist-info/METADATA,sha256=lCSBxvLdTxa4utHPT4xX9D4SIuaa-EMor-KerNyMaNU,2055
+nucliadb_utils-6.0.0.post2310.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+nucliadb_utils-6.0.0.post2310.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
+nucliadb_utils-6.0.0.post2310.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+nucliadb_utils-6.0.0.post2310.dist-info/RECORD,,