nucliadb-utils 6.8.1.post4926__py3-none-any.whl → 6.8.1.post4939__py3-none-any.whl

nucliadb_utils/storages/exceptions.py

@@ -21,10 +21,6 @@ class InvalidCloudFile(Exception):
     pass
 
 
-class CouldNotCreateBucket(Exception):
-    pass
-
-
 class InvalidOffset(Exception):
     def __init__(self, range_header, offset):
         self.range_header = range_header

nucliadb_utils/storages/gcs.py

@@ -43,7 +43,6 @@ from nucliadb_utils import logger
 from nucliadb_utils.storages import CHUNK_SIZE
 from nucliadb_utils.storages.exceptions import (
     CouldNotCopyNotFound,
-    CouldNotCreateBucket,
     InvalidOffset,
     ResumableUploadGone,
 )
@@ -97,7 +96,6 @@ RETRIABLE_EXCEPTIONS = (
     aiohttp.client_exceptions.ClientOSError,
     aiohttp.client_exceptions.ServerConnectionError,
     aiohttp.client_exceptions.ServerDisconnectedError,
-    CouldNotCreateBucket,
     socket.gaierror,
 )
 
@@ -131,9 +129,6 @@ class GCSStorageField(StorageField):
         origin_bucket_name: str,
         destination_bucket_name: str,
     ):
-        if self.storage.session is None:
-            raise AttributeError()
-
         url = "{}/{}/o/{}/rewriteTo/b/{}/o/{}".format(
             self.storage.object_base_url,
             origin_bucket_name,
@@ -231,9 +226,6 @@ class GCSStorageField(StorageField):
 
         cf: New file to upload
         """
-        if self.storage.session is None:
-            raise AttributeError()
-
         if self.field is not None and self.field.upload_uri != "":
             # If there is a temporal url
             await self.storage.delete_upload(self.field.upload_uri, self.field.bucket_name)
@@ -308,10 +300,6 @@ class GCSStorageField(StorageField):
     async def _append(self, cf: CloudFile, data: bytes):
         if self.field is None:
             raise AttributeError()
-
-        if self.storage.session is None:
-            raise AttributeError()
-
         # size = 0 ==> size may be unset, as 0 is the default protobuffer value
         # Makes no sense to assume a file with size = 0 in upload
         if cf.size > 0:
@@ -394,8 +382,6 @@ class GCSStorageField(StorageField):
         Existence can be checked either with a CloudFile data in the field attribute
         or own StorageField key and bucket. Field takes precendece
         """
-        if self.storage.session is None:
-            raise AttributeError()
         key = None
         bucket = None
         if self.field is not None and self.field.uri != "":
@@ -439,7 +425,7 @@ class GCSStorageField(StorageField):
 
 class GCSStorage(Storage):
     field_klass = GCSStorageField
-    session: Optional[aiohttp.ClientSession] = None
+    _session: Optional[aiohttp.ClientSession] = None
     _credentials = None
     _json_credentials = None
     chunk_size = CHUNK_SIZE
@@ -484,7 +470,7 @@ class GCSStorage(Storage):
         self.source = CloudFile.GCS
         self.deadletter_bucket = deadletter_bucket
         self.indexing_bucket = indexing_bucket
-        self.bucket = bucket
+        self.bucket = bucket or "{kbid}"
         self._location = location
         self._project = project
         # https://cloud.google.com/storage/docs/bucket-locations
@@ -492,7 +478,7 @@ class GCSStorage(Storage):
         self._executor = executor
         self._upload_url = url + "/upload/storage/v1/b/{bucket}/o"
         self.object_base_url = url + "/storage/v1/b"
-        self._client = None
+        self._session = None
 
     def _get_access_token(self):
         if self._credentials.expired or self._credentials.valid is False:
@@ -501,11 +487,17 @@ class GCSStorage(Storage):
 
         return self._credentials.token
 
+    @property
+    def session(self) -> aiohttp.ClientSession:
+        if self._session is None:  # pragma: no cover
+            raise AttributeError("Session not initialized, call initialize first")
+        return self._session
+
     @storage_ops_observer.wrap({"type": "initialize"})
     async def initialize(self):
         loop = asyncio.get_event_loop()
 
-        self.session = aiohttp.ClientSession(
+        self._session = aiohttp.ClientSession(
             loop=loop, connector=aiohttp.TCPConnector(ttl_dns_cache=60 * 5), timeout=TIMEOUT
         )
         try:
@@ -521,7 +513,9 @@ class GCSStorage(Storage):
             logger.exception(f"Could not create bucket {self.indexing_bucket}", exc_info=True)
 
     async def finalize(self):
-        await self.session.close()
+        if self._session is not None:
+            await self._session.close()
+            self._session = None
 
     async def get_access_headers(self):
         if self._credentials is None:
@@ -538,8 +532,6 @@ class GCSStorage(Storage):
     )
     @storage_ops_observer.wrap({"type": "delete"})
     async def delete_upload(self, uri: str, bucket_name: str):
-        if self.session is None:
-            raise AttributeError()
         if uri:
             url = "{}/{}/o/{}".format(self.object_base_url, bucket_name, quote_plus(uri))
             headers = await self.get_access_headers()
@@ -557,9 +549,6 @@ class GCSStorage(Storage):
 
     @storage_ops_observer.wrap({"type": "check_bucket_exists"})
     async def check_exists(self, bucket_name: str):
-        if self.session is None:
-            raise AttributeError()
-
         headers = await self.get_access_headers()
         # Using object access url instead of bucket access to avoid
         # giving admin permission to the SA, needed to GET a bucket
@@ -573,10 +562,14 @@ class GCSStorage(Storage):
                 return True
         return False
 
+    @backoff.on_exception(
+        backoff.expo,
+        RETRIABLE_EXCEPTIONS,
+        jitter=backoff.random_jitter,
+        max_tries=MAX_TRIES,
+    )
+    @storage_ops_observer.wrap({"type": "create_bucket"})
     async def create_bucket(self, bucket_name: str, kbid: Optional[str] = None):
-        if self.session is None:
-            raise AttributeError()
-
         if await self.check_exists(bucket_name=bucket_name):
             return
 
@@ -586,10 +579,7 @@ class GCSStorage(Storage):
         labels = deepcopy(self._bucket_labels)
         if kbid is not None:
             labels["kbid"] = kbid.lower()
-        await self._create_bucket(url, headers, bucket_name, labels)
 
-    @storage_ops_observer.wrap({"type": "create_bucket"})
-    async def _create_bucket(self, url, headers, bucket_name, labels):
         async with self.session.post(
             url,
             headers=headers,
@@ -605,20 +595,12 @@ class GCSStorage(Storage):
                 },
             },
         ) as resp:
-            if resp.status != 200:  # pragma: no cover
-                logger.info(f"Creation of bucket error: {resp.status}")
-                text = await resp.text()
-                logger.info(f"Bucket : {bucket_name}")
-                logger.info(f"Location : {self._location}")
-                logger.info(f"Labels : {labels}")
-                logger.info(f"URL : {url}")
-                logger.info(text)
-
-                raise CouldNotCreateBucket(text)
+            if resp.status == 200:
+                return
+            text = await resp.text()
+            raise GoogleCloudException(f"Bucket creation failed with status {resp.status}: {text}")
 
     def get_bucket_name(self, kbid: str):
-        if self.bucket is None:
-            raise AttributeError()
         bucket_name = self.bucket.format(
             kbid=kbid,
         )
@@ -631,13 +613,12 @@ class GCSStorage(Storage):
             await self.create_bucket(bucket_name, kbid)
             created = True
         except Exception as e:
+            errors.capture_exception(e)
             logger.exception(f"Could not create bucket {kbid}", exc_info=e)
         return created
 
     @storage_ops_observer.wrap({"type": "schedule_delete"})
     async def schedule_delete_kb(self, kbid: str):
-        if self.session is None:
-            raise AttributeError()
         bucket_name = self.get_bucket_name(kbid)
         headers = await self.get_access_headers()
         url = f"{self.object_base_url}/{bucket_name}?fields=lifecycle"
@@ -664,8 +645,6 @@ class GCSStorage(Storage):
 
     @storage_ops_observer.wrap({"type": "delete"})
     async def delete_kb(self, kbid: str) -> tuple[bool, bool]:
-        if self.session is None:
-            raise AttributeError()
         bucket_name = self.get_bucket_name(kbid)
         headers = await self.get_access_headers()
         url = f"{self.object_base_url}/{bucket_name}"
@@ -694,8 +673,6 @@ class GCSStorage(Storage):
     async def iterate_objects(
         self, bucket: str, prefix: str, start: Optional[str] = None
     ) -> AsyncGenerator[ObjectInfo, None]:
-        if self.session is None:
-            raise AttributeError()
         url = "{}/{}/o".format(self.object_base_url, bucket)
         headers = await self.get_access_headers()
         params = {"prefix": prefix}
@@ -743,8 +720,6 @@ class GCSStorage(Storage):
         """
         Put an object in the storage without any metadata.
         """
-        if self.session is None:  # pragma: no cover
-            raise AttributeError()
         bucket_upload_url = self._upload_url.format(bucket=bucket_name)
         url = f"{bucket_upload_url}?uploadType=media&name={quote_plus(key)}"
         headers = await self.get_access_headers()

nucliadb_utils/storages/s3.py

@@ -522,6 +522,12 @@ async def bucket_exists(client: AioSession, bucket_name: str) -> bool:
     return exists
 
 
+@backoff.on_exception(
+    backoff.expo,
+    RETRIABLE_EXCEPTIONS,
+    jitter=backoff.random_jitter,
+    max_tries=MAX_TRIES,
+)
 async def create_bucket(
     client: AioSession,
     bucket_name: str,

nucliadb_utils/tests/gcs.py

@@ -119,7 +119,7 @@ async def gcs_storage(gcs, gcs_storage_settings: dict[str, Any]):
     storage = GCSStorage(
         url=storage_settings.gcs_endpoint_url,
         account_credentials=storage_settings.gcs_base64_creds,
-        bucket=storage_settings.gcs_bucket,
+        bucket=storage_settings.gcs_bucket,  # type: ignore
         location=storage_settings.gcs_location,
         project=storage_settings.gcs_project,
         executor=ThreadPoolExecutor(1),

nucliadb_utils/utilities.py

@@ -155,6 +155,9 @@ async def _create_storage(gcs_scopes: Optional[List[str]] = None) -> Storage:
     elif storage_settings.file_backend == FileBackendConfig.GCS:
         from nucliadb_utils.storages.gcs import GCSStorage
 
+        if storage_settings.gcs_bucket is None:
+            raise ConfigurationError("GCS_BUCKET env var not configured")
+
         gcsutil = GCSStorage(
             url=storage_settings.gcs_endpoint_url,
             account_credentials=storage_settings.gcs_base64_creds,

{nucliadb_utils-6.8.1.post4926.dist-info → nucliadb_utils-6.8.1.post4939.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb_utils
-Version: 6.8.1.post4926
+Version: 6.8.1.post4939
 Summary: NucliaDB util library
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -27,8 +27,8 @@ Requires-Dist: nats-py[nkeys]>=2.6.0
 Requires-Dist: PyNaCl
 Requires-Dist: pyjwt>=2.4.0
 Requires-Dist: mrflagly>=0.2.9
-Requires-Dist: nucliadb-protos>=6.8.1.post4926
-Requires-Dist: nucliadb-telemetry>=6.8.1.post4926
+Requires-Dist: nucliadb-protos>=6.8.1.post4939
+Requires-Dist: nucliadb-telemetry>=6.8.1.post4939
 Provides-Extra: cache
 Requires-Dist: redis>=4.3.4; extra == "cache"
 Requires-Dist: orjson>=3.6.7; extra == "cache"

{nucliadb_utils-6.8.1.post4926.dist-info → nucliadb_utils-6.8.1.post4939.dist-info}/RECORD

@@ -15,7 +15,7 @@ nucliadb_utils/settings.py,sha256=lZUCliwNKYfk_Tt0KiYeHsT4jRBG0gLAompuHWu9fBI,82
 nucliadb_utils/signals.py,sha256=lo_Mk12NIX5Au--3H3WObvDOXq_OMurql2qiC2TnAao,2676
 nucliadb_utils/store.py,sha256=kQ35HemE0v4_Qg6xVqNIJi8vSFAYQtwI3rDtMsNy62Y,890
 nucliadb_utils/transaction.py,sha256=l3ZvrITYMnAs_fv1OOC-1nDZxWPG5qmbBhzvuC3DUzQ,8039
-nucliadb_utils/utilities.py,sha256=-7iNa3My7gpIrpJDgswgQ9rx8pumkv-wyWUiG6pJxVo,15659
+nucliadb_utils/utilities.py,sha256=D07dg5Ycm1sYkO65OqxHG9TqDfvBoy91143iln8pOhA,15782
 nucliadb_utils/aiopynecone/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb_utils/aiopynecone/client.py,sha256=MPyHnDXwhukJr7U3CJh7BpsekfSuOkyM4g5b9LLtzc8,22941
 nucliadb_utils/aiopynecone/exceptions.py,sha256=fUErx3ceKQK1MUbOnYcZhIzpNe8UVAptZE9JIRDLXDE,4000
@@ -40,12 +40,12 @@ nucliadb_utils/nuclia_usage/utils/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZ
 nucliadb_utils/nuclia_usage/utils/kb_usage_report.py,sha256=6lLuxCCPQVn3dOuZNL5ThPjl2yws-1TJ_7duhQSWkPU,3934
 nucliadb_utils/storages/__init__.py,sha256=5Qc8AUWiJv9_JbGCBpAn88AIJhwDlm0OPQpg2ZdRL4U,872
 nucliadb_utils/storages/azure.py,sha256=EEUyi-2c69FQz8iPhKixkZDp8xVMgMFGEPaZDVuillc,17429
-nucliadb_utils/storages/exceptions.py,sha256=GOPKH-F3dPTfHEkwGNfVkSfF70eWJJXjI83yccw9WpA,2501
-nucliadb_utils/storages/gcs.py,sha256=XbtX0Lt3GO7kzuJ1E5CdazlpSqjU46Bhhezq32VQUok,29041
+nucliadb_utils/storages/exceptions.py,sha256=07Isip18qxkEynGz28AkO2BBC34b_zjL5dEUzeSC2OU,2451
+nucliadb_utils/storages/gcs.py,sha256=VyT72My34N4pEMmrQc5wdAMNLiuqpYl8OW3d50cJfSA,28222
 nucliadb_utils/storages/local.py,sha256=2aCHpZymORG_dUc1FDq0VFcgQulu0w2pZiUaj9dphFs,11686
 nucliadb_utils/storages/nuclia.py,sha256=vEv94xAT7QM2g80S25QyrOw2pzvP2BAX-ADgZLtuCVc,2097
 nucliadb_utils/storages/object_store.py,sha256=2PueRP5Q3XOuWgKhj6B9Kp2fyBql5np0T400YRUbqn4,4535
-nucliadb_utils/storages/s3.py,sha256=1mMXfC0hCJLlVnw-B_WWPWZrNyeYzW3bqQm3u-EE9T8,21707
+nucliadb_utils/storages/s3.py,sha256=eFFVRgNTIxTz1Hpmd6ofRz9KQhPJAmiyetW4EmWN8EM,21835
 nucliadb_utils/storages/settings.py,sha256=ugCPy1zxBOmA2KosT-4tsjpvP002kg5iQyi42yCGCJA,1285
 nucliadb_utils/storages/storage.py,sha256=2EIgnaCN5XzKpienounOjQ2AX3ANtQA2Xgl6hnMpHr4,21951
 nucliadb_utils/storages/utils.py,sha256=F4Iboa_0_bhDQr-JOKD9sGPld_-hKwJW5ptyZdn9Oag,1505
@@ -53,11 +53,11 @@ nucliadb_utils/tests/__init__.py,sha256=Oo9CAE7B0eW5VHn8sHd6o30SQzOWUhktLPRXdlDO
 nucliadb_utils/tests/asyncbenchmark.py,sha256=vrX_x9ifCXi18PfNShc23w9x_VUiB_Ph-2nuolh9z3Q,10707
 nucliadb_utils/tests/azure.py,sha256=NvMrPG6gfbpDE0m_aZgaa7eorbmA1r9rhAsAANhMlJk,4494
 nucliadb_utils/tests/fixtures.py,sha256=4lzz-khYvbGzdbT18IG6KKg40f7CVex2q3ho88I-jL8,3799
-nucliadb_utils/tests/gcs.py,sha256=MBMzn_UHU5SU6iILuCsB5zU4umhNcaCw_MKrxZhwvOc,4705
+nucliadb_utils/tests/gcs.py,sha256=JNqp5ymeNNU9Ci8rNYTh7-VqP4fjybElhyB3ap7EV1c,4721
 nucliadb_utils/tests/local.py,sha256=z9E11_ol1mu7N8Y6PkjKl-WMPPMl7JqQbDj3uhVa1A0,1933
 nucliadb_utils/tests/nats.py,sha256=RWHjwqq5esuO7OFbP24yYX1cXnpPLcWJwDUdmwCpH28,1897
 nucliadb_utils/tests/s3.py,sha256=kz9ULxrAYLVslZ59I8dtweZ9DJz5R8Ioy2XYrveZzHw,3829
-nucliadb_utils-6.8.1.post4926.dist-info/METADATA,sha256=knYfMAcN1DAT8940phgY66M4QnJy7uv1icsDc09IJkg,2180
-nucliadb_utils-6.8.1.post4926.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb_utils-6.8.1.post4926.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
-nucliadb_utils-6.8.1.post4926.dist-info/RECORD,,
+nucliadb_utils-6.8.1.post4939.dist-info/METADATA,sha256=_Hc2_6N9wNVxBxy-GAJpiK4wZDy-1Xw5av_UvYeVBwI,2180
+nucliadb_utils-6.8.1.post4939.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb_utils-6.8.1.post4939.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
+nucliadb_utils-6.8.1.post4939.dist-info/RECORD,,