nucliadb 6.3.1.post3574__py3-none-any.whl → 6.3.1.post3584__py3-none-any.whl

nucliadb/backups/create.py

@@ -18,6 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import asyncio
+import logging
 import tarfile
 from datetime import datetime, timezone
 from typing import AsyncIterator, Optional
@@ -29,6 +30,7 @@ from nucliadb.backups.const import (
 )
 from nucliadb.backups.models import BackupMetadata, CreateBackupRequest
 from nucliadb.backups.settings import settings
+from nucliadb.backups.utils import exists_in_storge
 from nucliadb.common import datamanagers
 from nucliadb.common.context import ApplicationContext
 from nucliadb.export_import.utils import (
@@ -44,6 +46,8 @@ from nucliadb_utils.audit.stream import StreamAuditStorage
 from nucliadb_utils.storages.storage import StorageField
 from nucliadb_utils.utilities import get_audit
 
+logger = logging.getLogger(__name__)
+
 
 async def backup_kb_task(context: ApplicationContext, msg: CreateBackupRequest):
     kbid = msg.kb_id
@@ -101,6 +105,10 @@ async def backup_resources(context: ApplicationContext, kbid: str, backup_id: st
             await set_metadata(context, kbid, backup_id, metadata)
             tasks = []
             backing_up = []
+            logger.info(
+                f"Backup resources: {len(metadata.missing_resources)} remaining",
+                extra={"kbid": kbid, "backup_id": backup_id},
+            )
     if len(tasks) > 0:
         resources_bytes = await asyncio.gather(*tasks)
         metadata.total_size += sum(resources_bytes)
@@ -108,6 +116,7 @@ async def backup_resources(context: ApplicationContext, kbid: str, backup_id: st
         await set_metadata(context, kbid, backup_id, metadata)
         tasks = []
         backing_up = []
+        logger.info(f"Backup resources: completed", extra={"kbid": kbid, "backup_id": backup_id})
 
 
 async def backup_resource(context: ApplicationContext, backup_id: str, kbid: str, rid: str) -> int:
@@ -163,6 +172,13 @@ async def backup_resource_with_binaries(
         nonlocal total_size
 
         for cloud_file in get_cloud_files(bm):
+            if not await exists_cf(context, cloud_file):
+                logger.warning(
+                    "Cloud file not found in storage, skipping",
+                    extra={"kbid": kbid, "rid": rid, "cf_uri": cloud_file.uri},
+                )
+                continue
+
             serialized_cf = cloud_file.SerializeToString()
 
             async def cf_iterator():
@@ -244,6 +260,10 @@ async def delete_metadata(context: ApplicationContext, kbid: str, backup_id: str
         await txn.commit()
 
 
+async def exists_cf(context: ApplicationContext, cf: resources_pb2.CloudFile) -> bool:
+    return await exists_in_storge(context.blob_storage, cf.bucket_name, cf.uri)
+
+
 async def upload_to_bucket(context: ApplicationContext, bytes_iterator: AsyncIterator[bytes], key: str):
     storage = context.blob_storage
     bucket = settings.backups_bucket

nucliadb/backups/restore.py

@@ -21,6 +21,7 @@
 
 import asyncio
 import functools
+import logging
 import tarfile
 from typing import AsyncIterator, Callable, Optional, Union
 
@@ -39,6 +40,8 @@ from nucliadb_protos import knowledgebox_pb2 as kb_pb2
 from nucliadb_protos.resources_pb2 import CloudFile
 from nucliadb_protos.writer_pb2 import BrokerMessage
 
+logger = logging.getLogger(__name__)
+
 
 async def restore_kb_task(context: ApplicationContext, msg: RestoreBackupRequest):
     kbid = msg.kb_id
@@ -193,7 +196,7 @@ class ResourceBackupReader:
         elif tarinfo.name.startswith("cloud-files"):
             raw_cf = await self.read_data(tarinfo)
             cf = CloudFile()
-            cf.FromString(raw_cf)
+            cf.ParseFromString(raw_cf)
             return cf
         elif tarinfo.name.startswith("binaries"):
             uri = tarinfo.name.lstrip("binaries/")
@@ -219,14 +222,19 @@ async def restore_resource(context: ApplicationContext, kbid: str, backup_id: st
             bm = item
             bm.kbid = kbid
             break
-
-        # Read the cloud file and its binary
-        cf = await reader.read_item()
-        assert isinstance(cf, CloudFile)
-        cf_binary = await reader.read_item()
-        assert isinstance(cf_binary, CloudFileBinary)
-        assert cf.uri == cf_binary.uri
-        await import_binary(context, kbid, cf, cf_binary.read)
+        elif isinstance(item, CloudFile):
+            # Read its binary and import it
+            cf = item
+            cf_binary = await reader.read_item()
+            assert isinstance(cf_binary, CloudFileBinary)
+            assert cf.uri == cf_binary.uri
+            await import_binary(context, kbid, cf, cf_binary.read)
+        else:
+            logger.error(
+                "Unexpected item in resource backup. Backup may be corrupted",
+                extra={"item_type": type(item), kbid: kbid, resource_id: resource_id},
+            )
+            continue
 
     await import_broker_message(context, kbid, bm)
 

nucliadb/backups/utils.py

@@ -24,9 +24,12 @@ from nucliadb_utils.storages.storage import Storage
 
 
 async def exists_backup(storage: Storage, backup_id: str) -> bool:
-    async for _ in storage.iterate_objects(
-        bucket=settings.backups_bucket,
-        prefix=StorageKeys.BACKUP_PREFIX.format(backup_id=backup_id),
-    ):
+    return await exists_in_storge(
+        storage, settings.backups_bucket, StorageKeys.BACKUP_PREFIX.format(backup_id=backup_id)
+    )
+
+
+async def exists_in_storge(storage: Storage, bucket: str, key: str) -> bool:
+    async for _ in storage.iterate_objects(bucket=bucket, prefix=key):
         return True
     return False

nucliadb/tasks/retries.py

@@ -20,7 +20,7 @@
 import functools
 import logging
 from enum import Enum
-from typing import Optional
+from typing import Callable, Optional
 
 from pydantic import BaseModel
 
@@ -73,7 +73,8 @@ class TaskRetryHandler:
         self.kbid = kbid
         self.task_type = task_type
         self.task_id = task_id
-        self.max_retries = max_retries
+        # Limit max retries to 50
+        self.max_retries = min(max_retries, 50)
         self.context = context
 
     @property
@@ -94,7 +95,7 @@ class TaskRetryHandler:
             await txn.set(self.metadata_key, metadata.model_dump_json().encode())
             await txn.commit()
 
-    def wrap(self, func):
+    def wrap(self, func: Callable) -> Callable:
         @functools.wraps(func)
         async def wrapper(*args, **kwargs):
             func_result = None
@@ -110,7 +111,7 @@ class TaskRetryHandler:
 
             if metadata.status in (TaskMetadata.Status.COMPLETED, TaskMetadata.Status.FAILED):
                 logger.info(
-                    f"{self.type} task is {metadata.status.value}. Skipping",
+                    f"{self.task_type} task is {metadata.status.value}. Skipping",
                     extra={"kbid": self.kbid, "task_type": self.task_type, "task_id": self.task_id},
                 )
                 return
@@ -130,7 +131,7 @@ class TaskRetryHandler:
             except Exception as ex:
                 metadata.retries += 1
                 metadata.error_messages.append(str(ex))
-                logger.info(
+                logger.exception(
                     f"Task failed. Will be retried",
                     extra={"kbid": self.kbid, "task_type": self.task_type, "task_id": self.task_id},
                 )

{nucliadb-6.3.1.post3574.dist-info → nucliadb-6.3.1.post3584.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.3.1.post3574
+Version: 6.3.1.post3584
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.1.post3574
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.1.post3574
-Requires-Dist: nucliadb-protos>=6.3.1.post3574
-Requires-Dist: nucliadb-models>=6.3.1.post3574
-Requires-Dist: nidx-protos>=6.3.1.post3574
+Requires-Dist: nucliadb-telemetry[all]>=6.3.1.post3584
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.1.post3584
+Requires-Dist: nucliadb-protos>=6.3.1.post3584
+Requires-Dist: nucliadb-models>=6.3.1.post3584
+Requires-Dist: nidx-protos>=6.3.1.post3584
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.3.1.post3574.dist-info → nucliadb-6.3.1.post3584.dist-info}/RECORD

@@ -41,13 +41,13 @@ nucliadb/openapi.py,sha256=wDiw0dVEvTpJvbatkJ0JZLkKm9RItZT5PWRHjqRfqTA,2272
 nucliadb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/backups/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/backups/const.py,sha256=9vPAhLxQO_gNAjSdPxWuv3V66s9WcdpjOQ89CZlfmuk,1894
-nucliadb/backups/create.py,sha256=AM_nC7TgHOX0EFGaTXClS28jBSK28fHrKNZi14z2wek,10442
+nucliadb/backups/create.py,sha256=mvirguMbtxNgSDGG81l0kkgHWJSZPk4GFyra9nAkBZM,11275
 nucliadb/backups/delete.py,sha256=1rnBhVUGYYZJXSZUrrgYMDZ5NyswEWkIA-G-crRCyHk,2404
 nucliadb/backups/models.py,sha256=-hITU4Mv6AxePu12toBu_fjpEv6vVGcwNVxV22O9jQA,1273
-nucliadb/backups/restore.py,sha256=xhslVvTf4H8VmDucZpjrEFpKj6csPIWBadCPMVJYKQ8,9703
+nucliadb/backups/restore.py,sha256=wepEgv4vBN5yeiZU-f17PbuFV4xT4_SVKplNr8xSJrE,10001
 nucliadb/backups/settings.py,sha256=SyzsInj1BRbBI0atg5IXWbMbOZ_eVg4eSQ3IcnUhCxQ,1357
 nucliadb/backups/tasks.py,sha256=4_kOVJ2yCwMvDEpzJgTuTt75TNlpq5woyw9sTAcaSkw,4194
-nucliadb/backups/utils.py,sha256=ayDaxfWP5cPnAkQH-tF4M6cnowsPQgU2ljYz_iL1CbE,1249
+nucliadb/backups/utils.py,sha256=b1hi0gEp90tNrWHejNVoUgRpa4D6uKGhbACq0yeLkJY,1375
 nucliadb/common/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/constants.py,sha256=QpigxJh_CtD85Evy0PtV5cVq6x0U_f9xfIcXz1ymkUg,869
 nucliadb/common/counters.py,sha256=8lOi3A2HeLDDlcNaS2QT1SfD3350VPBjiY3FkmHH1V8,977
@@ -272,7 +272,7 @@ nucliadb/tasks/consumer.py,sha256=xc0Ql3N1Iq52dJ3t4YYGJFj1NCQAly0J5W_brfLa_F8,68
 nucliadb/tasks/logger.py,sha256=C7keOEO_mjLVp5VbqAZ2QXfqVB2Hot7NgBlUP_SDSMw,924
 nucliadb/tasks/models.py,sha256=qrZKi5DNDQ07waMsp5L4_Fi7WRs57YiO-kmXlrBzEAA,1168
 nucliadb/tasks/producer.py,sha256=UnpJAzhj_GElsCoO5G6T4m6MshsgOaqR2tVzJmEta64,2625
-nucliadb/tasks/retries.py,sha256=Zv-3Hys-SKayG9VQ7_7EIflkegE5j-xPGrf-nwaxsfY,5075
+nucliadb/tasks/retries.py,sha256=tLNtwAutGEfFV3Adr4a9ew-Wg4vuxBd72GGDE_Mma8s,5160
 nucliadb/tasks/utils.py,sha256=tV1AbWdFc3qfIULX44Veqj41FCD1B6XYjG6brULBeiw,1459
 nucliadb/tests/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/tests/config.py,sha256=JN_Jhgj-fwM9_8IeO9pwxr6C1PiwRDrXxm67Y38rU30,2080
@@ -347,8 +347,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.3.1.post3574.dist-info/METADATA,sha256=JiB_eKqj0pPsq4nf9VvqU9gm6CEEQkYvElBlLSISd7c,4291
-nucliadb-6.3.1.post3574.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-nucliadb-6.3.1.post3574.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.3.1.post3574.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.3.1.post3574.dist-info/RECORD,,
+nucliadb-6.3.1.post3584.dist-info/METADATA,sha256=1LFsE32EtmIRlKK9EAN1-kK6P5pfLq3pIRewpO0k2Ug,4291
+nucliadb-6.3.1.post3584.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+nucliadb-6.3.1.post3584.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.1.post3584.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.1.post3584.dist-info/RECORD,,