nucliadb 6.3.1.post3574__py3-none-any.whl → 6.3.1.post3577__py3-none-any.whl

nucliadb/backups/create.py

@@ -18,6 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import asyncio
+import logging
 import tarfile
 from datetime import datetime, timezone
 from typing import AsyncIterator, Optional
@@ -29,6 +30,7 @@ from nucliadb.backups.const import (
 )
 from nucliadb.backups.models import BackupMetadata, CreateBackupRequest
 from nucliadb.backups.settings import settings
+from nucliadb.backups.utils import exists_in_storge
 from nucliadb.common import datamanagers
 from nucliadb.common.context import ApplicationContext
 from nucliadb.export_import.utils import (
@@ -44,6 +46,8 @@ from nucliadb_utils.audit.stream import StreamAuditStorage
 from nucliadb_utils.storages.storage import StorageField
 from nucliadb_utils.utilities import get_audit
 
+logger = logging.getLogger(__name__)
+
 
 async def backup_kb_task(context: ApplicationContext, msg: CreateBackupRequest):
     kbid = msg.kb_id
@@ -101,6 +105,10 @@ async def backup_resources(context: ApplicationContext, kbid: str, backup_id: st
             await set_metadata(context, kbid, backup_id, metadata)
             tasks = []
             backing_up = []
+            logger.info(
+                f"Backup resources: {len(metadata.missing_resources)} remaining",
+                extra={"kbid": kbid, "backup_id": backup_id},
+            )
     if len(tasks) > 0:
         resources_bytes = await asyncio.gather(*tasks)
         metadata.total_size += sum(resources_bytes)
@@ -108,6 +116,7 @@ async def backup_resources(context: ApplicationContext, kbid: str, backup_id: st
         await set_metadata(context, kbid, backup_id, metadata)
         tasks = []
         backing_up = []
+        logger.info(f"Backup resources: completed", extra={"kbid": kbid, "backup_id": backup_id})
 
 
 async def backup_resource(context: ApplicationContext, backup_id: str, kbid: str, rid: str) -> int:
@@ -163,6 +172,13 @@ async def backup_resource_with_binaries(
         nonlocal total_size
 
         for cloud_file in get_cloud_files(bm):
+            if not await exists_cf(context, cloud_file):
+                logger.warning(
+                    "Cloud file not found in storage, skipping",
+                    extra={"kbid": kbid, "rid": rid, "cf_uri": cloud_file.uri},
+                )
+                continue
+
             serialized_cf = cloud_file.SerializeToString()
 
             async def cf_iterator():
@@ -244,6 +260,10 @@ async def delete_metadata(context: ApplicationContext, kbid: str, backup_id: str
         await txn.commit()
 
 
+async def exists_cf(context: ApplicationContext, cf: resources_pb2.CloudFile) -> bool:
+    return await exists_in_storge(context.blob_storage, cf.bucket_name, cf.uri)
+
+
 async def upload_to_bucket(context: ApplicationContext, bytes_iterator: AsyncIterator[bytes], key: str):
     storage = context.blob_storage
     bucket = settings.backups_bucket

nucliadb/backups/restore.py

@@ -21,6 +21,7 @@
 
 import asyncio
 import functools
+import logging
 import tarfile
 from typing import AsyncIterator, Callable, Optional, Union
 
@@ -39,6 +40,8 @@ from nucliadb_protos import knowledgebox_pb2 as kb_pb2
 from nucliadb_protos.resources_pb2 import CloudFile
 from nucliadb_protos.writer_pb2 import BrokerMessage
 
+logger = logging.getLogger(__name__)
+
 
 async def restore_kb_task(context: ApplicationContext, msg: RestoreBackupRequest):
     kbid = msg.kb_id
@@ -193,7 +196,7 @@ class ResourceBackupReader:
         elif tarinfo.name.startswith("cloud-files"):
             raw_cf = await self.read_data(tarinfo)
             cf = CloudFile()
-            cf.FromString(raw_cf)
+            cf.ParseFromString(raw_cf)
             return cf
         elif tarinfo.name.startswith("binaries"):
             uri = tarinfo.name.lstrip("binaries/")
@@ -219,14 +222,19 @@ async def restore_resource(context: ApplicationContext, kbid: str, backup_id: st
             bm = item
             bm.kbid = kbid
             break
-
-        # Read the cloud file and its binary
-        cf = await reader.read_item()
-        assert isinstance(cf, CloudFile)
-        cf_binary = await reader.read_item()
-        assert isinstance(cf_binary, CloudFileBinary)
-        assert cf.uri == cf_binary.uri
-        await import_binary(context, kbid, cf, cf_binary.read)
+        elif isinstance(item, CloudFile):
+            # Read its binary and import it
+            cf = item
+            cf_binary = await reader.read_item()
+            assert isinstance(cf_binary, CloudFileBinary)
+            assert cf.uri == cf_binary.uri
+            await import_binary(context, kbid, cf, cf_binary.read)
+        else:
+            logger.error(
+                "Unexpected item in resource backup. Backup may be corrupted",
+                extra={"item_type": type(item), kbid: kbid, resource_id: resource_id},
+            )
+            continue
 
     await import_broker_message(context, kbid, bm)
 

nucliadb/backups/utils.py

@@ -24,9 +24,12 @@ from nucliadb_utils.storages.storage import Storage
 
 
 async def exists_backup(storage: Storage, backup_id: str) -> bool:
-    async for _ in storage.iterate_objects(
-        bucket=settings.backups_bucket,
-        prefix=StorageKeys.BACKUP_PREFIX.format(backup_id=backup_id),
-    ):
+    return await exists_in_storge(
+        storage, settings.backups_bucket, StorageKeys.BACKUP_PREFIX.format(backup_id=backup_id)
+    )
+
+
+async def exists_in_storge(storage: Storage, bucket: str, key: str) -> bool:
+    async for _ in storage.iterate_objects(bucket=bucket, prefix=key):
         return True
     return False

{nucliadb-6.3.1.post3574.dist-info → nucliadb-6.3.1.post3577.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.3.1.post3574
+Version: 6.3.1.post3577
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.1.post3574
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.1.post3574
-Requires-Dist: nucliadb-protos>=6.3.1.post3574
-Requires-Dist: nucliadb-models>=6.3.1.post3574
-Requires-Dist: nidx-protos>=6.3.1.post3574
+Requires-Dist: nucliadb-telemetry[all]>=6.3.1.post3577
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.1.post3577
+Requires-Dist: nucliadb-protos>=6.3.1.post3577
+Requires-Dist: nucliadb-models>=6.3.1.post3577
+Requires-Dist: nidx-protos>=6.3.1.post3577
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.3.1.post3574.dist-info → nucliadb-6.3.1.post3577.dist-info}/RECORD

@@ -41,13 +41,13 @@ nucliadb/openapi.py,sha256=wDiw0dVEvTpJvbatkJ0JZLkKm9RItZT5PWRHjqRfqTA,2272
 nucliadb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/backups/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/backups/const.py,sha256=9vPAhLxQO_gNAjSdPxWuv3V66s9WcdpjOQ89CZlfmuk,1894
-nucliadb/backups/create.py,sha256=AM_nC7TgHOX0EFGaTXClS28jBSK28fHrKNZi14z2wek,10442
+nucliadb/backups/create.py,sha256=mvirguMbtxNgSDGG81l0kkgHWJSZPk4GFyra9nAkBZM,11275
 nucliadb/backups/delete.py,sha256=1rnBhVUGYYZJXSZUrrgYMDZ5NyswEWkIA-G-crRCyHk,2404
 nucliadb/backups/models.py,sha256=-hITU4Mv6AxePu12toBu_fjpEv6vVGcwNVxV22O9jQA,1273
-nucliadb/backups/restore.py,sha256=xhslVvTf4H8VmDucZpjrEFpKj6csPIWBadCPMVJYKQ8,9703
+nucliadb/backups/restore.py,sha256=wepEgv4vBN5yeiZU-f17PbuFV4xT4_SVKplNr8xSJrE,10001
 nucliadb/backups/settings.py,sha256=SyzsInj1BRbBI0atg5IXWbMbOZ_eVg4eSQ3IcnUhCxQ,1357
 nucliadb/backups/tasks.py,sha256=4_kOVJ2yCwMvDEpzJgTuTt75TNlpq5woyw9sTAcaSkw,4194
-nucliadb/backups/utils.py,sha256=ayDaxfWP5cPnAkQH-tF4M6cnowsPQgU2ljYz_iL1CbE,1249
+nucliadb/backups/utils.py,sha256=b1hi0gEp90tNrWHejNVoUgRpa4D6uKGhbACq0yeLkJY,1375
 nucliadb/common/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/constants.py,sha256=QpigxJh_CtD85Evy0PtV5cVq6x0U_f9xfIcXz1ymkUg,869
 nucliadb/common/counters.py,sha256=8lOi3A2HeLDDlcNaS2QT1SfD3350VPBjiY3FkmHH1V8,977
@@ -347,8 +347,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.3.1.post3574.dist-info/METADATA,sha256=JiB_eKqj0pPsq4nf9VvqU9gm6CEEQkYvElBlLSISd7c,4291
-nucliadb-6.3.1.post3574.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-nucliadb-6.3.1.post3574.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.3.1.post3574.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.3.1.post3574.dist-info/RECORD,,
+nucliadb-6.3.1.post3577.dist-info/METADATA,sha256=7cMll6LH15F3_kAg1yrlOSDK8XFk77amc4TEa0kApug,4291
+nucliadb-6.3.1.post3577.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+nucliadb-6.3.1.post3577.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.1.post3577.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.1.post3577.dist-info/RECORD,,