nucliadb 6.3.3.post3613__py3-none-any.whl → 6.3.3.post3626__py3-none-any.whl

nucliadb/backups/delete.py

@@ -20,12 +20,15 @@
 
 
 import asyncio
+import logging
 
 from nucliadb.backups.const import StorageKeys
 from nucliadb.backups.models import DeleteBackupRequest
 from nucliadb.backups.settings import settings
 from nucliadb.common.context import ApplicationContext
 
+logger = logging.getLogger(__name__)
+
 
 async def delete_backup_task(context: ApplicationContext, msg: DeleteBackupRequest):
     """
@@ -35,12 +38,15 @@ async def delete_backup_task(context: ApplicationContext, msg: DeleteBackupReque
 
 
 async def delete_backup(context: ApplicationContext, backup_id: str):
+    total_deleted = 0
     while True:
         deleted = await delete_n(context, backup_id, n=1000)
         if deleted == 0:
             # No more objects to delete
             break
-        await asyncio.sleep(1)
+        total_deleted += deleted
+        logger.info(f"Deleted {total_deleted} objects from backup", extra={"backup_id": backup_id})
+    logger.info(f"Backup deletion completed", extra={"backup_id": backup_id})
 
 
 async def delete_n(context: ApplicationContext, backup_id: str, n: int):

nucliadb/backups/tasks.py

@@ -21,7 +21,7 @@ from typing import Awaitable, Callable
 
 from nucliadb.backups.const import BackupsNatsConfig
 from nucliadb.backups.create import backup_kb_task
-from nucliadb.backups.delete import delete_backup
+from nucliadb.backups.delete import delete_backup_task
 from nucliadb.backups.models import CreateBackupRequest, DeleteBackupRequest, RestoreBackupRequest
 from nucliadb.backups.restore import restore_kb_task
 from nucliadb.common.context import ApplicationContext
@@ -83,11 +83,11 @@ async def restore(kbid: str, backup_id: str) -> None:
 
 
 def deleter_consumer() -> NatsTaskConsumer[DeleteBackupRequest]:
-    consumer: NatsTaskConsumer = create_consumer(
+    consumer: NatsTaskConsumer[DeleteBackupRequest] = create_consumer(
         name="backup_deleter",
         stream=BackupsNatsConfig.stream,
         consumer=BackupsNatsConfig.delete_consumer,
-        callback=delete_backup,
+        callback=delete_backup_task,
         msg_type=DeleteBackupRequest,
         max_concurrent_messages=2,
     )

nucliadb/purge/__init__.py

@@ -21,6 +21,7 @@ import asyncio
 import importlib.metadata
 from typing import AsyncGenerator
 
+from nucliadb.common import datamanagers
 from nucliadb.common.cluster.exceptions import NodeError, ShardNotFound
 from nucliadb.common.cluster.utils import setup_cluster, teardown_cluster
 from nucliadb.common.maindb.driver import Driver
@@ -185,7 +186,11 @@ async def _purge_resources_storage_batch(driver: Driver, storage: Storage, batch
     tasks = []
     for key in to_delete_batch:
         kbid, resource_id = key.split("/")[-2:]
-        tasks.append(asyncio.create_task(storage.delete_resource(kbid, resource_id)))
+        # Check if resource exists in maindb. This can happen if a file is deleted (marked for purge) and immediately
+        # reuploaded. Without this check, we will delete the data of the newly uploaded copy of the resource.
+        if not await datamanagers.atomic.resources.resource_exists(kbid=kbid, rid=resource_id):
+            tasks.append(asyncio.create_task(storage.delete_resource(kbid, resource_id)))
+
     await asyncio.gather(*tasks)
 
     # Delete the schedule-to-delete keys

{nucliadb-6.3.3.post3613.dist-info → nucliadb-6.3.3.post3626.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.3.3.post3613
+Version: 6.3.3.post3626
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.3.post3613
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.3.post3613
-Requires-Dist: nucliadb-protos>=6.3.3.post3613
-Requires-Dist: nucliadb-models>=6.3.3.post3613
-Requires-Dist: nidx-protos>=6.3.3.post3613
+Requires-Dist: nucliadb-telemetry[all]>=6.3.3.post3626
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.3.post3626
+Requires-Dist: nucliadb-protos>=6.3.3.post3626
+Requires-Dist: nucliadb-models>=6.3.3.post3626
+Requires-Dist: nidx-protos>=6.3.3.post3626
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.3.3.post3613.dist-info → nucliadb-6.3.3.post3626.dist-info}/RECORD

@@ -42,11 +42,11 @@ nucliadb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/backups/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/backups/const.py,sha256=9vPAhLxQO_gNAjSdPxWuv3V66s9WcdpjOQ89CZlfmuk,1894
 nucliadb/backups/create.py,sha256=TJtYewhD0jkYV_h3rNUhKzhqB2QHAhLWYOgRVlGysGs,11450
-nucliadb/backups/delete.py,sha256=1rnBhVUGYYZJXSZUrrgYMDZ5NyswEWkIA-G-crRCyHk,2404
+nucliadb/backups/delete.py,sha256=AAs-WP-ujejj6c1LJgiMn7Ht67N_j0a1sKQlUepHpEA,2659
 nucliadb/backups/models.py,sha256=-hITU4Mv6AxePu12toBu_fjpEv6vVGcwNVxV22O9jQA,1273
 nucliadb/backups/restore.py,sha256=YD3Bbo9ry4YLMM6imB-DXbOAMXfGxVzJtTAAUFDvB0I,10153
 nucliadb/backups/settings.py,sha256=SyzsInj1BRbBI0atg5IXWbMbOZ_eVg4eSQ3IcnUhCxQ,1357
-nucliadb/backups/tasks.py,sha256=4_kOVJ2yCwMvDEpzJgTuTt75TNlpq5woyw9sTAcaSkw,4194
+nucliadb/backups/tasks.py,sha256=WkL1LgdYBHbV_A5ilyYv5p3zmXwxH68TDudytN5f7zk,4225
 nucliadb/backups/utils.py,sha256=_Vogjqcru5oqNZM-bZ0q7Ju79Bv1PD-LVFEa7Z-Q13I,1261
 nucliadb/common/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/constants.py,sha256=QpigxJh_CtD85Evy0PtV5cVq6x0U_f9xfIcXz1ymkUg,869
@@ -165,7 +165,7 @@ nucliadb/migrator/settings.py,sha256=jOUX0ZMunCXN8HpF9xXN0aunJYRhu4Vdr_ffjRIqwtw
 nucliadb/migrator/utils.py,sha256=NgUreUvON8_nWEzTxELBMWlfV7E6-6qi-g0DMEbVEz4,2885
 nucliadb/models/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/models/responses.py,sha256=qnuOoc7TrVSUnpikfTwHLKez47_DE4mSFzpxrwtqijA,1599
-nucliadb/purge/__init__.py,sha256=p-DxJ3vGtEcCuiSsWY22bQn99vTMHk32BQdQJVy5b0k,11983
+nucliadb/purge/__init__.py,sha256=UXbto56EWYLwZj6uEc-flQVe3gDDNFtM6EV-aIkryPU,12353
 nucliadb/purge/orphan_shards.py,sha256=AU1Jfc4qtQFasB6dkuGz0a_Zjs-i7liUTf7Xpl8R8ng,7939
 nucliadb/reader/__init__.py,sha256=C5Efic7WlGm2U2C5WOyquMFbIj2Pojwe_8mwzVYnOzE,1304
 nucliadb/reader/app.py,sha256=Se-BFTE6d1v1msLzQn4q5XIhjnSxa2ckDSHdvm7NRf8,3096
@@ -351,8 +351,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.3.3.post3613.dist-info/METADATA,sha256=NjyQDQSB_zbOUyfe9Zcekfh2SMCNZHggFv3GkrVI6gE,4291
-nucliadb-6.3.3.post3613.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-nucliadb-6.3.3.post3613.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.3.3.post3613.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.3.3.post3613.dist-info/RECORD,,
+nucliadb-6.3.3.post3626.dist-info/METADATA,sha256=IoTcS-I-HSFWU6Be7gUPbWE3I60TUvgAfOP3jRQ-LUs,4291
+nucliadb-6.3.3.post3626.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+nucliadb-6.3.3.post3626.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.3.post3626.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.3.post3626.dist-info/RECORD,,