nucliadb 6.2.0.post2679__py3-none-any.whl → 6.2.1.post2701__py3-none-any.whl

nucliadb/ingest/orm/knowledgebox.py

@@ -49,7 +49,6 @@ from nucliadb.ingest.orm.exceptions import (
 from nucliadb.ingest.orm.metrics import processor_observer
 from nucliadb.ingest.orm.resource import Resource
 from nucliadb.ingest.orm.utils import choose_matryoshka_dimension, compute_paragraph_key
-from nucliadb.ingest.settings import settings
 from nucliadb.migrator.utils import get_latest_version
 from nucliadb_protos import knowledgebox_pb2, noderesources_pb2, nodewriter_pb2, writer_pb2
 from nucliadb_protos.knowledgebox_pb2 import (
@@ -60,6 +59,7 @@ from nucliadb_protos.knowledgebox_pb2 import (
     StoredExternalIndexProviderMetadata,
 )
 from nucliadb_protos.resources_pb2 import Basic
+from nucliadb_utils.settings import is_onprem_nucliadb
 from nucliadb_utils.storages.storage import Storage
 from nucliadb_utils.utilities import (
     get_audit,
@@ -74,6 +74,9 @@ KB_KEYS = "/kbs/{kbid}/"
 KB_TO_DELETE_BASE = "/kbtodelete/"
 KB_TO_DELETE_STORAGE_BASE = "/storagetodelete/"
 
+RESOURCE_TO_DELETE_STORAGE_BASE = "/resourcestoragetodelete"
+RESOURCE_TO_DELETE_STORAGE = f"{RESOURCE_TO_DELETE_STORAGE_BASE}/{{kbid}}/{{uuid}}"
+
 KB_TO_DELETE = f"{KB_TO_DELETE_BASE}{{kbid}}"
 KB_TO_DELETE_STORAGE = f"{KB_TO_DELETE_STORAGE_BASE}{{kbid}}"
 
@@ -415,9 +418,16 @@ class KnowledgeBox:
                 logger.exception("Error deleting slug")
 
     async def storage_delete_resource(self, uuid: str):
-        await self.storage.delete_resource(
-            self.kbid, uuid, max_parallel=settings.ingest_delete_resource_storage_max_parallel
-        )
+        if is_onprem_nucliadb():
+            await self.storage.delete_resource(self.kbid, uuid)
+        else:
+            # Deleting from storage can be slow, so we schedule its deletion and the purge cronjob
+            # will take care of it
+            await self.schedule_delete_resource(self.kbid, uuid)
+
+    async def schedule_delete_resource(self, kbid: str, uuid: str):
+        key = RESOURCE_TO_DELETE_STORAGE.format(kbid=kbid, uuid=uuid)
+        await self.txn.set(key, b"")
 
     async def delete_resource(self, uuid: str):
         with processor_observer({"type": "delete_resource_maindb"}):

nucliadb/ingest/settings.py

@@ -85,8 +85,5 @@ class Settings(DriverSettings):
 
     max_concurrent_ingest_processing: int = 5
 
-    # Ingest processor settings
-    ingest_delete_resource_storage_max_parallel: int = 20
-
 
 settings = Settings()

nucliadb/purge/__init__.py

@@ -32,6 +32,7 @@ from nucliadb.ingest.orm.knowledgebox import (
     KB_TO_DELETE_STORAGE_BASE,
     KB_VECTORSET_TO_DELETE,
     KB_VECTORSET_TO_DELETE_BASE,
+    RESOURCE_TO_DELETE_STORAGE_BASE,
     KnowledgeBox,
 )
 from nucliadb_telemetry import errors
@@ -131,6 +132,67 @@ async def purge_kb_storage(driver: Driver, storage: Storage):
     logger.info("FINISH PURGING KB STORAGE")
 
 
+async def purge_deleted_resource_storage(driver: Driver, storage: Storage) -> None:
+    """
+    Remove from storage all resources marked as deleted.
+
+    Returns the number of resources purged.
+    """
+    logger.info("Starting purge of deleted resource storage")
+    to_purge = await _count_resources_storage_to_purge(driver)
+    logger.info(f"Found {to_purge} resources to purge")
+    while True:
+        try:
+            purged = await _purge_resources_storage_batch(driver, storage, batch_size=100)
+            if not purged:
+                logger.info("No more resources to purge found")
+                return
+            logger.info(f"Purged {purged} resources")
+
+        except asyncio.CancelledError:
+            logger.info("Purge of deleted resource storage was cancelled")
+            return
+
+
+async def _count_resources_storage_to_purge(driver: Driver) -> int:
+    """
+    Count the number of resources marked as deleted in storage.
+    """
+    async with driver.transaction(read_only=True) as txn:
+        return await txn.count(match=RESOURCE_TO_DELETE_STORAGE_BASE)
+
+
+async def _purge_resources_storage_batch(driver: Driver, storage: Storage, batch_size: int = 100) -> int:
+    """
+    Remove from storage a batch of resources marked as deleted. Returns the
+    number of resources purged.
+    """
+    # Get the keys of the resources to delete in batches of 100
+    to_delete_batch = []
+    async with driver.transaction(read_only=True) as txn:
+        async for key in txn.keys(match=RESOURCE_TO_DELETE_STORAGE_BASE, count=batch_size):
+            to_delete_batch.append(key)
+
+    if not to_delete_batch:
+        return 0
+
+    # Delete the resources blobs from storage
+    logger.info(f"Purging {len(to_delete_batch)} deleted resources")
+    tasks = []
+    for key in to_delete_batch:
+        kbid, resource_id = key.split("/")[-2:]
+        tasks.append(asyncio.create_task(storage.delete_resource(kbid, resource_id)))
+    await asyncio.gather(*tasks)
+
+    # Delete the schedule-to-delete keys
+    async with driver.transaction() as txn:
+        for key in to_delete_batch:
+            await txn.delete(key)
+        await txn.commit()
+
+    return len(to_delete_batch)
+
+
 async def purge_kb_vectorsets(driver: Driver, storage: Storage):
     """Vectors for a vectorset are stored in a key inside each resource. Iterate
     through all resources of the KB and remove any storage object containing
@@ -186,14 +248,19 @@ async def main():
         service_name=SERVICE_NAME,
     )
     try:
+        purge_resources_storage_task = asyncio.create_task(
+            purge_deleted_resource_storage(driver, storage)
+        )
         await purge_kb(driver)
         await purge_kb_storage(driver, storage)
         await purge_kb_vectorsets(driver, storage)
+        await purge_resources_storage_task
     except Exception as ex:  # pragma: no cover
         logger.exception("Unhandled exception on purge command")
         errors.capture_exception(ex)
     finally:
         try:
+            purge_resources_storage_task.cancel()
             await storage.finalize()
             await teardown_driver()
             await teardown_cluster()

{nucliadb-6.2.0.post2679.dist-info → nucliadb-6.2.1.post2701.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nucliadb
-Version: 6.2.0.post2679
+Version: 6.2.1.post2701
 Home-page: https://docs.nuclia.dev/docs/management/nucliadb/intro
 Author: NucliaDB Community
 Author-email: nucliadb@nuclia.com
@@ -22,10 +22,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.9, <4
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.2.0.post2679
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.0.post2679
-Requires-Dist: nucliadb-protos>=6.2.0.post2679
-Requires-Dist: nucliadb-models>=6.2.0.post2679
+Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post2701
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post2701
+Requires-Dist: nucliadb-protos>=6.2.1.post2701
+Requires-Dist: nucliadb-models>=6.2.1.post2701
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nucliadb-node-binding>=2.26.0
 Requires-Dist: nuclia-models>=0.24.2

{nucliadb-6.2.0.post2679.dist-info → nucliadb-6.2.1.post2701.dist-info}/RECORD

@@ -111,7 +111,7 @@ nucliadb/ingest/partitions.py,sha256=2NIhMYbNT0TNBL6bX1UMSi7vxFGICstCKEqsB0TXHOE
 nucliadb/ingest/processing.py,sha256=x8FGnq2epsGl0QEzdYlgCys9MpxtV5_WO09hc7Wy150,20254
 nucliadb/ingest/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/ingest/serialize.py,sha256=GSDfrO4JLm-QLKw8LJ7TD1JFcXXvwm-ugXzbCfGh3Fk,15492
-nucliadb/ingest/settings.py,sha256=SDQpMRsTsNyi6IDxCJy6BZVUSKUzwAMuxf6ktp31VMM,3130
+nucliadb/ingest/settings.py,sha256=AFeqM2Sq2lHCrW745S4VjUmxfwkB2H9T_bncVRvLNRA,3039
 nucliadb/ingest/utils.py,sha256=l1myURu3r8oA11dx3GpHw-gNTUc1AFX8xdPm9Lgl2rA,2275
 nucliadb/ingest/consumer/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/ingest/consumer/auditing.py,sha256=EJoqRRr4dk2eUMK0GOY6b9xHO0YLQ0LjoP_xZBLACZo,7280
@@ -135,7 +135,7 @@ nucliadb/ingest/orm/brain.py,sha256=TtHPKZzv_Yz-tYkB1QjzPWUxA7Z_naMVXAfo8iov9Gw,
 nucliadb/ingest/orm/broker_message.py,sha256=JYYUJIZEL_EqovQuw6u-FmEkjyoYlxIXJq9hFekOiks,6441
 nucliadb/ingest/orm/entities.py,sha256=2PslT1FZ6yCvJtjR0UpKTSzxJrtS-C_gZx4ZTWHunTc,15759
 nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
-nucliadb/ingest/orm/knowledgebox.py,sha256=loH4PBxL2aJYMpbumrYpWq1kzqt-HNUeAok2-kOdqz8,22736
+nucliadb/ingest/orm/knowledgebox.py,sha256=dBetjoJBYT6JuGmMHiqjcfJeD8qJrK3MQt9X03IrHRA,23228
 nucliadb/ingest/orm/metrics.py,sha256=OkwMSPKLZcKba0ZTwtTiIxwBgaLMX5ydhGieKvi2y7E,1096
 nucliadb/ingest/orm/resource.py,sha256=_m4B14dSpO-lszpoqlhXYL3LrplB9p3NrDZC5kQbXHs,53860
 nucliadb/ingest/orm/utils.py,sha256=vCe_9UxHu26JDFGLwQ0wH-XyzJIpQCTK-Ow9dtZR5Vg,2716
@@ -160,7 +160,7 @@ nucliadb/migrator/settings.py,sha256=jOUX0ZMunCXN8HpF9xXN0aunJYRhu4Vdr_ffjRIqwtw
 nucliadb/migrator/utils.py,sha256=NgUreUvON8_nWEzTxELBMWlfV7E6-6qi-g0DMEbVEz4,2885
 nucliadb/models/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/models/responses.py,sha256=qnuOoc7TrVSUnpikfTwHLKez47_DE4mSFzpxrwtqijA,1599
-nucliadb/purge/__init__.py,sha256=Y_PcRfvqccUJwXSFZO4Q9uogBGe1_pH4MyS8RvyCPgA,7941
+nucliadb/purge/__init__.py,sha256=tcXwO99714cqflLVJyZzOv6_64H9pt7r6V0UogDd4oA,10389
 nucliadb/purge/orphan_shards.py,sha256=fA5yqRRN-M50OIk8dkAi1_ShFVjwDYEYqzMA9dYP0eU,9227
 nucliadb/reader/__init__.py,sha256=C5Efic7WlGm2U2C5WOyquMFbIj2Pojwe_8mwzVYnOzE,1304
 nucliadb/reader/app.py,sha256=Se-BFTE6d1v1msLzQn4q5XIhjnSxa2ckDSHdvm7NRf8,3096
@@ -332,9 +332,9 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.2.0.post2679.dist-info/METADATA,sha256=Ys9PJ1dTfL7VBWnMzfhtgnsPA9GbhaJUTi5EcSwivnU,4429
-nucliadb-6.2.0.post2679.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-nucliadb-6.2.0.post2679.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.2.0.post2679.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.2.0.post2679.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-nucliadb-6.2.0.post2679.dist-info/RECORD,,
+nucliadb-6.2.1.post2701.dist-info/METADATA,sha256=hdFV2cB9AAEk1AfUWZeiT_VI91OuRsNXvEe0fSMVN5M,4429
+nucliadb-6.2.1.post2701.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+nucliadb-6.2.1.post2701.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.2.1.post2701.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.2.1.post2701.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+nucliadb-6.2.1.post2701.dist-info/RECORD,,