nucliadb 6.3.1.post3526__py3-none-any.whl → 6.3.1.post3544__py3-none-any.whl

nucliadb/backups/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#

nucliadb/backups/const.py

@@ -0,0 +1,49 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+class MaindbKeys:
+    METADATA = "kbs/{kbid}/backups/{backup_id}"
+    LAST_RESTORED = "kbs/{kbid}/backup/{backup_id}/last_restored"
+
+
+class StorageKeys:
+    """
+    Defines the key templates used to store backup files in the backups bucket of the storage.
+    """
+
+    BACKUP_PREFIX = "backups/{backup_id}/"
+    RESOURCES_PREFIX = "backups/{backup_id}/resources/"
+    RESOURCE = "backups/{backup_id}/resources/{resource_id}.tar"
+    ENTITIES = "backups/{backup_id}/entities.pb"
+    LABELS = "backups/{backup_id}/labels.pb"
+
+
+class BackupFinishedStream:
+    name = "backups"
+    subject = "backups.creation_finished"
+
+
+class BackupsNatsStream:
+    name = "ndb-backups"
+    stream_subjects = ["ndb-backups.>"]
+    create_subject = "ndb-backups.create"
+    delete_subject = "ndb-backups.delete"
+    restore_subject = "ndb-backups.restore"

nucliadb/backups/create.py

@@ -0,0 +1,277 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import asyncio
+import tarfile
+from datetime import datetime, timezone
+from typing import AsyncIterator, Optional
+
+from nucliadb.backups.const import (
+    BackupFinishedStream,
+    MaindbKeys,
+    StorageKeys,
+)
+from nucliadb.backups.models import BackupMetadata, CreateBackupRequest
+from nucliadb.backups.settings import settings
+from nucliadb.common import datamanagers
+from nucliadb.common.context import ApplicationContext
+from nucliadb.export_import.utils import (
+    download_binary,
+    get_broker_message,
+    get_cloud_files,
+    get_entities,
+    get_labels,
+)
+from nucliadb.tasks.retries import TaskRetryHandler
+from nucliadb_protos import backups_pb2, resources_pb2, writer_pb2
+from nucliadb_utils.audit.stream import StreamAuditStorage
+from nucliadb_utils.storages.storage import StorageField
+from nucliadb_utils.utilities import get_audit
+
+
+async def backup_kb_task(context: ApplicationContext, msg: CreateBackupRequest):
+    kbid = msg.kbid
+    backup_id = msg.backup_id
+
+    retry_handler = TaskRetryHandler(
+        kbid=kbid,
+        task_type="backup",
+        task_id=backup_id,
+        context=context,
+        max_retries=5,
+    )
+
+    @retry_handler.wrap
+    async def _backup_kb(context: ApplicationContext, kbid: str, backup_id: str):
+        await backup_kb(context, kbid, backup_id)
+
+    await _backup_kb(context, kbid, backup_id)
+
+
+async def backup_kb(context: ApplicationContext, kbid: str, backup_id: str):
+    """
+    Backs up a KB to the cloud storage.
+    """
+    await backup_resources(context, kbid, backup_id)
+    await backup_labels(context, kbid, backup_id)
+    await backup_entities(context, kbid, backup_id)
+    await notify_backup_completed(context, kbid, backup_id)
+    await delete_metadata(context, kbid, backup_id)
+
+
+async def backup_resources(context: ApplicationContext, kbid: str, backup_id: str):
+    metadata = await get_metadata(context, kbid, backup_id)
+    if metadata is None:
+        metadata = BackupMetadata(
+            kbid=kbid,
+            requested_at=datetime.now(tz=timezone.utc),
+        )
+        async for rid in datamanagers.resources.iterate_resource_ids(kbid=kbid):
+            metadata.total_resources += 1
+            metadata.missing_resources.append(rid)
+        metadata.missing_resources.sort()
+        await set_metadata(context, kbid, backup_id, metadata)
+    tasks = []
+    backing_up = []
+    for rid in metadata.missing_resources:
+        tasks.append(asyncio.create_task(backup_resource(context, backup_id, kbid, rid)))
+        backing_up.append(rid)
+        if len(tasks) >= settings.backup_resources_concurrency:
+            resources_bytes = await asyncio.gather(*tasks)
+            metadata.total_size += sum(resources_bytes)
+            metadata.missing_resources = [
+                rid for rid in metadata.missing_resources if rid not in backing_up
+            ]
+            await set_metadata(context, kbid, backup_id, metadata)
+            tasks = []
+            backing_up = []
+    if len(tasks) > 0:
+        resources_bytes = await asyncio.gather(*tasks)
+        metadata.total_size += sum(resources_bytes)
+        metadata.missing_resources = [rid for rid in metadata.missing_resources if rid not in backing_up]
+        await set_metadata(context, kbid, backup_id, metadata)
+        tasks = []
+        backing_up = []
+
+
+async def backup_resource(context: ApplicationContext, backup_id: str, kbid: str, rid: str) -> int:
+    """
+    Backs up a resource to the blob storage service.
+    Returns the size of the resource in bytes.
+    """
+    bm = await get_broker_message(context, kbid, rid)
+    if bm is None:
+        # Resource not found. May have been deleted while the backup was running.
+        return 0
+    return await backup_resource_with_binaries(context, backup_id, kbid, rid, bm)
+
+
+async def to_tar(name: str, size: int, chunks: AsyncIterator[bytes]) -> AsyncIterator[bytes]:
+    """
+    This function is a generator that adds tar header and padding to the end of the chunks
+    to be compatible with the tar format.
+    """
+    tarinfo = tarfile.TarInfo(name)
+    tarinfo.size = size
+    tarinfo.mtime = int(datetime.now().timestamp())
+    tarinfo.mode = 0o644
+    tarinfo.type = tarfile.REGTYPE
+    header_bytes = tarinfo.tobuf(format=tarfile.GNU_FORMAT)
+    yield header_bytes
+    async for chunk in chunks:
+        yield chunk
+    if size % 512 != 0:
+        yield b"\x00" * (512 - (size % 512))
+
+
+async def backup_resource_with_binaries(
+    context, backup_id: str, kbid: str, rid: str, bm: writer_pb2.BrokerMessage
+) -> int:
+    """
+    Generate a tar file dynamically with the resource broker message and all its binary files,
+    and stream it to the blob storage service. Returns the total size of the tar file in bytes.
+    """
+    total_size = 0
+
+    async def resource_data_iterator():
+        """
+        Each tar file will have the following structure:
+
+        - cloud-files/{cloud_file.uri}  (serialized resources_pb2.CloudFile)
+        - binaries/{cloud_file.uri} (the actual binary content of the cloud file)
+        - broker-message.pb
+
+        The order is important because the restore process depends on it (needs to import
+        the cloud files and its binaries first before the broker message).
+        """
+        nonlocal total_size
+
+        for cloud_file in get_cloud_files(bm):
+            serialized_cf = cloud_file.SerializeToString()
+
+            async def cf_iterator():
+                yield serialized_cf
+
+            async for chunk in to_tar(
+                name=f"cloud-files/{cloud_file.uri}", size=len(serialized_cf), chunks=cf_iterator()
+            ):
+                yield chunk
+                total_size += len(chunk)
+
+            async for chunk in to_tar(
+                name=f"binaries/{cloud_file.uri}",
+                size=cloud_file.size,
+                chunks=download_binary(context, cloud_file),
+            ):
+                yield chunk
+                total_size += len(chunk)
+
+        bm_serialized = bm.SerializeToString()
+
+        async def bm_iterator():
+            yield bm_serialized
+
+        async for chunk in to_tar(
+            name="broker-message.pb", size=len(bm_serialized), chunks=bm_iterator()
+        ):
+            yield chunk
+            total_size += len(chunk)
+
+    await upload_to_bucket(
+        context,
+        resource_data_iterator(),
+        key=StorageKeys.RESOURCE.format(kbid=kbid, backup_id=backup_id, resource_id=rid),
+    )
+    return total_size
+
+
+async def backup_labels(context: ApplicationContext, kbid: str, backup_id: str):
+    labels = await get_labels(context, kbid)
+    await context.blob_storage.upload_object(
+        bucket=settings.backups_bucket,
+        key=StorageKeys.LABELS.format(kbid=kbid, backup_id=backup_id),
+        data=labels.SerializeToString(),
+    )
+
+
+async def backup_entities(context: ApplicationContext, kbid: str, backup_id: str):
+    entities = await get_entities(context, kbid)
+    await context.blob_storage.upload_object(
+        bucket=settings.backups_bucket,
+        key=StorageKeys.ENTITIES.format(kbid=kbid, backup_id=backup_id),
+        data=entities.SerializeToString(),
+    )
+
+
+async def get_metadata(
+    context: ApplicationContext, kbid: str, backup_id: str
+) -> Optional[BackupMetadata]:
+    async with context.kv_driver.transaction(read_only=True) as txn:
+        metadata_raw = await txn.get(MaindbKeys.METADATA.format(kbid=kbid, backup_id=backup_id))
+        if metadata_raw is None:
+            return None
+        return BackupMetadata.model_validate_json(metadata_raw)
+
+
+async def set_metadata(context: ApplicationContext, kbid: str, backup_id: str, metadata: BackupMetadata):
+    async with context.kv_driver.transaction() as txn:
+        await txn.set(
+            MaindbKeys.METADATA.format(kbid=kbid, backup_id=backup_id),
+            metadata.model_dump_json().encode(),
+        )
+        await txn.commit()
+
+
+async def delete_metadata(context: ApplicationContext, kbid: str, backup_id: str):
+    async with context.kv_driver.transaction() as txn:
+        await txn.delete(MaindbKeys.METADATA.format(kbid=kbid, backup_id=backup_id))
+        await txn.commit()
+
+
+async def upload_to_bucket(context: ApplicationContext, bytes_iterator: AsyncIterator[bytes], key: str):
+    storage = context.blob_storage
+    bucket = settings.backups_bucket
+    cf = resources_pb2.CloudFile()
+    cf.bucket_name = bucket
+    cf.content_type = "binary/octet-stream"
+    cf.source = resources_pb2.CloudFile.Source.EXPORT
+    field: StorageField = storage.field_klass(storage=storage, bucket=bucket, fullkey=key, field=cf)
+    await storage.uploaditerator(bytes_iterator, field, cf)
+
+
+async def notify_backup_completed(context: ApplicationContext, kbid: str, backup_id: str):
+    audit = get_audit()
+    if audit is None or not isinstance(audit, StreamAuditStorage):
+        # We rely on the stream audit utility as it already holds a connection
+        # to the idp nats server. If it's not available, we can't send the notification.
+        return
+    metadata = await get_metadata(context, kbid, backup_id)
+    if metadata is None:  # pragma: no cover
+        raise ValueError("Backup metadata not found")
+    notification = backups_pb2.BackupCreatedNotification()
+    notification.finished_at.FromDatetime(datetime.now(tz=timezone.utc))
+    notification.kb_id = kbid
+    notification.backup_id = backup_id
+    notification.size = metadata.total_size
+    notification.resources = metadata.total_resources
+    await audit.js.publish(
+        BackupFinishedStream.subject,
+        notification.SerializeToString(),
+        stream=BackupFinishedStream.name,
+    )

nucliadb/backups/delete.py

@@ -0,0 +1,72 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+import asyncio
+
+from nucliadb.backups.const import StorageKeys
+from nucliadb.backups.models import DeleteBackupRequest
+from nucliadb.backups.settings import settings
+from nucliadb.common.context import ApplicationContext
+
+
+async def delete_backup_task(context: ApplicationContext, msg: DeleteBackupRequest):
+    """
+    Deletes the backup files from the cloud storage.
+    """
+    await delete_backup(context, msg.backup_id)
+
+
+async def delete_backup(context: ApplicationContext, backup_id: str):
+    while True:
+        deleted = await delete_n(context, backup_id, n=1000)
+        if deleted == 0:
+            # No more objects to delete
+            break
+        await asyncio.sleep(1)
+
+
+async def delete_n(context: ApplicationContext, backup_id: str, n: int):
+    concurrent_batch_size = 50
+    deleted = 0
+    tasks = []
+    async for object_info in context.blob_storage.iterate_objects(
+        bucket=settings.backups_bucket,
+        prefix=StorageKeys.BACKUP_PREFIX.format(backup_id=backup_id),
+    ):
+        if deleted >= n:
+            # Deleted enough objects
+            break
+        tasks.append(
+            asyncio.create_task(
+                context.blob_storage.delete_upload(
+                    uri=object_info.name,
+                    bucket_name=settings.backups_bucket,
+                )
+            )
+        )
+        deleted += 1
+        if len(tasks) > concurrent_batch_size:
+            await asyncio.gather(*tasks)
+            tasks = []
+    if len(tasks) > 0:
+        await asyncio.gather(*tasks)
+        tasks = []
+    return deleted

nucliadb/backups/models.py

@@ -0,0 +1,44 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from datetime import datetime
+
+from pydantic import BaseModel
+
+
+class CreateBackupRequest(BaseModel):
+    kbid: str
+    backup_id: str
+
+
+class RestoreBackupRequest(BaseModel):
+    kbid: str
+    backup_id: str
+
+
+class DeleteBackupRequest(BaseModel):
+    backup_id: str
+
+
+class BackupMetadata(BaseModel):
+    kbid: str
+    requested_at: datetime
+    total_resources: int = 0
+    missing_resources: list[str] = []
+    total_size: int = 0

nucliadb/backups/restore.py

@@ -0,0 +1,251 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+import asyncio
+import functools
+import tarfile
+from typing import AsyncIterator, Callable, Optional, Union
+
+from nucliadb.backups.const import MaindbKeys, StorageKeys
+from nucliadb.backups.models import RestoreBackupRequest
+from nucliadb.backups.settings import settings
+from nucliadb.common.context import ApplicationContext
+from nucliadb.export_import.utils import (
+    import_binary,
+    import_broker_message,
+    set_entities_groups,
+    set_labels,
+)
+from nucliadb.tasks.retries import TaskRetryHandler
+from nucliadb_protos import knowledgebox_pb2 as kb_pb2
+from nucliadb_protos.resources_pb2 import CloudFile
+from nucliadb_protos.writer_pb2 import BrokerMessage
+
+
+async def restore_kb_task(context: ApplicationContext, msg: RestoreBackupRequest):
+    kbid = msg.kbid
+    backup_id = msg.backup_id
+
+    retry_handler = TaskRetryHandler(
+        kbid=kbid,
+        task_type="restore",
+        task_id=backup_id,
+        context=context,
+        max_retries=3,
+    )
+
+    @retry_handler.wrap
+    async def _restore_kb(context: ApplicationContext, kbid: str, backup_id: str):
+        await restore_kb(context, kbid, backup_id)
+
+    await _restore_kb(context, kbid, backup_id)
+
+
+async def restore_kb(context: ApplicationContext, kbid: str, backup_id: str):
+    """
+    Downloads the backup files from the cloud storage and imports them into the KB.
+    """
+    await restore_resources(context, kbid, backup_id)
+    await restore_labels(context, kbid, backup_id)
+    await restore_entities(context, kbid, backup_id)
+    await delete_last_restored_resource_key(context, kbid, backup_id)
+
+
+async def restore_resources(context: ApplicationContext, kbid: str, backup_id: str):
+    last_restored = await get_last_restored_resource_key(context, kbid, backup_id)
+    tasks = []
+    async for object_info in context.blob_storage.iterate_objects(
+        bucket=settings.backups_bucket,
+        prefix=StorageKeys.RESOURCES_PREFIX.format(kbid=kbid, backup_id=backup_id),
+        start=last_restored,
+    ):
+        key = object_info.name
+        resource_id = key.split("/")[-1].rstrip(".tar")
+        tasks.append(asyncio.create_task(restore_resource(context, kbid, backup_id, resource_id)))
+        if len(tasks) > settings.restore_resources_concurrency:
+            await asyncio.gather(*tasks)
+            tasks = []
+            await set_last_restored_resource_key(context, kbid, backup_id, key)
+    if len(tasks) > 0:
+        await asyncio.gather(*tasks)
+        tasks = []
+        await set_last_restored_resource_key(context, kbid, backup_id, key)
+
+
+async def get_last_restored_resource_key(
+    context: ApplicationContext, kbid: str, backup_id: str
+) -> Optional[str]:
+    key = MaindbKeys.LAST_RESTORED.format(kbid=kbid, backup_id=backup_id)
+    async with context.kv_driver.transaction(read_only=True) as txn:
+        raw = await txn.get(key)
+        if raw is None:
+            return None
+        return raw.decode()
+
+
+async def set_last_restored_resource_key(
+    context: ApplicationContext, kbid: str, backup_id: str, resource_id: str
+):
+    key = MaindbKeys.LAST_RESTORED.format(kbid=kbid, backup_id=backup_id)
+    async with context.kv_driver.transaction() as txn:
+        await txn.set(key, resource_id.encode())
+        await txn.commit()
+
+
+async def delete_last_restored_resource_key(context: ApplicationContext, kbid: str, backup_id: str):
+    key = MaindbKeys.LAST_RESTORED.format(kbid=kbid, backup_id=backup_id)
+    async with context.kv_driver.transaction() as txn:
+        await txn.delete(key)
+        await txn.commit()
+
+
+class CloudFileBinary:
+    def __init__(self, uri: str, download_stream: Callable[[int], AsyncIterator[bytes]]):
+        self.uri = uri
+        self.download_stream = download_stream
+
+    async def read(self, chunk_size: int) -> AsyncIterator[bytes]:
+        async for chunk in self.download_stream(chunk_size):
+            yield chunk
+
+
+class ResourceBackupReader:
+    def __init__(self, download_stream: AsyncIterator[bytes]):
+        self.download_stream = download_stream
+        self.buffer = b""
+
+    async def read(self, size: int) -> bytes:
+        while len(self.buffer) < size:
+            chunk = await self.download_stream.__anext__()
+            if not chunk:
+                continue
+            self.buffer += chunk
+        result = self.buffer[:size]
+        self.buffer = self.buffer[size:]
+        return result
+
+    async def iter_data(self, total_bytes: int, chunk_size: int = 1024 * 1024) -> AsyncIterator[bytes]:
+        padding_bytes = 0
+        if total_bytes % 512 != 0:
+            # We need to read the padding bytes and then discard them
+            padding_bytes = 512 - (total_bytes % 512)
+        read_bytes = 0
+        padding_reached = False
+        async for chunk in self._iter(total_bytes + padding_bytes, chunk_size):
+            if padding_reached:
+                # Skip padding bytes. We can't break here because we need
+                # to read the padding bytes from the stream
+                continue
+            padding_reached = read_bytes + len(chunk) >= total_bytes
+            if padding_reached:
+                chunk = chunk[: total_bytes - read_bytes]
+            else:
+                read_bytes += len(chunk)
+            yield chunk
+
+    async def _iter(self, total_bytes: int, chunk_size: int = 1024 * 1024) -> AsyncIterator[bytes]:
+        remaining_bytes = total_bytes
+        while remaining_bytes > 0:
+            to_read = min(chunk_size, remaining_bytes)
+            chunk = await self.read(to_read)
+            yield chunk
+            remaining_bytes -= len(chunk)
+        assert remaining_bytes == 0
+
+    async def read_tarinfo(self):
+        raw_tar_header = await self.read(512)
+        return tarfile.TarInfo.frombuf(raw_tar_header, encoding="utf-8", errors="strict")
+
+    async def read_data(self, tarinfo: tarfile.TarInfo) -> bytes:
+        tarinfo_size = tarinfo.size
+        padding_bytes = 0
+        if tarinfo_size % 512 != 0:
+            # We need to read the padding bytes and then discard them
+            padding_bytes = 512 - (tarinfo_size % 512)
+        data = await self.read(tarinfo_size + padding_bytes)
+        return data[:tarinfo_size]
+
+    async def read_item(self) -> Union[BrokerMessage, CloudFile, CloudFileBinary]:
+        tarinfo = await self.read_tarinfo()
+        if tarinfo.name.startswith("broker-message"):
+            raw_bm = await self.read_data(tarinfo)
+            bm = BrokerMessage()
+            bm.ParseFromString(raw_bm)
+            return bm
+        elif tarinfo.name.startswith("cloud-files"):
+            raw_cf = await self.read_data(tarinfo)
+            cf = CloudFile()
+            cf.FromString(raw_cf)
+            return cf
+        elif tarinfo.name.startswith("binaries"):
+            uri = tarinfo.name.lstrip("binaries/")
+            size = tarinfo.size
+            download_stream = functools.partial(self.iter_data, size)
+            return CloudFileBinary(uri, download_stream)
+        else:  # pragma: no cover
+            raise ValueError(f"Unknown tar entry: {tarinfo.name}")
+
+
+async def restore_resource(context: ApplicationContext, kbid: str, backup_id: str, resource_id: str):
+    download_stream = context.blob_storage.download(
+        bucket=settings.backups_bucket,
+        key=StorageKeys.RESOURCE.format(kbid=kbid, backup_id=backup_id, resource_id=resource_id),
+    )
+    reader = ResourceBackupReader(download_stream)
+    bm = None
+    while True:
+        item = await reader.read_item()
+        if isinstance(item, BrokerMessage):
+            # When the broker message is read, this means all cloud files
+            # and binaries of that resource have been read and imported
+            bm = item
+            bm.kbid = kbid
+            break
+
+        # Read the cloud file and its binary
+        cf = await reader.read_item()
+        assert isinstance(cf, CloudFile)
+        cf_binary = await reader.read_item()
+        assert isinstance(cf_binary, CloudFileBinary)
+        assert cf.uri == cf_binary.uri
+        await import_binary(context, kbid, cf, cf_binary.read)
+
+    await import_broker_message(context, kbid, bm)
+
+
+async def restore_labels(context: ApplicationContext, kbid: str, backup_id: str):
+    raw = await context.blob_storage.downloadbytes(
+        bucket=settings.backups_bucket,
+        key=StorageKeys.LABELS.format(kbid=kbid, backup_id=backup_id),
+    )
+    labels = kb_pb2.Labels()
+    labels.ParseFromString(raw.getvalue())
+    await set_labels(context, kbid, labels)
+
+
+async def restore_entities(context: ApplicationContext, kbid: str, backup_id: str):
+    raw = await context.blob_storage.downloadbytes(
+        bucket=settings.backups_bucket,
+        key=StorageKeys.ENTITIES.format(kbid=kbid, backup_id=backup_id),
+    )
+    entities = kb_pb2.EntitiesGroups()
+    entities.ParseFromString(raw.getvalue())
+    await set_entities_groups(context, kbid, entities)

nucliadb/backups/settings.py

@@ -0,0 +1,37 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class BackupSettings(BaseSettings):
+    backups_bucket: str = Field(
+        default="backups", description="The bucket where the backups are stored."
+    )
+    restore_resources_concurrency: int = Field(
+        default=10, description="The number of concurrent resource restores."
+    )
+    backup_resources_concurrency: int = Field(
+        default=10, description="The number of concurrent resource backups."
+    )
+
+
+settings = BackupSettings()

nucliadb/backups/tasks.py

@@ -0,0 +1,127 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from typing import Awaitable, Callable
+
+from nucliadb.backups.const import BackupsNatsStream
+from nucliadb.backups.create import backup_kb_task
+from nucliadb.backups.delete import delete_backup
+from nucliadb.backups.models import CreateBackupRequest, DeleteBackupRequest, RestoreBackupRequest
+from nucliadb.backups.restore import restore_kb_task
+from nucliadb.common.context import ApplicationContext
+from nucliadb.tasks import create_consumer, create_producer
+from nucliadb.tasks.consumer import NatsTaskConsumer
+from nucliadb.tasks.producer import NatsTaskProducer
+
+
+def creator_consumer() -> NatsTaskConsumer[CreateBackupRequest]:
+    consumer: NatsTaskConsumer = create_consumer(
+        name="backup_creator",
+        stream=BackupsNatsStream.name,
+        stream_subjects=BackupsNatsStream.stream_subjects,
+        consumer_subject=BackupsNatsStream.create_subject,
+        callback=backup_kb_task,
+        msg_type=CreateBackupRequest,
+        max_concurrent_messages=10,
+    )
+    return consumer
+
+
+async def create(kbid: str, backup_id: str) -> None:
+    producer: NatsTaskProducer[CreateBackupRequest] = create_producer(
+        name="backup_creator",
+        stream=BackupsNatsStream.name,
+        stream_subjects=BackupsNatsStream.stream_subjects,
+        producer_subject=BackupsNatsStream.create_subject,
+        msg_type=CreateBackupRequest,
+    )
+    msg = CreateBackupRequest(
+        kbid=kbid,
+        backup_id=backup_id,
+    )
+    await producer.send(msg)
+
+
+def restorer_consumer() -> NatsTaskConsumer[RestoreBackupRequest]:
+    consumer: NatsTaskConsumer = create_consumer(
+        name="backup_restorer",
+        stream=BackupsNatsStream.name,
+        stream_subjects=BackupsNatsStream.stream_subjects,
+        consumer_subject=BackupsNatsStream.restore_subject,
+        callback=restore_kb_task,
+        msg_type=RestoreBackupRequest,
+        max_concurrent_messages=10,
+    )
+    return consumer
+
+
+async def restore(kbid: str, backup_id: str) -> None:
+    producer: NatsTaskProducer[RestoreBackupRequest] = create_producer(
+        name="backup_restorer",
+        stream=BackupsNatsStream.name,
+        stream_subjects=BackupsNatsStream.stream_subjects,
+        producer_subject=BackupsNatsStream.restore_subject,
+        msg_type=RestoreBackupRequest,
+    )
+    msg = RestoreBackupRequest(
+        kbid=kbid,
+        backup_id=backup_id,
+    )
+    await producer.send(msg)
+
+
+def deleter_consumer() -> NatsTaskConsumer[DeleteBackupRequest]:
+    consumer: NatsTaskConsumer = create_consumer(
+        name="backup_deleter",
+        stream=BackupsNatsStream.name,
+        stream_subjects=BackupsNatsStream.stream_subjects,
+        consumer_subject=BackupsNatsStream.delete_subject,
+        callback=delete_backup,
+        msg_type=DeleteBackupRequest,
+        max_concurrent_messages=2,
+    )
+    return consumer
+
+
+async def delete(backup_id: str) -> None:
+    producer: NatsTaskProducer[DeleteBackupRequest] = create_producer(
+        name="backup_deleter",
+        stream=BackupsNatsStream.name,
+        stream_subjects=BackupsNatsStream.stream_subjects,
+        producer_subject=BackupsNatsStream.delete_subject,
+        msg_type=DeleteBackupRequest,
+    )
+    msg = DeleteBackupRequest(
+        backup_id=backup_id,
+    )
+    await producer.send(msg)
+
+
+async def initialize_consumers(context: ApplicationContext) -> list[Callable[[], Awaitable[None]]]:
+    creator = creator_consumer()
+    restorer = restorer_consumer()
+    deleter = deleter_consumer()
+    await creator.initialize(context)
+    await restorer.initialize(context)
+    await deleter.initialize(context)
+    return [
+        creator.finalize,
+        restorer.finalize,
+        deleter.finalize,
+    ]

nucliadb/backups/utils.py

@@ -0,0 +1,32 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from nucliadb.backups.const import StorageKeys
+from nucliadb.backups.settings import settings
+from nucliadb_utils.storages.storage import Storage
+
+
+async def exists_backup(storage: Storage, backup_id: str) -> bool:
+    async for _ in storage.iterate_objects(
+        bucket=settings.backups_bucket,
+        prefix=StorageKeys.BACKUP_PREFIX.format(backup_id=backup_id),
+    ):
+        return True
+    return False

nucliadb/export_import/utils.py

@@ -40,7 +40,7 @@ from nucliadb_protos import resources_pb2, writer_pb2
 from nucliadb_utils.const import Streams
 from nucliadb_utils.transaction import MaxTransactionSizeExceededError
 
-BinaryStream = AsyncGenerator[bytes, None]
+BinaryStream = AsyncIterator[bytes]
 BinaryStreamGenerator = Callable[[int], BinaryStream]
 
 
@@ -237,8 +237,11 @@ async def download_binary(
     context: ApplicationContext, cf: resources_pb2.CloudFile
 ) -> AsyncGenerator[bytes, None]:
     bucket_name = context.blob_storage.get_bucket_name_from_cf(cf)
+    downloaded_bytes = 0
     async for data in context.blob_storage.download(bucket_name, cf.uri):
         yield data
+        downloaded_bytes += len(data)
+    assert downloaded_bytes == cf.size, "Downloaded bytes do not match the expected size"
 
 
 async def get_entities(context: ApplicationContext, kbid: str) -> kb_pb2.EntitiesGroups:
@@ -416,6 +419,8 @@ class ExportStreamReader:
 class TaskRetryHandler:
     """
     Class that wraps an import/export task and adds retry logic to it.
+
+    TODO: This should be refactored to use generic task retry logic at tasks/retries.py::TaskRetryHandler
     """
 
     def __init__(

nucliadb/ingest/app.py

@@ -22,6 +22,7 @@ import importlib.metadata
 from typing import Awaitable, Callable
 
 from nucliadb import health
+from nucliadb.backups.tasks import initialize_consumers as initialize_backup_consumers
 from nucliadb.common.cluster.utils import setup_cluster, teardown_cluster
 from nucliadb.common.context import ApplicationContext
 from nucliadb.common.nidx import start_nidx_utility
@@ -154,6 +155,7 @@ async def main_subscriber_workers():  # pragma: no cover
     await exports_consumer.initialize(context)
     imports_consumer = get_imports_consumer()
     await imports_consumer.initialize(context)
+    backup_consumers_finalizers = await initialize_backup_consumers(context)
 
     await run_until_exit(
         [
@@ -165,7 +167,10 @@ async def main_subscriber_workers():  # pragma: no cover
             metrics_server.shutdown,
             grpc_health_finalizer,
             context.finalize,
+            exports_consumer.finalize,
+            imports_consumer.finalize,
         ]
+        + backup_consumers_finalizers
         + finalizers
     )
 
@@ -216,6 +221,7 @@ def run_subscriber_workers() -> None:  # pragma: no cover
         - audit fields subscriber
         - export/import subscriber
         - materializer subscriber
+        - backups subscribers
     """
     setup_configuration()
     asyncio.run(main_subscriber_workers())

nucliadb/ingest/service/writer.py

@@ -20,6 +20,8 @@
 import uuid
 from typing import AsyncIterator
 
+from nucliadb.backups import tasks as backup_tasks
+from nucliadb.backups import utils as backup_utils
 from nucliadb.common import datamanagers
 from nucliadb.common.cluster.exceptions import AlreadyExists, EntitiesGroupNotFound
 from nucliadb.common.cluster.manager import get_nidx_fake_node
@@ -471,14 +473,37 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
     async def CreateBackup(
         self, request: backups_pb2.CreateBackupRequest, context=None
     ) -> backups_pb2.CreateBackupResponse:
-        return backups_pb2.CreateBackupResponse()
+        if not await exists_kb(request.kbid):
+            return backups_pb2.CreateBackupResponse(
+                status=backups_pb2.CreateBackupResponse.Status.KB_NOT_FOUND
+            )
+        await backup_tasks.create(request.kbid, request.backup_id)
+        return backups_pb2.CreateBackupResponse(status=backups_pb2.CreateBackupResponse.Status.OK)
 
     async def DeleteBackup(
         self, request: backups_pb2.DeleteBackupRequest, context=None
     ) -> backups_pb2.DeleteBackupResponse:
-        return backups_pb2.DeleteBackupResponse()
+        if not await backup_utils.exists_backup(self.storage, request.backup_id):
+            return backups_pb2.DeleteBackupResponse(
+                status=backups_pb2.DeleteBackupResponse.Status.OK,
+            )
+        await backup_tasks.delete(request.backup_id)
+        return backups_pb2.DeleteBackupResponse(status=backups_pb2.DeleteBackupResponse.Status.OK)
 
     async def RestoreBackup(
         self, request: backups_pb2.RestoreBackupRequest, context=None
     ) -> backups_pb2.RestoreBackupResponse:
-        return backups_pb2.RestoreBackupResponse()
+        if not await exists_kb(request.kbid):
+            return backups_pb2.RestoreBackupResponse(
+                status=backups_pb2.RestoreBackupResponse.Status.NOT_FOUND
+            )
+        if not await backup_utils.exists_backup(self.storage, request.backup_id):
+            return backups_pb2.RestoreBackupResponse(
+                status=backups_pb2.RestoreBackupResponse.Status.NOT_FOUND
+            )
+        await backup_tasks.restore(request.kbid, request.backup_id)
+        return backups_pb2.RestoreBackupResponse(status=backups_pb2.RestoreBackupResponse.Status.OK)
+
+
+async def exists_kb(kbid: str) -> bool:
+    return await datamanagers.atomic.kb.exists_kb(kbid=kbid)

nucliadb/tasks/__init__.py

@@ -17,5 +17,5 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from .consumer import NatsTaskConsumer, create_consumer  # noqa
-from .producer import NatsTaskProducer, create_producer  # noqa
+from .consumer import NatsTaskConsumer, create_consumer  # noqa: F401
+from .producer import NatsTaskProducer, create_producer  # noqa: F401

nucliadb/tasks/retries.py

@@ -0,0 +1,148 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import functools
+import logging
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel
+
+from nucliadb.common.context import ApplicationContext
+
+logger = logging.getLogger(__name__)
+
+TASK_METADATA_MAINDB = "/kbs/{kbid}/tasks/{task_type}/{task_id}"
+
+
+class TaskMetadata(BaseModel):
+    class Status(Enum):
+        RUNNING = "running"
+        FAILED = "failed"
+        COMPLETED = "completed"
+
+    task_id: str
+    status: Status
+    retries: int = 0
+    error_messages: list[str] = []
+
+
+class TaskRetryHandler:
+    """
+    Class that wraps a task consumer function and handles retries by storing metadata in the KV store.
+
+    Example:
+
+        retry_handler = TaskRetryHandler(
+            kbid="kbid",
+            task_type="work",
+            task_id="task_id",
+            context=context,
+        )
+
+        @retry_handler.wrap
+        async def my_task_consumer_func(kbid: str, task_id: str):
+            pass
+
+    """
+
+    def __init__(
+        self,
+        kbid: str,
+        task_type: str,
+        task_id: str,
+        context: ApplicationContext,
+        max_retries: int = 5,
+    ):
+        self.kbid = kbid
+        self.task_type = task_type
+        self.task_id = task_id
+        self.max_retries = max_retries
+        self.context = context
+
+    @property
+    def metadata_key(self):
+        return TASK_METADATA_MAINDB.format(
+            kbid=self.kbid, task_type=self.task_type, task_id=self.task_id
+        )
+
+    async def get_metadata(self) -> Optional[TaskMetadata]:
+        async with self.context.kv_driver.transaction(read_only=True) as txn:
+            metadata = await txn.get(self.metadata_key)
+            if metadata is None:
+                return None
+            return TaskMetadata.model_validate_json(metadata)
+
+    async def set_metadata(self, metadata: TaskMetadata) -> None:
+        async with self.context.kv_driver.transaction() as txn:
+            await txn.set(self.metadata_key, metadata.model_dump_json().encode())
+            await txn.commit()
+
+    def wrap(self, func):
+        @functools.wraps(func)
+        async def wrapper(*args, **kwargs):
+            func_result = None
+            metadata = await self.get_metadata()
+            if metadata is None:
+                # Task is not scheduled yet
+                metadata = TaskMetadata(
+                    task_id=self.task_id,
+                    status=TaskMetadata.Status.RUNNING,
+                    retries=0,
+                )
+                await self.set_metadata(metadata)
+
+            if metadata.status in (TaskMetadata.Status.COMPLETED, TaskMetadata.Status.FAILED):
+                logger.info(
+                    f"{self.type} task is {metadata.status.value}. Skipping",
+                    extra={"kbid": self.kbid, "task_type": self.task_type, "task_id": self.task_id},
+                )
+                return
+
+            if metadata.retries >= self.max_retries:
+                metadata.status = TaskMetadata.Status.FAILED
+                metadata.error_messages.append("Max retries reached")
+                logger.info(
+                    f"Task reached max retries. Setting to FAILED state",
+                    extra={"kbid": self.kbid, "task_type": self.task_type, "task_id": self.task_id},
+                )
+                await self.set_metadata(metadata)
+                return
+            try:
+                metadata.status = TaskMetadata.Status.RUNNING
+                func_result = await func(*args, **kwargs)
+            except Exception as ex:
+                metadata.retries += 1
+                metadata.error_messages.append(str(ex))
+                logger.info(
+                    f"Task failed. Will be retried",
+                    extra={"kbid": self.kbid, "task_type": self.task_type, "task_id": self.task_id},
+                )
+                raise
+            else:
+                logger.info(
+                    f"Task finished successfully",
+                    extra={"kbid": self.kbid, "task_type": self.task_type, "task_id": self.task_id},
+                )
+                metadata.status = TaskMetadata.Status.COMPLETED
+                return func_result
+            finally:
+                await self.set_metadata(metadata)
+
+        return wrapper

{nucliadb-6.3.1.post3526.dist-info → nucliadb-6.3.1.post3544.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.3.1.post3526
+Version: 6.3.1.post3544
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.1.post3526
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.1.post3526
-Requires-Dist: nucliadb-protos>=6.3.1.post3526
-Requires-Dist: nucliadb-models>=6.3.1.post3526
-Requires-Dist: nidx-protos>=6.3.1.post3526
+Requires-Dist: nucliadb-telemetry[all]>=6.3.1.post3544
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.1.post3544
+Requires-Dist: nucliadb-protos>=6.3.1.post3544
+Requires-Dist: nucliadb-models>=6.3.1.post3544
+Requires-Dist: nidx-protos>=6.3.1.post3544
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.3.1.post3526.dist-info → nucliadb-6.3.1.post3544.dist-info}/RECORD

@@ -39,6 +39,15 @@ nucliadb/learning_proxy.py,sha256=Gf76qXxjl1lrHEFaCpOUfjjf0ab6eGLNxLMJz3-M_mo,19
 nucliadb/metrics_exporter.py,sha256=6u0geEYFxgE5I2Fhl_sxsvGN-ZkaFZNGutSXwrzrsVs,5624
 nucliadb/openapi.py,sha256=wDiw0dVEvTpJvbatkJ0JZLkKm9RItZT5PWRHjqRfqTA,2272
 nucliadb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nucliadb/backups/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
+nucliadb/backups/const.py,sha256=llb5TaC53Ce6BMmlPKPUrVhVGl7uQrqv_Vle-P4GET4,1673
+nucliadb/backups/create.py,sha256=4RsvwY808X22kr06_uiASz6hEqmMirWQZgA_RoTbkqw,10440
+nucliadb/backups/delete.py,sha256=1rnBhVUGYYZJXSZUrrgYMDZ5NyswEWkIA-G-crRCyHk,2404
+nucliadb/backups/models.py,sha256=13-Z4p-Ypjdtg5NuDE2m-09CTdFYHh-W6U9FyWSEhPA,1270
+nucliadb/backups/restore.py,sha256=X-Ai5HjujNWIjqxegDaJp33dSUIDaTzJ3K8n_heeDeo,9702
+nucliadb/backups/settings.py,sha256=SyzsInj1BRbBI0atg5IXWbMbOZ_eVg4eSQ3IcnUhCxQ,1357
+nucliadb/backups/tasks.py,sha256=e0J85c7RjqYO92hcG9GT_g_LK-enyisWuSWxAUl5IZE,4528
+nucliadb/backups/utils.py,sha256=ayDaxfWP5cPnAkQH-tF4M6cnowsPQgU2ljYz_iL1CbE,1249
 nucliadb/common/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/constants.py,sha256=QpigxJh_CtD85Evy0PtV5cVq6x0U_f9xfIcXz1ymkUg,869
 nucliadb/common/counters.py,sha256=8lOi3A2HeLDDlcNaS2QT1SfD3350VPBjiY3FkmHH1V8,977
@@ -101,9 +110,9 @@ nucliadb/export_import/exporter.py,sha256=k2QVx1EjqFlDYiggriWiEJzwtMXzHbldsqWdpG
 nucliadb/export_import/importer.py,sha256=v5cq9Nn8c2zrY_K_00mydR52f8mdFxR7tLdtNLQ0qvk,4229
 nucliadb/export_import/models.py,sha256=dbjScNkiMRv4X3Ktudy1JRliD25bfoDTy3JmEZgQSCc,2121
 nucliadb/export_import/tasks.py,sha256=yPNdBdvTD7eGc7zvV9Rp7UZ0-mDhA34OOsLqHvns_v0,2975
-nucliadb/export_import/utils.py,sha256=t7xLA3f5W3zGq3HNXe3noOQnY7gRO8TAoe8S8BG3_so,19733
+nucliadb/export_import/utils.py,sha256=iAQAjYuNx0dhM2b5-1A0NEs8tSRsznuT-izysUrTwS0,19986
 nucliadb/ingest/__init__.py,sha256=fsw3C38VP50km3R-nHL775LNGPpJ4JxqXJ2Ib1f5SqE,1011
-nucliadb/ingest/app.py,sha256=ErNd3q44xbBcUOl-Ae_lvcKPAsfFMSzb8zqWAjekrM4,7097
+nucliadb/ingest/app.py,sha256=rX1KE5vsAzG9hlArBk8WE2SOlvdYylcb-jNkMQNPJdQ,7407
 nucliadb/ingest/cache.py,sha256=w7jMMzamOmQ7gwXna6Dqm6isRNBVv6l5BTBlTxaYWjE,1005
 nucliadb/ingest/partitions.py,sha256=2NIhMYbNT0TNBL6bX1UMSi7vxFGICstCKEqsB0TXHOE,2410
 nucliadb/ingest/processing.py,sha256=8OggvuxNzktTTKDTUwsIuazhDParEWhn46CBZaMYAy8,20659
@@ -145,7 +154,7 @@ nucliadb/ingest/orm/processor/processor.py,sha256=oaiZ9HUszhUbvNMCmDq5Xj_jtXiCDj
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=uqEphtI1Ir_yk9jRl2gPf7BlzzXWovbARY5MNZSBI_8,1704
 nucliadb/ingest/service/__init__.py,sha256=MME_G_ERxzJR6JW_hfE2qcfXpmpH1kdG-S0a-M0qRm8,2043
 nucliadb/ingest/service/exceptions.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/ingest/service/writer.py,sha256=69xsQgoWngfZfWFTKPbePi-ya4fP5T7dTiomLr1gSYw,20953
+nucliadb/ingest/service/writer.py,sha256=XISw3m5joj30gKe9CfpyybXMwgWOueAcgfiHRnX4Cqc,22311
 nucliadb/middleware/__init__.py,sha256=A8NBlBuEkunCFMKpR9gnfNELsVn0Plc55BIQMbWDM8Q,2202
 nucliadb/migrator/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/migrator/command.py,sha256=dKbJ1tAmP6X4lMVRSSlz351euaqs2wBPpOczLjATUes,2089
@@ -260,11 +269,12 @@ nucliadb/standalone/versions.py,sha256=8CxNMNt2NgWM8ct50UsR4d44-ae7wtQI-sV-yGiFq
 nucliadb/standalone/static/favicon.ico,sha256=96pKGp6Sx457JkTfjy1dtApMhkitixfU6invCUGAYOU,2285
 nucliadb/standalone/static/index.html,sha256=PEZfuEQFYnYACAL1ceN8xC0im8lBrUx838RkE8tbvgA,3833
 nucliadb/standalone/static/logo.svg,sha256=-wQqSvPGTdlKjUP6pHE6kiq005pgYjDzp9nPl0X71Mk,2639
-nucliadb/tasks/__init__.py,sha256=vruCOMmCu0BcAZQrEinlgiQtiR1WYxSxvI5UsydAopc,963
+nucliadb/tasks/__init__.py,sha256=oFJ3A8HD7w11mBu-IixYE_KxA7juMGlYQb7YD_y6WPM,975
 nucliadb/tasks/consumer.py,sha256=x-999Nsw6lBcKvyGyCGPiGP_naANVYMfl9M-u0U3mhY,7052
 nucliadb/tasks/logger.py,sha256=C7keOEO_mjLVp5VbqAZ2QXfqVB2Hot7NgBlUP_SDSMw,924
 nucliadb/tasks/models.py,sha256=qrZKi5DNDQ07waMsp5L4_Fi7WRs57YiO-kmXlrBzEAA,1168
 nucliadb/tasks/producer.py,sha256=w4R1YXgXtmCPGcoNNOr3qkqJYcHJtSmix-xjt7vsPqk,3261
+nucliadb/tasks/retries.py,sha256=Zv-3Hys-SKayG9VQ7_7EIflkegE5j-xPGrf-nwaxsfY,5075
 nucliadb/tasks/utils.py,sha256=6tQVckqyzxv8PhVAd3ZqcMYpGcn73ZY6p1cpm1FxagA,1214
 nucliadb/tests/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/tests/config.py,sha256=JN_Jhgj-fwM9_8IeO9pwxr6C1PiwRDrXxm67Y38rU30,2080
@@ -339,8 +349,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.3.1.post3526.dist-info/METADATA,sha256=eyuN6ZoyLypgZCawSpKtbge-d3fzg3KatkY51mDYxAc,4291
-nucliadb-6.3.1.post3526.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-nucliadb-6.3.1.post3526.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.3.1.post3526.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.3.1.post3526.dist-info/RECORD,,
+nucliadb-6.3.1.post3544.dist-info/METADATA,sha256=vKxvrIe5oh3QKwI-JxIAA8NJ2V0bwXCdg6K_O4wOCFg,4291
+nucliadb-6.3.1.post3544.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+nucliadb-6.3.1.post3544.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.1.post3544.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.1.post3544.dist-info/RECORD,,