nucliadb 6.3.4.post3645__py3-none-any.whl → 6.3.4.post3663__py3-none-any.whl

nucliadb/backups/const.py

@@ -37,6 +37,8 @@ class StorageKeys:
     RESOURCE = "backups/{backup_id}/resources/{resource_id}.tar"
     ENTITIES = "backups/{backup_id}/entities.pb"
     LABELS = "backups/{backup_id}/labels.pb"
+    SYNONYMS = "backups/{backup_id}/synonyms.pb"
+    SEARCH_CONFIGURATIONS = "backups/{backup_id}/search_configurations.pb"
 
 
 class BackupFinishedStream:

nucliadb/backups/create.py

@@ -18,6 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import asyncio
+import json
 import logging
 import tarfile
 from datetime import datetime, timezone
@@ -38,6 +39,8 @@ from nucliadb.export_import.utils import (
     get_cloud_files,
     get_entities,
     get_labels,
+    get_search_configurations,
+    get_synonyms,
 )
 from nucliadb.tasks.retries import TaskRetryHandler
 from nucliadb_protos import backups_pb2, resources_pb2, writer_pb2
@@ -74,6 +77,8 @@ async def backup_kb(context: ApplicationContext, kbid: str, backup_id: str):
     await backup_resources(context, kbid, backup_id)
     await backup_labels(context, kbid, backup_id)
     await backup_entities(context, kbid, backup_id)
+    await backup_synonyms(context, kbid, backup_id)
+    await backup_search_configurations(context, kbid, backup_id)
     await notify_backup_completed(context, kbid, backup_id)
     await delete_metadata(context, kbid, backup_id)
 
@@ -216,7 +221,7 @@ async def backup_resource_with_binaries(
     await upload_to_bucket(
         context,
         resource_data_iterator(),
-        key=StorageKeys.RESOURCE.format(kbid=kbid, backup_id=backup_id, resource_id=rid),
+        key=StorageKeys.RESOURCE.format(backup_id=backup_id, resource_id=rid),
     )
     return total_size
 
@@ -225,7 +230,7 @@ async def backup_labels(context: ApplicationContext, kbid: str, backup_id: str):
     labels = await get_labels(context, kbid)
     await context.blob_storage.upload_object(
         bucket=settings.backups_bucket,
-        key=StorageKeys.LABELS.format(kbid=kbid, backup_id=backup_id),
+        key=StorageKeys.LABELS.format(backup_id=backup_id),
         data=labels.SerializeToString(),
     )
 
@@ -234,11 +239,33 @@ async def backup_entities(context: ApplicationContext, kbid: str, backup_id: str
     entities = await get_entities(context, kbid)
     await context.blob_storage.upload_object(
         bucket=settings.backups_bucket,
-        key=StorageKeys.ENTITIES.format(kbid=kbid, backup_id=backup_id),
+        key=StorageKeys.ENTITIES.format(backup_id=backup_id),
         data=entities.SerializeToString(),
     )
 
 
+async def backup_synonyms(context: ApplicationContext, kbid: str, backup_id: str):
+    synonyms = await get_synonyms(context, kbid)
+    await context.blob_storage.upload_object(
+        bucket=settings.backups_bucket,
+        key=StorageKeys.SYNONYMS.format(backup_id=backup_id),
+        data=synonyms.SerializeToString(),
+    )
+
+
+async def backup_search_configurations(context: ApplicationContext, kbid: str, backup_id: str):
+    search_configurations = await get_search_configurations(context, kbid=kbid)
+    serialized_search_configs = {
+        config_id: config.model_dump(mode="python", exclude_unset=True)
+        for config_id, config in search_configurations.items()
+    }
+    await context.blob_storage.upload_object(
+        bucket=settings.backups_bucket,
+        key=StorageKeys.SEARCH_CONFIGURATIONS.format(backup_id=backup_id),
+        data=json.dumps(serialized_search_configs).encode(),
+    )
+
+
 async def get_metadata(
     context: ApplicationContext, kbid: str, backup_id: str
 ) -> Optional[BackupMetadata]:

nucliadb/backups/restore.py

@@ -21,9 +21,12 @@
 
 import asyncio
 import functools
+import json
 import logging
 import tarfile
-from typing import AsyncIterator, Callable, Optional, Union
+from typing import Any, AsyncIterator, Callable, Optional, Union
+
+from pydantic import TypeAdapter
 
 from nucliadb.backups.const import MaindbKeys, StorageKeys
 from nucliadb.backups.models import RestoreBackupRequest
@@ -34,8 +37,11 @@ from nucliadb.export_import.utils import (
     restore_broker_message,
     set_entities_groups,
     set_labels,
+    set_search_configurations,
+    set_synonyms,
 )
 from nucliadb.tasks.retries import TaskRetryHandler
+from nucliadb_models.configuration import SearchConfiguration
 from nucliadb_protos import knowledgebox_pb2 as kb_pb2
 from nucliadb_protos.resources_pb2 import CloudFile
 from nucliadb_protos.writer_pb2 import BrokerMessage
@@ -69,6 +75,8 @@ async def restore_kb(context: ApplicationContext, kbid: str, backup_id: str):
     await restore_resources(context, kbid, backup_id)
     await restore_labels(context, kbid, backup_id)
     await restore_entities(context, kbid, backup_id)
+    await restore_synonyms(context, kbid, backup_id)
+    await restore_search_configurations(context, kbid, backup_id)
     await delete_last_restored(context, kbid, backup_id)
 
 
@@ -77,7 +85,7 @@ async def restore_resources(context: ApplicationContext, kbid: str, backup_id: s
     tasks = []
     async for object_info in context.blob_storage.iterate_objects(
         bucket=settings.backups_bucket,
-        prefix=StorageKeys.RESOURCES_PREFIX.format(kbid=kbid, backup_id=backup_id),
+        prefix=StorageKeys.RESOURCES_PREFIX.format(backup_id=backup_id),
         start=last_restored,
     ):
         key = object_info.name
@@ -210,7 +218,7 @@ class ResourceBackupReader:
 async def restore_resource(context: ApplicationContext, kbid: str, backup_id: str, resource_id: str):
     download_stream = context.blob_storage.download(
         bucket=settings.backups_bucket,
-        key=StorageKeys.RESOURCE.format(kbid=kbid, backup_id=backup_id, resource_id=resource_id),
+        key=StorageKeys.RESOURCE.format(backup_id=backup_id, resource_id=resource_id),
     )
     reader = ResourceBackupReader(download_stream)
     bm = None
@@ -242,7 +250,7 @@ async def restore_resource(context: ApplicationContext, kbid: str, backup_id: st
 async def restore_labels(context: ApplicationContext, kbid: str, backup_id: str):
     raw = await context.blob_storage.downloadbytes(
         bucket=settings.backups_bucket,
-        key=StorageKeys.LABELS.format(kbid=kbid, backup_id=backup_id),
+        key=StorageKeys.LABELS.format(backup_id=backup_id),
     )
     labels = kb_pb2.Labels()
     labels.ParseFromString(raw.getvalue())
@@ -252,8 +260,31 @@ async def restore_labels(context: ApplicationContext, kbid: str, backup_id: str)
 async def restore_entities(context: ApplicationContext, kbid: str, backup_id: str):
     raw = await context.blob_storage.downloadbytes(
         bucket=settings.backups_bucket,
-        key=StorageKeys.ENTITIES.format(kbid=kbid, backup_id=backup_id),
+        key=StorageKeys.ENTITIES.format(backup_id=backup_id),
     )
     entities = kb_pb2.EntitiesGroups()
     entities.ParseFromString(raw.getvalue())
     await set_entities_groups(context, kbid, entities)
+
+
+async def restore_synonyms(context: ApplicationContext, kbid: str, backup_id: str):
+    raw = await context.blob_storage.downloadbytes(
+        bucket=settings.backups_bucket,
+        key=StorageKeys.SYNONYMS.format(backup_id=backup_id),
+    )
+    synonyms = kb_pb2.Synonyms()
+    synonyms.ParseFromString(raw.getvalue())
+    await set_synonyms(context, kbid, synonyms)
+
+
+async def restore_search_configurations(context: ApplicationContext, kbid: str, backup_id: str):
+    raw = await context.blob_storage.downloadbytes(
+        bucket=settings.backups_bucket,
+        key=StorageKeys.SEARCH_CONFIGURATIONS.format(backup_id=backup_id),
+    )
+    as_dict: dict[str, dict[str, Any]] = json.loads(raw.getvalue())
+    search_configurations: dict[str, SearchConfiguration] = {}
+    for name, data in as_dict.items():
+        config: SearchConfiguration = TypeAdapter(SearchConfiguration).validate_python(data)
+        search_configurations[name] = config
+    await set_search_configurations(context, kbid, search_configurations)

nucliadb/export_import/utils.py

@@ -34,6 +34,7 @@ from nucliadb.export_import.exceptions import (
 )
 from nucliadb.export_import.models import ExportedItemType, ExportItem, Metadata
 from nucliadb.ingest.orm.broker_message import generate_broker_message
+from nucliadb_models.configuration import SearchConfiguration
 from nucliadb_models.export_import import Status
 from nucliadb_protos import knowledgebox_pb2 as kb_pb2
 from nucliadb_protos import resources_pb2, writer_pb2
@@ -45,23 +46,61 @@ BinaryStream = AsyncIterator[bytes]
 BinaryStreamGenerator = Callable[[int], BinaryStream]
 
 
-# Broker message fields that are populated by the processing pipeline
-PROCESSING_BM_FIELDS = [
-    "link_extracted_data",
-    "file_extracted_data",
-    "extracted_text",
-    "field_metadata",
-    "field_vectors",
-    "field_large_metadata",
-]
-
-# Broker message fields that are populated by the nucliadb writer component
-WRITER_BM_FIELDS = [
-    "links",
-    "files",
-    "texts",
-    "conversations",
-]
+# Map that indicates which fields are written by the writer
+# and which are written by the processor.
+BM_FIELDS = {
+    "common": [
+        "kbid",
+        "uuid",
+        "type",
+        "source",
+        "reindex",
+    ],
+    "writer": [
+        "slug",
+        "basic",
+        "origin",
+        "user_relations",
+        "conversations",
+        "texts",
+        "links",
+        "files",
+        "extra",
+        "security",
+    ],
+    "processor": [
+        "link_extracted_data",
+        "file_extracted_data",
+        "extracted_text",
+        "field_metadata",
+        "field_vectors",
+        "field_large_metadata",
+        "question_answers",
+        "relations",
+        "field_statuses",
+    ],
+    # These fields are mostly used for internal purposes and they are not part of
+    # the representation of the exported resource as broker message.
+    "ignored": [
+        "audit",
+        "multiid",
+        "origin_seq",
+        "slow_processing_time",
+        "pre_processing_time",
+        "done_time",
+        "processing_id",
+        "account_seq",
+        "delete_fields",
+        "delete_question_answers",
+        "errors",
+        "generated_by",
+    ],
+    # No longer used fields
+    "deprecated": [
+        "txseqid",
+        "user_vectors",
+    ],
+}
 
 
 async def import_broker_message(
@@ -125,7 +164,7 @@ async def transaction_commit(
 def get_writer_bm(bm: writer_pb2.BrokerMessage) -> writer_pb2.BrokerMessage:
     wbm = writer_pb2.BrokerMessage()
     wbm.CopyFrom(bm)
-    for field in PROCESSING_BM_FIELDS:
+    for field in BM_FIELDS["processor"]:
         wbm.ClearField(field)  # type: ignore
     wbm.type = writer_pb2.BrokerMessage.MessageType.AUTOCOMMIT
     wbm.source = writer_pb2.BrokerMessage.MessageSource.WRITER
@@ -135,7 +174,7 @@ def get_writer_bm(bm: writer_pb2.BrokerMessage) -> writer_pb2.BrokerMessage:
 def get_processor_bm(bm: writer_pb2.BrokerMessage) -> writer_pb2.BrokerMessage:
     pbm = writer_pb2.BrokerMessage()
     pbm.CopyFrom(bm)
-    for field in WRITER_BM_FIELDS:
+    for field in BM_FIELDS["writer"]:
         pbm.ClearField(field)  # type: ignore
     pbm.type = writer_pb2.BrokerMessage.MessageType.AUTOCOMMIT
     pbm.source = writer_pb2.BrokerMessage.MessageSource.PROCESSOR
@@ -173,6 +212,21 @@ async def set_entities_groups(
         await txn.commit()
 
 
+async def set_synonyms(context: ApplicationContext, kbid: str, synonyms: kb_pb2.Synonyms) -> None:
+    async with datamanagers.with_transaction() as txn:
+        await datamanagers.synonyms.set(txn, kbid=kbid, synonyms=synonyms)
+        await txn.commit()
+
+
+async def set_search_configurations(
+    context: ApplicationContext, kbid: str, search_configurations: dict[str, SearchConfiguration]
+) -> None:
+    async with datamanagers.with_transaction() as txn:
+        for name, config in search_configurations.items():
+            await datamanagers.search_configurations.set(txn, kbid=kbid, name=name, config=config)
+        await txn.commit()
+
+
 async def set_labels(context: ApplicationContext, kbid: str, labels: kb_pb2.Labels) -> None:
     async with datamanagers.with_transaction() as txn:
         await datamanagers.labels.set_labels(txn, kbid=kbid, labels=labels)
@@ -273,6 +327,18 @@ async def get_labels(context: ApplicationContext, kbid: str) -> kb_pb2.Labels:
         return await datamanagers.labels.get_labels(txn, kbid=kbid)
 
 
+async def get_synonyms(context: ApplicationContext, kbid: str) -> kb_pb2.Synonyms:
+    async with datamanagers.with_ro_transaction() as txn:
+        return await datamanagers.synonyms.get(txn, kbid=kbid) or kb_pb2.Synonyms()
+
+
+async def get_search_configurations(
+    context: ApplicationContext, kbid: str
+) -> dict[str, SearchConfiguration]:
+    async with datamanagers.with_ro_transaction() as txn:
+        return await datamanagers.search_configurations.list(txn, kbid=kbid)
+
+
 class EndOfStream(Exception): ...
 
 

nucliadb/ingest/orm/broker_message.py

@@ -26,6 +26,7 @@ from nucliadb.ingest.fields.conversation import Conversation
 from nucliadb.ingest.fields.file import File
 from nucliadb.ingest.fields.link import Link
 from nucliadb.ingest.orm.resource import Resource
+from nucliadb_protos import writer_pb2
 from nucliadb_protos.knowledgebox_pb2 import VectorSetConfig
 from nucliadb_protos.resources_pb2 import (
     ExtractedTextWrapper,
@@ -34,7 +35,7 @@ from nucliadb_protos.resources_pb2 import (
     FieldType,
     LargeComputedMetadataWrapper,
 )
-from nucliadb_protos.writer_pb2 import BrokerMessage
+from nucliadb_protos.writer_pb2 import BrokerMessage, FieldIDStatus
 
 
 async def generate_broker_message(resource: Resource) -> BrokerMessage:
@@ -102,8 +103,25 @@ class _BrokerMessageBuilder:
             # Large metadata
             await self.generate_field_large_computed_metadata(type_id, field_id, field)
 
+            # Field status
+            await self.generate_field_status(type_id, field_id, field)
+
         return self.bm
 
+    async def generate_field_status(
+        self,
+        type_id: FieldType.ValueType,
+        field_id: str,
+        field: Field,
+    ):
+        fid = writer_pb2.FieldID(field_type=type_id, field=field_id)
+        status = await field.get_status()
+        if status is not None:
+            field_id_status = FieldIDStatus()
+            field_id_status.id.CopyFrom(fid)
+            field_id_status.status = status.status
+            self.bm.field_statuses.append(field_id_status)
+
     async def generate_field(
         self,
         type_id: FieldType.ValueType,

nucliadb/ingest/processing.py

@@ -19,6 +19,7 @@
 #
 import base64
 import datetime
+import json
 import logging
 import uuid
 from collections import defaultdict
@@ -32,6 +33,7 @@ import jwt
 from pydantic import BaseModel, Field
 
 import nucliadb_models as models
+from nucliadb_models.labels import ClassificationLabel
 from nucliadb_models.resource import QueueType
 from nucliadb_protos.resources_pb2 import CloudFile
 from nucliadb_protos.resources_pb2 import FieldFile as FieldFilePB
@@ -93,7 +95,10 @@ class PushPayload(BaseModel):
     genericfield: dict[str, models.Text] = {}
 
     # New File
-    filefield: dict[str, str] = {}
+    filefield: dict[str, str] = Field(
+        default={},
+        description="Map of each file field to the jwt token computed in ProcessingEngine methods",
+    )
 
     # New Link
     linkfield: dict[str, models.LinkUpload] = {}
@@ -238,7 +243,9 @@ class ProcessingEngine:
         }
         return jwt.encode(payload, self.nuclia_jwt_key, algorithm="HS256")
 
-    def generate_file_token_from_fieldfile(self, file: FieldFilePB) -> str:
+    def generate_file_token_from_fieldfile(
+        self, file: FieldFilePB, classif_labels: Optional[list[ClassificationLabel]] = None
+    ) -> str:
         if self.nuclia_jwt_key is None:
             raise AttributeError("Nuclia JWT key not set")
         now = datetime.datetime.now(tz=datetime.timezone.utc)
@@ -263,6 +270,8 @@ class ProcessingEngine:
             "language": file.language,
             "extract_strategy": file.extract_strategy,
         }
+        if classif_labels:
+            payload["classification_labels"] = self.encode_classif_labels(classif_labels)
         return jwt.encode(payload, self.nuclia_jwt_key, algorithm="HS256")
 
     @backoff.on_exception(
@@ -272,7 +281,9 @@ class ProcessingEngine:
         max_tries=MAX_TRIES,
     )
     @processing_observer.wrap({"type": "file_field_upload"})
-    async def convert_filefield_to_str(self, file: models.FileField) -> str:
+    async def convert_filefield_to_str(
+        self, file: models.FileField, classif_labels: Optional[list[ClassificationLabel]] = None
+    ) -> str:
         # Upload file without storing on Nuclia DB
         headers = {}
         headers["X-PASSWORD"] = file.password
@@ -281,6 +292,8 @@ class ProcessingEngine:
         headers["X-MD5"] = file.file.md5
         if file.extract_strategy is not None:
             headers["X-EXTRACT-STRATEGY"] = file.extract_strategy
+        if classif_labels:
+            headers["X-CLASSIFICATION-LABELS"] = self.encode_classif_labels(classif_labels)
         headers["CONTENT_TYPE"] = file.file.content_type
         headers["CONTENT-LENGTH"] = str(len(file.file.payload))  # type: ignore
         headers["X-STF-NUAKEY"] = f"Bearer {self.nuclia_service_account}"
@@ -299,7 +312,14 @@ class ProcessingEngine:
                 text = await resp.text()
                 raise Exception(f"STATUS: {resp.status} - {text}")
 
-    def convert_external_filefield_to_str(self, file_field: models.FileField) -> str:
+    def encode_classif_labels(self, classif_labels: list[ClassificationLabel]) -> str:
+        return base64.b64encode(
+            json.dumps([label.model_dump(mode="python") for label in classif_labels]).encode()
+        ).decode()
+
+    def convert_external_filefield_to_str(
+        self, file_field: models.FileField, classif_labels: Optional[list[ClassificationLabel]] = None
+    ) -> str:
         if self.nuclia_jwt_key is None:
             raise AttributeError("Nuclia JWT key not set")
 
@@ -322,6 +342,8 @@ class ProcessingEngine:
             "password": file_field.password,
             "extract_strategy": file_field.extract_strategy,
         }
+        if classif_labels:
+            payload["classification_labels"] = self.encode_classif_labels(classif_labels)
         return jwt.encode(payload, self.nuclia_jwt_key, algorithm="HS256")
 
     @backoff.on_exception(
@@ -331,11 +353,16 @@ class ProcessingEngine:
         max_tries=MAX_TRIES,
     )
     @processing_observer.wrap({"type": "file_field_upload_internal"})
-    async def convert_internal_filefield_to_str(self, file: FieldFilePB, storage: Storage) -> str:
+    async def convert_internal_filefield_to_str(
+        self,
+        file: FieldFilePB,
+        storage: Storage,
+        classif_labels: Optional[list[ClassificationLabel]] = None,
+    ) -> str:
         """It's already an internal file that needs to be uploaded"""
         if self.onprem is False:
             # Upload the file to processing upload
-            jwttoken = self.generate_file_token_from_fieldfile(file)
+            jwttoken = self.generate_file_token_from_fieldfile(file, classif_labels)
         else:
             headers = {}
             headers["X-PASSWORD"] = file.password
@@ -347,6 +374,8 @@ class ProcessingEngine:
                 headers["CONTENT-LENGTH"] = str(file.file.size)
             if file.extract_strategy != "":
                 headers["X-EXTRACT-STRATEGY"] = file.extract_strategy
+            if classif_labels:
+                headers["X-CLASSIFICATION-LABELS"] = self.encode_classif_labels(classif_labels)
             headers["X-STF-NUAKEY"] = f"Bearer {self.nuclia_service_account}"
 
             iterator = storage.downloadbytescf_iterator(file.file)
@@ -488,22 +517,31 @@ class DummyProcessingEngine(ProcessingEngine):
     async def finalize(self):
         pass
 
-    async def convert_filefield_to_str(self, file: models.FileField) -> str:
+    async def convert_filefield_to_str(
+        self, file: models.FileField, classif_labels: Optional[list[ClassificationLabel]] = None
+    ) -> str:
         self.calls.append([file])
         index = len(self.values["convert_filefield_to_str"])
-        self.values["convert_filefield_to_str"].append(file)
+        self.values["convert_filefield_to_str"].append((file, classif_labels))
         return f"convert_filefield_to_str,{index}"
 
-    def convert_external_filefield_to_str(self, file_field: models.FileField) -> str:
+    def convert_external_filefield_to_str(
+        self, file_field: models.FileField, classif_labels: Optional[list[ClassificationLabel]] = None
+    ) -> str:
         self.calls.append([file_field])
         index = len(self.values["convert_external_filefield_to_str"])
-        self.values["convert_external_filefield_to_str"].append(file_field)
+        self.values["convert_external_filefield_to_str"].append((file_field, classif_labels))
         return f"convert_external_filefield_to_str,{index}"
 
-    async def convert_internal_filefield_to_str(self, file: FieldFilePB, storage: Storage) -> str:
+    async def convert_internal_filefield_to_str(
+        self,
+        file: FieldFilePB,
+        storage: Storage,
+        classif_labels: Optional[list[ClassificationLabel]] = None,
+    ) -> str:
         self.calls.append([file, storage])
         index = len(self.values["convert_internal_filefield_to_str"])
-        self.values["convert_internal_filefield_to_str"].append([file, storage])
+        self.values["convert_internal_filefield_to_str"].append((file, storage, classif_labels))
         return f"convert_internal_filefield_to_str,{index}"
 
     async def convert_internal_cf_to_str(self, cf: CloudFile, storage: Storage) -> str:

nucliadb/writer/api/v1/field.py

@@ -43,7 +43,10 @@ from nucliadb.writer.api.v1.router import KB_PREFIX, RESOURCE_PREFIX, RSLUG_PREF
 from nucliadb.writer.back_pressure import maybe_back_pressure
 from nucliadb.writer.resource.audit import parse_audit
 from nucliadb.writer.resource.field import (
+    ResourceClassifications,
+    atomic_get_stored_resource_classifications,
     extract_file_field,
+    get_stored_resource_classifications,
     parse_conversation_field,
     parse_file_field,
     parse_link_field,
@@ -114,11 +117,31 @@ async def add_field_to_resource(
 
     parse_audit(writer.audit, request)
 
+    resource_classifications = await atomic_get_stored_resource_classifications(kbid=kbid, rid=rid)
+
     parse_field = FIELD_PARSERS_MAP[type(field_payload)]
     if iscoroutinefunction(parse_field):
-        await parse_field(kbid, rid, field_id, field_payload, writer, toprocess, **parser_kwargs)
+        await parse_field(
+            kbid,
+            rid,
+            field_id,
+            field_payload,
+            writer,
+            toprocess,
+            resource_classifications,
+            **parser_kwargs,
+        )
     else:
-        parse_field(kbid, rid, field_id, field_payload, writer, toprocess, **parser_kwargs)
+        parse_field(
+            kbid,
+            rid,
+            field_id,
+            field_payload,
+            writer,
+            toprocess,
+            resource_classifications,
+            **parser_kwargs,
+        )
 
     processing = get_processing()
     await transaction.commit(writer, partition)
@@ -200,8 +223,9 @@ def parse_text_field_adapter(
     field_payload: models.TextField,
     writer: BrokerMessage,
     toprocess: PushPayload,
+    resource_classifications: ResourceClassifications,
 ):
-    return parse_text_field(field_id, field_payload, writer, toprocess)
+    return parse_text_field(field_id, field_payload, writer, toprocess, resource_classifications)
 
 
 def parse_link_field_adapter(
@@ -211,8 +235,9 @@ def parse_link_field_adapter(
     field_payload: models.LinkField,
     writer: BrokerMessage,
     toprocess: PushPayload,
+    resource_classifications: ResourceClassifications,
 ):
-    return parse_link_field(field_id, field_payload, writer, toprocess)
+    return parse_link_field(field_id, field_payload, writer, toprocess, resource_classifications)
 
 
 async def parse_conversation_field_adapter(
@@ -222,8 +247,11 @@ async def parse_conversation_field_adapter(
     field_payload: models.InputConversationField,
     writer: BrokerMessage,
     toprocess: PushPayload,
+    resource_classifications: ResourceClassifications,
 ):
-    return await parse_conversation_field(field_id, field_payload, writer, toprocess, kbid, rid)
+    return await parse_conversation_field(
+        field_id, field_payload, writer, toprocess, kbid, rid, resource_classifications
+    )
 
 
 async def parse_file_field_adapter(
@@ -233,14 +261,22 @@ async def parse_file_field_adapter(
     field_payload: models.FileField,
     writer: BrokerMessage,
     toprocess: PushPayload,
+    resource_classifications: ResourceClassifications,
     skip_store: bool,
 ):
     return await parse_file_field(
-        field_id, field_payload, writer, toprocess, kbid, rid, skip_store=skip_store
+        field_id,
+        field_payload,
+        writer,
+        toprocess,
+        kbid,
+        rid,
+        resource_classifications,
+        skip_store=skip_store,
     )
 
 
-FIELD_PARSERS_MAP: dict[Type, Union[Callable]] = {
+FIELD_PARSERS_MAP: dict[Type, Callable] = {
     models.TextField: parse_text_field_adapter,
     models.LinkField: parse_link_field_adapter,
     models.InputConversationField: parse_conversation_field_adapter,
@@ -537,12 +573,15 @@ async def reprocess_file_field(
         if resource.basic is not None:
             toprocess.title = resource.basic.title
 
+        rclassif = await get_stored_resource_classifications(txn, kbid=kbid, rid=rid)
+
         try:
             await extract_file_field(
                 field_id,
                 resource=resource,
                 toprocess=toprocess,
                 password=x_file_password,
+                resource_classifications=rclassif,
             )
         except KeyError:
             raise HTTPException(status_code=404, detail="Field does not exist")

nucliadb/writer/api/v1/resource.py

@@ -50,10 +50,16 @@ from nucliadb.writer.resource.audit import parse_audit
 from nucliadb.writer.resource.basic import (
     parse_basic_creation,
     parse_basic_modify,
+    parse_user_classifications,
     set_status,
     set_status_modify,
 )
-from nucliadb.writer.resource.field import extract_fields, parse_fields
+from nucliadb.writer.resource.field import (
+    ResourceClassifications,
+    atomic_get_stored_resource_classifications,
+    extract_fields,
+    parse_fields,
+)
 from nucliadb.writer.resource.origin import parse_extra, parse_origin
 from nucliadb.writer.utilities import get_processing
 from nucliadb_models.resource import NucliaDBRoles
@@ -139,6 +145,11 @@ async def create_resource(
         if item.extra is not None:
             parse_extra(writer.extra, item.extra)
 
+        # Since this is a resource creation, we need to care only about the user-provided
+        # classifications in the request.
+        resource_classifications = ResourceClassifications(
+            resource_level=set(parse_user_classifications(item))
+        )
         await parse_fields(
             writer=writer,
             item=item,
@@ -146,6 +157,7 @@ async def create_resource(
             kbid=kbid,
             uuid=uuid,
             x_skip_store=x_skip_store,
+            resource_classifications=resource_classifications,
         )
 
         set_status(writer.basic, item)
@@ -296,6 +308,15 @@ async def modify_resource(
     if item.extra is not None:
         parse_extra(writer.extra, item.extra)
 
+    if item.usermetadata is not None:
+        # If usermetadata is set in the request payload, this means that stored resource classifications
+        # are not valid and we need to use the ones provided by the user in the request
+        resource_classifications = ResourceClassifications(
+            resource_level=set(parse_user_classifications(item))
+        )
+    else:
+        resource_classifications = await atomic_get_stored_resource_classifications(kbid, rid)
+
     await parse_fields(
         writer=writer,
         item=item,
@@ -303,6 +324,7 @@ async def modify_resource(
         kbid=kbid,
         uuid=rid,
         x_skip_store=x_skip_store,
+        resource_classifications=resource_classifications,
     )
     set_status_modify(writer.basic, item)
 

nucliadb/writer/api/v1/upload.py

@@ -46,7 +46,10 @@ from nucliadb.writer.api.v1.slug import ensure_slug_uniqueness, noop_context_man
 from nucliadb.writer.back_pressure import maybe_back_pressure
 from nucliadb.writer.resource.audit import parse_audit
 from nucliadb.writer.resource.basic import parse_basic_creation
-from nucliadb.writer.resource.field import parse_fields
+from nucliadb.writer.resource.field import (
+    atomic_get_stored_resource_classifications,
+    parse_fields,
+)
 from nucliadb.writer.resource.origin import parse_extra, parse_origin
 from nucliadb.writer.tus import TUSUPLOAD, UPLOAD, get_dm, get_storage_manager
 from nucliadb.writer.tus.exceptions import (
@@ -64,6 +67,7 @@ from nucliadb_models import content_types
 from nucliadb_models.resource import NucliaDBRoles
 from nucliadb_models.utils import FieldIdString
 from nucliadb_models.writer import CreateResourcePayload, ResourceFileUploaded
+from nucliadb_protos import resources_pb2
 from nucliadb_protos.resources_pb2 import CloudFile, FieldFile, FieldID, FieldType, Metadata
 from nucliadb_protos.writer_pb2 import BrokerMessage, FieldIDStatus, FieldStatus
 from nucliadb_utils.authentication import requires_one
@@ -864,7 +868,6 @@ async def store_file_on_nuclia_db(
     partitioning = get_partitioning()
     processing = get_processing()
     storage = await get_storage(service_name=SERVICE_NAME)
-
     partition = partitioning.generate_partition(kbid, rid)
 
     writer = BrokerMessage()
@@ -901,6 +904,7 @@ async def store_file_on_nuclia_db(
 
         toprocess.title = writer.basic.title
 
+        resource_classifications = await atomic_get_stored_resource_classifications(kbid, rid)
         await parse_fields(
             writer=writer,
             item=item,
@@ -908,6 +912,7 @@ async def store_file_on_nuclia_db(
             kbid=kbid,
             uuid=rid,
             x_skip_store=False,
+            resource_classifications=resource_classifications,
         )
     else:
         # Use defaults for everything, but don't forget hidden which depends on KB config
@@ -953,8 +958,10 @@ async def store_file_on_nuclia_db(
             )
         )
 
+        rclassif = await atomic_get_stored_resource_classifications(kbid, rid)
+        classif_labels = rclassif.for_field(field, resources_pb2.FieldType.FILE)
         toprocess.filefield[field] = await processing.convert_internal_filefield_to_str(
-            file_field, storage=storage
+            file_field, storage=storage, classif_labels=classif_labels
         )
 
         writer.source = BrokerMessage.MessageSource.WRITER

nucliadb/writer/resource/basic.py

@@ -18,7 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 from datetime import datetime
-from typing import Optional
+from typing import Optional, Union
 
 from fastapi import HTTPException
 
@@ -31,6 +31,7 @@ from nucliadb.ingest.orm.utils import set_title
 from nucliadb.ingest.processing import PushPayload
 from nucliadb_models.content_types import GENERIC_MIME_TYPE
 from nucliadb_models.file import FileField
+from nucliadb_models.labels import ClassificationLabel
 from nucliadb_models.link import LinkField
 from nucliadb_models.metadata import (
     ParagraphAnnotation,
@@ -290,3 +291,20 @@ def build_question_answer_annotation_pb(
         answer.ids_paragraphs.extend(answer_annotation.ids_paragraphs)
         pb.question_answer.answers.append(answer)
     return pb
+
+
+def parse_user_classifications(
+    item: Union[CreateResourcePayload, UpdateResourcePayload],
+) -> list[ClassificationLabel]:
+    return (
+        [
+            ClassificationLabel(
+                labelset=classification.labelset,
+                label=classification.label,
+            )
+            for classification in item.usermetadata.classifications
+            if classification.cancelled_by_user is False
+        ]
+        if item.usermetadata is not None
+        else []
+    )

nucliadb/writer/resource/field.py

@@ -17,12 +17,15 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import dataclasses
 from datetime import datetime
 from typing import Optional, Union
 
 from google.protobuf.json_format import MessageToDict
 
 import nucliadb_models as models
+from nucliadb.common import datamanagers
+from nucliadb.common.maindb.driver import Transaction
 from nucliadb.common.models_utils import from_proto, to_proto
 from nucliadb.ingest.fields.conversation import Conversation
 from nucliadb.ingest.orm.resource import Resource as ORMResource
@@ -32,6 +35,7 @@ from nucliadb.writer.utilities import get_processing
 from nucliadb_models.common import FieldTypeName
 from nucliadb_models.content_types import GENERIC_MIME_TYPE
 from nucliadb_models.conversation import PushConversation
+from nucliadb_models.labels import ClassificationLabel
 from nucliadb_models.writer import (
     CreateResourcePayload,
     UpdateResourcePayload,
@@ -42,9 +46,29 @@ from nucliadb_utils.storages.storage import StorageField
 from nucliadb_utils.utilities import get_storage
 
 
-async def extract_file_field_from_pb(field_pb: resources_pb2.FieldFile) -> str:
-    processing = get_processing()
+@dataclasses.dataclass
+class ResourceClassifications:
+    resource_level: set[ClassificationLabel] = dataclasses.field(default_factory=set)
+    field_level: dict[tuple[resources_pb2.FieldType.ValueType, str], set[ClassificationLabel]] = (
+        dataclasses.field(default_factory=dict)
+    )
 
+    def for_field(
+        self, field_key: str, field_type: resources_pb2.FieldType.ValueType
+    ) -> list[ClassificationLabel]:
+        """
+        Returns a list of unique classification labels for a given field, including those inherited from the resource.
+        """
+        field_id = (field_type, field_key)
+        resource_level = self.resource_level
+        field_level = self.field_level.get(field_id, set())
+        return list(resource_level.union(field_level))
+
+
+async def extract_file_field_from_pb(
+    field_pb: resources_pb2.FieldFile, classif_labels: list[ClassificationLabel]
+) -> str:
+    processing = get_processing()
     if field_pb.file.source == resources_pb2.CloudFile.Source.EXTERNAL:
         file_field = models.FileField(
             language=field_pb.language,
@@ -52,16 +76,17 @@ async def extract_file_field_from_pb(field_pb: resources_pb2.FieldFile) -> str:
             file=models.File(payload=None, uri=field_pb.file.uri),
             extract_strategy=field_pb.extract_strategy,
         )
-        return processing.convert_external_filefield_to_str(file_field)
+        return processing.convert_external_filefield_to_str(file_field, classif_labels)
     else:
         storage = await get_storage(service_name=SERVICE_NAME)
-        return await processing.convert_internal_filefield_to_str(field_pb, storage)
+        return await processing.convert_internal_filefield_to_str(field_pb, storage, classif_labels)
 
 
 async def extract_file_field(
     field_id: str,
     resource: ORMResource,
     toprocess: PushPayload,
+    resource_classifications: ResourceClassifications,
     password: Optional[str] = None,
 ):
     field_type = resources_pb2.FieldType.FILE
@@ -73,13 +98,19 @@ async def extract_file_field(
     if password is not None:
         field_pb.password = password
 
-    toprocess.filefield[field_id] = await extract_file_field_from_pb(field_pb)
+    classif_labels = resource_classifications.for_field(field_id, resources_pb2.FieldType.FILE)
+    toprocess.filefield[field_id] = await extract_file_field_from_pb(field_pb, classif_labels)
 
 
 async def extract_fields(resource: ORMResource, toprocess: PushPayload):
     processing = get_processing()
     storage = await get_storage(service_name=SERVICE_NAME)
     await resource.get_fields()
+
+    resource_classifications = await atomic_get_stored_resource_classifications(
+        kbid=toprocess.kbid,
+        rid=toprocess.uuid,
+    )
     for (field_type, field_id), field in resource.fields.items():
         field_type_name = from_proto.field_type_name(field_type)
 
@@ -92,9 +123,9 @@ async def extract_fields(resource: ORMResource, toprocess: PushPayload):
             continue
 
         field_pb = await field.get_value()
-
+        classif_labels = resource_classifications.for_field(field_id, field_type)
         if field_type_name is FieldTypeName.FILE:
-            toprocess.filefield[field_id] = await extract_file_field_from_pb(field_pb)
+            toprocess.filefield[field_id] = await extract_file_field_from_pb(field_pb, classif_labels)
 
         if field_type_name is FieldTypeName.LINK:
             parsed_link = MessageToDict(
@@ -104,6 +135,7 @@ async def extract_fields(resource: ORMResource, toprocess: PushPayload):
             )
             parsed_link["link"] = parsed_link.pop("uri", None)
             toprocess.linkfield[field_id] = models.LinkUpload(**parsed_link)
+            toprocess.linkfield[field_id].classification_labels = classif_labels
 
         if field_type_name is FieldTypeName.TEXT:
             parsed_text = MessageToDict(
@@ -113,6 +145,7 @@ async def extract_fields(resource: ORMResource, toprocess: PushPayload):
             )
             parsed_text["format"] = models.PushTextFormat[parsed_text["format"]]
             toprocess.textfield[field_id] = models.Text(**parsed_text)
+            toprocess.textfield[field_id].classification_labels = classif_labels
 
         if field_type_name is FieldTypeName.CONVERSATION and isinstance(field, Conversation):
             metadata = await field.get_metadata()
@@ -143,6 +176,7 @@ async def extract_fields(resource: ORMResource, toprocess: PushPayload):
                     )
                     full_conversation.messages.append(models.PushMessage(**parsed_message))
             toprocess.conversationfield[field_id] = full_conversation
+            toprocess.conversationfield[field_id].classification_labels = classif_labels
 
 
 async def parse_fields(
@@ -152,18 +186,48 @@ async def parse_fields(
     kbid: str,
     uuid: str,
     x_skip_store: bool,
+    resource_classifications: ResourceClassifications,
 ):
     for key, file_field in item.files.items():
-        await parse_file_field(key, file_field, writer, toprocess, kbid, uuid, skip_store=x_skip_store)
+        await parse_file_field(
+            key,
+            file_field,
+            writer,
+            toprocess,
+            kbid,
+            uuid,
+            resource_classifications,
+            skip_store=x_skip_store,
+        )
 
     for key, link_field in item.links.items():
-        parse_link_field(key, link_field, writer, toprocess)
+        parse_link_field(
+            key,
+            link_field,
+            writer,
+            toprocess,
+            resource_classifications,
+        )
 
     for key, text_field in item.texts.items():
-        parse_text_field(key, text_field, writer, toprocess)
+        parse_text_field(
+            key,
+            text_field,
+            writer,
+            toprocess,
+            resource_classifications,
+        )
 
     for key, conversation_field in item.conversations.items():
-        await parse_conversation_field(key, conversation_field, writer, toprocess, kbid, uuid)
+        await parse_conversation_field(
+            key,
+            conversation_field,
+            writer,
+            toprocess,
+            kbid,
+            uuid,
+            resource_classifications,
+        )
 
 
 def parse_text_field(
@@ -171,7 +235,9 @@ def parse_text_field(
     text_field: models.TextField,
     writer: BrokerMessage,
     toprocess: PushPayload,
+    resource_classifications: ResourceClassifications,
 ) -> None:
+    classif_labels = resource_classifications.for_field(key, resources_pb2.FieldType.TEXT)
     if text_field.extract_strategy is not None:
         writer.texts[key].extract_strategy = text_field.extract_strategy
     writer.texts[key].body = text_field.body
@@ -185,6 +251,7 @@ def parse_text_field(
         body=text_field.body,
         format=getattr(models.PushTextFormat, text_field.format.value),
         extract_strategy=text_field.extract_strategy,
+        classification_labels=classif_labels,
     )
     writer.field_statuses.append(
         FieldIDStatus(
@@ -201,13 +268,21 @@ async def parse_file_field(
     toprocess: PushPayload,
     kbid: str,
     uuid: str,
+    resource_classifications: ResourceClassifications,
     skip_store: bool = False,
 ):
     if file_field.file.is_external:
-        parse_external_file_field(key, file_field, writer, toprocess)
+        parse_external_file_field(key, file_field, writer, toprocess, resource_classifications)
     else:
         await parse_internal_file_field(
-            key, file_field, writer, toprocess, kbid, uuid, skip_store=skip_store
+            key,
+            file_field,
+            writer,
+            toprocess,
+            kbid,
+            uuid,
+            resource_classifications,
+            skip_store=skip_store,
         )
 
     writer.field_statuses.append(
@@ -225,8 +300,10 @@ async def parse_internal_file_field(
     toprocess: PushPayload,
     kbid: str,
     uuid: str,
+    resource_classifications: ResourceClassifications,
     skip_store: bool = False,
 ) -> None:
+    classif_labels = resource_classifications.for_field(key, resources_pb2.FieldType.FILE)
     writer.files[key].added.FromDatetime(datetime.now())
     if file_field.language:
         writer.files[key].language = file_field.language
@@ -234,10 +311,9 @@ async def parse_internal_file_field(
         writer.files[key].extract_strategy = file_field.extract_strategy
 
     processing = get_processing()
-
     if skip_store:
         # Does not store file on nuclia's blob storage. Only sends it to process
-        toprocess.filefield[key] = await processing.convert_filefield_to_str(file_field)
+        toprocess.filefield[key] = await processing.convert_filefield_to_str(file_field, classif_labels)
 
     else:
         # Store file on nuclia's blob storage
@@ -254,7 +330,7 @@ async def parse_internal_file_field(
         )
         # Send the pointer of the new blob to processing
         toprocess.filefield[key] = await processing.convert_internal_filefield_to_str(
-            writer.files[key], storage
+            writer.files[key], storage, classif_labels
         )
 
 
@@ -263,7 +339,9 @@ def parse_external_file_field(
     file_field: models.FileField,
     writer: BrokerMessage,
     toprocess: PushPayload,
+    resource_classifications: ResourceClassifications,
 ) -> None:
+    classif_labels = resource_classifications.for_field(key, resources_pb2.FieldType.FILE)
     writer.files[key].added.FromDatetime(datetime.now())
     if file_field.language:
         writer.files[key].language = file_field.language
@@ -276,9 +354,8 @@ def parse_external_file_field(
     writer.files[key].file.content_type = file_field.file.content_type
     if file_field.file.content_type and writer.basic.icon == GENERIC_MIME_TYPE:
         writer.basic.icon = file_field.file.content_type
-
     processing = get_processing()
-    toprocess.filefield[key] = processing.convert_external_filefield_to_str(file_field)
+    toprocess.filefield[key] = processing.convert_external_filefield_to_str(file_field, classif_labels)
 
 
 def parse_link_field(
@@ -286,7 +363,9 @@ def parse_link_field(
     link_field: models.LinkField,
     writer: BrokerMessage,
     toprocess: PushPayload,
+    resource_classifications: ResourceClassifications,
 ) -> None:
+    classif_labels = resource_classifications.for_field(key, resources_pb2.FieldType.LINK)
     writer.links[key].added.FromDatetime(datetime.now())
 
     writer.links[key].uri = link_field.uri
@@ -322,6 +401,7 @@ def parse_link_field(
         css_selector=link_field.css_selector,
         xpath=link_field.xpath,
         extract_strategy=link_field.extract_strategy,
+        classification_labels=classif_labels,
     )
     writer.field_statuses.append(
         FieldIDStatus(
@@ -338,7 +418,9 @@ async def parse_conversation_field(
     toprocess: PushPayload,
     kbid: str,
     uuid: str,
+    resource_classifications: ResourceClassifications,
 ) -> None:
+    classif_labels = resource_classifications.for_field(key, resources_pb2.FieldType.CONVERSATION)
     storage = await get_storage(service_name=SERVICE_NAME)
     processing = get_processing()
     field_value = resources_pb2.Conversation()
@@ -401,7 +483,7 @@ async def parse_conversation_field(
             processing_message.to.append(to)
         convs.messages.append(processing_message)
         field_value.messages.append(cm)
-
+    convs.classification_labels = classif_labels
     toprocess.conversationfield[key] = convs
     writer.conversations[key].CopyFrom(field_value)
     writer.field_statuses.append(
@@ -410,3 +492,37 @@ async def parse_conversation_field(
             status=FieldStatus.Status.PENDING,
         )
     )
+
+
+async def atomic_get_stored_resource_classifications(
+    kbid: str,
+    rid: str,
+) -> ResourceClassifications:
+    async with datamanagers.with_ro_transaction() as txn:
+        return await get_stored_resource_classifications(txn, kbid=kbid, rid=rid)
+
+
+async def get_stored_resource_classifications(
+    txn: Transaction,
+    *,
+    kbid: str,
+    rid: str,
+) -> ResourceClassifications:
+    rc = ResourceClassifications()
+    basic = await datamanagers.resources.get_basic(txn, kbid=kbid, rid=rid)
+    if basic is None:
+        # Resource not found
+        return rc
+
+    # User resource-level classifications
+    for u_classif in basic.usermetadata.classifications:
+        classif = ClassificationLabel(labelset=u_classif.labelset, label=u_classif.label)
+        rc.resource_level.add(classif)
+
+    # Processor-computed field-level classifications. These are not user-defined and are immutable.
+    for field_classif in basic.computedmetadata.field_classifications:
+        fid = (field_classif.field.field_type, field_classif.field.field)
+        for f_classif in field_classif.classifications:
+            classif = ClassificationLabel(labelset=f_classif.labelset, label=f_classif.label)
+            rc.field_level.setdefault(fid, set()).add(classif)
+    return rc

{nucliadb-6.3.4.post3645.dist-info → nucliadb-6.3.4.post3663.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.3.4.post3645
+Version: 6.3.4.post3663
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.4.post3645
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.4.post3645
-Requires-Dist: nucliadb-protos>=6.3.4.post3645
-Requires-Dist: nucliadb-models>=6.3.4.post3645
-Requires-Dist: nidx-protos>=6.3.4.post3645
+Requires-Dist: nucliadb-telemetry[all]>=6.3.4.post3663
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.4.post3663
+Requires-Dist: nucliadb-protos>=6.3.4.post3663
+Requires-Dist: nucliadb-models>=6.3.4.post3663
+Requires-Dist: nidx-protos>=6.3.4.post3663
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.3.4.post3645.dist-info → nucliadb-6.3.4.post3663.dist-info}/RECORD

@@ -40,11 +40,11 @@ nucliadb/metrics_exporter.py,sha256=6u0geEYFxgE5I2Fhl_sxsvGN-ZkaFZNGutSXwrzrsVs,
 nucliadb/openapi.py,sha256=wDiw0dVEvTpJvbatkJ0JZLkKm9RItZT5PWRHjqRfqTA,2272
 nucliadb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/backups/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/backups/const.py,sha256=9vPAhLxQO_gNAjSdPxWuv3V66s9WcdpjOQ89CZlfmuk,1894
-nucliadb/backups/create.py,sha256=TJtYewhD0jkYV_h3rNUhKzhqB2QHAhLWYOgRVlGysGs,11450
+nucliadb/backups/const.py,sha256=AaIsBB04WerR-V6t8NoCUScxO1ojMYJzfHgdkF2qh4M,2018
+nucliadb/backups/create.py,sha256=D0MEpIYu74AhbZXeybinQo8cm_A2-T9JxDGLrp5lavA,12507
 nucliadb/backups/delete.py,sha256=AAs-WP-ujejj6c1LJgiMn7Ht67N_j0a1sKQlUepHpEA,2659
 nucliadb/backups/models.py,sha256=-hITU4Mv6AxePu12toBu_fjpEv6vVGcwNVxV22O9jQA,1273
-nucliadb/backups/restore.py,sha256=YD3Bbo9ry4YLMM6imB-DXbOAMXfGxVzJtTAAUFDvB0I,10153
+nucliadb/backups/restore.py,sha256=KAly9iTXUP32mFFnW6neJm7qyNSZJ7fO5LGbC46vSAM,11416
 nucliadb/backups/settings.py,sha256=SyzsInj1BRbBI0atg5IXWbMbOZ_eVg4eSQ3IcnUhCxQ,1357
 nucliadb/backups/tasks.py,sha256=WkL1LgdYBHbV_A5ilyYv5p3zmXwxH68TDudytN5f7zk,4225
 nucliadb/backups/utils.py,sha256=_Vogjqcru5oqNZM-bZ0q7Ju79Bv1PD-LVFEa7Z-Q13I,1261
@@ -110,11 +110,11 @@ nucliadb/export_import/exporter.py,sha256=k2QVx1EjqFlDYiggriWiEJzwtMXzHbldsqWdpG
 nucliadb/export_import/importer.py,sha256=v5cq9Nn8c2zrY_K_00mydR52f8mdFxR7tLdtNLQ0qvk,4229
 nucliadb/export_import/models.py,sha256=dbjScNkiMRv4X3Ktudy1JRliD25bfoDTy3JmEZgQSCc,2121
 nucliadb/export_import/tasks.py,sha256=DWbdqY97ffoyfipelGXz3Jqz1iam6JCjQSh367Fc3NA,2947
-nucliadb/export_import/utils.py,sha256=DlGUHaqT43b3jG9U-vZ48GpC4O2OgD2WSP_0-hrYW9k,20774
+nucliadb/export_import/utils.py,sha256=aBBB7p05GfKknpb9LQa8Krtz0LlFoP5NUTiPy7PwPBY,22840
 nucliadb/ingest/__init__.py,sha256=fsw3C38VP50km3R-nHL775LNGPpJ4JxqXJ2Ib1f5SqE,1011
 nucliadb/ingest/app.py,sha256=TaVgh5B2riFVmcsrbPb7a5YCzmnybjx-NK0BXgTwGAY,7535
 nucliadb/ingest/partitions.py,sha256=2NIhMYbNT0TNBL6bX1UMSi7vxFGICstCKEqsB0TXHOE,2410
-nucliadb/ingest/processing.py,sha256=8OggvuxNzktTTKDTUwsIuazhDParEWhn46CBZaMYAy8,20659
+nucliadb/ingest/processing.py,sha256=7NNoVxbSwsRdbo5goqVSrUc_QXZRVfOT_jZPzrmbxJQ,22207
 nucliadb/ingest/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/ingest/serialize.py,sha256=42zNKu-O6g9EmLnQOXwhfagD76FSmWD6xRf69LrJxfA,16415
 nucliadb/ingest/settings.py,sha256=0B-wQNa8FLqtNcQgRzh-fuIuGptM816XHcbH1NQKfmE,3050
@@ -138,7 +138,7 @@ nucliadb/ingest/fields/link.py,sha256=kN_gjRUEEj5cy8K_BwPijYg3TiWhedc24apXYlTbRJ
 nucliadb/ingest/fields/text.py,sha256=tFvSQJAe0W7ePpp2_WDfLiE2yglR1OTU0Zht9acvOFw,1594
 nucliadb/ingest/orm/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/ingest/orm/brain.py,sha256=JwHRneyE2sAo6PwYJnnyUMXKiGBWXLv4JI_aemodB3U,28479
-nucliadb/ingest/orm/broker_message.py,sha256=ZEMueoGuuRKO4tHgzc0P0AM1Ls1TTYey_4UvRQf0BpY,6915
+nucliadb/ingest/orm/broker_message.py,sha256=vFDdfo_kz_GRai2MGq_3qKRynRXOGR7lawogNx6ZsfA,7553
 nucliadb/ingest/orm/entities.py,sha256=3_n6lKhBy2GsdmNmkh0_mvxP8md20OZsbtTNEmfJ8Hg,14888
 nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
 nucliadb/ingest/orm/knowledgebox.py,sha256=IGOPvBR1qXqDxE5DeiOdYCLdPgjzOVVpsASJ2zYvWwQ,23651
@@ -328,20 +328,20 @@ nucliadb/writer/api/constants.py,sha256=qWEDjFUycrEZnSJyLnNK4PQNodU2oVmkO4NycaEZ
 nucliadb/writer/api/utils.py,sha256=wIQHlU8RQiIGVLI72suvyVIKlCU44Unh0Ae0IiN6Qwo,1313
 nucliadb/writer/api/v1/__init__.py,sha256=akI9A_jloNLb0dU4T5zjfdyvmSAiDeIdjAlzNx74FlU,1128
 nucliadb/writer/api/v1/export_import.py,sha256=elf-EQY5DD3mhw8kWb9tQpDcbrF9sY6VFYqxQOjuVP0,8201
-nucliadb/writer/api/v1/field.py,sha256=OsWOYA0WQ6onE5Rkl20QIEdtrSi7Jgnu62fUt90Ziy8,17503
+nucliadb/writer/api/v1/field.py,sha256=FySCMpcruSAKGeepeAlOihjwxyUPcDO73Uilq5VDWRk,18514
 nucliadb/writer/api/v1/knowledgebox.py,sha256=MLeIuym4jPrJgfy1NTcN9CpUGwuBiqDHMcx0hY9DR7g,9530
 nucliadb/writer/api/v1/learning_config.py,sha256=CKBjqcbewkfPwGUPLDWzZSpro6XkmCaVppe5Qtpu5Go,3117
-nucliadb/writer/api/v1/resource.py,sha256=A8fAHlN5XFsg6XFYKhfWJS8czgNH6yXr-PsnUqz2WUE,18757
+nucliadb/writer/api/v1/resource.py,sha256=jV9HM-ID1PPYypfy4Sl4_9aSPF87v7gSJZUSzHjHcQ4,19740
 nucliadb/writer/api/v1/router.py,sha256=RjuoWLpZer6Kl2BW_wznpNo6XL3BOpdTGqXZCn3QrrQ,1034
 nucliadb/writer/api/v1/services.py,sha256=HLQW18AEC5zQp5azpeAtRi7ZTzQSwG6SbmkHlmjTIFA,13165
 nucliadb/writer/api/v1/slug.py,sha256=xlVBDBpRi9bNulpBHZwhyftVvulfE0zFm1XZIWl-AKY,2389
 nucliadb/writer/api/v1/transaction.py,sha256=d2Vbgnkk_-FLGSTt3vfldwiJIUf0XoyD0wP1jQNz_DY,2430
-nucliadb/writer/api/v1/upload.py,sha256=VOeqNTrZx1_z8iaKjM7p8fVlVcIYMtnQNK1dm72ct6k,33161
+nucliadb/writer/api/v1/upload.py,sha256=iFHD3nB1ZXiSNyoOzIdzJ0PNLBnnlfOWPba1qWieT4o,33608
 nucliadb/writer/api/v1/vectorsets.py,sha256=mESaXkkI9f-jWWMW61ZZgv7E5YWXKemyc6vwT0lFXns,6747
 nucliadb/writer/resource/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/writer/resource/audit.py,sha256=FvxMZPzrNHtd31HgpZEvxzwAkbxJTZRhPLqRYYJi3tA,1426
-nucliadb/writer/resource/basic.py,sha256=cHhh5hQRHFIoKd-6fEteHuGWW6fGN56ornIWPBuSpHg,11214
-nucliadb/writer/resource/field.py,sha256=HsOERELyAsb9e0dx2IkSQ9lk0SThALFRcDKCVBw8ifU,15478
+nucliadb/writer/resource/basic.py,sha256=_zdAr110C7rtEzOKoBRMzPjAnQ0pAtRfGjB8qCzodvI,11767
+nucliadb/writer/resource/field.py,sha256=qnj31lM9F0AFlj3QhPcPj90vHg7SMbbYW098fMtYt9o,20053
 nucliadb/writer/resource/origin.py,sha256=pvhUDdU0mlWPUcpoQi4LDUJaRtfjzVVrA8XcGVI_N8k,2021
 nucliadb/writer/tus/__init__.py,sha256=huWpKnDnjsrKlBBJk30ta5vamlA-4x0TbPs_2Up8hyM,5443
 nucliadb/writer/tus/azure.py,sha256=XhWAlWTM0vmXcXtuEPYjjeEhuZjiZXZu8q9WsJ7omFE,4107
@@ -352,8 +352,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.3.4.post3645.dist-info/METADATA,sha256=xz2Jc-1oMxDuyL7MTPcAu4jfc1gbBWL7-yoP6Vt2onw,4291
-nucliadb-6.3.4.post3645.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-nucliadb-6.3.4.post3645.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.3.4.post3645.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.3.4.post3645.dist-info/RECORD,,
+nucliadb-6.3.4.post3663.dist-info/METADATA,sha256=Olqrr9vrS1hKJrDcrbrYwfm62xUA3i4aj8WVRLxeePc,4291
+nucliadb-6.3.4.post3663.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+nucliadb-6.3.4.post3663.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.4.post3663.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.4.post3663.dist-info/RECORD,,