nucliadb 6.4.0.post4213__py3-none-any.whl → 6.4.0.post4227__py3-none-any.whl

nucliadb/ingest/consumer/service.py

@@ -59,6 +59,7 @@ async def _exit_tasks(tasks: list[asyncio.Task]) -> None:
 
 
 async def start_back_pressure() -> BackPressureMaterializer:
+    logger.info("Starting back pressure materializer")
     nats_manager = await start_nats_manager(
         SERVICE_NAME,
         indexing_settings.index_jetstream_servers,

nucliadb/ingest/orm/brain_v2.py

@@ -70,7 +70,7 @@ class ParagraphClassifications:
     denied: dict[str, list[str]]
 
 
-class ResourceBrainV2:
+class ResourceBrain:
     def __init__(self, rid: str):
         self.rid = rid
         self.brain: PBBrainResource = PBBrainResource(resource=ResourceID(uuid=rid))

nucliadb/ingest/orm/index_message.py

@@ -27,14 +27,12 @@ from nidx_protos.noderesources_pb2 import Resource as IndexMessage
 from nucliadb.common import datamanagers
 from nucliadb.ingest.fields.exceptions import FieldAuthorNotFound
 from nucliadb.ingest.fields.file import File
-from nucliadb.ingest.orm.brain_v2 import ResourceBrainV2 as ResourceBrain
+from nucliadb.ingest.orm.brain_v2 import ResourceBrain
 from nucliadb.ingest.orm.metrics import index_message_observer as observer
 from nucliadb.ingest.orm.resource import Resource, get_file_page_positions
 from nucliadb_protos.knowledgebox_pb2 import VectorSetConfig
 from nucliadb_protos.resources_pb2 import Basic, FieldID, FieldType
 from nucliadb_protos.writer_pb2 import BrokerMessage
-from nucliadb_utils import const
-from nucliadb_utils.utilities import has_feature
 
 
 class IndexMessageBuilder:
@@ -403,14 +401,5 @@ async def get_resource_index_message(
     """
     Get the full index message for a resource.
     """
-    if has_feature(
-        const.Features.INDEX_MESSAGE_GENERATION_V2,
-        context={
-            "kbid": resource.kb.kbid,
-        },
-    ):
-        im_builder = IndexMessageBuilder(resource)
-        return await im_builder.full(reindex=reindex)
-    else:
-        # TODO: remove this code when we remove the old index message generation
-        return (await resource.generate_index_message(reindex=reindex)).brain
+    im_builder = IndexMessageBuilder(resource)
+    return await im_builder.full(reindex=reindex)

nucliadb/ingest/orm/processor/processor.py

@@ -461,8 +461,8 @@ class Processor:
                 source=source,
             )
 
-    @processor_observer.wrap({"type": "generate_index_message_v2"})
-    async def generate_index_message_v2(
+    @processor_observer.wrap({"type": "generate_index_message"})
+    async def generate_index_message(
         self,
         resource: Resource,
         messages: list[writer_pb2.BrokerMessage],
@@ -477,40 +477,6 @@ class Processor:
         else:  # pragma: no cover
             raise InvalidBrokerMessage(f"Unknown broker message source: {message_source}")
 
-    @processor_observer.wrap({"type": "generate_index_message_v1"})
-    async def generate_index_message_v1(
-        self,
-        resource: Resource,
-        messages: list[writer_pb2.BrokerMessage],
-    ) -> PBBrainResource:
-        if any(needs_reindex(m) for m in messages):
-            # when reindexing, let's just generate full new index message
-            # TODO - This should be improved in the future as it's not optimal for very large resources:
-            # As of now, there are some API operations that require fully reindexing all the fields of a resource.
-            # An example of this is classification label changes - we need to reindex all the fields of a resource to
-            # propagate the label changes to the index.
-            resource.replace_indexer(await resource.generate_index_message(reindex=True))
-        else:
-            # TODO - Ideally we should only update the fields that have been changed in the current transaction.
-            await resource.compute_global_text()
-            await resource.compute_global_tags(resource.indexer)
-            await resource.compute_security(resource.indexer)
-        return resource.indexer.brain
-
-    async def generate_index_message(
-        self,
-        resource: Resource,
-        messages: list[writer_pb2.BrokerMessage],
-        resource_created: bool = False,
-    ) -> PBBrainResource:
-        if has_feature(
-            const.Features.INDEX_MESSAGE_GENERATION_V2,
-            context={"kbid": resource.kb.kbid},
-        ):
-            return await self.generate_index_message_v2(resource, messages, resource_created)
-        else:
-            return await self.generate_index_message_v1(resource, messages)
-
     async def external_index_delete_resource(
         self, external_index_manager: ExternalIndexManager, resource_uuid: str
     ):

nucliadb/ingest/orm/resource.py

@@ -19,11 +19,9 @@
 #
 from __future__ import annotations
 
-import asyncio
 import logging
 from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor
-from functools import partial
 from typing import TYPE_CHECKING, Any, Optional, Sequence, Type
 
 from nucliadb.common import datamanagers
@@ -32,12 +30,11 @@ from nucliadb.common.ids import FIELD_TYPE_PB_TO_STR, FieldId
 from nucliadb.common.maindb.driver import Transaction
 from nucliadb.ingest.fields.base import Field
 from nucliadb.ingest.fields.conversation import Conversation
-from nucliadb.ingest.fields.exceptions import FieldAuthorNotFound
 from nucliadb.ingest.fields.file import File
 from nucliadb.ingest.fields.generic import VALID_GENERIC_FIELDS, Generic
 from nucliadb.ingest.fields.link import Link
 from nucliadb.ingest.fields.text import Text
-from nucliadb.ingest.orm.brain import FilePagePositions, ResourceBrain
+from nucliadb.ingest.orm.brain_v2 import FilePagePositions
 from nucliadb.ingest.orm.metrics import processor_observer
 from nucliadb_models import content_types
 from nucliadb_models.common import CloudLink
@@ -69,9 +66,7 @@ from nucliadb_protos.resources_pb2 import Metadata as PBMetadata
 from nucliadb_protos.resources_pb2 import Origin as PBOrigin
 from nucliadb_protos.resources_pb2 import Relations as PBRelations
 from nucliadb_protos.writer_pb2 import BrokerMessage
-from nucliadb_utils import const
 from nucliadb_utils.storages.storage import Storage
-from nucliadb_utils.utilities import has_feature
 
 if TYPE_CHECKING:  # pragma: no cover
     from nucliadb.ingest.orm.knowledgebox import KnowledgeBox
@@ -121,7 +116,6 @@ class Resource:
         self.extra: Optional[PBExtra] = None
         self.security: Optional[utils_pb2.Security] = None
         self.modified: bool = False
-        self._indexer: Optional[ResourceBrain] = None
         self._modified_extracted_text: list[FieldID] = []
 
         self.txn = txn
@@ -133,15 +127,6 @@ class Resource:
         self._previous_status: Optional[Metadata.Status.ValueType] = None
         self.user_relations: Optional[PBRelations] = None
 
-    @property
-    def indexer(self) -> ResourceBrain:
-        if self._indexer is None:
-            self._indexer = ResourceBrain(rid=self.uuid)
-        return self._indexer
-
-    def replace_indexer(self, indexer: ResourceBrain) -> None:
-        self._indexer = indexer
-
     async def set_slug(self):
         basic = await self.get_basic()
         new_key = KB_RESOURCE_SLUG.format(kbid=self.kb.kbid, slug=basic.slug)
@@ -159,14 +144,6 @@ class Resource:
         if basic_in_payload.HasField("metadata") and basic_in_payload.metadata.useful:
             current_basic.metadata.status = basic_in_payload.metadata.status
 
-    def has_index_message_v2_feature(self) -> bool:
-        return has_feature(
-            const.Features.INDEX_MESSAGE_GENERATION_V2,
-            context={
-                "kbid": self.kb.kbid,
-            },
-        )
-
     @processor_observer.wrap({"type": "set_basic"})
     async def set_basic(
         self,
@@ -219,30 +196,6 @@ class Resource:
                 del self.basic.fieldmetadata[:]
                 self.basic.fieldmetadata.extend(updated)
 
-                if not self.has_index_message_v2_feature():
-                    # TODO: Remove this when we remove the old indexer is removed
-                    # All modified field metadata should be indexed
-                    # TODO: could be improved to only index the diff
-                    for user_field_metadata in self.basic.fieldmetadata:
-                        field_id = self.generate_field_id(fieldmetadata.field)
-                        field_obj = await self.get_field(
-                            fieldmetadata.field.field, fieldmetadata.field.field_type
-                        )
-                        field_metadata = await field_obj.get_field_metadata()
-                        if field_metadata is not None:
-                            page_positions: Optional[FilePagePositions] = None
-                            if isinstance(field_obj, File):
-                                page_positions = await get_file_page_positions(field_obj)
-
-                            self.indexer.apply_field_metadata(
-                                field_id,
-                                field_metadata,
-                                page_positions=page_positions,
-                                extracted_text=await field_obj.get_extracted_text(),
-                                basic_user_field_metadata=user_field_metadata,
-                                replace_field=True,
-                            )
-
         # Some basic fields are computed off field metadata.
         # This means we need to recompute upon field deletions.
         if deleted_fields is not None and len(deleted_fields) > 0:
@@ -313,66 +266,6 @@ class Resource:
         self.modified = True
         self.user_relations = payload
 
-    @processor_observer.wrap({"type": "generate_index_message_old"})
-    async def generate_index_message(self, reindex: bool = False) -> ResourceBrain:
-        brain = ResourceBrain(rid=self.uuid)
-        basic = await self.get_basic()
-        await self.compute_security(brain)
-        await self.compute_global_tags(brain)
-        fields = await self.get_fields(force=True)
-        for (type_id, field_id), field in fields.items():
-            fieldid = FieldID(field_type=type_id, field=field_id)
-            await self.compute_global_text_field(fieldid, brain)
-
-            field_metadata = await field.get_field_metadata()
-            field_key = self.generate_field_id(fieldid)
-            if field_metadata is not None:
-                page_positions: Optional[FilePagePositions] = None
-                if type_id == FieldType.FILE and isinstance(field, File):
-                    page_positions = await get_file_page_positions(field)
-
-                user_field_metadata = None
-                if basic is not None:
-                    user_field_metadata = next(
-                        (
-                            fm
-                            for fm in basic.fieldmetadata
-                            if fm.field.field == field_id and fm.field.field_type == type_id
-                        ),
-                        None,
-                    )
-                brain.apply_field_metadata(
-                    field_key,
-                    field_metadata,
-                    page_positions=page_positions,
-                    extracted_text=await field.get_extracted_text(),
-                    basic_user_field_metadata=user_field_metadata,
-                    replace_field=reindex,
-                )
-
-            if self.disable_vectors is False:
-                vectorset_configs = []
-                async for vectorset_id, vectorset_config in datamanagers.vectorsets.iter(
-                    self.txn, kbid=self.kb.kbid
-                ):
-                    vectorset_configs.append(vectorset_config)
-
-                for vectorset_config in vectorset_configs:
-                    vo = await field.get_vectors(
-                        vectorset=vectorset_config.vectorset_id,
-                        storage_key_kind=vectorset_config.storage_key_kind,
-                    )
-                    if vo is not None:
-                        dimension = vectorset_config.vectorset_index_config.vector_dimension
-                        brain.apply_field_vectors(
-                            field_key,
-                            vo,
-                            vectorset=vectorset_config.vectorset_id,
-                            vector_dimension=dimension,
-                            replace_field=reindex,
-                        )
-        return brain
-
     # Fields
     async def get_fields(self, force: bool = False) -> dict[tuple[FieldType.ValueType, str], Field]:
         # Get all fields
@@ -445,11 +338,6 @@ class Resource:
             if field in self.all_fields_keys:
                 self.all_fields_keys.remove(field)
 
-        # TODO: Remove this when we remove the old indexer
-        if not self.has_index_message_v2_feature():
-            field_key = self.generate_field_id(FieldID(field_type=type, field=key))
-            self.indexer.delete_field(field_key=field_key)
-
         await field_obj.delete()
 
     def has_field(self, type: FieldType.ValueType, field: str) -> bool:
@@ -668,7 +556,6 @@ class Resource:
         update_basic_languages(self.basic, extracted_languages)
 
         # Upload to binary storage
-        # Vector indexing
         if self.disable_vectors is False:
             await self._apply_extracted_vectors(message.field_vectors)
 
@@ -828,38 +715,7 @@ class Resource:
             field_metadata.field.field_type,
             load=False,
         )
-        metadata = await field_obj.set_field_metadata(field_metadata)
-
-        # TODO: Remove this when we remove the old indexer
-        if not self.has_index_message_v2_feature():
-            field_key = self.generate_field_id(field_metadata.field)
-
-            page_positions: Optional[FilePagePositions] = None
-            if field_metadata.field.field_type == FieldType.FILE and isinstance(field_obj, File):
-                page_positions = await get_file_page_positions(field_obj)
-
-            user_field_metadata = next(
-                (
-                    fm
-                    for fm in self.basic.fieldmetadata
-                    if fm.field.field == field_metadata.field.field
-                    and fm.field.field_type == field_metadata.field.field_type
-                ),
-                None,
-            )
-
-            extracted_text = await field_obj.get_extracted_text()
-            apply_field_metadata = partial(
-                self.indexer.apply_field_metadata,
-                field_key,
-                metadata,
-                page_positions=page_positions,
-                extracted_text=extracted_text,
-                basic_user_field_metadata=user_field_metadata,
-                replace_field=True,
-            )
-            loop = asyncio.get_running_loop()
-            await loop.run_in_executor(_executor, apply_field_metadata)
+        await field_obj.set_field_metadata(field_metadata)
 
         maybe_update_basic_thumbnail(self.basic, field_metadata.metadata.metadata.thumbnail)
 
@@ -913,27 +769,6 @@ class Resource:
             if vo is None:
                 raise AttributeError("Vector object not found on set_vectors")
 
-            if self.has_index_message_v2_feature():
-                continue
-
-            # TODO: Remove this when we remove the old indexer
-            # Prepare vectors to be indexed
-            field_key = self.generate_field_id(field_vectors.field)
-            dimension = vectorset.vectorset_index_config.vector_dimension
-            if not dimension:
-                raise ValueError(f"Vector dimension not set for vectorset '{vectorset.vectorset_id}'")
-
-            apply_field_vectors_partial = partial(
-                self.indexer.apply_field_vectors,
-                field_key,
-                vo,
-                vectorset=vectorset.vectorset_id,
-                replace_field=True,
-                vector_dimension=dimension,
-            )
-            loop = asyncio.get_running_loop()
-            await loop.run_in_executor(_executor, apply_field_vectors_partial)
-
     async def _apply_field_large_metadata(self, field_large_metadata: LargeComputedMetadataWrapper):
         field_obj = await self.get_field(
             field_large_metadata.field.field,
@@ -946,67 +781,7 @@ class Resource:
     def generate_field_id(self, field: FieldID) -> str:
         return f"{FIELD_TYPE_PB_TO_STR[field.field_type]}/{field.field}"
 
-    async def compute_security(self, brain: ResourceBrain):
-        security = await self.get_security()
-        if security is None:
-            return
-        brain.set_security(security)
-
-    @processor_observer.wrap({"type": "compute_global_tags"})
-    async def compute_global_tags(self, brain: ResourceBrain):
-        origin = await self.get_origin()
-        basic = await self.get_basic()
-        user_relations = await self.get_user_relations()
-        if basic is None:
-            raise KeyError("Resource not found")
-
-        brain.set_processing_status(basic=basic, previous_status=self._previous_status)
-        brain.set_resource_metadata(basic=basic, origin=origin, user_relations=user_relations)
-        for type, field in await self.get_fields_ids(force=True):
-            fieldobj = await self.get_field(field, type, load=False)
-            fieldid = FieldID(field_type=type, field=field)
-            fieldkey = self.generate_field_id(fieldid)
-            extracted_metadata = await fieldobj.get_field_metadata()
-            valid_user_field_metadata = None
-            for user_field_metadata in basic.fieldmetadata:
-                if (
-                    user_field_metadata.field.field == field
-                    and user_field_metadata.field.field_type == type
-                ):
-                    valid_user_field_metadata = user_field_metadata
-                    break
-            try:
-                generated_by = await fieldobj.generated_by()
-            except FieldAuthorNotFound:
-                generated_by = None
-            brain.apply_field_labels(
-                fieldkey,
-                extracted_metadata,
-                self.uuid,
-                generated_by,
-                basic.usermetadata,
-                valid_user_field_metadata,
-            )
-
-    @processor_observer.wrap({"type": "compute_global_text"})
-    async def compute_global_text(self):
-        for type, field in await self.get_fields_ids(force=True):
-            fieldid = FieldID(field_type=type, field=field)
-            await self.compute_global_text_field(fieldid, self.indexer)
-
-    async def compute_global_text_field(self, fieldid: FieldID, brain: ResourceBrain):
-        fieldobj = await self.get_field(fieldid.field, fieldid.field_type, load=False)
-        fieldkey = self.generate_field_id(fieldid)
-        extracted_text = await fieldobj.get_extracted_text()
-        if extracted_text is None:
-            return
-        field_text = extracted_text.text
-        for _, split in extracted_text.split_text.items():
-            field_text += f" {split} "
-        brain.apply_field_text(fieldkey, field_text, replace_field=True)
-
     def clean(self):
-        self._indexer = None
         self.txn = None
 
 

nucliadb/search/search/chat/query.py

@@ -178,6 +178,7 @@ def add_resource_filter(request: Union[FindRequest, AskRequest], resources: list
 
 def find_request_from_ask_request(item: AskRequest, query: str) -> FindRequest:
     find_request = FindRequest()
+    find_request.filter_expression = item.filter_expression
     find_request.resource_filters = item.resource_filters
     find_request.features = []
     if ChatOptions.SEMANTIC in item.features:

{nucliadb-6.4.0.post4213.dist-info → nucliadb-6.4.0.post4227.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.4.0.post4213
+Version: 6.4.0.post4227
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.4.0.post4213
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.4.0.post4213
-Requires-Dist: nucliadb-protos>=6.4.0.post4213
-Requires-Dist: nucliadb-models>=6.4.0.post4213
-Requires-Dist: nidx-protos>=6.4.0.post4213
+Requires-Dist: nucliadb-telemetry[all]>=6.4.0.post4227
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.4.0.post4227
+Requires-Dist: nucliadb-protos>=6.4.0.post4227
+Requires-Dist: nucliadb-models>=6.4.0.post4227
+Requires-Dist: nidx-protos>=6.4.0.post4227
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn[standard]

{nucliadb-6.4.0.post4213.dist-info → nucliadb-6.4.0.post4227.dist-info}/RECORD

@@ -135,7 +135,7 @@ nucliadb/ingest/consumer/consumer.py,sha256=OgS1fr5Yo55u-XbC6zypTH1aJ562Y1vZHnPD
 nucliadb/ingest/consumer/materializer.py,sha256=tgD_rDI2twQzcz8kKNiW_L4YIth16IGh9mUfD5wiSD4,3858
 nucliadb/ingest/consumer/metrics.py,sha256=ji1l_4cKiHJthQd8YNem1ft4iMbw9KThmVvJmLcv3Xg,1075
 nucliadb/ingest/consumer/pull.py,sha256=vv1AyN0EhVgbgnZyT0D_1_IB4hWy7jPd4lAWPAOHGNc,10374
-nucliadb/ingest/consumer/service.py,sha256=mWzMQS1QkWmJNrkIahEZsn7jb8NbY9FRvPz89NeTT-4,7842
+nucliadb/ingest/consumer/service.py,sha256=GhuqlK-9Lvhzd8kBox8wOlKlJgM3W_gssKoWSfVVdoI,7897
 nucliadb/ingest/consumer/shard_creator.py,sha256=w0smEu01FU_2cjZnsfBRNqT_Ntho11X17zTMST-vKbc,4359
 nucliadb/ingest/consumer/utils.py,sha256=jpX8D4lKzuPCpArQLZeX_Zczq3pfen_zAf8sPJfOEZU,2642
 nucliadb/ingest/fields/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
@@ -147,21 +147,20 @@ nucliadb/ingest/fields/generic.py,sha256=elgtqv15aJUq3zY7X_g0bli_2BpcwPArVvzhe54
 nucliadb/ingest/fields/link.py,sha256=kN_gjRUEEj5cy8K_BwPijYg3TiWhedc24apXYlTbRJs,4172
 nucliadb/ingest/fields/text.py,sha256=2grxo8twWbpXEd_iwUMBw9q0dWorVmlPONmY5d1ThwQ,1684
 nucliadb/ingest/orm/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/ingest/orm/brain.py,sha256=q8iW7TWgABcGG5gIrR4SN5Flv_hOy6uuSv3VL8aWgAk,29098
-nucliadb/ingest/orm/brain_v2.py,sha256=0OYqH9srWghajGh0l1oqTFPBh1Jtlw3ui3Qpww6IC7A,33573
+nucliadb/ingest/orm/brain_v2.py,sha256=qX81wvU-KCcEZ-hNgkQOskMOlZmdbJqDyAfe7eXbGLw,33571
 nucliadb/ingest/orm/broker_message.py,sha256=XWaiZgDOz94NPOPT-hqbRr5ZkpVimUw6PjUJNftfoVw,7514
 nucliadb/ingest/orm/entities.py,sha256=kXyeF6XOpFKhEsGLcY-GLIk21Exp0cJst4XQQ9jJoug,14791
 nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
-nucliadb/ingest/orm/index_message.py,sha256=7Pl2qtqoI3b3NAjWfgoiLQktayngdsJ_NfDH0wpTJBw,16041
+nucliadb/ingest/orm/index_message.py,sha256=hI85nSNVChNLLdEFuEJvOt61Tsir-Gq-2_WZoayAdvk,15617
 nucliadb/ingest/orm/knowledgebox.py,sha256=_rkeTMIXMhR64gbYtZpFHoUHghV2DTJ2lUBqZsoqC_4,23898
 nucliadb/ingest/orm/metrics.py,sha256=OiuggTh-n3kZHA2G73NEUdIlh8c3yFrbusI88DK-Mko,1273
-nucliadb/ingest/orm/resource.py,sha256=nk-aT9mEPenmZ4blyqTbTOXCpJaSSi1AE-wTJ01V2vA,47007
+nucliadb/ingest/orm/resource.py,sha256=hGELQgnzK2wIWgD478bR5OiVDyAxHn6WrFSq2YuHANU,36896
 nucliadb/ingest/orm/utils.py,sha256=fCQRuyecgqhaY7mcBG93oaXMkzkKb9BFjOcy4-ZiSNw,2693
 nucliadb/ingest/orm/processor/__init__.py,sha256=Aqd9wCNTvggkMkCY3WvoI8spdr94Jnqk-0iq9XpLs18,922
 nucliadb/ingest/orm/processor/auditing.py,sha256=TeYhXGJRyQ7ROytbb2u8R0fIh_FYi3HgTu3S1ribY3U,4623
 nucliadb/ingest/orm/processor/data_augmentation.py,sha256=v-pj4GbBWSuO8dQyahs5UDr5ghsyfhCZDS0ftKd6ZYc,5179
 nucliadb/ingest/orm/processor/pgcatalog.py,sha256=ht9_I5WlPc6sSFTY8PsxHlpjN-EsaBaChwqsLlMXwUk,3100
-nucliadb/ingest/orm/processor/processor.py,sha256=jLGLyfj6TIXoLubmNvQvkj5K32MxDDSoktY6M8z5pVk,33149
+nucliadb/ingest/orm/processor/processor.py,sha256=jaEBwbv--WyoC8zcdxWAyF0dAzVA5crVDJl56Bqv1eI,31444
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=uqEphtI1Ir_yk9jRl2gPf7BlzzXWovbARY5MNZSBI_8,1704
 nucliadb/ingest/service/__init__.py,sha256=LHQFUkdmNBOWqBG0Md9sMMI7g5TQZ-hLAnhw6ZblrJg,2002
 nucliadb/ingest/service/exceptions.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
@@ -260,7 +259,7 @@ nucliadb/search/search/chat/ask.py,sha256=jYOGh2rySV4aFx_D2KlNVbPXHBsbkcy0Ve-eBS
 nucliadb/search/search/chat/exceptions.py,sha256=Siy4GXW2L7oPhIR86H3WHBhE9lkV4A4YaAszuGGUf54,1356
 nucliadb/search/search/chat/images.py,sha256=PA8VWxT5_HUGfW1ULhKTK46UBsVyINtWWqEM1ulzX1E,3095
 nucliadb/search/search/chat/prompt.py,sha256=Jnja-Ss7skgnnDY8BymVfdeYsFPnIQFL8tEvcRXTKUE,47356
-nucliadb/search/search/chat/query.py,sha256=S6bRvCWcvU5sxsbUdamNAc8JFpuLAo7mggXm0MQ0p8g,16818
+nucliadb/search/search/chat/query.py,sha256=IdVPeKLUbq4hWJ81LePWdUrljeyehnIXg-Ars-37msQ,16878
 nucliadb/search/search/query_parser/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/search/query_parser/exceptions.py,sha256=szAOXUZ27oNY-OSa9t2hQ5HHkQQC0EX1FZz_LluJHJE,1224
 nucliadb/search/search/query_parser/fetcher.py,sha256=SkvBRDfSKmuz-QygNKLAU4AhZhhDo1dnOZmt1zA28RA,16851
@@ -369,8 +368,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.4.0.post4213.dist-info/METADATA,sha256=PAI_c9PMh-wJWIS4SmAAltmQcXStRUi6tKINdrNKJRM,4223
-nucliadb-6.4.0.post4213.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-nucliadb-6.4.0.post4213.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.4.0.post4213.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.4.0.post4213.dist-info/RECORD,,
+nucliadb-6.4.0.post4227.dist-info/METADATA,sha256=eZVBfu03hI3bpE7r2yLzoiPcUtqH82BsKHO51-dFfN0,4223
+nucliadb-6.4.0.post4227.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+nucliadb-6.4.0.post4227.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.4.0.post4227.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.4.0.post4227.dist-info/RECORD,,

nucliadb/ingest/orm/brain.py

@@ -1,695 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import logging
-from copy import deepcopy
-from dataclasses import dataclass
-from typing import Optional
-
-from nidx_protos.noderesources_pb2 import IndexParagraph as BrainParagraph
-from nidx_protos.noderesources_pb2 import (
-    IndexRelation,
-    ParagraphMetadata,
-    Representation,
-    ResourceID,
-)
-from nidx_protos.noderesources_pb2 import Position as TextPosition
-from nidx_protos.noderesources_pb2 import Resource as PBBrainResource
-
-from nucliadb.common import ids
-from nucliadb.ingest import logger
-from nucliadb.ingest.orm.utils import compute_paragraph_key
-from nucliadb_models.labels import BASE_LABELS, LABEL_HIDDEN, flatten_resource_labels
-from nucliadb_models.metadata import ResourceProcessingStatus
-from nucliadb_protos import utils_pb2
-from nucliadb_protos.resources_pb2 import (
-    Basic,
-    ExtractedText,
-    FieldAuthor,
-    FieldComputedMetadata,
-    FieldMetadata,
-    Metadata,
-    Origin,
-    Paragraph,
-    Relations,
-    UserFieldMetadata,
-    UserMetadata,
-)
-from nucliadb_protos.utils_pb2 import Relation, RelationNode
-
-FilePagePositions = dict[int, tuple[int, int]]
-
-METADATA_STATUS_PB_TYPE_TO_NAME_MAP = {
-    Metadata.Status.ERROR: ResourceProcessingStatus.ERROR.name,
-    Metadata.Status.PROCESSED: ResourceProcessingStatus.PROCESSED.name,
-    Metadata.Status.PENDING: ResourceProcessingStatus.PENDING.name,
-    Metadata.Status.BLOCKED: ResourceProcessingStatus.BLOCKED.name,
-    Metadata.Status.EXPIRED: ResourceProcessingStatus.EXPIRED.name,
-}
-
-
-@dataclass
-class ParagraphClassifications:
-    valid: dict[str, list[str]]
-    denied: dict[str, list[str]]
-
-
-class ResourceBrain:
-    def __init__(self, rid: str):
-        self.rid = rid
-        ridobj = ResourceID(uuid=rid)
-        self.brain: PBBrainResource = PBBrainResource(resource=ridobj)
-        self.labels: dict[str, set[str]] = deepcopy(BASE_LABELS)
-
-    def apply_field_text(self, field_key: str, text: str, replace_field: bool):
-        self.brain.texts[field_key].text = text
-        if replace_field:
-            field_type, field_name = field_key.split("/")
-            full_field_id = ids.FieldId(rid=self.rid, type=field_type, key=field_name).full()
-            self.brain.texts_to_delete.append(full_field_id)
-
-    def _get_paragraph_user_classifications(
-        self, basic_user_field_metadata: Optional[UserFieldMetadata]
-    ) -> ParagraphClassifications:
-        pc = ParagraphClassifications(valid={}, denied={})
-        if basic_user_field_metadata is None:
-            return pc
-        for annotated_paragraph in basic_user_field_metadata.paragraphs:
-            for classification in annotated_paragraph.classifications:
-                paragraph_key = compute_paragraph_key(self.rid, annotated_paragraph.key)
-                classif_label = f"/l/{classification.labelset}/{classification.label}"
-                if classification.cancelled_by_user:
-                    pc.denied.setdefault(paragraph_key, []).append(classif_label)
-                else:
-                    pc.valid.setdefault(paragraph_key, []).append(classif_label)
-        return pc
-
-    def apply_field_metadata(
-        self,
-        field_key: str,
-        metadata: FieldComputedMetadata,
-        page_positions: Optional[FilePagePositions],
-        extracted_text: Optional[ExtractedText],
-        basic_user_field_metadata: Optional[UserFieldMetadata] = None,
-        *,
-        replace_field: bool = False,
-    ):
-        # To check for duplicate paragraphs
-        unique_paragraphs: set[str] = set()
-
-        # Expose also user classifications
-        user_paragraph_classifications = self._get_paragraph_user_classifications(
-            basic_user_field_metadata
-        )
-
-        # We should set paragraphs and labels
-        paragraph_pages = ParagraphPages(page_positions) if page_positions else None
-        for subfield, metadata_split in metadata.split_metadata.items():
-            extracted_text_str = extracted_text.split_text[subfield] if extracted_text else None
-
-            # For each split of this field
-            for index, paragraph in enumerate(metadata_split.paragraphs):
-                key = f"{self.rid}/{field_key}/{subfield}/{paragraph.start}-{paragraph.end}"
-
-                denied_classifications = set(user_paragraph_classifications.denied.get(key, []))
-                position = TextPosition(
-                    index=index,
-                    start=paragraph.start,
-                    end=paragraph.end,
-                    start_seconds=paragraph.start_seconds,
-                    end_seconds=paragraph.end_seconds,
-                )
-                page_with_visual = False
-                if paragraph.HasField("page"):
-                    position.page_number = paragraph.page.page
-                    page_with_visual = paragraph.page.page_with_visual
-                    position.in_page = True
-                elif paragraph_pages:
-                    position.page_number = paragraph_pages.get(paragraph.start)
-                    position.in_page = True
-                else:
-                    position.in_page = False
-
-                representation = Representation()
-                if paragraph.HasField("representation"):
-                    representation.file = paragraph.representation.reference_file
-                    representation.is_a_table = paragraph.representation.is_a_table
-
-                p = BrainParagraph(
-                    start=paragraph.start,
-                    end=paragraph.end,
-                    field=field_key,
-                    split=subfield,
-                    index=index,
-                    repeated_in_field=is_paragraph_repeated_in_field(
-                        paragraph,
-                        extracted_text_str,
-                        unique_paragraphs,
-                    ),
-                    metadata=ParagraphMetadata(
-                        position=position,
-                        page_with_visual=page_with_visual,
-                        representation=representation,
-                    ),
-                )
-                paragraph_kind_label = f"/k/{Paragraph.TypeParagraph.Name(paragraph.kind).lower()}"
-                paragraph_labels = {paragraph_kind_label}
-                paragraph_labels.update(
-                    f"/l/{classification.labelset}/{classification.label}"
-                    for classification in paragraph.classifications
-                )
-                paragraph_labels.update(set(user_paragraph_classifications.valid.get(key, [])))
-                paragraph_labels.difference_update(denied_classifications)
-                p.labels.extend(list(paragraph_labels))
-
-                self.brain.paragraphs[field_key].paragraphs[key].CopyFrom(p)
-
-        extracted_text_str = extracted_text.text if extracted_text else None
-        for index, paragraph in enumerate(metadata.metadata.paragraphs):
-            key = f"{self.rid}/{field_key}/{paragraph.start}-{paragraph.end}"
-            denied_classifications = set(user_paragraph_classifications.denied.get(key, []))
-            position = TextPosition(
-                index=index,
-                start=paragraph.start,
-                end=paragraph.end,
-                start_seconds=paragraph.start_seconds,
-                end_seconds=paragraph.end_seconds,
-            )
-            page_with_visual = False
-            if paragraph.HasField("page"):
-                position.page_number = paragraph.page.page
-                position.in_page = True
-                page_with_visual = paragraph.page.page_with_visual
-            elif paragraph_pages:
-                position.page_number = paragraph_pages.get(paragraph.start)
-                position.in_page = True
-            else:
-                position.in_page = False
-
-            representation = Representation()
-            if paragraph.HasField("representation"):
-                representation.file = paragraph.representation.reference_file
-                representation.is_a_table = paragraph.representation.is_a_table
-
-            p = BrainParagraph(
-                start=paragraph.start,
-                end=paragraph.end,
-                field=field_key,
-                index=index,
-                repeated_in_field=is_paragraph_repeated_in_field(
-                    paragraph, extracted_text_str, unique_paragraphs
-                ),
-                metadata=ParagraphMetadata(
-                    position=position,
-                    page_with_visual=page_with_visual,
-                    representation=representation,
-                ),
-            )
-            paragraph_kind_label = f"/k/{Paragraph.TypeParagraph.Name(paragraph.kind).lower()}"
-            paragraph_labels = {paragraph_kind_label}
-            paragraph_labels.update(
-                f"/l/{classification.labelset}/{classification.label}"
-                for classification in paragraph.classifications
-            )
-            paragraph_labels.update(set(user_paragraph_classifications.valid.get(key, [])))
-            paragraph_labels.difference_update(denied_classifications)
-            p.labels.extend(list(paragraph_labels))
-
-            self.brain.paragraphs[field_key].paragraphs[key].CopyFrom(p)
-
-        if replace_field:
-            field_type, field_name = field_key.split("/")
-            full_field_id = ids.FieldId(rid=self.rid, type=field_type, key=field_name).full()
-            self.brain.paragraphs_to_delete.append(full_field_id)
-
-        field_relations = self.brain.field_relations[field_key].relations
-        for relations in metadata.metadata.relations:
-            for relation in relations.relations:
-                index_relation = IndexRelation(relation=relation)
-                if relation.metadata.HasField("data_augmentation_task_id"):
-                    index_relation.facets.append(f"/g/da/{relation.metadata.data_augmentation_task_id}")
-                field_relations.append(index_relation)
-
-    def delete_field(self, field_key: str):
-        ftype, fkey = field_key.split("/")
-        full_field_id = ids.FieldId(rid=self.rid, type=ftype, key=fkey).full()
-        self.brain.texts_to_delete.append(full_field_id)
-        self.brain.paragraphs_to_delete.append(full_field_id)
-        self.brain.sentences_to_delete.append(full_field_id)
-        self.brain.relation_fields_to_delete.append(field_key)
-
-    def apply_field_vectors(
-        self,
-        field_id: str,
-        vo: utils_pb2.VectorObject,
-        *,
-        vectorset: str,
-        replace_field: bool = False,
-        # cut to specific dimension if specified
-        vector_dimension: Optional[int] = None,
-    ):
-        fid = ids.FieldId.from_string(f"{self.rid}/{field_id}")
-        for subfield, vectors in vo.split_vectors.items():
-            _field_id = ids.FieldId(
-                rid=fid.rid,
-                type=fid.type,
-                key=fid.key,
-                subfield_id=subfield,
-            )
-            # For each split of this field
-            for index, vector in enumerate(vectors.vectors):
-                paragraph_key = ids.ParagraphId(
-                    field_id=_field_id,
-                    paragraph_start=vector.start_paragraph,
-                    paragraph_end=vector.end_paragraph,
-                )
-                sentence_key = ids.VectorId(
-                    field_id=_field_id,
-                    index=index,
-                    vector_start=vector.start,
-                    vector_end=vector.end,
-                )
-                self._apply_field_vector(
-                    field_id,
-                    paragraph_key,
-                    sentence_key,
-                    vector,
-                    vectorset=vectorset,
-                    vector_dimension=vector_dimension,
-                )
-
-        _field_id = ids.FieldId(
-            rid=fid.rid,
-            type=fid.type,
-            key=fid.key,
-        )
-        for index, vector in enumerate(vo.vectors.vectors):
-            paragraph_key = ids.ParagraphId(
-                field_id=_field_id,
-                paragraph_start=vector.start_paragraph,
-                paragraph_end=vector.end_paragraph,
-            )
-            sentence_key = ids.VectorId(
-                field_id=_field_id,
-                index=index,
-                vector_start=vector.start,
-                vector_end=vector.end,
-            )
-            self._apply_field_vector(
-                field_id,
-                paragraph_key,
-                sentence_key,
-                vector,
-                vectorset=vectorset,
-                vector_dimension=vector_dimension,
-            )
-
-        if replace_field:
-            full_field_id = ids.FieldId(rid=self.rid, type=fid.type, key=fid.key).full()
-            self.brain.vector_prefixes_to_delete[vectorset].items.append(full_field_id)
-
-    def _apply_field_vector(
-        self,
-        field_id: str,
-        paragraph_key: ids.ParagraphId,
-        sentence_key: ids.VectorId,
-        vector: utils_pb2.Vector,
-        *,
-        vectorset: str,
-        # cut vectors if a specific dimension is specified
-        vector_dimension: Optional[int] = None,
-    ):
-        paragraph_pb = self.brain.paragraphs[field_id].paragraphs[paragraph_key.full()]
-        sentence_pb = paragraph_pb.vectorsets_sentences[vectorset].sentences[sentence_key.full()]
-
-        sentence_pb.ClearField("vector")  # clear first to prevent duplicates
-        sentence_pb.vector.extend(vector.vector[:vector_dimension])
-
-        # we only care about start/stop position of the paragraph for a given sentence here
-        # the key has the sentence position
-        sentence_pb.metadata.position.start = vector.start_paragraph
-        sentence_pb.metadata.position.end = vector.end_paragraph
-
-        # does it make sense to copy forward paragraph values here?
-        sentence_pb.metadata.position.page_number = paragraph_pb.metadata.position.page_number
-        sentence_pb.metadata.position.in_page = paragraph_pb.metadata.position.in_page
-
-        sentence_pb.metadata.page_with_visual = paragraph_pb.metadata.page_with_visual
-
-        sentence_pb.metadata.representation.file = paragraph_pb.metadata.representation.file
-
-        sentence_pb.metadata.representation.is_a_table = paragraph_pb.metadata.representation.is_a_table
-
-        sentence_pb.metadata.position.index = paragraph_pb.metadata.position.index
-
-    def set_processing_status(self, basic: Basic, previous_status: Optional[Metadata.Status.ValueType]):
-        """
-        We purposefully overwrite what we index as a status and DO NOT reflect
-        actual status with what we index.
-
-        This seems to be is on purpose so the frontend of the product can operate
-        on 2 statuses only -- PENDING and PROCESSED.
-        """
-        # The value of brain.status will either be PROCESSED or PENDING
-        status = basic.metadata.status
-        if previous_status is not None and previous_status != Metadata.Status.PENDING:
-            # Already processed once, so it stays as PROCESSED
-            self.brain.status = PBBrainResource.PROCESSED
-            return
-        # previos_status is None or PENDING
-        if status == Metadata.Status.PENDING:
-            # Stays in pending
-            self.brain.status = PBBrainResource.PENDING
-        else:
-            # Means it has just been processed
-            self.brain.status = PBBrainResource.PROCESSED
-
-    def set_security(self, security: utils_pb2.Security):
-        self.brain.security.CopyFrom(security)
-
-    def get_processing_status_tag(self, metadata: Metadata) -> str:
-        if not metadata.useful:
-            return "EMPTY"
-        return METADATA_STATUS_PB_TYPE_TO_NAME_MAP[metadata.status]
-
-    def set_resource_metadata(self, basic: Basic, origin: Optional[Origin], user_relations: Relations):
-        self._set_resource_dates(basic, origin)
-        self._set_resource_labels(basic, origin)
-        self._set_resource_relations(basic, origin, user_relations)
-
-    def _set_resource_dates(self, basic: Basic, origin: Optional[Origin]):
-        if basic.created.seconds > 0:
-            self.brain.metadata.created.CopyFrom(basic.created)
-        else:
-            logging.warning(f"Basic metadata has no created field for {self.rid}")
-            self.brain.metadata.created.GetCurrentTime()
-        if basic.modified.seconds > 0:
-            self.brain.metadata.modified.CopyFrom(basic.modified)
-        else:
-            if basic.created.seconds > 0:
-                self.brain.metadata.modified.CopyFrom(basic.created)
-            else:
-                self.brain.metadata.modified.GetCurrentTime()
-
-        if origin is not None:
-            # overwrite created/modified if provided on origin
-            if origin.HasField("created") and origin.created.seconds > 0:
-                self.brain.metadata.created.CopyFrom(origin.created)
-            if origin.HasField("modified") and origin.modified.seconds > 0:
-                self.brain.metadata.modified.CopyFrom(origin.modified)
-
-    def _set_resource_relations(self, basic: Basic, origin: Optional[Origin], user_relations: Relations):
-        relationnodedocument = RelationNode(value=self.rid, ntype=RelationNode.NodeType.RESOURCE)
-        if origin is not None:
-            # origin contributors
-            for contrib in origin.colaborators:
-                relationnodeuser = RelationNode(value=contrib, ntype=RelationNode.NodeType.USER)
-                relation = Relation(
-                    relation=Relation.COLAB,
-                    source=relationnodedocument,
-                    to=relationnodeuser,
-                )
-                self.brain.field_relations["a/metadata"].relations.append(
-                    IndexRelation(relation=relation)
-                )
-
-        # labels
-        for classification in basic.usermetadata.classifications:
-            relation_node_label = RelationNode(
-                value=f"{classification.labelset}/{classification.label}",
-                ntype=RelationNode.NodeType.LABEL,
-            )
-            relation = Relation(
-                relation=Relation.ABOUT,
-                source=relationnodedocument,
-                to=relation_node_label,
-            )
-            self.brain.field_relations["a/metadata"].relations.append(IndexRelation(relation=relation))
-
-        # relations
-        for relation in user_relations.relations:
-            self.brain.field_relations["a/metadata"].relations.append(
-                IndexRelation(relation=relation, facets=["/g/u"])
-            )
-
-        self.brain.relation_fields_to_delete.append("a/metadata")
-
-    def _set_resource_labels(self, basic: Basic, origin: Optional[Origin]):
-        if origin is not None:
-            if origin.source_id:
-                self.labels["o"] = {origin.source_id}
-            # origin tags
-            for tag in origin.tags:
-                self.labels["t"].add(tag)
-            # origin source
-            if origin.source_id != "":
-                self.labels["u"].add(f"s/{origin.source_id}")
-
-            if origin.path:
-                self.labels["p"].add(origin.path.lstrip("/"))
-
-            # origin contributors
-            for contrib in origin.colaborators:
-                self.labels["u"].add(f"o/{contrib}")
-
-            for key, value in origin.metadata.items():
-                self.labels["m"].add(f"{key[:255]}/{value[:255]}")
-
-        # icon
-        self.labels["n"].add(f"i/{basic.icon}")
-
-        # processing status
-        status_tag = self.get_processing_status_tag(basic.metadata)
-        self.labels["n"].add(f"s/{status_tag}")
-
-        # main language
-        if basic.metadata.language:
-            self.labels["s"].add(f"p/{basic.metadata.language}")
-
-        # all language
-        for lang in basic.metadata.languages:
-            self.labels["s"].add(f"s/{lang}")
-
-        # labels
-        for classification in basic.usermetadata.classifications:
-            self.labels["l"].add(f"{classification.labelset}/{classification.label}")
-
-        # hidden
-        if basic.hidden:
-            _, p1, p2 = LABEL_HIDDEN.split("/")
-            self.labels[p1].add(p2)
-
-        self.brain.ClearField("labels")
-        self.brain.labels.extend(flatten_resource_labels(self.labels))
-
-    def process_field_metadata(
-        self,
-        field_key: str,
-        metadata: FieldMetadata,
-        labels: dict[str, set[str]],
-        relation_node_document: RelationNode,
-        user_canceled_labels: set[str],
-    ):
-        if metadata.mime_type != "":
-            labels["mt"].add(metadata.mime_type)
-
-        base_classification_relation = Relation(
-            relation=Relation.ABOUT,
-            source=relation_node_document,
-            to=RelationNode(
-                ntype=RelationNode.NodeType.LABEL,
-            ),
-        )
-        for classification in metadata.classifications:
-            label = f"{classification.labelset}/{classification.label}"
-            if label not in user_canceled_labels:
-                labels["l"].add(label)
-                relation = Relation()
-                relation.CopyFrom(base_classification_relation)
-                relation.to.value = label
-                self.brain.field_relations[field_key].relations.append(IndexRelation(relation=relation))
-
-        # Data Augmentation + Processor entities
-        base_entity_relation = Relation(
-            relation=Relation.ENTITY,
-            source=relation_node_document,
-            to=RelationNode(ntype=RelationNode.NodeType.ENTITY),
-        )
-        use_legacy_entities = True
-        for data_augmentation_task_id, entities in metadata.entities.items():
-            # If we recieved the entities from the processor here, we don't want to use the legacy entities
-            # TODO: Remove this when processor doesn't use this anymore
-            if data_augmentation_task_id == "processor":
-                use_legacy_entities = False
-
-            for ent in entities.entities:
-                entity_text = ent.text
-                entity_label = ent.label
-                # Seems like we don't care about where the entity is in the text
-                # entity_positions = entity.positions
-                labels["e"].add(
-                    f"{entity_label}/{entity_text}"
-                )  # Add data_augmentation_task_id as a prefix?
-                relation = Relation()
-                relation.CopyFrom(base_entity_relation)
-                relation.to.value = entity_text
-                relation.to.subtype = entity_label
-                self.brain.field_relations[field_key].relations.append(IndexRelation(relation=relation))
-
-        # Legacy processor entities
-        # TODO: Remove once processor doesn't use this anymore and remove the positions and ner fields from the message
-        def _parse_entity(klass_entity: str) -> tuple[str, str]:
-            try:
-                klass, entity = klass_entity.split("/", 1)
-                return klass, entity
-            except ValueError:
-                raise AttributeError(f"Entity should be with type {klass_entity}")
-
-        if use_legacy_entities:
-            for klass_entity in metadata.positions.keys():
-                labels["e"].add(klass_entity)
-                klass, entity = _parse_entity(klass_entity)
-                relation = Relation()
-                relation.CopyFrom(base_entity_relation)
-                relation.to.value = entity
-                relation.to.subtype = klass
-                self.brain.field_relations[field_key].relations.append(IndexRelation(relation=relation))
-
-    def apply_field_labels(
-        self,
-        field_key: str,
-        metadata: Optional[FieldComputedMetadata],
-        uuid: str,
-        generated_by: Optional[FieldAuthor],
-        basic_user_metadata: Optional[UserMetadata] = None,
-        basic_user_fieldmetadata: Optional[UserFieldMetadata] = None,
-    ):
-        user_canceled_labels: set[str] = set()
-        if basic_user_metadata is not None:
-            user_canceled_labels.update(
-                f"{classification.labelset}/{classification.label}"
-                for classification in basic_user_metadata.classifications
-                if classification.cancelled_by_user
-            )
-        relation_node_resource = RelationNode(value=uuid, ntype=RelationNode.NodeType.RESOURCE)
-        labels: dict[str, set[str]] = {
-            "l": set(),  # classification labels
-            "e": set(),  # entities
-            "mt": set(),  # mime type
-            "g/da": set(),  # generated by
-        }
-        if metadata is not None:
-            for meta in metadata.split_metadata.values():
-                self.process_field_metadata(
-                    field_key,
-                    meta,
-                    labels,
-                    relation_node_resource,
-                    user_canceled_labels,
-                )
-            self.process_field_metadata(
-                field_key,
-                metadata.metadata,
-                labels,
-                relation_node_resource,
-                user_canceled_labels,
-            )
-
-        if basic_user_fieldmetadata is not None:
-            for paragraph_annotation in basic_user_fieldmetadata.paragraphs:
-                for classification in paragraph_annotation.classifications:
-                    if not classification.cancelled_by_user:
-                        label = f"/l/{classification.labelset}/{classification.label}"
-                        # FIXME: this condition avoid adding duplicate labels
-                        # while importing a kb. We shouldn't add duplicates on
-                        # the first place
-                        if (
-                            label
-                            not in self.brain.paragraphs[field_key]
-                            .paragraphs[paragraph_annotation.key]
-                            .labels
-                        ):
-                            self.brain.paragraphs[field_key].paragraphs[
-                                paragraph_annotation.key
-                            ].labels.append(label)
-
-        if generated_by is not None and generated_by.WhichOneof("author") == "data_augmentation":
-            field_type, field_id = field_key.split("/")
-            da_task_id = ids.extract_data_augmentation_id(field_id)
-            if da_task_id is None:  # pragma: nocover
-                logger.warning(
-                    "Data augmentation field id has an unexpected format! Skipping label",
-                    extra={
-                        "rid": uuid,
-                        "field_id": field_id,
-                    },
-                )
-            else:
-                labels["g/da"].add(da_task_id)
-
-        flat_labels = flatten_resource_labels(labels)
-        if len(flat_labels) > 0:
-            self.brain.texts[field_key].labels.extend(flat_labels)
-
-
-def is_paragraph_repeated_in_field(
-    paragraph: Paragraph,
-    extracted_text: Optional[str],
-    unique_paragraphs: set[str],
-) -> bool:
-    if extracted_text is None:
-        return False
-
-    paragraph_text = extracted_text[paragraph.start : paragraph.end]
-    if len(paragraph_text) == 0:
-        return False
-
-    if paragraph_text in unique_paragraphs:
-        repeated_in_field = True
-    else:
-        repeated_in_field = False
-        unique_paragraphs.add(paragraph_text)
-    return repeated_in_field
-
-
-class ParagraphPages:
-    """
-    Class to get the page number for a given paragraph in an optimized way.
-    """
-
-    def __init__(self, positions: FilePagePositions):
-        self.positions = positions
-        self._materialized = self._materialize_page_numbers(positions)
-
-    def _materialize_page_numbers(self, positions: FilePagePositions) -> list[int]:
-        page_numbers_by_index = []
-        for page_number, (page_start, page_end) in positions.items():
-            page_numbers_by_index.extend([page_number] * (page_end - page_start + 1))
-        return page_numbers_by_index
-
-    def get(self, paragraph_start_index: int) -> int:
-        try:
-            return self._materialized[paragraph_start_index]
-        except IndexError:
-            logger.error(
-                f"Could not find a page for the given index: {paragraph_start_index}. Page positions: {self.positions}"  # noqa
-            )
-            if len(self._materialized) > 0:
-                return self._materialized[-1]
-            return 0