nucliadb 6.2.0.post2675__py3-none-any.whl → 6.2.1__py3-none-any.whl

nucliadb/ingest/orm/resource.py

@@ -21,13 +21,14 @@ from __future__ import annotations
 
 import asyncio
 import logging
+from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
-from typing import TYPE_CHECKING, Any, AsyncIterator, MutableMapping, Optional, Type
+from typing import TYPE_CHECKING, Any, Optional, Sequence, Type
 
 from nucliadb.common import datamanagers
 from nucliadb.common.datamanagers.resources import KB_RESOURCE_SLUG
-from nucliadb.common.ids import FIELD_TYPE_PB_TO_STR
+from nucliadb.common.ids import FIELD_TYPE_PB_TO_STR, FieldId
 from nucliadb.common.maindb.driver import Transaction
 from nucliadb.ingest.fields.base import Field
 from nucliadb.ingest.fields.conversation import Conversation
@@ -49,8 +50,8 @@ from nucliadb_protos.resources_pb2 import (
     ExtractedVectorsWrapper,
     FieldClassifications,
     FieldComputedMetadataWrapper,
+    FieldFile,
     FieldID,
-    FieldMetadata,
     FieldQuestionAnswerWrapper,
     FieldText,
     FieldType,
@@ -59,7 +60,6 @@ from nucliadb_protos.resources_pb2 import (
     LinkExtractedData,
     Metadata,
     Paragraph,
-    ParagraphAnnotation,
 )
 from nucliadb_protos.resources_pb2 import Basic as PBBasic
 from nucliadb_protos.resources_pb2 import Conversation as PBConversation
@@ -67,18 +67,11 @@ from nucliadb_protos.resources_pb2 import Extra as PBExtra
 from nucliadb_protos.resources_pb2 import Metadata as PBMetadata
 from nucliadb_protos.resources_pb2 import Origin as PBOrigin
 from nucliadb_protos.resources_pb2 import Relations as PBRelations
-from nucliadb_protos.train_pb2 import (
-    EnabledMetadata,
-    TrainField,
-    TrainMetadata,
-    TrainParagraph,
-    TrainResource,
-    TrainSentence,
-)
-from nucliadb_protos.train_pb2 import Position as TrainPosition
 from nucliadb_protos.utils_pb2 import Relation as PBRelation
 from nucliadb_protos.writer_pb2 import BrokerMessage
+from nucliadb_utils import const
 from nucliadb_utils.storages.storage import Storage
+from nucliadb_utils.utilities import has_feature
 
 if TYPE_CHECKING:  # pragma: no cover
     from nucliadb.ingest.orm.knowledgebox import KnowledgeBox
@@ -226,6 +219,7 @@ class Resource:
                             page_positions=page_positions,
                             extracted_text=await field_obj.get_extracted_text(),
                             basic_user_field_metadata=user_field_metadata,
+                            replace_field=True,
                         )
 
         # Some basic fields are computed off field metadata.
@@ -336,39 +330,28 @@ class Resource:
                     page_positions=page_positions,
                     extracted_text=await field.get_extracted_text(),
                     basic_user_field_metadata=user_field_metadata,
+                    replace_field=reindex,
                 )
 
             if self.disable_vectors is False:
-                # XXX: while we don't remove the "default" vectorset concept, we
-                # need to do use None as the default one
-                vo = await field.get_vectors()
-                if vo is not None:
-                    async with datamanagers.with_ro_transaction() as ro_txn:
-                        dimension = await datamanagers.kb.get_matryoshka_vector_dimension(
-                            ro_txn, kbid=self.kb.kbid
-                        )
-                    brain.apply_field_vectors(
-                        field_key,
-                        vo,
-                        matryoshka_vector_dimension=dimension,
-                        replace_field=reindex,
-                    )
-
                 vectorset_configs = []
-                async with datamanagers.with_ro_transaction() as ro_txn:
-                    async for vectorset_id, vectorset_config in datamanagers.vectorsets.iter(
-                        ro_txn, kbid=self.kb.kbid
-                    ):
-                        vectorset_configs.append(vectorset_config)
+                async for vectorset_id, vectorset_config in datamanagers.vectorsets.iter(
+                    self.txn, kbid=self.kb.kbid
+                ):
+                    vectorset_configs.append(vectorset_config)
+
                 for vectorset_config in vectorset_configs:
-                    vo = await field.get_vectors(vectorset=vectorset_config.vectorset_id)
+                    vo = await field.get_vectors(
+                        vectorset=vectorset_config.vectorset_id,
+                        storage_key_kind=vectorset_config.storage_key_kind,
+                    )
                     if vo is not None:
                         dimension = vectorset_config.vectorset_index_config.vector_dimension
                         brain.apply_field_vectors(
                             field_key,
                             vo,
                             vectorset=vectorset_config.vectorset_id,
-                            matryoshka_vector_dimension=dimension,
+                            vector_dimension=dimension,
                             replace_field=reindex,
                         )
         return brain
@@ -501,7 +484,6 @@ class Resource:
     @processor_observer.wrap({"type": "apply_fields"})
     async def apply_fields(self, message: BrokerMessage):
         message_updated_fields = []
-
         for field, text in message.texts.items():
             fid = FieldID(field_type=FieldType.TEXT, field=field)
             await self.set_field(fid.field_type, fid.field, text)
@@ -532,14 +514,99 @@ class Resource:
                 errors=message.errors,  # type: ignore
             )
 
+    @processor_observer.wrap({"type": "apply_fields_status"})
+    async def apply_fields_status(self, message: BrokerMessage, updated_fields: list[FieldID]):
+        # Dictionary of all errors per field (we may have several due to DA tasks)
+        errors_by_field: dict[tuple[FieldType.ValueType, str], list[writer_pb2.Error]] = defaultdict(
+            list
+        )
+
+        # Make sure if a file is updated without errors, it ends up in errors_by_field
+        for field_id in updated_fields:
+            errors_by_field[(field_id.field_type, field_id.field)] = []
+        for fs in message.field_statuses:
+            errors_by_field[(fs.id.field_type, fs.id.field)] = []
+
+        for error in message.errors:
+            errors_by_field[(error.field_type, error.field)].append(error)
+
+        # If this message comes from the processor (not a DA worker), we clear all previous errors
+        # TODO: When generated_by is populated with DA tasks by processor, remove only related errors
+        from_processor = any((x.WhichOneof("generator") == "processor" for x in message.generated_by))
+
+        for (field_type, field), errors in errors_by_field.items():
+            field_obj = await self.get_field(field, field_type, load=False)
+            if from_processor:
+                # Create a new field status to clear all errors
+                status = writer_pb2.FieldStatus()
+            else:
+                status = await field_obj.get_status() or writer_pb2.FieldStatus()
+
+            for error in errors:
+                field_error = writer_pb2.FieldError(
+                    source_error=error,
+                )
+                field_error.created.GetCurrentTime()
+                status.errors.append(field_error)
+
+            # We infer the status for processor messages
+            if message.source == BrokerMessage.MessageSource.PROCESSOR:
+                if len(status.errors) > 0:
+                    status.status = writer_pb2.FieldStatus.Status.ERROR
+                else:
+                    status.status = writer_pb2.FieldStatus.Status.PROCESSED
+            else:
+                field_status = next(
+                    (
+                        fs.status
+                        for fs in message.field_statuses
+                        if fs.id.field_type == field_type and fs.id.field == field
+                    ),
+                    None,
+                )
+                if field_status is not None:
+                    status.status = field_status
+                # If the field was not found and the message comes from the writer, this implicitly sets the
+                # status to the default value, which is PROCESSING. This covers the case of new field creation.
+
+            await field_obj.set_status(status)
+
+    async def update_status(self):
+        field_ids = await self.get_all_field_ids(for_update=False)
+        if field_ids is None:
+            return
+        field_statuses = await datamanagers.fields.get_statuses(
+            self.txn, kbid=self.kb.kbid, rid=self.uuid, fields=field_ids.fields
+        )
+
+        # If any field is processing -> PENDING
+        if any((f.status == writer_pb2.FieldStatus.Status.PENDING for f in field_statuses)):
+            self.basic.metadata.status = PBMetadata.Status.PENDING
+        # If we have any non-DA error -> ERROR
+        elif any(
+            (
+                f.status == writer_pb2.FieldStatus.Status.ERROR
+                and any(
+                    (
+                        e.source_error.code != writer_pb2.Error.ErrorCode.DATAAUGMENTATION
+                        for e in f.errors
+                    )
+                )
+                for f in field_statuses
+            )
+        ):
+            self.basic.metadata.status = PBMetadata.Status.ERROR
+        # Otherwise (everything processed or we only have DA errors) -> PROCESSED
+        else:
+            self.basic.metadata.status = PBMetadata.Status.PROCESSED
+
     @processor_observer.wrap({"type": "apply_extracted"})
     async def apply_extracted(self, message: BrokerMessage):
-        errors = False
-        field_obj: Field
-        for error in message.errors:
-            field_obj = await self.get_field(error.field, error.field_type, load=False)
-            await field_obj.set_error(error)
-            errors = True
+        if not has_feature(const.Features.FIELD_STATUS):
+            field_obj: Field
+            for error in message.errors:
+                field_obj = await self.get_field(error.field, error.field_type, load=False)
+                await field_obj.set_error(error)
 
         await self.get_basic()
         if self.basic is None:
@@ -548,19 +615,29 @@ class Resource:
         previous_basic = Basic()
         previous_basic.CopyFrom(self.basic)
 
-        if errors:
-            self.basic.metadata.status = PBMetadata.Status.ERROR
-        elif errors is False and message.source is message.MessageSource.PROCESSOR:
-            self.basic.metadata.status = PBMetadata.Status.PROCESSED
-
         maybe_update_basic_icon(self.basic, get_text_field_mimetype(message))
 
         for question_answers in message.question_answers:
             await self._apply_question_answers(question_answers)
 
+        for field_id in message.delete_question_answers:
+            await self._delete_question_answers(field_id)
+
         for extracted_text in message.extracted_text:
             await self._apply_extracted_text(extracted_text)
 
+        # Update field and resource status depending on processing results
+        await self.apply_fields_status(message, self._modified_extracted_text)
+        if has_feature(const.Features.FIELD_STATUS):
+            # Compute resource status based on all fields statuses
+            await self.update_status()
+        else:
+            # Old code path, compute resource status based on the presence of errors in this BrokerMessage
+            if message.errors:
+                self.basic.metadata.status = PBMetadata.Status.ERROR
+            elif message.source is message.MessageSource.PROCESSOR:
+                self.basic.metadata.status = PBMetadata.Status.PROCESSED
+
         extracted_languages = []
 
         for link_extracted_data in message.link_extracted_data:
@@ -584,8 +661,7 @@ class Resource:
         # Upload to binary storage
         # Vector indexing
         if self.disable_vectors is False:
-            for field_vectors in message.field_vectors:
-                await self._apply_extracted_vectors(field_vectors)
+            await self._apply_extracted_vectors(message.field_vectors)
 
         # Only uploading to binary storage
         for field_large_metadata in message.field_large_metadata:
@@ -614,6 +690,10 @@ class Resource:
         field_obj = await self.get_field(field.field, field.field_type, load=False)
         await field_obj.set_question_answers(question_answers)
 
+    async def _delete_question_answers(self, field_id: FieldID):
+        field_obj = await self.get_field(field_id.field, field_id.field_type, load=False)
+        await field_obj.delete_question_answers()
+
     async def _apply_link_extracted_data(self, link_extracted_data: LinkExtractedData):
         assert self.basic is not None
         field_link: Link = await self.get_field(
@@ -679,15 +759,52 @@ class Resource:
         maybe_update_basic_icon(self.basic, file_extracted_data.icon)
         maybe_update_basic_thumbnail(self.basic, file_extracted_data.file_thumbnail)
 
+    async def _should_update_resource_title_from_file_metadata(self) -> bool:
+        """
+        We only want to update resource title from file metadata if the title is empty,
+        equal to the resource uuid or equal to any of the file filenames in the resource.
+        """
+        basic = await self.get_basic()
+        if basic is None:
+            return True
+        current_title = basic.title
+        if current_title == "":
+            # If the title is empty, we should update it
+            return True
+        if current_title == self.uuid:
+            # If the title is the same as the resource uuid, we should update it
+            return True
+        fields = await self.get_fields(force=True)
+        filenames = set()
+        for (field_type, _), field_obj in fields.items():
+            if field_type == FieldType.FILE:
+                field_value: Optional[FieldFile] = await field_obj.get_value()
+                if field_value is not None:
+                    if field_value.file.filename not in ("", None):
+                        filenames.add(field_value.file.filename)
+        if current_title in filenames:
+            # If the title is equal to any of the file filenames, we should update it
+            return True
+        return False
+
     async def maybe_update_resource_title_from_file_extracted_data(self, message: BrokerMessage):
         """
         Update the resource title with the first file that has a title extracted.
         """
-        for file_extracted_data in message.file_extracted_data:
-            if file_extracted_data.title != "":
-                await self.update_resource_title(file_extracted_data.title)
-                # Break after the first file with a title is found
-                break
+        if not await self._should_update_resource_title_from_file_metadata():
+            return
+        for fed in message.file_extracted_data:
+            if fed.title == "":
+                # Skip if the extracted title is empty
+                continue
+            fid = FieldId.from_pb(rid=self.uuid, field_type=FieldType.FILE, key=fed.field)
+            logger.info(
+                "Updating resource title from file extracted data",
+                extra={"kbid": self.kb.kbid, "field": fid.full(), "new_title": fed.title},
+            )
+            await self.update_resource_title(fed.title)
+            # Break after the first file with a title is found
+            break
 
     async def _apply_field_computed_metadata(self, field_metadata: FieldComputedMetadataWrapper):
         assert self.basic is not None
@@ -723,6 +840,7 @@ class Resource:
             page_positions=page_positions,
             extracted_text=extracted_text,
             basic_user_field_metadata=user_field_metadata,
+            replace_field=True,
         )
         loop = asyncio.get_running_loop()
         await loop.run_in_executor(_executor, apply_field_metadata)
@@ -731,55 +849,69 @@ class Resource:
 
         add_field_classifications(self.basic, field_metadata)
 
-    async def _apply_extracted_vectors(self, field_vectors: ExtractedVectorsWrapper):
-        # Store vectors in the resource
-
-        if not self.has_field(field_vectors.field.field_type, field_vectors.field.field):
-            # skipping because field does not exist
-            logger.warning(f'Field "{field_vectors.field.field}" does not exist, skipping vectors')
-            return
-
-        field_obj = await self.get_field(
-            field_vectors.field.field,
-            field_vectors.field.field_type,
-            load=False,
-        )
-        vo = await field_obj.set_vectors(field_vectors)
-
-        # Prepare vectors to be indexed
+    async def _apply_extracted_vectors(
+        self,
+        fields_vectors: Sequence[ExtractedVectorsWrapper],
+    ):
+        await self.get_fields(force=True)
+        vectorsets = {
+            vectorset_id: vs
+            async for vectorset_id, vs in datamanagers.vectorsets.iter(self.txn, kbid=self.kb.kbid)
+        }
+
+        for field_vectors in fields_vectors:
+            # Bw/c with extracted vectors without vectorsets
+            if not field_vectors.vectorset_id:
+                assert (
+                    len(vectorsets) == 1
+                ), "Invalid broker message, can't ingest vectors from unknown vectorset to KB with multiple vectorsets"
+                vectorset = list(vectorsets.values())[0]
 
-        field_key = self.generate_field_id(field_vectors.field)
-        if vo is not None:
-            vectorset_id = field_vectors.vectorset_id or None
-            if vectorset_id is None:
-                dimension = await datamanagers.kb.get_matryoshka_vector_dimension(
-                    self.txn, kbid=self.kb.kbid
-                )
             else:
-                config = await datamanagers.vectorsets.get(
-                    self.txn, kbid=self.kb.kbid, vectorset_id=vectorset_id
-                )
-                if config is None:
+                if field_vectors.vectorset_id not in vectorsets:
                     logger.warning(
-                        f"Trying to apply a resource on vectorset '{vectorset_id}' that doesn't exist."
+                        "Dropping extracted vectors for unknown vectorset",
+                        extra={"kbid": self.kb.kbid, "vectorset": field_vectors.vectorset_id},
                     )
-                    return
-                dimension = config.vectorset_index_config.vector_dimension
-                if not dimension:
-                    raise ValueError(f"Vector dimension not set for vectorset '{vectorset_id}'")
+                    continue
+
+                vectorset = vectorsets[field_vectors.vectorset_id]
+
+            # Store vectors in the resource
+
+            if not self.has_field(field_vectors.field.field_type, field_vectors.field.field):
+                # skipping because field does not exist
+                logger.warning(f'Field "{field_vectors.field.field}" does not exist, skipping vectors')
+                return
+
+            field_obj = await self.get_field(
+                field_vectors.field.field,
+                field_vectors.field.field_type,
+                load=False,
+            )
+            vo = await field_obj.set_vectors(
+                field_vectors, vectorset.vectorset_id, vectorset.storage_key_kind
+            )
+            if vo is None:
+                raise AttributeError("Vector object not found on set_vectors")
+
+            # Prepare vectors to be indexed
+
+            field_key = self.generate_field_id(field_vectors.field)
+            dimension = vectorset.vectorset_index_config.vector_dimension
+            if not dimension:
+                raise ValueError(f"Vector dimension not set for vectorset '{vectorset.vectorset_id}'")
 
             apply_field_vectors_partial = partial(
                 self.indexer.apply_field_vectors,
                 field_key,
                 vo,
-                vectorset=vectorset_id,
+                vectorset=vectorset.vectorset_id,
                 replace_field=True,
-                matryoshka_vector_dimension=dimension,
+                vector_dimension=dimension,
             )
             loop = asyncio.get_running_loop()
             await loop.run_in_executor(_executor, apply_field_vectors_partial)
-        else:
-            raise AttributeError("VO not found on set")
 
     async def _apply_field_large_metadata(self, field_large_metadata: LargeComputedMetadataWrapper):
         field_obj = await self.get_field(
@@ -852,291 +984,6 @@ class Resource:
         self._indexer = None
         self.txn = None
 
-    async def iterate_sentences(
-        self, enabled_metadata: EnabledMetadata
-    ) -> AsyncIterator[TrainSentence]:  # pragma: no cover
-        fields = await self.get_fields(force=True)
-        metadata = TrainMetadata()
-        userdefinedparagraphclass: dict[str, ParagraphAnnotation] = {}
-        if enabled_metadata.labels:
-            if self.basic is None:
-                self.basic = await self.get_basic()
-            if self.basic is not None:
-                metadata.labels.resource.extend(self.basic.usermetadata.classifications)
-                for fieldmetadata in self.basic.fieldmetadata:
-                    field_id = self.generate_field_id(fieldmetadata.field)
-                    for annotationparagraph in fieldmetadata.paragraphs:
-                        userdefinedparagraphclass[annotationparagraph.key] = annotationparagraph
-
-        for (type_id, field_id), field in fields.items():
-            fieldid = FieldID(field_type=type_id, field=field_id)
-            field_key = self.generate_field_id(fieldid)
-            fm = await field.get_field_metadata()
-            extracted_text = None
-            vo = None
-            text = None
-
-            if enabled_metadata.vector:
-                vo = await field.get_vectors()
-
-            extracted_text = await field.get_extracted_text()
-
-            if fm is None:
-                continue
-
-            field_metadatas: list[tuple[Optional[str], FieldMetadata]] = [(None, fm.metadata)]
-            for subfield_metadata, splitted_metadata in fm.split_metadata.items():
-                field_metadatas.append((subfield_metadata, splitted_metadata))
-
-            for subfield, field_metadata in field_metadatas:
-                if enabled_metadata.labels:
-                    metadata.labels.ClearField("field")
-                    metadata.labels.field.extend(field_metadata.classifications)
-
-                entities: dict[str, str] = {}
-                if enabled_metadata.entities:
-                    _update_entities_dict(entities, field_metadata)
-
-                precomputed_vectors = {}
-                if vo is not None:
-                    if subfield is not None:
-                        vectors = vo.split_vectors[subfield]
-                        base_vector_key = f"{self.uuid}/{field_key}/{subfield}"
-                    else:
-                        vectors = vo.vectors
-                        base_vector_key = f"{self.uuid}/{field_key}"
-                    for index, vector in enumerate(vectors.vectors):
-                        vector_key = f"{base_vector_key}/{index}/{vector.start}-{vector.end}"
-                        precomputed_vectors[vector_key] = vector.vector
-
-                if extracted_text is not None:
-                    if subfield is not None:
-                        text = extracted_text.split_text[subfield]
-                    else:
-                        text = extracted_text.text
-
-                for paragraph in field_metadata.paragraphs:
-                    if subfield is not None:
-                        paragraph_key = (
-                            f"{self.uuid}/{field_key}/{subfield}/{paragraph.start}-{paragraph.end}"
-                        )
-                    else:
-                        paragraph_key = f"{self.uuid}/{field_key}/{paragraph.start}-{paragraph.end}"
-
-                    if enabled_metadata.labels:
-                        metadata.labels.ClearField("field")
-                        metadata.labels.paragraph.extend(paragraph.classifications)
-                        if paragraph_key in userdefinedparagraphclass:
-                            metadata.labels.paragraph.extend(
-                                userdefinedparagraphclass[paragraph_key].classifications
-                            )
-
-                    for index, sentence in enumerate(paragraph.sentences):
-                        if subfield is not None:
-                            sentence_key = f"{self.uuid}/{field_key}/{subfield}/{index}/{sentence.start}-{sentence.end}"
-                        else:
-                            sentence_key = (
-                                f"{self.uuid}/{field_key}/{index}/{sentence.start}-{sentence.end}"
-                            )
-
-                        if vo is not None:
-                            metadata.ClearField("vector")
-                            vector_tmp = precomputed_vectors.get(sentence_key)
-                            if vector_tmp:
-                                metadata.vector.extend(vector_tmp)
-
-                        if extracted_text is not None and text is not None:
-                            metadata.text = text[sentence.start : sentence.end]
-
-                        metadata.ClearField("entities")
-                        metadata.ClearField("entity_positions")
-                        if enabled_metadata.entities and text is not None:
-                            local_text = text[sentence.start : sentence.end]
-                            add_entities_to_metadata(entities, local_text, metadata)
-
-                        pb_sentence = TrainSentence()
-                        pb_sentence.uuid = self.uuid
-                        pb_sentence.field.CopyFrom(fieldid)
-                        pb_sentence.paragraph = paragraph_key
-                        pb_sentence.sentence = sentence_key
-                        pb_sentence.metadata.CopyFrom(metadata)
-                        yield pb_sentence
-
-    async def iterate_paragraphs(
-        self, enabled_metadata: EnabledMetadata
-    ) -> AsyncIterator[TrainParagraph]:
-        fields = await self.get_fields(force=True)
-        metadata = TrainMetadata()
-        userdefinedparagraphclass: dict[str, ParagraphAnnotation] = {}
-        if enabled_metadata.labels:
-            if self.basic is None:
-                self.basic = await self.get_basic()
-            if self.basic is not None:
-                metadata.labels.resource.extend(self.basic.usermetadata.classifications)
-                for fieldmetadata in self.basic.fieldmetadata:
-                    field_id = self.generate_field_id(fieldmetadata.field)
-                    for annotationparagraph in fieldmetadata.paragraphs:
-                        userdefinedparagraphclass[annotationparagraph.key] = annotationparagraph
-
-        for (type_id, field_id), field in fields.items():
-            fieldid = FieldID(field_type=type_id, field=field_id)
-            field_key = self.generate_field_id(fieldid)
-            fm = await field.get_field_metadata()
-            extracted_text = None
-            text = None
-
-            extracted_text = await field.get_extracted_text()
-
-            if fm is None:
-                continue
-
-            field_metadatas: list[tuple[Optional[str], FieldMetadata]] = [(None, fm.metadata)]
-            for subfield_metadata, splitted_metadata in fm.split_metadata.items():
-                field_metadatas.append((subfield_metadata, splitted_metadata))
-
-            for subfield, field_metadata in field_metadatas:
-                if enabled_metadata.labels:
-                    metadata.labels.ClearField("field")
-                    metadata.labels.field.extend(field_metadata.classifications)
-
-                entities: dict[str, str] = {}
-                if enabled_metadata.entities:
-                    _update_entities_dict(entities, field_metadata)
-
-                if extracted_text is not None:
-                    if subfield is not None:
-                        text = extracted_text.split_text[subfield]
-                    else:
-                        text = extracted_text.text
-
-                for paragraph in field_metadata.paragraphs:
-                    if subfield is not None:
-                        paragraph_key = (
-                            f"{self.uuid}/{field_key}/{subfield}/{paragraph.start}-{paragraph.end}"
-                        )
-                    else:
-                        paragraph_key = f"{self.uuid}/{field_key}/{paragraph.start}-{paragraph.end}"
-
-                    if enabled_metadata.labels:
-                        metadata.labels.ClearField("paragraph")
-                        metadata.labels.paragraph.extend(paragraph.classifications)
-
-                        if extracted_text is not None and text is not None:
-                            metadata.text = text[paragraph.start : paragraph.end]
-
-                        metadata.ClearField("entities")
-                        metadata.ClearField("entity_positions")
-                        if enabled_metadata.entities and text is not None:
-                            local_text = text[paragraph.start : paragraph.end]
-                            add_entities_to_metadata(entities, local_text, metadata)
-
-                        if paragraph_key in userdefinedparagraphclass:
-                            metadata.labels.paragraph.extend(
-                                userdefinedparagraphclass[paragraph_key].classifications
-                            )
-
-                        pb_paragraph = TrainParagraph()
-                        pb_paragraph.uuid = self.uuid
-                        pb_paragraph.field.CopyFrom(fieldid)
-                        pb_paragraph.paragraph = paragraph_key
-                        pb_paragraph.metadata.CopyFrom(metadata)
-
-                        yield pb_paragraph
-
-    async def iterate_fields(self, enabled_metadata: EnabledMetadata) -> AsyncIterator[TrainField]:
-        fields = await self.get_fields(force=True)
-        metadata = TrainMetadata()
-        if enabled_metadata.labels:
-            if self.basic is None:
-                self.basic = await self.get_basic()
-            if self.basic is not None:
-                metadata.labels.resource.extend(self.basic.usermetadata.classifications)
-
-        for (type_id, field_id), field in fields.items():
-            fieldid = FieldID(field_type=type_id, field=field_id)
-            fm = await field.get_field_metadata()
-            extracted_text = None
-
-            if enabled_metadata.text:
-                extracted_text = await field.get_extracted_text()
-
-            if fm is None:
-                continue
-
-            field_metadatas: list[tuple[Optional[str], FieldMetadata]] = [(None, fm.metadata)]
-            for subfield_metadata, splitted_metadata in fm.split_metadata.items():
-                field_metadatas.append((subfield_metadata, splitted_metadata))
-
-            for subfield, splitted_metadata in field_metadatas:
-                if enabled_metadata.labels:
-                    metadata.labels.ClearField("field")
-                    metadata.labels.field.extend(splitted_metadata.classifications)
-
-                if extracted_text is not None:
-                    if subfield is not None:
-                        metadata.text = extracted_text.split_text[subfield]
-                    else:
-                        metadata.text = extracted_text.text
-
-                if enabled_metadata.entities:
-                    metadata.ClearField("entities")
-                    _update_entities_dict(metadata.entities, splitted_metadata)
-
-                pb_field = TrainField()
-                pb_field.uuid = self.uuid
-                pb_field.field.CopyFrom(fieldid)
-                pb_field.metadata.CopyFrom(metadata)
-                yield pb_field
-
-    async def generate_train_resource(self, enabled_metadata: EnabledMetadata) -> TrainResource:
-        fields = await self.get_fields(force=True)
-        metadata = TrainMetadata()
-        if enabled_metadata.labels:
-            if self.basic is None:
-                self.basic = await self.get_basic()
-            if self.basic is not None:
-                metadata.labels.resource.extend(self.basic.usermetadata.classifications)
-
-        metadata.labels.ClearField("field")
-        metadata.ClearField("entities")
-
-        for (_, _), field in fields.items():
-            extracted_text = None
-            fm = await field.get_field_metadata()
-
-            if enabled_metadata.text:
-                extracted_text = await field.get_extracted_text()
-
-            if extracted_text is not None:
-                metadata.text += extracted_text.text
-                for text in extracted_text.split_text.values():
-                    metadata.text += f" {text}"
-
-            if fm is None:
-                continue
-
-            field_metadatas: list[tuple[Optional[str], FieldMetadata]] = [(None, fm.metadata)]
-            for subfield_metadata, splitted_metadata in fm.split_metadata.items():
-                field_metadatas.append((subfield_metadata, splitted_metadata))
-
-            for _, splitted_metadata in field_metadatas:
-                if enabled_metadata.labels:
-                    metadata.labels.field.extend(splitted_metadata.classifications)
-
-                if enabled_metadata.entities:
-                    _update_entities_dict(metadata.entities, splitted_metadata)
-
-        pb_resource = TrainResource()
-        pb_resource.uuid = self.uuid
-        if self.basic is not None:
-            pb_resource.title = self.basic.title
-            pb_resource.icon = self.basic.icon
-            pb_resource.slug = self.basic.slug
-            pb_resource.modified.CopyFrom(self.basic.modified)
-            pb_resource.created.CopyFrom(self.basic.created)
-        pb_resource.metadata.CopyFrom(metadata)
-        return pb_resource
-
 
 async def get_file_page_positions(field: File) -> FilePagePositions:
     positions: FilePagePositions = {}
@@ -1181,24 +1028,6 @@ def add_field_classifications(basic: PBBasic, fcmw: FieldComputedMetadataWrapper
     return True
 
 
-def add_entities_to_metadata(entities: dict[str, str], local_text: str, metadata: TrainMetadata) -> None:
-    for entity_key, entity_value in entities.items():
-        if entity_key not in local_text:
-            # Add the entity only if found in text
-            continue
-        metadata.entities[entity_key] = entity_value
-
-        # Add positions for the entity relative to the local text
-        poskey = f"{entity_value}/{entity_key}"
-        metadata.entity_positions[poskey].entity = entity_key
-        last_occurrence_end = 0
-        for _ in range(local_text.count(entity_key)):
-            start = local_text.index(entity_key, last_occurrence_end)
-            end = start + len(entity_key)
-            metadata.entity_positions[poskey].positions.append(TrainPosition(start=start, end=end))
-            last_occurrence_end = end
-
-
 def maybe_update_basic_summary(basic: PBBasic, summary_text: str) -> bool:
     if basic.summary or not summary_text:
         return False
@@ -1267,23 +1096,3 @@ def extract_field_metadata_languages(
     for _, splitted_metadata in field_metadata.metadata.split_metadata.items():
         languages.add(splitted_metadata.language)
     return list(languages)
-
-
-def _update_entities_dict(target_entites_dict: MutableMapping[str, str], field_metadata: FieldMetadata):
-    """
-    Update the entities dict with the entities from the field metadata.
-    Method created to ease the transition from legacy ner field to new entities field.
-    """
-    # Data Augmentation + Processor entities
-    # This will overwrite entities detected from more than one data augmentation task
-    # TODO: Change TrainMetadata proto to accept multiple entities with the same text
-    entity_map = {
-        entity.text: entity.label
-        for data_augmentation_task_id, entities_wrapper in field_metadata.entities.items()
-        for entity in entities_wrapper.entities
-    }
-    target_entites_dict.update(entity_map)
-
-    # Legacy processor entities
-    # TODO: Remove once processor doesn't use this anymore and remove the positions and ner fields from the message
-    target_entites_dict.update(field_metadata.ner)