nucliadb 6.7.2.post4874__py3-none-any.whl → 6.10.0.post5705__py3-none-any.whl

nucliadb/ingest/fields/conversation.py

@@ -18,16 +18,19 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import uuid
-from typing import Any, Optional
+from typing import Any
 
 from nucliadb.ingest.fields.base import Field
-from nucliadb_protos.resources_pb2 import CloudFile, FieldConversation
+from nucliadb_protos.resources_pb2 import CloudFile, FieldConversation, SplitsMetadata
 from nucliadb_protos.resources_pb2 import Conversation as PBConversation
 from nucliadb_utils.storages.storage import StorageField
 
+MAX_CONVERSATION_MESSAGES = None  # No limit
+
 PAGE_SIZE = 200
 
 CONVERSATION_PAGE_VALUE = "/kbs/{kbid}/r/{uuid}/f/{type}/{field}/{page}"
+CONVERSATION_SPLITS_METADATA = "/kbs/{kbid}/r/{uuid}/f/{type}/{field}/splits_metadata"
 CONVERSATION_METADATA = "/kbs/{kbid}/r/{uuid}/f/{type}/{field}"
 
 
@@ -39,7 +42,7 @@ class Conversation(Field[PBConversation]):
     pbklass = PBConversation
     type: str = "c"
     value: dict[int, PBConversation]
-    metadata: Optional[FieldConversation]
+    metadata: FieldConversation | None
 
     _created: bool = False
 
@@ -47,20 +50,33 @@ class Conversation(Field[PBConversation]):
         self,
         id: str,
         resource: Any,
-        pb: Optional[Any] = None,
-        value: Optional[dict[int, PBConversation]] = None,
+        pb: Any | None = None,
+        value: dict[int, PBConversation] | None = None,
     ):
-        super(Conversation, self).__init__(id, resource, pb, value)
+        super().__init__(id, resource, pb, value)
         self.value = {}
+        self._splits_metadata: SplitsMetadata | None = None
+        self.metadata = None
+
+    async def delete_value(self):
+        await self.resource.txn.delete_by_prefix(
+            CONVERSATION_METADATA.format(kbid=self.kbid, uuid=self.uuid, type=self.type, field=self.id)
+        )
+        self._split_metadata = None
         self.metadata = None
+        self.value.clear()
 
     async def set_value(self, payload: PBConversation):
+        if payload.replace_field:
+            # As we need to overwrite the value of the conversation, first delete any previous data.
+            await self.delete_value()
+
         metadata = await self.get_metadata()
         metadata.extract_strategy = payload.extract_strategy
         metadata.split_strategy = payload.split_strategy
 
         # Get the last page if it exists
-        last_page: Optional[PBConversation] = None
+        last_page: PBConversation | None = None
         if self._created is False and metadata.pages > 0:
             try:
                 last_page = await self.db_get_value(page=metadata.pages)
@@ -70,10 +86,13 @@ class Conversation(Field[PBConversation]):
             last_page = PBConversation()
             metadata.pages += 1
 
+        self._splits_metadata = await self.get_splits_metadata()
+
         # Make sure message attachment files are on our region. This is needed
         # to support the hybrid-onprem deployment as the attachments must be stored
         # at the storage services of the client's premises.
         for message in payload.messages:
+            self._splits_metadata.metadata.get_or_create(message.ident)
             new_message_files = []
             for idx, file in enumerate(message.content.attachments):
                 if self.storage.needs_move(file, self.kbid):
@@ -117,8 +136,9 @@ class Conversation(Field[PBConversation]):
 
         # Finally, set the metadata
         await self.db_set_metadata(metadata)
+        await self.set_splits_metadata(self._splits_metadata)
 
-    async def get_value(self, page: Optional[int] = None) -> Optional[PBConversation]:
+    async def get_value(self, page: int | None = None) -> PBConversation | None:
         # If no page was requested, force fetch of metadata
         # and set the page to the last page
         if page is None and self.metadata is None:
@@ -133,7 +153,7 @@ class Conversation(Field[PBConversation]):
         except PageNotFound:
             return None
 
-    async def get_full_conversation(self) -> Optional[PBConversation]:
+    async def get_full_conversation(self) -> PBConversation | None:
         """
         Messages of a conversations may be stored across several pages.
         This method fetches them all and returns a single complete conversation.
@@ -203,3 +223,29 @@ class Conversation(Field[PBConversation]):
         self.metadata = payload
         self.resource.modified = True
         self._created = False
+
+    async def get_splits_metadata(self) -> SplitsMetadata:
+        if self._splits_metadata is None:
+            field_key = CONVERSATION_SPLITS_METADATA.format(
+                kbid=self.kbid,
+                uuid=self.uuid,
+                type=self.type,
+                field=self.id,
+            )
+            payload = await self.resource.txn.get(field_key)
+            if payload is None:
+                return SplitsMetadata()
+            self._splits_metadata = SplitsMetadata()
+            self._splits_metadata.ParseFromString(payload)
+        return self._splits_metadata
+
+    async def set_splits_metadata(self, payload: SplitsMetadata) -> None:
+        key = CONVERSATION_SPLITS_METADATA.format(
+            kbid=self.kbid,
+            uuid=self.uuid,
+            type=self.type,
+            field=self.id,
+        )
+        await self.resource.txn.set(key, payload.SerializeToString())
+        self._split_metadata = payload
+        self.resource.modified = True

nucliadb/ingest/fields/exceptions.py

@@ -17,7 +17,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from typing import Type
 
 
 class InvalidFieldClass(Exception):
@@ -25,7 +24,7 @@ class InvalidFieldClass(Exception):
 
 
 class InvalidPBClass(Exception):
-    def __init__(self, source: Type, destination: Type):
+    def __init__(self, source: type, destination: type):
         self.source = source
         self.destination = destination
         super().__init__(f"Source and destination does not match {self.source} - {self.destination}")

nucliadb/ingest/fields/file.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from typing import Any, Optional
+from typing import Any
 
 from nucliadb.ingest.fields.base import Field
 from nucliadb_protos.resources_pb2 import CloudFile, FieldFile, FileExtractedData
@@ -30,22 +30,22 @@ class File(Field[FieldFile]):
     pbklass = FieldFile
     value: FieldFile
     type: str = "f"
-    file_extracted_data: Optional[FileExtractedData]
+    file_extracted_data: FileExtractedData | None
 
     def __init__(
         self,
         id: str,
         resource: Any,
-        pb: Optional[Any] = None,
-        value: Optional[str] = None,
+        pb: Any | None = None,
+        value: str | None = None,
     ):
-        super(File, self).__init__(id, resource, pb, value)
+        super().__init__(id, resource, pb, value)
         self.file_extracted_data = None
 
     async def set_value(self, payload: FieldFile):
         old_file = await self.get_value()
         if old_file is None:
-            old_cf: Optional[CloudFile] = None
+            old_cf: CloudFile | None = None
         else:
             old_cf = old_file.file
 
@@ -57,7 +57,7 @@ class File(Field[FieldFile]):
 
         await self.db_set_value(payload)
 
-    async def get_value(self) -> Optional[FieldFile]:
+    async def get_value(self) -> FieldFile | None:
         return await self.db_get_value()
 
     async def set_file_extracted_data(self, file_extracted_data: FileExtractedData):
@@ -101,10 +101,24 @@ class File(Field[FieldFile]):
         await self.storage.upload_pb(sf, file_extracted_data)
         self.file_extracted_data = file_extracted_data
 
-    async def get_file_extracted_data(self) -> Optional[FileExtractedData]:
+    async def get_file_extracted_data(self) -> FileExtractedData | None:
         if self.file_extracted_data is None:
             sf: StorageField = self.storage.file_extracted(
                 self.kbid, self.uuid, self.type, self.id, FILE_METADATA
             )
             self.file_extracted_data = await self.storage.download_pb(sf, FileExtractedData)
         return self.file_extracted_data
+
+    async def thumbnail(self) -> StorageField | None:
+        """Access the file field thumbnail."""
+        fed = await self.get_file_extracted_data()
+        if fed is None:
+            return None
+        if not fed.HasField("file_thumbnail"):
+            return None
+
+        sf: StorageField = self.storage.file_extracted(
+            self.kbid, self.uuid, self.type, self.id, "file_thumbnail"
+        )
+        sf.field = fed.file_thumbnail
+        return sf

nucliadb/ingest/fields/link.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from typing import Any, Optional
+from typing import Any
 
 from nucliadb.ingest.fields.base import Field
 from nucliadb_protos.resources_pb2 import CloudFile, FieldLink, LinkExtractedData
@@ -30,22 +30,22 @@ class Link(Field[FieldLink]):
     pbklass = FieldLink
     value: FieldLink
     type: str = "u"
-    link_extracted_data: Optional[LinkExtractedData]
+    link_extracted_data: LinkExtractedData | None
 
     def __init__(
         self,
         id: str,
         resource: Any,
-        pb: Optional[Any] = None,
-        value: Optional[str] = None,
+        pb: Any | None = None,
+        value: str | None = None,
     ):
-        super(Link, self).__init__(id, resource, pb, value)
+        super().__init__(id, resource, pb, value)
         self.link_extracted_data = None
 
     async def set_value(self, payload: FieldLink):
         await self.db_set_value(payload)
 
-    async def get_value(self) -> Optional[FieldLink]:
+    async def get_value(self) -> FieldLink | None:
         return await self.db_get_value()
 
     async def set_link_extracted_data(self, link_extracted_data: LinkExtractedData):
@@ -88,7 +88,7 @@ class Link(Field[FieldLink]):
         await self.storage.upload_pb(sf, link_extracted_data)
         self.link_extracted_data = link_extracted_data
 
-    async def get_link_extracted_data(self) -> Optional[LinkExtractedData]:
+    async def get_link_extracted_data(self) -> LinkExtractedData | None:
         if self.link_extracted_data is None:
             sf: StorageField = self.storage.file_extracted(
                 self.kbid, self.uuid, self.type, self.id, LINK_METADATA

nucliadb/ingest/fields/text.py

@@ -19,7 +19,6 @@
 #
 
 import hashlib
-from typing import Optional
 
 from nucliadb.ingest.fields.base import Field
 from nucliadb.ingest.fields.exceptions import FieldAuthorNotFound
@@ -39,8 +38,8 @@ class Text(Field[FieldText]):
 
     async def set_value(self, payload: FieldText):
         if payload.md5 == "":
-            payload.md5 = hashlib.md5(payload.body.encode()).hexdigest()
+            payload.md5 = hashlib.md5(payload.body.encode(), usedforsecurity=False).hexdigest()
         await self.db_set_value(payload)
 
-    async def get_value(self) -> Optional[FieldText]:
+    async def get_value(self) -> FieldText | None:
         return await self.db_get_value()

nucliadb/ingest/orm/brain_v2.py

@@ -18,9 +18,9 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import logging
+from collections.abc import Iterator
 from copy import deepcopy
 from dataclasses import dataclass
-from typing import Optional
 
 from nidx_protos.noderesources_pb2 import IndexParagraph as BrainParagraph
 from nidx_protos.noderesources_pb2 import (
@@ -81,9 +81,9 @@ class ResourceBrain:
         self,
         basic: Basic,
         user_relations: Relations,
-        origin: Optional[Origin],
-        previous_processing_status: Optional[Metadata.Status.ValueType],
-        security: Optional[utils_pb2.Security],
+        origin: Origin | None,
+        previous_processing_status: Metadata.Status.ValueType | None,
+        security: utils_pb2.Security | None,
     ) -> None:
         self._set_resource_status(basic, previous_processing_status)
         self._set_resource_dates(basic, origin)
@@ -97,9 +97,9 @@ class ResourceBrain:
         self,
         field_key: str,
         extracted_text: ExtractedText,
-        field_computed_metadata: Optional[FieldComputedMetadata],
-        basic_user_metadata: Optional[UserMetadata],
-        field_author: Optional[FieldAuthor],
+        field_computed_metadata: FieldComputedMetadata | None,
+        basic_user_metadata: UserMetadata | None,
+        field_author: FieldAuthor | None,
         replace_field: bool,
         skip_index: bool,
     ) -> None:
@@ -122,13 +122,17 @@ class ResourceBrain:
         field_key: str,
         extracted_text: ExtractedText,
         replace_field: bool,
-        skip_texts: Optional[bool],
+        skip_texts: bool | None,
     ):
         if skip_texts is not None:
             self.brain.skip_texts = skip_texts
+
         field_text = extracted_text.text
-        for _, split in extracted_text.split_text.items():
-            field_text += f" {split} "
+
+        for split_id in self.sorted_splits(extracted_text):
+            split_text = extracted_text.split_text[split_id]
+            field_text += f"{split_text} "
+
         self.brain.texts[field_key].text = field_text
 
         if replace_field:
@@ -140,18 +144,16 @@ class ResourceBrain:
     def apply_field_labels(
         self,
         field_key: str,
-        field_computed_metadata: Optional[FieldComputedMetadata],
-        field_author: Optional[FieldAuthor],
-        basic_user_metadata: Optional[UserMetadata] = None,
+        field_computed_metadata: FieldComputedMetadata | None,
+        field_author: FieldAuthor | None,
+        basic_user_metadata: UserMetadata | None = None,
     ):
         user_cancelled_labels: set[str] = (
-            set(
-                [
-                    f"{classification.labelset}/{classification.label}"
-                    for classification in basic_user_metadata.classifications
-                    if classification.cancelled_by_user
-                ]
-            )
+            {
+                f"{classification.labelset}/{classification.label}"
+                for classification in basic_user_metadata.classifications
+                if classification.cancelled_by_user
+            }
             if basic_user_metadata
             else set()
         )
@@ -193,7 +195,7 @@ class ResourceBrain:
         if field_author is not None and field_author.WhichOneof("author") == "data_augmentation":
             field_type, field_id = field_key.split("/")
             da_task_id = ids.extract_data_augmentation_id(field_id)
-            if da_task_id is None:  # pragma: nocover
+            if da_task_id is None:  # pragma: no cover
                 logger.warning(
                     "Data augmentation field id has an unexpected format! Skipping label",
                     extra={
@@ -212,12 +214,17 @@ class ResourceBrain:
         field_key: str,
         field_computed_metadata: FieldComputedMetadata,
         extracted_text: ExtractedText,
-        page_positions: Optional[FilePagePositions],
-        user_field_metadata: Optional[UserFieldMetadata],
+        page_positions: FilePagePositions | None,
+        user_field_metadata: UserFieldMetadata | None,
         replace_field: bool,
-        skip_paragraphs_index: Optional[bool],
-        skip_texts_index: Optional[bool],
+        skip_paragraphs_index: bool | None,
+        skip_texts_index: bool | None,
+        append_splits: set[str] | None = None,
     ) -> None:
+        """
+        append_splits: when provided, only the splits in this set will be indexed. This is used for conversation appends, to
+                       avoid reindexing all previous messages of the conversation.
+        """
         # We need to add the extracted text to the texts section of the Resource so that
         # the paragraphs can be indexed
         self.apply_field_text(
@@ -234,27 +241,45 @@ class ResourceBrain:
             user_field_metadata,
             replace_field=replace_field,
             skip_paragraphs=skip_paragraphs_index,
+            append_splits=append_splits,
         )
 
+    def sorted_splits(self, extracted_text: ExtractedText) -> Iterator[str]:
+        yield from sorted(extracted_text.split_text.keys())
+
     @observer.wrap({"type": "apply_field_paragraphs"})
     def apply_field_paragraphs(
         self,
         field_key: str,
         field_computed_metadata: FieldComputedMetadata,
         extracted_text: ExtractedText,
-        page_positions: Optional[FilePagePositions],
-        user_field_metadata: Optional[UserFieldMetadata],
+        page_positions: FilePagePositions | None,
+        user_field_metadata: UserFieldMetadata | None,
         replace_field: bool,
-        skip_paragraphs: Optional[bool],
+        skip_paragraphs: bool | None,
+        append_splits: set[str] | None = None,
     ) -> None:
         if skip_paragraphs is not None:
             self.brain.skip_paragraphs = skip_paragraphs
         unique_paragraphs: set[str] = set()
         user_paragraph_classifications = self._get_paragraph_user_classifications(user_field_metadata)
         paragraph_pages = ParagraphPages(page_positions) if page_positions else None
+
         # Splits of the field
-        for subfield, field_metadata in field_computed_metadata.split_metadata.items():
-            extracted_text_str = extracted_text.split_text[subfield] if extracted_text else None
+
+        # Used to adjust the paragraph start/end when indexing splits, as they are all
+        # concatenated in the main text part of the brain Resource.
+        split_offset = 0
+        for subfield in self.sorted_splits(extracted_text):
+            if subfield not in field_computed_metadata.split_metadata or should_skip_split_indexing(
+                subfield, replace_field, append_splits
+            ):
+                # We're skipping this split but we need to adjust the offset as we have added the text
+                # of this split to the main text
+                split_offset += len(extracted_text.split_text[subfield]) + 1  # +1 for the space
+                continue
+            field_metadata = field_computed_metadata.split_metadata[subfield]
+            extracted_text_str = extracted_text.split_text[subfield]
             for idx, paragraph in enumerate(field_metadata.paragraphs):
                 key = f"{self.rid}/{field_key}/{subfield}/{paragraph.start}-{paragraph.end}"
                 denied_classifications = set(user_paragraph_classifications.denied.get(key, []))
@@ -280,8 +305,8 @@ class ResourceBrain:
                     representation.file = paragraph.representation.reference_file
                     representation.is_a_table = paragraph.representation.is_a_table
                 p = BrainParagraph(
-                    start=paragraph.start,
-                    end=paragraph.end,
+                    start=paragraph.start + split_offset,
+                    end=paragraph.end + split_offset,
                     field=field_key,
                     split=subfield,
                     index=idx,
@@ -296,6 +321,7 @@ class ResourceBrain:
                         representation=representation,
                     ),
                 )
+                split_offset = p.end + 1  # +1 for the space
                 paragraph_kind_label = f"/k/{Paragraph.TypeParagraph.Name(paragraph.kind).lower()}"
                 paragraph_labels = {paragraph_kind_label}
                 paragraph_labels.update(
@@ -308,7 +334,7 @@ class ResourceBrain:
                 self.brain.paragraphs[field_key].paragraphs[key].CopyFrom(p)
 
         # Main field
-        extracted_text_str = extracted_text.text if extracted_text else None
+        extracted_text_str = extracted_text.text
         for idx, paragraph in enumerate(field_computed_metadata.metadata.paragraphs):
             key = f"{self.rid}/{field_key}/{paragraph.start}-{paragraph.end}"
             denied_classifications = set(user_paragraph_classifications.denied.get(key, []))
@@ -364,7 +390,7 @@ class ResourceBrain:
             self.brain.paragraphs_to_delete.append(full_field_id)
 
     def _get_paragraph_user_classifications(
-        self, basic_user_field_metadata: Optional[UserFieldMetadata]
+        self, basic_user_field_metadata: UserFieldMetadata | None
     ) -> ParagraphClassifications:
         pc = ParagraphClassifications(valid={}, denied={})
         if basic_user_field_metadata is None:
@@ -383,18 +409,16 @@ class ResourceBrain:
     def generate_relations(
         self,
         field_key: str,
-        field_computed_metadata: Optional[FieldComputedMetadata],
-        basic_user_metadata: Optional[UserMetadata],
+        field_computed_metadata: FieldComputedMetadata | None,
+        basic_user_metadata: UserMetadata | None,
         replace_field: bool,
     ) -> None:
         user_cancelled_labels: set[str] = (
-            set(
-                [
-                    f"{classification.labelset}/{classification.label}"
-                    for classification in basic_user_metadata.classifications
-                    if classification.cancelled_by_user
-                ]
-            )
+            {
+                f"{classification.labelset}/{classification.label}"
+                for classification in basic_user_metadata.classifications
+                if classification.cancelled_by_user
+            }
             if basic_user_metadata
             else set()
         )
@@ -483,7 +507,7 @@ class ResourceBrain:
         full_field_id = ids.FieldId(rid=self.rid, type=ftype, key=fkey).full()
         self.brain.texts_to_delete.append(full_field_id)
         self.brain.paragraphs_to_delete.append(full_field_id)
-        self.brain.sentences_to_delete.append(full_field_id)
+        self.brain.vectors_to_delete_in_all_vectorsets.append(full_field_id)
         self.brain.relation_fields_to_delete.append(field_key)
 
     @observer.wrap({"type": "generate_vectors"})
@@ -495,10 +519,13 @@ class ResourceBrain:
         vectorset: str,
         replace_field: bool = False,
         # cut to specific dimension if specified
-        vector_dimension: Optional[int] = None,
+        vector_dimension: int | None = None,
+        append_splits: set[str] | None = None,
     ):
         fid = ids.FieldId.from_string(f"{self.rid}/{field_id}")
         for subfield, vectors in vo.split_vectors.items():
+            if should_skip_split_indexing(subfield, replace_field, append_splits):
+                continue
             _field_id = ids.FieldId(
                 rid=fid.rid,
                 type=fid.type,
@@ -567,7 +594,7 @@ class ResourceBrain:
         *,
         vectorset: str,
         # cut vectors if a specific dimension is specified
-        vector_dimension: Optional[int] = None,
+        vector_dimension: int | None = None,
     ):
         paragraph_pb = self.brain.paragraphs[field_id].paragraphs[paragraph_key.full()]
         sentence_pb = paragraph_pb.vectorsets_sentences[vectorset].sentences[sentence_key.full()]
@@ -592,7 +619,7 @@ class ResourceBrain:
 
         sentence_pb.metadata.position.index = paragraph_pb.metadata.position.index
 
-    def _set_resource_status(self, basic: Basic, previous_status: Optional[Metadata.Status.ValueType]):
+    def _set_resource_status(self, basic: Basic, previous_status: Metadata.Status.ValueType | None):
         """
         We purposefully overwrite what we index as a status and DO NOT reflect
         actual status with what we index.
@@ -622,32 +649,32 @@ class ResourceBrain:
             return "EMPTY"
         return METADATA_STATUS_PB_TYPE_TO_NAME_MAP[metadata.status]
 
-    def _set_resource_dates(self, basic: Basic, origin: Optional[Origin]):
+    def _set_resource_dates(self, basic: Basic, origin: Origin | None):
         """
         Adds the user-defined dates to the brain object. This is at resource level and applies to
         all fields of the resource.
         """
-        if basic.created.seconds > 0:
+        if basic.created.seconds != 0:
             self.brain.metadata.created.CopyFrom(basic.created)
         else:
             logging.warning(f"Basic metadata has no created field for {self.rid}")
             self.brain.metadata.created.GetCurrentTime()
-        if basic.modified.seconds > 0:
+        if basic.modified.seconds != 0:
             self.brain.metadata.modified.CopyFrom(basic.modified)
         else:
-            if basic.created.seconds > 0:
+            if basic.created.seconds != 0:
                 self.brain.metadata.modified.CopyFrom(basic.created)
             else:
                 self.brain.metadata.modified.GetCurrentTime()
 
         if origin is not None:
             # overwrite created/modified if provided on origin
-            if origin.HasField("created") and origin.created.seconds > 0:
+            if origin.HasField("created") and origin.created.seconds != 0:
                 self.brain.metadata.created.CopyFrom(origin.created)
-            if origin.HasField("modified") and origin.modified.seconds > 0:
+            if origin.HasField("modified") and origin.modified.seconds != 0:
                 self.brain.metadata.modified.CopyFrom(origin.modified)
 
-    def _set_resource_relations(self, basic: Basic, origin: Optional[Origin], user_relations: Relations):
+    def _set_resource_relations(self, basic: Basic, origin: Origin | None, user_relations: Relations):
         """
         Adds the relations to the brain object corresponding to the user-defined metadata at the resource level:
         - Contributors of the document
@@ -691,7 +718,7 @@ class ResourceBrain:
 
         self.brain.relation_fields_to_delete.append("a/metadata")
 
-    def _set_resource_labels(self, basic: Basic, origin: Optional[Origin]):
+    def _set_resource_labels(self, basic: Basic, origin: Origin | None):
         """
         Adds the resource-level labels to the brain object.
         These levels are user-defined in basic or origin metadata.
@@ -748,7 +775,7 @@ class ResourceBrain:
 
 def is_paragraph_repeated_in_field(
     paragraph: Paragraph,
-    extracted_text: Optional[str],
+    extracted_text: str | None,
     unique_paragraphs: set[str],
 ) -> bool:
     if extracted_text is None:
@@ -787,8 +814,13 @@ class ParagraphPages:
             return self._materialized[paragraph_start_index]
         except IndexError:
             logger.error(
-                f"Could not find a page for the given index: {paragraph_start_index}. Page positions: {self.positions}"  # noqa
+                f"Could not find a page for the given index: {paragraph_start_index}. Page positions: {self.positions}"
             )
             if len(self._materialized) > 0:
                 return self._materialized[-1]
             return 0
+
+
+def should_skip_split_indexing(split: str, replace_field: bool, append_splits: set[str] | None) -> bool:
+    # When replacing the whole field, reindex all splits. Otherwise, we're only indexing the splits that are appended
+    return not replace_field and append_splits is not None and split not in append_splits

nucliadb/ingest/orm/broker_message.py

@@ -56,7 +56,7 @@ class _BrokerMessageBuilder:
         # clear the state and generate a new broker message
         self.bm.Clear()
 
-        self.bm.kbid = resource.kb.kbid
+        self.bm.kbid = resource.kbid
         self.bm.uuid = resource.uuid
         basic = await resource.get_basic()
         if basic is not None:
@@ -93,9 +93,7 @@ class _BrokerMessageBuilder:
                     self.bm.link_extracted_data.append(link_extracted_data)
 
             # Field vectors
-            async for vectorset_id, vs in datamanagers.vectorsets.iter(
-                resource.txn, kbid=resource.kb.kbid
-            ):
+            async for vectorset_id, vs in datamanagers.vectorsets.iter(resource.txn, kbid=resource.kbid):
                 await self.generate_field_vectors(
                     type_id, field_id, field, vectorset_id, vs.storage_key_kind
                 )