nucliadb 6.7.2.post4874__py3-none-any.whl → 6.10.0.post5705__py3-none-any.whl

nucliadb/ingest/orm/entities.py

@@ -18,8 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-import asyncio
-from typing import AsyncGenerator, Optional
+from collections.abc import AsyncGenerator
 
 from nidx_protos.nodereader_pb2 import (
     Faceted,
@@ -29,23 +28,12 @@ from nidx_protos.nodereader_pb2 import (
     SearchResponse,
 )
 
-from nucliadb.common import datamanagers
-from nucliadb.common.cluster.exceptions import (
-    AlreadyExists,
-    EntitiesGroupNotFound,
-)
 from nucliadb.common.cluster.utils import get_shard_manager
-from nucliadb.common.datamanagers.entities import (
-    KB_DELETED_ENTITIES_GROUPS,
-    KB_ENTITIES,
-    KB_ENTITIES_GROUP,
-)
 from nucliadb.common.maindb.driver import Transaction
 from nucliadb.ingest.orm.knowledgebox import KnowledgeBox
 from nucliadb.ingest.settings import settings
 from nucliadb.search.search.shards import graph_search_shard, query_shard
 from nucliadb_protos.knowledgebox_pb2 import (
-    DeletedEntitiesGroups,
     EntitiesGroup,
     EntitiesGroupSummary,
     Entity,
@@ -53,8 +41,6 @@ from nucliadb_protos.knowledgebox_pb2 import (
 from nucliadb_protos.utils_pb2 import RelationNode
 from nucliadb_protos.writer_pb2 import GetEntitiesResponse
 
-from .exceptions import EntityManagementException
-
 MAX_DUPLICATES = 300
 MAX_DELETED = 300
 
@@ -69,20 +55,11 @@ class EntitiesManager:
         self.txn = txn
         self.kbid = self.kb.kbid
 
-    async def create_entities_group(self, group: str, entities: EntitiesGroup):
-        if await self.entities_group_exists(group):
-            raise AlreadyExists(f"Entities group {group} already exists")
-
-        await self.store_entities_group(group, entities)
-
     async def get_entities(self, entities: GetEntitiesResponse):
         async for group, eg in self.iterate_entities_groups(exclude_deleted=True):
             entities.groups[group].CopyFrom(eg)
 
-    async def get_entities_group(self, group: str) -> Optional[EntitiesGroup]:
-        deleted = await self.is_entities_group_deleted(group)
-        if deleted:
-            return None
+    async def get_entities_group(self, group: str) -> EntitiesGroup | None:
         return await self.get_entities_group_inner(group)
 
     async def get_entities_groups(self) -> dict[str, EntitiesGroup]:
@@ -93,113 +70,18 @@ class EntitiesManager:
 
     async def list_entities_groups(self) -> dict[str, EntitiesGroupSummary]:
         groups = {}
-        max_simultaneous = asyncio.Semaphore(10)
 
-        async def _composition(group: str):
-            async with max_simultaneous:
-                stored = await self.get_stored_entities_group(group)
-                if stored is not None:
-                    groups[group] = EntitiesGroupSummary(
-                        title=stored.title, color=stored.color, custom=stored.custom
-                    )
-                else:
-                    # We don't want to search for each indexed group, as we are
-                    # providing a quick summary
-                    groups[group] = EntitiesGroupSummary()
+        async for group in self.iterate_entities_groups_names(exclude_deleted=True):
+            groups[group] = EntitiesGroupSummary()
 
-        tasks = [
-            asyncio.create_task(_composition(group))
-            async for group in self.iterate_entities_groups_names(exclude_deleted=True)
-        ]
-        if tasks:
-            await asyncio.wait(tasks)
         return groups
 
-    async def update_entities(self, group: str, entities: dict[str, Entity]):
-        """Update entities on an entity group. New entities are appended and existing
-        are overwriten. Existing entities not appearing in `entities` are left
-        intact. Use `delete_entities` to delete them instead.
-
-        """
-        if not await self.entities_group_exists(group):
-            raise EntitiesGroupNotFound(f"Entities group '{group}' doesn't exist")
-
-        entities_group = await self.get_stored_entities_group(group)
-        if entities_group is None:
-            entities_group = EntitiesGroup()
-
-        for name, entity in entities.items():
-            entities_group.entities[name].CopyFrom(entity)
-
-        await self.store_entities_group(group, entities_group)
-
-    async def set_entities_group(self, group: str, entities: EntitiesGroup):
-        indexed = await self.get_indexed_entities_group(group)
-        if indexed is None:
-            updated = entities
-        else:
-            updated = EntitiesGroup()
-            updated.CopyFrom(entities)
-
-            for name, entity in indexed.entities.items():
-                if name not in updated.entities:
-                    updated.entities[name].CopyFrom(entity)
-                    updated.entities[name].deleted = True
-
-        await self.store_entities_group(group, updated)
-
-    async def set_entities_group_force(self, group: str, entitiesgroup: EntitiesGroup):
-        await self.store_entities_group(group, entitiesgroup)
-
-    async def set_entities_group_metadata(
-        self, group: str, *, title: Optional[str] = None, color: Optional[str] = None
-    ):
-        entities_group = await self.get_stored_entities_group(group)
-        if entities_group is None:
-            entities_group = EntitiesGroup()
-
-        if title:
-            entities_group.title = title
-        if color:
-            entities_group.color = color
-
-        await self.store_entities_group(group, entities_group)
-
-    async def delete_entities(self, group: str, delete: list[str]):
-        stored = await self.get_stored_entities_group(group)
-
-        stored = stored or EntitiesGroup()
-        for name in delete:
-            if name not in stored.entities:
-                entity = stored.entities[name]
-                entity.value = name
-            else:
-                entity = stored.entities[name]
-            entity.deleted = True
-        await self.store_entities_group(group, stored)
-
-    async def delete_entities_group(self, group: str):
-        await self.delete_stored_entities_group(group)
-        await self.mark_entities_group_as_deleted(group)
-
     # Private API
 
-    async def get_entities_group_inner(self, group: str) -> Optional[EntitiesGroup]:
-        stored = await self.get_stored_entities_group(group)
-        indexed = await self.get_indexed_entities_group(group)
-        if stored is None and indexed is None:
-            # Entity group does not exist
-            return None
-        elif stored is not None and indexed is not None:
-            entities_group = self.merge_entities_groups(indexed, stored)
-        else:
-            entities_group = stored or indexed
-        return entities_group
-
-    async def get_stored_entities_group(self, group: str) -> Optional[EntitiesGroup]:
-        return await datamanagers.entities.get_entities_group(self.txn, kbid=self.kbid, group=group)
+    async def get_entities_group_inner(self, group: str) -> EntitiesGroup | None:
+        return await self.get_indexed_entities_group(group)
 
-    async def get_indexed_entities_group(self, group: str) -> Optional[EntitiesGroup]:
+    async def get_indexed_entities_group(self, group: str) -> EntitiesGroup | None:
         shard_manager = get_shard_manager()
 
         async def do_entities_search(shard_id: str) -> GraphSearchResponse:
@@ -228,26 +110,9 @@ class EntitiesManager:
         eg = EntitiesGroup(entities=entities)
         return eg
 
-    async def get_deleted_entities_groups(self) -> set[str]:
-        deleted: set[str] = set()
-        key = KB_DELETED_ENTITIES_GROUPS.format(kbid=self.kbid)
-        payload = await self.txn.get(key)
-        if payload:
-            deg = DeletedEntitiesGroups()
-            deg.ParseFromString(payload)
-            deleted.update(deg.entities_groups)
-        return deleted
-
     async def entities_group_exists(self, group: str) -> bool:
-        stored = await self.get_stored_entities_group(group)
-        if stored is not None:
-            return True
-
         indexed = await self.get_indexed_entities_group(group)
-        if indexed is not None:
-            return True
-
-        return False
+        return indexed is not None
 
     async def iterate_entities_groups(
         self, exclude_deleted: bool
@@ -262,27 +127,10 @@ class EntitiesManager:
         self,
         exclude_deleted: bool,
     ) -> AsyncGenerator[str, None]:
-        # Start the task to get indexed groups
-        indexed_task = asyncio.create_task(self.get_indexed_entities_groups_names())
-
-        if exclude_deleted:
-            deleted_groups = await self.get_deleted_entities_groups()
-
         visited_groups = set()
-
-        # stored groups
-        entities_key = KB_ENTITIES.format(kbid=self.kbid)
-        async for key in self.txn.keys(entities_key):
-            group = key.split("/")[-1]
-            if exclude_deleted and group in deleted_groups:
-                continue
-            yield group
-            visited_groups.add(group)
-
-        # indexed groups
-        indexed_groups = await indexed_task
+        indexed_groups = await self.get_indexed_entities_groups_names()
         for group in indexed_groups:
-            if (exclude_deleted and group in deleted_groups) or group in visited_groups:
+            if group in visited_groups:
                 continue
             yield group
             visited_groups.add(group)
@@ -319,53 +167,6 @@ class EntitiesManager:
             return set()
         return set.union(*results)
 
-    async def store_entities_group(self, group: str, eg: EntitiesGroup):
-        meta_cache = await datamanagers.entities.get_entities_meta_cache(self.txn, kbid=self.kbid)
-        duplicates = {}
-        deleted = []
-        duplicate_count = 0
-        for entity in eg.entities.values():
-            if entity.deleted:
-                deleted.append(entity.value)
-                continue
-            if len(entity.represents) == 0:
-                continue
-            duplicates[entity.value] = list(entity.represents)
-            duplicate_count += len(duplicates[entity.value])
-
-        if duplicate_count > MAX_DUPLICATES:
-            raise EntityManagementException(
-                f"Too many duplicates: {duplicate_count}. Max of {MAX_DUPLICATES} currently allowed"
-            )
-        if len(deleted) > MAX_DELETED:
-            raise EntityManagementException(
-                f"Too many deleted entities: {len(deleted)}. Max of {MAX_DELETED} currently allowed"
-            )
-
-        meta_cache.set_duplicates(group, duplicates)
-        meta_cache.set_deleted(group, deleted)
-        await datamanagers.entities.set_entities_meta_cache(self.txn, kbid=self.kbid, cache=meta_cache)
-
-        await datamanagers.entities.set_entities_group(
-            self.txn, kbid=self.kbid, group_id=group, entities=eg
-        )
-        # if it was preivously deleted, we must unmark it
-        await self.unmark_entities_group_as_deleted(group)
-
-    async def is_entities_group_deleted(self, group: str):
-        deleted_groups = await self.get_deleted_entities_groups()
-        return group in deleted_groups
-
-    async def delete_stored_entities_group(self, group: str):
-        entities_key = KB_ENTITIES_GROUP.format(kbid=self.kbid, id=group)
-        await self.txn.delete(entities_key)
-
-    async def mark_entities_group_as_deleted(self, group: str):
-        await datamanagers.entities.mark_group_as_deleted(self.txn, kbid=self.kbid, group=group)
-
-    async def unmark_entities_group_as_deleted(self, group: str):
-        await datamanagers.entities.unmark_group_as_deleted(self.txn, kbid=self.kbid, group=group)
-
     @staticmethod
     def merge_entities_groups(indexed: EntitiesGroup, stored: EntitiesGroup):
         """Create a new EntitiesGroup with the merged entities from `stored` and

nucliadb/ingest/orm/index_message.py

@@ -20,11 +20,12 @@
 
 
 import asyncio
-from typing import Optional
+from collections.abc import Sequence
 
 from nidx_protos.noderesources_pb2 import Resource as IndexMessage
 
 from nucliadb.common import datamanagers
+from nucliadb.ingest.fields.conversation import Conversation
 from nucliadb.ingest.fields.exceptions import FieldAuthorNotFound
 from nucliadb.ingest.fields.file import File
 from nucliadb.ingest.orm.brain_v2 import ResourceBrain
@@ -32,6 +33,7 @@ from nucliadb.ingest.orm.metrics import index_message_observer as observer
 from nucliadb.ingest.orm.resource import Resource, get_file_page_positions
 from nucliadb_protos.knowledgebox_pb2 import VectorSetConfig
 from nucliadb_protos.resources_pb2 import Basic, FieldID, FieldType
+from nucliadb_protos.utils_pb2 import ExtractedText
 from nucliadb_protos.writer_pb2 import BrokerMessage
 
 
@@ -68,7 +70,8 @@ class IndexMessageBuilder:
         vectors: bool = True,
         relations: bool = True,
         replace: bool = True,
-        vectorset_configs: Optional[list[VectorSetConfig]] = None,
+        vectorset_configs: list[VectorSetConfig] | None = None,
+        append_splits: set[str] | None = None,
     ):
         field = await self.resource.get_field(fieldid.field, fieldid.field_type)
         extracted_text = await field.get_extracted_text()
@@ -120,6 +123,7 @@ class IndexMessageBuilder:
                     replace_field=replace_paragraphs,
                     skip_paragraphs_index=skip_paragraphs_index,
                     skip_texts_index=skip_texts_index,
+                    append_splits=append_splits,
                 )
         if vectors:
             assert vectorset_configs is not None
@@ -137,6 +141,7 @@ class IndexMessageBuilder:
                         vectorset=vectorset_config.vectorset_id,
                         replace_field=replace,
                         vector_dimension=dimension,
+                        append_splits=append_splits,
                     )
         if relations:
             await asyncio.to_thread(
@@ -150,7 +155,7 @@ class IndexMessageBuilder:
     def _apply_field_deletions(
         self,
         brain: ResourceBrain,
-        field_ids: list[FieldID],
+        field_ids: Sequence[FieldID],
     ) -> None:
         for field_id in field_ids:
             brain.delete_field(self.resource.generate_field_id(field_id))
@@ -158,20 +163,19 @@ class IndexMessageBuilder:
     @observer.wrap({"type": "writer_bm"})
     async def for_writer_bm(
         self,
-        messages: list[BrokerMessage],
+        message: BrokerMessage,
         resource_created: bool,
     ) -> IndexMessage:
         """
-        Builds the index message for the broker messages coming from the writer.
+        Builds the index message for the broker message coming from the writer.
         The writer messages are not adding new vectors to the index.
         """
-        assert all(message.source == BrokerMessage.MessageSource.WRITER for message in messages)
+        assert message.source == BrokerMessage.MessageSource.WRITER
 
-        deleted_fields = get_bm_deleted_fields(messages)
-        self._apply_field_deletions(self.brain, deleted_fields)
+        self._apply_field_deletions(self.brain, message.delete_fields)
         await self._apply_resource_index_data(self.brain)
         basic = await self.get_basic()
-        prefilter_update = needs_prefilter_update(messages)
+        prefilter_update = needs_prefilter_update(message)
         if prefilter_update:
             # Changes on some metadata at the resource level that is used for filtering require that we reindex all the fields
             # in the texts index (as it is the one used for prefiltering).
@@ -181,16 +185,16 @@ class IndexMessageBuilder:
             ]
         else:
             # Simply process the fields that are in the message
-            fields_to_index = get_bm_modified_fields(messages)
+            fields_to_index = get_bm_modified_fields(message)
         for fieldid in fields_to_index:
-            if fieldid in deleted_fields:
+            if fieldid in message.delete_fields:
                 continue
             await self._apply_field_index_data(
                 self.brain,
                 fieldid,
                 basic,
-                texts=prefilter_update or needs_texts_update(fieldid, messages),
-                paragraphs=needs_paragraphs_update(fieldid, messages),
+                texts=prefilter_update or needs_texts_update(fieldid, message),
+                paragraphs=needs_paragraphs_update(fieldid, message),
                 relations=False,  # Relations at the field level are not modified by the writer
                 vectors=False,  # Vectors are never added by the writer
                 replace=not resource_created,
@@ -200,32 +204,45 @@ class IndexMessageBuilder:
     @observer.wrap({"type": "processor_bm"})
     async def for_processor_bm(
         self,
-        messages: list[BrokerMessage],
+        message: BrokerMessage,
     ) -> IndexMessage:
         """
         Builds the index message for the broker messages coming from the processor.
         The processor can index new data to any index.
         """
-        assert all(message.source == BrokerMessage.MessageSource.PROCESSOR for message in messages)
-        deleted_fields = get_bm_deleted_fields(messages)
-        self._apply_field_deletions(self.brain, deleted_fields)
+        assert message.source == BrokerMessage.MessageSource.PROCESSOR
+        self._apply_field_deletions(self.brain, message.delete_fields)
         await self._apply_resource_index_data(self.brain)
         basic = await self.get_basic()
-        fields_to_index = get_bm_modified_fields(messages)
+        fields_to_index = get_bm_modified_fields(message)
         vectorsets_configs = await self.get_vectorsets_configs()
         for fieldid in fields_to_index:
-            if fieldid in deleted_fields:
+            if fieldid in message.delete_fields:
                 continue
+
+            # For conversation fields, we only replace the full field if it is not an append messages operation.
+            # All other fields are always replaced upon modification.
+            replace_field = True
+            modified_splits = None
+            if fieldid.field_type == FieldType.CONVERSATION:
+                modified_splits = await get_bm_modified_split_ids(fieldid, message, self.resource)
+                stored_splits = await get_stored_split_ids(fieldid, self.resource)
+                is_append_messages_op = modified_splits.issubset(stored_splits) and 0 < len(
+                    modified_splits
+                ) < len(stored_splits)
+                replace_field = not is_append_messages_op
+
             await self._apply_field_index_data(
                 self.brain,
                 fieldid,
                 basic,
-                texts=needs_texts_update(fieldid, messages),
-                paragraphs=needs_paragraphs_update(fieldid, messages),
-                relations=needs_relations_update(fieldid, messages),
-                vectors=needs_vectors_update(fieldid, messages),
-                replace=True,
+                texts=needs_texts_update(fieldid, message),
+                paragraphs=needs_paragraphs_update(fieldid, message),
+                relations=needs_relations_update(fieldid, message),
+                vectors=needs_vectors_update(fieldid, message),
+                replace=replace_field,
                 vectorset_configs=vectorsets_configs,
+                append_splits=modified_splits,
             )
         return self.brain.brain
 
@@ -264,136 +281,134 @@ class IndexMessageBuilder:
         vectorset_configs = [
             vectorset_config
             async for _, vectorset_config in datamanagers.vectorsets.iter(
-                self.resource.txn, kbid=self.resource.kb.kbid
+                self.resource.txn, kbid=self.resource.kbid
             )
         ]
         return vectorset_configs
 
 
-def get_bm_deleted_fields(
-    messages: list[BrokerMessage],
-) -> list[FieldID]:
-    deleted = []
-    for message in messages:
-        for field in message.delete_fields:
-            if field not in deleted:
-                deleted.append(field)
-    return deleted
-
-
-def get_bm_modified_fields(messages: list[BrokerMessage]) -> list[FieldID]:
-    message_source = get_messages_source(messages)
+def get_bm_modified_fields(message: BrokerMessage) -> list[FieldID]:
     modified = set()
-    for message in messages:
-        # Added or modified fields need indexing
-        for link in message.links:
-            modified.add((link, FieldType.LINK))
-        for file in message.files:
-            modified.add((file, FieldType.FILE))
-        for conv in message.conversations:
-            modified.add((conv, FieldType.CONVERSATION))
-        for text in message.texts:
-            modified.add((text, FieldType.TEXT))
+    # Added or modified fields need indexing
+    for link in message.links:
+        modified.add((link, FieldType.LINK))
+    for file in message.files:
+        modified.add((file, FieldType.FILE))
+    for conv in message.conversations:
+        modified.add((conv, FieldType.CONVERSATION))
+    for text in message.texts:
+        modified.add((text, FieldType.TEXT))
+    if message.HasField("basic"):
+        # Add title and summary only if they have changed
+        if message.basic.title != "":
+            modified.add(("title", FieldType.GENERIC))
+        if message.basic.summary != "":
+            modified.add(("summary", FieldType.GENERIC))
+
+    if message.source == BrokerMessage.MessageSource.PROCESSOR:
+        # Messages with field metadata, extracted text or field vectors need indexing
+        for fm in message.field_metadata:
+            modified.add((fm.field.field, fm.field.field_type))
+        for et in message.extracted_text:
+            modified.add((et.field.field, et.field.field_type))
+        for fv in message.field_vectors:
+            modified.add((fv.field.field, fv.field.field_type))
+
+    if message.source == BrokerMessage.MessageSource.WRITER:
+        # Any field that has fieldmetadata annotations should be considered as modified
+        # and needs to be reindexed
         if message.HasField("basic"):
-            # Add title and summary only if they have changed
-            if message.basic.title != "":
-                modified.add(("title", FieldType.GENERIC))
-            if message.basic.summary != "":
-                modified.add(("summary", FieldType.GENERIC))
-
-        if message_source == BrokerMessage.MessageSource.PROCESSOR:
-            # Messages with field metadata, extracted text or field vectors need indexing
-            for fm in message.field_metadata:
-                modified.add((fm.field.field, fm.field.field_type))
-            for et in message.extracted_text:
-                modified.add((et.field.field, et.field.field_type))
-            for fv in message.field_vectors:
-                modified.add((fv.field.field, fv.field.field_type))
-
-        if message_source == BrokerMessage.MessageSource.WRITER:
-            # Any field that has fieldmetadata annotations should be considered as modified
-            # and needs to be reindexed
-            if message.HasField("basic"):
-                for ufm in message.basic.fieldmetadata:
-                    modified.add((ufm.field.field, ufm.field.field_type))
+            for ufm in message.basic.fieldmetadata:
+                modified.add((ufm.field.field, ufm.field.field_type))
     return [FieldID(field=field, field_type=field_type) for field, field_type in modified]
 
 
-def get_messages_source(messages: list[BrokerMessage]) -> BrokerMessage.MessageSource.ValueType:
-    assert len(set(message.source for message in messages)) == 1
-    return messages[0].source
+def needs_prefilter_update(message: BrokerMessage) -> bool:
+    return message.reindex
 
 
-def needs_prefilter_update(messages: list[BrokerMessage]) -> bool:
-    return any(message.reindex for message in messages)
-
-
-def needs_paragraphs_update(field_id: FieldID, messages: list[BrokerMessage]) -> bool:
+def needs_paragraphs_update(field_id: FieldID, message: BrokerMessage) -> bool:
     return (
-        has_paragraph_annotations(field_id, messages)
-        or has_new_extracted_text(field_id, messages)
-        or has_new_field_metadata(field_id, messages)
+        has_paragraph_annotations(field_id, message)
+        or has_new_extracted_text(field_id, message)
+        or has_new_field_metadata(field_id, message)
     )
 
 
-def has_paragraph_annotations(field_id: FieldID, messages: list[BrokerMessage]) -> bool:
-    for message in messages:
-        ufm = next(
-            (fm for fm in message.basic.fieldmetadata if fm.field == field_id),
-            None,
-        )
-        if ufm is None:
-            continue
-        if len(ufm.paragraphs) > 0:
-            return True
-    return False
+def has_paragraph_annotations(field_id: FieldID, message: BrokerMessage) -> bool:
+    ufm = next(
+        (fm for fm in message.basic.fieldmetadata if fm.field == field_id),
+        None,
+    )
+    if ufm is None:
+        return False
+    return len(ufm.paragraphs) > 0
 
 
 def has_new_field_metadata(
     field_id: FieldID,
-    messages: list[BrokerMessage],
+    message: BrokerMessage,
 ) -> bool:
-    for message in messages:
-        for field_metadata in message.field_metadata:
-            if field_metadata.field == field_id:
-                return True
-    return False
+    return any(field_metadata.field == field_id for field_metadata in message.field_metadata)
 
 
 def has_new_extracted_text(
     field_id: FieldID,
-    messages: list[BrokerMessage],
+    message: BrokerMessage,
 ) -> bool:
-    for message in messages:
-        for extracted_text in message.extracted_text:
-            if extracted_text.field == field_id:
-                return True
-    return False
+    return any(extracted_text.field == field_id for extracted_text in message.extracted_text)
 
 
 def needs_texts_update(
     field_id: FieldID,
-    messages: list[BrokerMessage],
+    message: BrokerMessage,
 ) -> bool:
-    return has_new_extracted_text(field_id, messages) or has_new_field_metadata(field_id, messages)
+    return has_new_extracted_text(field_id, message) or has_new_field_metadata(field_id, message)
 
 
 def needs_vectors_update(
     field_id: FieldID,
-    messages: list[BrokerMessage],
+    message: BrokerMessage,
 ) -> bool:
-    for message in messages:
-        for field_vectors in message.field_vectors:
-            if field_vectors.field == field_id:
-                return True
-    return False
+    return any(field_vectors.field == field_id for field_vectors in message.field_vectors)
+
+
+async def get_bm_modified_split_ids(
+    conversation_field_id: FieldID,
+    message: BrokerMessage,
+    resource: Resource,
+) -> set[str]:
+    message_etw = next(
+        (etw for etw in message.extracted_text if etw.field == conversation_field_id), None
+    )
+    if message_etw is None:
+        return set()
+    storage = resource.storage
+    if message_etw.HasField("file"):
+        raw_payload = await storage.downloadbytescf(message_etw.file)
+        message_extracted_text = ExtractedText()
+        message_extracted_text.ParseFromString(raw_payload.read())
+        raw_payload.flush()
+    else:
+        message_extracted_text = message_etw.body
+    return set(message_extracted_text.split_text.keys())
+
+
+async def get_stored_split_ids(
+    conversation_field_id: FieldID,
+    resource: Resource,
+) -> set[str]:
+    fid = conversation_field_id
+    conv: Conversation = await resource.get_field(fid.field, fid.field_type, load=False)
+    splits_metadata = await conv.get_splits_metadata()
+    return set(splits_metadata.metadata)
 
 
 def needs_relations_update(
     field_id: FieldID,
-    messages: list[BrokerMessage],
+    message: BrokerMessage,
 ) -> bool:
-    return has_new_field_metadata(field_id, messages) or has_new_extracted_text(field_id, messages)
+    return has_new_field_metadata(field_id, message) or has_new_extracted_text(field_id, message)
 
 
 async def get_resource_index_message(