nucliadb 6.9.1.post5192__py3-none-any.whl → 6.10.0.post5705__py3-none-any.whl

nucliadb/ingest/orm/brain_v2.py

@@ -18,9 +18,9 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import logging
+from collections.abc import Iterator
 from copy import deepcopy
 from dataclasses import dataclass
-from typing import Optional
 
 from nidx_protos.noderesources_pb2 import IndexParagraph as BrainParagraph
 from nidx_protos.noderesources_pb2 import (
@@ -81,9 +81,9 @@ class ResourceBrain:
         self,
         basic: Basic,
         user_relations: Relations,
-        origin: Optional[Origin],
-        previous_processing_status: Optional[Metadata.Status.ValueType],
-        security: Optional[utils_pb2.Security],
+        origin: Origin | None,
+        previous_processing_status: Metadata.Status.ValueType | None,
+        security: utils_pb2.Security | None,
     ) -> None:
         self._set_resource_status(basic, previous_processing_status)
         self._set_resource_dates(basic, origin)
@@ -97,9 +97,9 @@ class ResourceBrain:
         self,
         field_key: str,
         extracted_text: ExtractedText,
-        field_computed_metadata: Optional[FieldComputedMetadata],
-        basic_user_metadata: Optional[UserMetadata],
-        field_author: Optional[FieldAuthor],
+        field_computed_metadata: FieldComputedMetadata | None,
+        basic_user_metadata: UserMetadata | None,
+        field_author: FieldAuthor | None,
         replace_field: bool,
         skip_index: bool,
     ) -> None:
@@ -122,13 +122,17 @@ class ResourceBrain:
         field_key: str,
         extracted_text: ExtractedText,
         replace_field: bool,
-        skip_texts: Optional[bool],
+        skip_texts: bool | None,
     ):
         if skip_texts is not None:
             self.brain.skip_texts = skip_texts
+
         field_text = extracted_text.text
-        for _, split in extracted_text.split_text.items():
-            field_text += f" {split} "
+
+        for split_id in self.sorted_splits(extracted_text):
+            split_text = extracted_text.split_text[split_id]
+            field_text += f"{split_text} "
+
         self.brain.texts[field_key].text = field_text
 
         if replace_field:
@@ -140,18 +144,16 @@ class ResourceBrain:
     def apply_field_labels(
         self,
         field_key: str,
-        field_computed_metadata: Optional[FieldComputedMetadata],
-        field_author: Optional[FieldAuthor],
-        basic_user_metadata: Optional[UserMetadata] = None,
+        field_computed_metadata: FieldComputedMetadata | None,
+        field_author: FieldAuthor | None,
+        basic_user_metadata: UserMetadata | None = None,
     ):
         user_cancelled_labels: set[str] = (
-            set(
-                [
-                    f"{classification.labelset}/{classification.label}"
-                    for classification in basic_user_metadata.classifications
-                    if classification.cancelled_by_user
-                ]
-            )
+            {
+                f"{classification.labelset}/{classification.label}"
+                for classification in basic_user_metadata.classifications
+                if classification.cancelled_by_user
+            }
             if basic_user_metadata
             else set()
         )
@@ -212,13 +214,17 @@ class ResourceBrain:
         field_key: str,
         field_computed_metadata: FieldComputedMetadata,
         extracted_text: ExtractedText,
-        page_positions: Optional[FilePagePositions],
-        user_field_metadata: Optional[UserFieldMetadata],
+        page_positions: FilePagePositions | None,
+        user_field_metadata: UserFieldMetadata | None,
         replace_field: bool,
-        skip_paragraphs_index: Optional[bool],
-        skip_texts_index: Optional[bool],
-        append_splits: Optional[set[str]] = None,
+        skip_paragraphs_index: bool | None,
+        skip_texts_index: bool | None,
+        append_splits: set[str] | None = None,
     ) -> None:
+        """
+        append_splits: when provided, only the splits in this set will be indexed. This is used for conversation appends, to
+                       avoid reindexing all previous messages of the conversation.
+        """
         # We need to add the extracted text to the texts section of the Resource so that
         # the paragraphs can be indexed
         self.apply_field_text(
@@ -238,30 +244,41 @@ class ResourceBrain:
             append_splits=append_splits,
         )
 
+    def sorted_splits(self, extracted_text: ExtractedText) -> Iterator[str]:
+        yield from sorted(extracted_text.split_text.keys())
+
     @observer.wrap({"type": "apply_field_paragraphs"})
     def apply_field_paragraphs(
         self,
         field_key: str,
         field_computed_metadata: FieldComputedMetadata,
         extracted_text: ExtractedText,
-        page_positions: Optional[FilePagePositions],
-        user_field_metadata: Optional[UserFieldMetadata],
+        page_positions: FilePagePositions | None,
+        user_field_metadata: UserFieldMetadata | None,
         replace_field: bool,
-        skip_paragraphs: Optional[bool],
-        append_splits: Optional[set[str]] = None,
+        skip_paragraphs: bool | None,
+        append_splits: set[str] | None = None,
     ) -> None:
         if skip_paragraphs is not None:
             self.brain.skip_paragraphs = skip_paragraphs
         unique_paragraphs: set[str] = set()
         user_paragraph_classifications = self._get_paragraph_user_classifications(user_field_metadata)
         paragraph_pages = ParagraphPages(page_positions) if page_positions else None
+
         # Splits of the field
-        for subfield, field_metadata in field_computed_metadata.split_metadata.items():
-            if should_skip_split_indexing(subfield, replace_field, append_splits):
-                continue
-            if subfield not in extracted_text.split_text:
-                # No extracted text for this split
+
+        # Used to adjust the paragraph start/end when indexing splits, as they are all
+        # concatenated in the main text part of the brain Resource.
+        split_offset = 0
+        for subfield in self.sorted_splits(extracted_text):
+            if subfield not in field_computed_metadata.split_metadata or should_skip_split_indexing(
+                subfield, replace_field, append_splits
+            ):
+                # We're skipping this split but we need to adjust the offset as we have added the text
+                # of this split to the main text
+                split_offset += len(extracted_text.split_text[subfield]) + 1  # +1 for the space
                 continue
+            field_metadata = field_computed_metadata.split_metadata[subfield]
             extracted_text_str = extracted_text.split_text[subfield]
             for idx, paragraph in enumerate(field_metadata.paragraphs):
                 key = f"{self.rid}/{field_key}/{subfield}/{paragraph.start}-{paragraph.end}"
@@ -288,8 +305,8 @@ class ResourceBrain:
                     representation.file = paragraph.representation.reference_file
                     representation.is_a_table = paragraph.representation.is_a_table
                 p = BrainParagraph(
-                    start=paragraph.start,
-                    end=paragraph.end,
+                    start=paragraph.start + split_offset,
+                    end=paragraph.end + split_offset,
                     field=field_key,
                     split=subfield,
                     index=idx,
@@ -304,6 +321,7 @@ class ResourceBrain:
                         representation=representation,
                     ),
                 )
+                split_offset = p.end + 1  # +1 for the space
                 paragraph_kind_label = f"/k/{Paragraph.TypeParagraph.Name(paragraph.kind).lower()}"
                 paragraph_labels = {paragraph_kind_label}
                 paragraph_labels.update(
@@ -372,7 +390,7 @@ class ResourceBrain:
             self.brain.paragraphs_to_delete.append(full_field_id)
 
     def _get_paragraph_user_classifications(
-        self, basic_user_field_metadata: Optional[UserFieldMetadata]
+        self, basic_user_field_metadata: UserFieldMetadata | None
     ) -> ParagraphClassifications:
         pc = ParagraphClassifications(valid={}, denied={})
         if basic_user_field_metadata is None:
@@ -391,18 +409,16 @@ class ResourceBrain:
     def generate_relations(
         self,
         field_key: str,
-        field_computed_metadata: Optional[FieldComputedMetadata],
-        basic_user_metadata: Optional[UserMetadata],
+        field_computed_metadata: FieldComputedMetadata | None,
+        basic_user_metadata: UserMetadata | None,
         replace_field: bool,
     ) -> None:
         user_cancelled_labels: set[str] = (
-            set(
-                [
-                    f"{classification.labelset}/{classification.label}"
-                    for classification in basic_user_metadata.classifications
-                    if classification.cancelled_by_user
-                ]
-            )
+            {
+                f"{classification.labelset}/{classification.label}"
+                for classification in basic_user_metadata.classifications
+                if classification.cancelled_by_user
+            }
             if basic_user_metadata
             else set()
         )
@@ -491,7 +507,7 @@ class ResourceBrain:
         full_field_id = ids.FieldId(rid=self.rid, type=ftype, key=fkey).full()
         self.brain.texts_to_delete.append(full_field_id)
         self.brain.paragraphs_to_delete.append(full_field_id)
-        self.brain.sentences_to_delete.append(full_field_id)
+        self.brain.vectors_to_delete_in_all_vectorsets.append(full_field_id)
         self.brain.relation_fields_to_delete.append(field_key)
 
     @observer.wrap({"type": "generate_vectors"})
@@ -503,8 +519,8 @@ class ResourceBrain:
         vectorset: str,
         replace_field: bool = False,
         # cut to specific dimension if specified
-        vector_dimension: Optional[int] = None,
-        append_splits: Optional[set[str]] = None,
+        vector_dimension: int | None = None,
+        append_splits: set[str] | None = None,
     ):
         fid = ids.FieldId.from_string(f"{self.rid}/{field_id}")
         for subfield, vectors in vo.split_vectors.items():
@@ -578,7 +594,7 @@ class ResourceBrain:
         *,
         vectorset: str,
         # cut vectors if a specific dimension is specified
-        vector_dimension: Optional[int] = None,
+        vector_dimension: int | None = None,
     ):
         paragraph_pb = self.brain.paragraphs[field_id].paragraphs[paragraph_key.full()]
         sentence_pb = paragraph_pb.vectorsets_sentences[vectorset].sentences[sentence_key.full()]
@@ -603,7 +619,7 @@ class ResourceBrain:
 
         sentence_pb.metadata.position.index = paragraph_pb.metadata.position.index
 
-    def _set_resource_status(self, basic: Basic, previous_status: Optional[Metadata.Status.ValueType]):
+    def _set_resource_status(self, basic: Basic, previous_status: Metadata.Status.ValueType | None):
         """
         We purposefully overwrite what we index as a status and DO NOT reflect
         actual status with what we index.
@@ -633,32 +649,32 @@ class ResourceBrain:
             return "EMPTY"
         return METADATA_STATUS_PB_TYPE_TO_NAME_MAP[metadata.status]
 
-    def _set_resource_dates(self, basic: Basic, origin: Optional[Origin]):
+    def _set_resource_dates(self, basic: Basic, origin: Origin | None):
         """
         Adds the user-defined dates to the brain object. This is at resource level and applies to
         all fields of the resource.
         """
-        if basic.created.seconds > 0:
+        if basic.created.seconds != 0:
             self.brain.metadata.created.CopyFrom(basic.created)
         else:
             logging.warning(f"Basic metadata has no created field for {self.rid}")
             self.brain.metadata.created.GetCurrentTime()
-        if basic.modified.seconds > 0:
+        if basic.modified.seconds != 0:
             self.brain.metadata.modified.CopyFrom(basic.modified)
         else:
-            if basic.created.seconds > 0:
+            if basic.created.seconds != 0:
                 self.brain.metadata.modified.CopyFrom(basic.created)
             else:
                 self.brain.metadata.modified.GetCurrentTime()
 
         if origin is not None:
             # overwrite created/modified if provided on origin
-            if origin.HasField("created") and origin.created.seconds > 0:
+            if origin.HasField("created") and origin.created.seconds != 0:
                 self.brain.metadata.created.CopyFrom(origin.created)
-            if origin.HasField("modified") and origin.modified.seconds > 0:
+            if origin.HasField("modified") and origin.modified.seconds != 0:
                 self.brain.metadata.modified.CopyFrom(origin.modified)
 
-    def _set_resource_relations(self, basic: Basic, origin: Optional[Origin], user_relations: Relations):
+    def _set_resource_relations(self, basic: Basic, origin: Origin | None, user_relations: Relations):
         """
         Adds the relations to the brain object corresponding to the user-defined metadata at the resource level:
         - Contributors of the document
@@ -702,7 +718,7 @@ class ResourceBrain:
 
         self.brain.relation_fields_to_delete.append("a/metadata")
 
-    def _set_resource_labels(self, basic: Basic, origin: Optional[Origin]):
+    def _set_resource_labels(self, basic: Basic, origin: Origin | None):
         """
         Adds the resource-level labels to the brain object.
         These levels are user-defined in basic or origin metadata.
@@ -759,7 +775,7 @@ class ResourceBrain:
 
 def is_paragraph_repeated_in_field(
     paragraph: Paragraph,
-    extracted_text: Optional[str],
+    extracted_text: str | None,
     unique_paragraphs: set[str],
 ) -> bool:
     if extracted_text is None:
@@ -798,15 +814,13 @@ class ParagraphPages:
             return self._materialized[paragraph_start_index]
         except IndexError:
             logger.error(
-                f"Could not find a page for the given index: {paragraph_start_index}. Page positions: {self.positions}"  # noqa
+                f"Could not find a page for the given index: {paragraph_start_index}. Page positions: {self.positions}"
             )
             if len(self._materialized) > 0:
                 return self._materialized[-1]
             return 0
 
 
-def should_skip_split_indexing(
-    split: str, replace_field: bool, append_splits: Optional[set[str]]
-) -> bool:
+def should_skip_split_indexing(split: str, replace_field: bool, append_splits: set[str] | None) -> bool:
     # When replacing the whole field, reindex all splits. Otherwise, we're only indexing the splits that are appended
     return not replace_field and append_splits is not None and split not in append_splits

nucliadb/ingest/orm/broker_message.py

@@ -56,7 +56,7 @@ class _BrokerMessageBuilder:
         # clear the state and generate a new broker message
         self.bm.Clear()
 
-        self.bm.kbid = resource.kb.kbid
+        self.bm.kbid = resource.kbid
         self.bm.uuid = resource.uuid
         basic = await resource.get_basic()
         if basic is not None:
@@ -93,9 +93,7 @@ class _BrokerMessageBuilder:
                     self.bm.link_extracted_data.append(link_extracted_data)
 
             # Field vectors
-            async for vectorset_id, vs in datamanagers.vectorsets.iter(
-                resource.txn, kbid=resource.kb.kbid
-            ):
+            async for vectorset_id, vs in datamanagers.vectorsets.iter(resource.txn, kbid=resource.kbid):
                 await self.generate_field_vectors(
                     type_id, field_id, field, vectorset_id, vs.storage_key_kind
                 )

nucliadb/ingest/orm/entities.py

@@ -18,8 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-import asyncio
-from typing import AsyncGenerator, Optional
+from collections.abc import AsyncGenerator
 
 from nidx_protos.nodereader_pb2 import (
     Faceted,
@@ -29,23 +28,12 @@ from nidx_protos.nodereader_pb2 import (
     SearchResponse,
 )
 
-from nucliadb.common import datamanagers
-from nucliadb.common.cluster.exceptions import (
-    AlreadyExists,
-    EntitiesGroupNotFound,
-)
 from nucliadb.common.cluster.utils import get_shard_manager
-from nucliadb.common.datamanagers.entities import (
-    KB_DELETED_ENTITIES_GROUPS,
-    KB_ENTITIES,
-    KB_ENTITIES_GROUP,
-)
 from nucliadb.common.maindb.driver import Transaction
 from nucliadb.ingest.orm.knowledgebox import KnowledgeBox
 from nucliadb.ingest.settings import settings
 from nucliadb.search.search.shards import graph_search_shard, query_shard
 from nucliadb_protos.knowledgebox_pb2 import (
-    DeletedEntitiesGroups,
     EntitiesGroup,
     EntitiesGroupSummary,
     Entity,
@@ -53,8 +41,6 @@ from nucliadb_protos.knowledgebox_pb2 import (
 from nucliadb_protos.utils_pb2 import RelationNode
 from nucliadb_protos.writer_pb2 import GetEntitiesResponse
 
-from .exceptions import EntityManagementException
-
 MAX_DUPLICATES = 300
 MAX_DELETED = 300
 
@@ -69,20 +55,11 @@ class EntitiesManager:
         self.txn = txn
         self.kbid = self.kb.kbid
 
-    async def create_entities_group(self, group: str, entities: EntitiesGroup):
-        if await self.entities_group_exists(group):
-            raise AlreadyExists(f"Entities group {group} already exists")
-
-        await self.store_entities_group(group, entities)
-
     async def get_entities(self, entities: GetEntitiesResponse):
         async for group, eg in self.iterate_entities_groups(exclude_deleted=True):
             entities.groups[group].CopyFrom(eg)
 
-    async def get_entities_group(self, group: str) -> Optional[EntitiesGroup]:
-        deleted = await self.is_entities_group_deleted(group)
-        if deleted:
-            return None
+    async def get_entities_group(self, group: str) -> EntitiesGroup | None:
         return await self.get_entities_group_inner(group)
 
     async def get_entities_groups(self) -> dict[str, EntitiesGroup]:
@@ -93,113 +70,18 @@ class EntitiesManager:
 
     async def list_entities_groups(self) -> dict[str, EntitiesGroupSummary]:
         groups = {}
-        max_simultaneous = asyncio.Semaphore(10)
 
-        async def _composition(group: str):
-            async with max_simultaneous:
-                stored = await self.get_stored_entities_group(group)
-                if stored is not None:
-                    groups[group] = EntitiesGroupSummary(
-                        title=stored.title, color=stored.color, custom=stored.custom
-                    )
-                else:
-                    # We don't want to search for each indexed group, as we are
-                    # providing a quick summary
-                    groups[group] = EntitiesGroupSummary()
+        async for group in self.iterate_entities_groups_names(exclude_deleted=True):
+            groups[group] = EntitiesGroupSummary()
 
-        tasks = [
-            asyncio.create_task(_composition(group))
-            async for group in self.iterate_entities_groups_names(exclude_deleted=True)
-        ]
-        if tasks:
-            await asyncio.wait(tasks)
         return groups
 
-    async def update_entities(self, group: str, entities: dict[str, Entity]):
-        """Update entities on an entity group. New entities are appended and existing
-        are overwriten. Existing entities not appearing in `entities` are left
-        intact. Use `delete_entities` to delete them instead.
-
-        """
-        if not await self.entities_group_exists(group):
-            raise EntitiesGroupNotFound(f"Entities group '{group}' doesn't exist")
-
-        entities_group = await self.get_stored_entities_group(group)
-        if entities_group is None:
-            entities_group = EntitiesGroup()
-
-        for name, entity in entities.items():
-            entities_group.entities[name].CopyFrom(entity)
-
-        await self.store_entities_group(group, entities_group)
-
-    async def set_entities_group(self, group: str, entities: EntitiesGroup):
-        indexed = await self.get_indexed_entities_group(group)
-        if indexed is None:
-            updated = entities
-        else:
-            updated = EntitiesGroup()
-            updated.CopyFrom(entities)
-
-            for name, entity in indexed.entities.items():
-                if name not in updated.entities:
-                    updated.entities[name].CopyFrom(entity)
-                    updated.entities[name].deleted = True
-
-        await self.store_entities_group(group, updated)
-
-    async def set_entities_group_force(self, group: str, entitiesgroup: EntitiesGroup):
-        await self.store_entities_group(group, entitiesgroup)
-
-    async def set_entities_group_metadata(
-        self, group: str, *, title: Optional[str] = None, color: Optional[str] = None
-    ):
-        entities_group = await self.get_stored_entities_group(group)
-        if entities_group is None:
-            entities_group = EntitiesGroup()
-
-        if title:
-            entities_group.title = title
-        if color:
-            entities_group.color = color
-
-        await self.store_entities_group(group, entities_group)
-
-    async def delete_entities(self, group: str, delete: list[str]):
-        stored = await self.get_stored_entities_group(group)
-
-        stored = stored or EntitiesGroup()
-        for name in delete:
-            if name not in stored.entities:
-                entity = stored.entities[name]
-                entity.value = name
-            else:
-                entity = stored.entities[name]
-            entity.deleted = True
-        await self.store_entities_group(group, stored)
-
-    async def delete_entities_group(self, group: str):
-        await self.delete_stored_entities_group(group)
-        await self.mark_entities_group_as_deleted(group)
-
     # Private API
 
-    async def get_entities_group_inner(self, group: str) -> Optional[EntitiesGroup]:
-        stored = await self.get_stored_entities_group(group)
-        indexed = await self.get_indexed_entities_group(group)
-        if stored is None and indexed is None:
-            # Entity group does not exist
-            return None
-        elif stored is not None and indexed is not None:
-            entities_group = self.merge_entities_groups(indexed, stored)
-        else:
-            entities_group = stored or indexed
-        return entities_group
-
-    async def get_stored_entities_group(self, group: str) -> Optional[EntitiesGroup]:
-        return await datamanagers.entities.get_entities_group(self.txn, kbid=self.kbid, group=group)
+    async def get_entities_group_inner(self, group: str) -> EntitiesGroup | None:
+        return await self.get_indexed_entities_group(group)
 
-    async def get_indexed_entities_group(self, group: str) -> Optional[EntitiesGroup]:
+    async def get_indexed_entities_group(self, group: str) -> EntitiesGroup | None:
         shard_manager = get_shard_manager()
 
         async def do_entities_search(shard_id: str) -> GraphSearchResponse:
@@ -228,26 +110,9 @@ class EntitiesManager:
         eg = EntitiesGroup(entities=entities)
         return eg
 
-    async def get_deleted_entities_groups(self) -> set[str]:
-        deleted: set[str] = set()
-        key = KB_DELETED_ENTITIES_GROUPS.format(kbid=self.kbid)
-        payload = await self.txn.get(key)
-        if payload:
-            deg = DeletedEntitiesGroups()
-            deg.ParseFromString(payload)
-            deleted.update(deg.entities_groups)
-        return deleted
-
     async def entities_group_exists(self, group: str) -> bool:
-        stored = await self.get_stored_entities_group(group)
-        if stored is not None:
-            return True
-
         indexed = await self.get_indexed_entities_group(group)
-        if indexed is not None:
-            return True
-
-        return False
+        return indexed is not None
 
     async def iterate_entities_groups(
         self, exclude_deleted: bool
@@ -262,27 +127,10 @@ class EntitiesManager:
         self,
         exclude_deleted: bool,
     ) -> AsyncGenerator[str, None]:
-        # Start the task to get indexed groups
-        indexed_task = asyncio.create_task(self.get_indexed_entities_groups_names())
-
-        if exclude_deleted:
-            deleted_groups = await self.get_deleted_entities_groups()
-
         visited_groups = set()
-
-        # stored groups
-        entities_key = KB_ENTITIES.format(kbid=self.kbid)
-        async for key in self.txn.keys(entities_key):
-            group = key.split("/")[-1]
-            if exclude_deleted and group in deleted_groups:
-                continue
-            yield group
-            visited_groups.add(group)
-
-        # indexed groups
-        indexed_groups = await indexed_task
+        indexed_groups = await self.get_indexed_entities_groups_names()
         for group in indexed_groups:
-            if (exclude_deleted and group in deleted_groups) or group in visited_groups:
+            if group in visited_groups:
                 continue
             yield group
             visited_groups.add(group)
@@ -319,53 +167,6 @@ class EntitiesManager:
             return set()
         return set.union(*results)
 
-    async def store_entities_group(self, group: str, eg: EntitiesGroup):
-        meta_cache = await datamanagers.entities.get_entities_meta_cache(self.txn, kbid=self.kbid)
-        duplicates = {}
-        deleted = []
-        duplicate_count = 0
-        for entity in eg.entities.values():
-            if entity.deleted:
-                deleted.append(entity.value)
-                continue
-            if len(entity.represents) == 0:
-                continue
-            duplicates[entity.value] = list(entity.represents)
-            duplicate_count += len(duplicates[entity.value])
-
-        if duplicate_count > MAX_DUPLICATES:
-            raise EntityManagementException(
-                f"Too many duplicates: {duplicate_count}. Max of {MAX_DUPLICATES} currently allowed"
-            )
-        if len(deleted) > MAX_DELETED:
-            raise EntityManagementException(
-                f"Too many deleted entities: {len(deleted)}. Max of {MAX_DELETED} currently allowed"
-            )
-
-        meta_cache.set_duplicates(group, duplicates)
-        meta_cache.set_deleted(group, deleted)
-        await datamanagers.entities.set_entities_meta_cache(self.txn, kbid=self.kbid, cache=meta_cache)
-
-        await datamanagers.entities.set_entities_group(
-            self.txn, kbid=self.kbid, group_id=group, entities=eg
-        )
-        # if it was preivously deleted, we must unmark it
-        await self.unmark_entities_group_as_deleted(group)
-
-    async def is_entities_group_deleted(self, group: str):
-        deleted_groups = await self.get_deleted_entities_groups()
-        return group in deleted_groups
-
-    async def delete_stored_entities_group(self, group: str):
-        entities_key = KB_ENTITIES_GROUP.format(kbid=self.kbid, id=group)
-        await self.txn.delete(entities_key)
-
-    async def mark_entities_group_as_deleted(self, group: str):
-        await datamanagers.entities.mark_group_as_deleted(self.txn, kbid=self.kbid, group=group)
-
-    async def unmark_entities_group_as_deleted(self, group: str):
-        await datamanagers.entities.unmark_group_as_deleted(self.txn, kbid=self.kbid, group=group)
-
     @staticmethod
     def merge_entities_groups(indexed: EntitiesGroup, stored: EntitiesGroup):
         """Create a new EntitiesGroup with the merged entities from `stored` and

nucliadb/ingest/orm/index_message.py

@@ -20,7 +20,7 @@
 
 
 import asyncio
-from typing import Optional, Sequence
+from collections.abc import Sequence
 
 from nidx_protos.noderesources_pb2 import Resource as IndexMessage
 
@@ -70,8 +70,8 @@ class IndexMessageBuilder:
         vectors: bool = True,
         relations: bool = True,
         replace: bool = True,
-        vectorset_configs: Optional[list[VectorSetConfig]] = None,
-        append_splits: Optional[set[str]] = None,
+        vectorset_configs: list[VectorSetConfig] | None = None,
+        append_splits: set[str] | None = None,
     ):
         field = await self.resource.get_field(fieldid.field, fieldid.field_type)
         extracted_text = await field.get_extracted_text()
@@ -281,7 +281,7 @@ class IndexMessageBuilder:
         vectorset_configs = [
             vectorset_config
             async for _, vectorset_config in datamanagers.vectorsets.iter(
-                self.resource.txn, kbid=self.resource.kb.kbid
+                self.resource.txn, kbid=self.resource.kbid
             )
         ]
         return vectorset_configs