nucliadb 6.2.1.post2864__py3-none-any.whl → 6.2.1.post2869__py3-none-any.whl

nucliadb/train/nodes.py

@@ -28,6 +28,12 @@ from nucliadb.common.datamanagers.resources import KB_RESOURCE_SLUG_BASE
 from nucliadb.common.maindb.driver import Driver, Transaction
 from nucliadb.ingest.orm.entities import EntitiesManager
 from nucliadb.ingest.orm.knowledgebox import KnowledgeBox
+from nucliadb.train.resource import (
+    generate_train_resource,
+    iterate_fields,
+    iterate_paragraphs,
+    iterate_sentences,
+)
 from nucliadb_protos.train_pb2 import (
     GetFieldsRequest,
     GetParagraphsRequest,
@@ -87,11 +93,11 @@ class TrainShardManager(manager.KBShardManager):
                 # Filter by uuid
                 resource = await kb.get(request.uuid)
                 if resource:
-                    async for sentence in resource.iterate_sentences(request.metadata):
+                    async for sentence in iterate_sentences(resource, request.metadata):
                         yield sentence
             else:
                 async for resource in kb.iterate_resources():
-                    async for sentence in resource.iterate_sentences(request.metadata):
+                    async for sentence in iterate_sentences(resource, request.metadata):
                         yield sentence
 
     async def kb_paragraphs(self, request: GetParagraphsRequest) -> AsyncIterator[TrainParagraph]:
@@ -101,11 +107,11 @@ class TrainShardManager(manager.KBShardManager):
                 # Filter by uuid
                 resource = await kb.get(request.uuid)
                 if resource:
-                    async for paragraph in resource.iterate_paragraphs(request.metadata):
+                    async for paragraph in iterate_paragraphs(resource, request.metadata):
                         yield paragraph
             else:
                 async for resource in kb.iterate_resources():
-                    async for paragraph in resource.iterate_paragraphs(request.metadata):
+                    async for paragraph in iterate_paragraphs(resource, request.metadata):
                         yield paragraph
 
     async def kb_fields(self, request: GetFieldsRequest) -> AsyncIterator[TrainField]:
@@ -115,11 +121,11 @@ class TrainShardManager(manager.KBShardManager):
                 # Filter by uuid
                 resource = await kb.get(request.uuid)
                 if resource:
-                    async for field in resource.iterate_fields(request.metadata):
+                    async for field in iterate_fields(resource, request.metadata):
                         yield field
             else:
                 async for resource in kb.iterate_resources():
-                    async for field in resource.iterate_fields(request.metadata):
+                    async for field in iterate_fields(resource, request.metadata):
                         yield field
 
     async def kb_resources(self, request: GetResourcesRequest) -> AsyncIterator[TrainResource]:
@@ -132,4 +138,4 @@ class TrainShardManager(manager.KBShardManager):
                 if rid is not None:
                     resource = await kb.get(rid.decode())
                     if resource is not None:
-                        yield await resource.generate_train_resource(request.metadata)
+                        yield await generate_train_resource(resource, request.metadata)

nucliadb/train/resource.py

@@ -0,0 +1,380 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import annotations
+
+from typing import AsyncIterator, MutableMapping, Optional
+
+from nucliadb.common import datamanagers
+from nucliadb.ingest.orm.resource import Resource
+from nucliadb_protos.resources_pb2 import (
+    FieldID,
+    FieldMetadata,
+    ParagraphAnnotation,
+)
+from nucliadb_protos.train_pb2 import (
+    EnabledMetadata,
+    TrainField,
+    TrainMetadata,
+    TrainParagraph,
+    TrainResource,
+    TrainSentence,
+)
+from nucliadb_protos.train_pb2 import Position as TrainPosition
+
+
+async def iterate_sentences(
+    resource: Resource,
+    enabled_metadata: EnabledMetadata,
+) -> AsyncIterator[TrainSentence]:  # pragma: no cover
+    fields = await resource.get_fields(force=True)
+    metadata = TrainMetadata()
+    userdefinedparagraphclass: dict[str, ParagraphAnnotation] = {}
+    if enabled_metadata.labels:
+        if resource.basic is None:
+            resource.basic = await resource.get_basic()
+        if resource.basic is not None:
+            metadata.labels.resource.extend(resource.basic.usermetadata.classifications)
+            for fieldmetadata in resource.basic.fieldmetadata:
+                field_id = resource.generate_field_id(fieldmetadata.field)
+                for annotationparagraph in fieldmetadata.paragraphs:
+                    userdefinedparagraphclass[annotationparagraph.key] = annotationparagraph
+
+    for (type_id, field_id), field in fields.items():
+        fieldid = FieldID(field_type=type_id, field=field_id)
+        field_key = resource.generate_field_id(fieldid)
+        fm = await field.get_field_metadata()
+        extracted_text = None
+        vo = None
+        text = None
+
+        if enabled_metadata.vector:
+            # XXX: Given that nobody requested any particular vectorset, we'll
+            # return any
+            vectorset_id = None
+            async with datamanagers.with_ro_transaction() as txn:
+                async for vectorset_id, vs in datamanagers.vectorsets.iter(
+                    txn=txn, kbid=resource.kb.kbid
+                ):
+                    break
+            assert vectorset_id is not None, "All KBs must have at least a vectorset"
+            vo = await field.get_vectors(vectorset_id, vs.storage_key_kind)
+
+        extracted_text = await field.get_extracted_text()
+
+        if fm is None:
+            continue
+
+        field_metadatas: list[tuple[Optional[str], FieldMetadata]] = [(None, fm.metadata)]
+        for subfield_metadata, splitted_metadata in fm.split_metadata.items():
+            field_metadatas.append((subfield_metadata, splitted_metadata))
+
+        for subfield, field_metadata in field_metadatas:
+            if enabled_metadata.labels:
+                metadata.labels.ClearField("field")
+                metadata.labels.field.extend(field_metadata.classifications)
+
+            entities: dict[str, str] = {}
+            if enabled_metadata.entities:
+                _update_entities_dict(entities, field_metadata)
+
+            precomputed_vectors = {}
+            if vo is not None:
+                if subfield is not None:
+                    vectors = vo.split_vectors[subfield]
+                    base_vector_key = f"{resource.uuid}/{field_key}/{subfield}"
+                else:
+                    vectors = vo.vectors
+                    base_vector_key = f"{resource.uuid}/{field_key}"
+                for index, vector in enumerate(vectors.vectors):
+                    vector_key = f"{base_vector_key}/{index}/{vector.start}-{vector.end}"
+                    precomputed_vectors[vector_key] = vector.vector
+
+            if extracted_text is not None:
+                if subfield is not None:
+                    text = extracted_text.split_text[subfield]
+                else:
+                    text = extracted_text.text
+
+            for paragraph in field_metadata.paragraphs:
+                if subfield is not None:
+                    paragraph_key = (
+                        f"{resource.uuid}/{field_key}/{subfield}/{paragraph.start}-{paragraph.end}"
+                    )
+                else:
+                    paragraph_key = f"{resource.uuid}/{field_key}/{paragraph.start}-{paragraph.end}"
+
+                if enabled_metadata.labels:
+                    metadata.labels.ClearField("field")
+                    metadata.labels.paragraph.extend(paragraph.classifications)
+                    if paragraph_key in userdefinedparagraphclass:
+                        metadata.labels.paragraph.extend(
+                            userdefinedparagraphclass[paragraph_key].classifications
+                        )
+
+                for index, sentence in enumerate(paragraph.sentences):
+                    if subfield is not None:
+                        sentence_key = f"{resource.uuid}/{field_key}/{subfield}/{index}/{sentence.start}-{sentence.end}"
+                    else:
+                        sentence_key = (
+                            f"{resource.uuid}/{field_key}/{index}/{sentence.start}-{sentence.end}"
+                        )
+
+                    if vo is not None:
+                        metadata.ClearField("vector")
+                        vector_tmp = precomputed_vectors.get(sentence_key)
+                        if vector_tmp:
+                            metadata.vector.extend(vector_tmp)
+
+                    if extracted_text is not None and text is not None:
+                        metadata.text = text[sentence.start : sentence.end]
+
+                    metadata.ClearField("entities")
+                    metadata.ClearField("entity_positions")
+                    if enabled_metadata.entities and text is not None:
+                        local_text = text[sentence.start : sentence.end]
+                        add_entities_to_metadata(entities, local_text, metadata)
+
+                    pb_sentence = TrainSentence()
+                    pb_sentence.uuid = resource.uuid
+                    pb_sentence.field.CopyFrom(fieldid)
+                    pb_sentence.paragraph = paragraph_key
+                    pb_sentence.sentence = sentence_key
+                    pb_sentence.metadata.CopyFrom(metadata)
+                    yield pb_sentence
+
+
+async def iterate_paragraphs(
+    resource: Resource, enabled_metadata: EnabledMetadata
+) -> AsyncIterator[TrainParagraph]:
+    fields = await resource.get_fields(force=True)
+    metadata = TrainMetadata()
+    userdefinedparagraphclass: dict[str, ParagraphAnnotation] = {}
+    if enabled_metadata.labels:
+        if resource.basic is None:
+            resource.basic = await resource.get_basic()
+        if resource.basic is not None:
+            metadata.labels.resource.extend(resource.basic.usermetadata.classifications)
+            for fieldmetadata in resource.basic.fieldmetadata:
+                field_id = resource.generate_field_id(fieldmetadata.field)
+                for annotationparagraph in fieldmetadata.paragraphs:
+                    userdefinedparagraphclass[annotationparagraph.key] = annotationparagraph
+
+    for (type_id, field_id), field in fields.items():
+        fieldid = FieldID(field_type=type_id, field=field_id)
+        field_key = resource.generate_field_id(fieldid)
+        fm = await field.get_field_metadata()
+        extracted_text = None
+        text = None
+
+        extracted_text = await field.get_extracted_text()
+
+        if fm is None:
+            continue
+
+        field_metadatas: list[tuple[Optional[str], FieldMetadata]] = [(None, fm.metadata)]
+        for subfield_metadata, splitted_metadata in fm.split_metadata.items():
+            field_metadatas.append((subfield_metadata, splitted_metadata))
+
+        for subfield, field_metadata in field_metadatas:
+            if enabled_metadata.labels:
+                metadata.labels.ClearField("field")
+                metadata.labels.field.extend(field_metadata.classifications)
+
+            entities: dict[str, str] = {}
+            if enabled_metadata.entities:
+                _update_entities_dict(entities, field_metadata)
+
+            if extracted_text is not None:
+                if subfield is not None:
+                    text = extracted_text.split_text[subfield]
+                else:
+                    text = extracted_text.text
+
+            for paragraph in field_metadata.paragraphs:
+                if subfield is not None:
+                    paragraph_key = (
+                        f"{resource.uuid}/{field_key}/{subfield}/{paragraph.start}-{paragraph.end}"
+                    )
+                else:
+                    paragraph_key = f"{resource.uuid}/{field_key}/{paragraph.start}-{paragraph.end}"
+
+                if enabled_metadata.labels:
+                    metadata.labels.ClearField("paragraph")
+                    metadata.labels.paragraph.extend(paragraph.classifications)
+
+                    if extracted_text is not None and text is not None:
+                        metadata.text = text[paragraph.start : paragraph.end]
+
+                    metadata.ClearField("entities")
+                    metadata.ClearField("entity_positions")
+                    if enabled_metadata.entities and text is not None:
+                        local_text = text[paragraph.start : paragraph.end]
+                        add_entities_to_metadata(entities, local_text, metadata)
+
+                    if paragraph_key in userdefinedparagraphclass:
+                        metadata.labels.paragraph.extend(
+                            userdefinedparagraphclass[paragraph_key].classifications
+                        )
+
+                    pb_paragraph = TrainParagraph()
+                    pb_paragraph.uuid = resource.uuid
+                    pb_paragraph.field.CopyFrom(fieldid)
+                    pb_paragraph.paragraph = paragraph_key
+                    pb_paragraph.metadata.CopyFrom(metadata)
+
+                    yield pb_paragraph
+
+
+async def iterate_fields(
+    resource: Resource, enabled_metadata: EnabledMetadata
+) -> AsyncIterator[TrainField]:
+    fields = await resource.get_fields(force=True)
+    metadata = TrainMetadata()
+    if enabled_metadata.labels:
+        if resource.basic is None:
+            resource.basic = await resource.get_basic()
+        if resource.basic is not None:
+            metadata.labels.resource.extend(resource.basic.usermetadata.classifications)
+
+    for (type_id, field_id), field in fields.items():
+        fieldid = FieldID(field_type=type_id, field=field_id)
+        fm = await field.get_field_metadata()
+        extracted_text = None
+
+        if enabled_metadata.text:
+            extracted_text = await field.get_extracted_text()
+
+        if fm is None:
+            continue
+
+        field_metadatas: list[tuple[Optional[str], FieldMetadata]] = [(None, fm.metadata)]
+        for subfield_metadata, splitted_metadata in fm.split_metadata.items():
+            field_metadatas.append((subfield_metadata, splitted_metadata))
+
+        for subfield, splitted_metadata in field_metadatas:
+            if enabled_metadata.labels:
+                metadata.labels.ClearField("field")
+                metadata.labels.field.extend(splitted_metadata.classifications)
+
+            if extracted_text is not None:
+                if subfield is not None:
+                    metadata.text = extracted_text.split_text[subfield]
+                else:
+                    metadata.text = extracted_text.text
+
+            if enabled_metadata.entities:
+                metadata.ClearField("entities")
+                _update_entities_dict(metadata.entities, splitted_metadata)
+
+            pb_field = TrainField()
+            pb_field.uuid = resource.uuid
+            pb_field.field.CopyFrom(fieldid)
+            pb_field.metadata.CopyFrom(metadata)
+            yield pb_field
+
+
+async def generate_train_resource(
+    resource: Resource, enabled_metadata: EnabledMetadata
+) -> TrainResource:
+    fields = await resource.get_fields(force=True)
+    metadata = TrainMetadata()
+    if enabled_metadata.labels:
+        if resource.basic is None:
+            resource.basic = await resource.get_basic()
+        if resource.basic is not None:
+            metadata.labels.resource.extend(resource.basic.usermetadata.classifications)
+
+    metadata.labels.ClearField("field")
+    metadata.ClearField("entities")
+
+    for (_, _), field in fields.items():
+        extracted_text = None
+        fm = await field.get_field_metadata()
+
+        if enabled_metadata.text:
+            extracted_text = await field.get_extracted_text()
+
+        if extracted_text is not None:
+            metadata.text += extracted_text.text
+            for text in extracted_text.split_text.values():
+                metadata.text += f" {text}"
+
+        if fm is None:
+            continue
+
+        field_metadatas: list[tuple[Optional[str], FieldMetadata]] = [(None, fm.metadata)]
+        for subfield_metadata, splitted_metadata in fm.split_metadata.items():
+            field_metadatas.append((subfield_metadata, splitted_metadata))
+
+        for _, splitted_metadata in field_metadatas:
+            if enabled_metadata.labels:
+                metadata.labels.field.extend(splitted_metadata.classifications)
+
+            if enabled_metadata.entities:
+                _update_entities_dict(metadata.entities, splitted_metadata)
+
+    pb_resource = TrainResource()
+    pb_resource.uuid = resource.uuid
+    if resource.basic is not None:
+        pb_resource.title = resource.basic.title
+        pb_resource.icon = resource.basic.icon
+        pb_resource.slug = resource.basic.slug
+        pb_resource.modified.CopyFrom(resource.basic.modified)
+        pb_resource.created.CopyFrom(resource.basic.created)
+    pb_resource.metadata.CopyFrom(metadata)
+    return pb_resource
+
+
+def add_entities_to_metadata(entities: dict[str, str], local_text: str, metadata: TrainMetadata) -> None:
+    for entity_key, entity_value in entities.items():
+        if entity_key not in local_text:
+            # Add the entity only if found in text
+            continue
+        metadata.entities[entity_key] = entity_value
+
+        # Add positions for the entity relative to the local text
+        poskey = f"{entity_value}/{entity_key}"
+        metadata.entity_positions[poskey].entity = entity_key
+        last_occurrence_end = 0
+        for _ in range(local_text.count(entity_key)):
+            start = local_text.index(entity_key, last_occurrence_end)
+            end = start + len(entity_key)
+            metadata.entity_positions[poskey].positions.append(TrainPosition(start=start, end=end))
+            last_occurrence_end = end
+
+
+def _update_entities_dict(target_entites_dict: MutableMapping[str, str], field_metadata: FieldMetadata):
+    """
+    Update the entities dict with the entities from the field metadata.
+    Method created to ease the transition from legacy ner field to new entities field.
+    """
+    # Data Augmentation + Processor entities
+    # This will overwrite entities detected from more than one data augmentation task
+    # TODO: Change TrainMetadata proto to accept multiple entities with the same text
+    entity_map = {
+        entity.text: entity.label
+        for data_augmentation_task_id, entities_wrapper in field_metadata.entities.items()
+        for entity in entities_wrapper.entities
+    }
+    target_entites_dict.update(entity_map)
+
+    # Legacy processor entities
+    # TODO: Remove once processor doesn't use this anymore and remove the positions and ner fields from the message
+    target_entites_dict.update(field_metadata.ner)

{nucliadb-6.2.1.post2864.dist-info → nucliadb-6.2.1.post2869.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.2.1.post2864
+Version: 6.2.1.post2869
 Home-page: https://docs.nuclia.dev/docs/management/nucliadb/intro
 Author: NucliaDB Community
 Author-email: nucliadb@nuclia.com
@@ -22,10 +22,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.9, <4
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post2864
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post2864
-Requires-Dist: nucliadb-protos>=6.2.1.post2864
-Requires-Dist: nucliadb-models>=6.2.1.post2864
+Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post2869
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post2869
+Requires-Dist: nucliadb-protos>=6.2.1.post2869
+Requires-Dist: nucliadb-models>=6.2.1.post2869
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nucliadb-node-binding>=2.26.0
 Requires-Dist: nuclia-models>=0.24.2

{nucliadb-6.2.1.post2864.dist-info → nucliadb-6.2.1.post2869.dist-info}/RECORD

@@ -79,7 +79,7 @@ nucliadb/common/datamanagers/resources.py,sha256=5EJk7P-G4A_YiobiUexz_yuZUTuxS5z
 nucliadb/common/datamanagers/rollover.py,sha256=c_DE3jtZusNL_9aOVjHOB9PV5OSVg7GJ5J-Ny0goHBE,7833
 nucliadb/common/datamanagers/synonyms.py,sha256=zk3GEH38KF5vV_VcuL6DCg-2JwgXJfQl7Io6VPqv2cw,1566
 nucliadb/common/datamanagers/utils.py,sha256=McHlXvE4P3x-bBY3pr0n8djbTDQvI1G5WusJrnRdhLA,1827
-nucliadb/common/datamanagers/vectorsets.py,sha256=XgHNQRw13GpWWymE6qu_ymdzuwL6hDiBKq50fN_sEMM,4007
+nucliadb/common/datamanagers/vectorsets.py,sha256=ciYb5uD435Zo8ZbqgPUAszFW9Svp_-R2hY2FEhQ411Y,4304
 nucliadb/common/external_index_providers/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/external_index_providers/base.py,sha256=yfPkCigT4unXFvAyzy1tXSy2UgWC481GcZAS9bdE4NI,8871
 nucliadb/common/external_index_providers/exceptions.py,sha256=nDhhOIkb66hjCrBk4Spvl2vN1SuW5gbwrMCDmrdjHHE,1209
@@ -115,7 +115,7 @@ nucliadb/ingest/cache.py,sha256=w7jMMzamOmQ7gwXna6Dqm6isRNBVv6l5BTBlTxaYWjE,1005
 nucliadb/ingest/partitions.py,sha256=2NIhMYbNT0TNBL6bX1UMSi7vxFGICstCKEqsB0TXHOE,2410
 nucliadb/ingest/processing.py,sha256=gg1DqbMFwqdOsmCSGsZc2abRdYz86xOZJun9vrHOCzs,20618
 nucliadb/ingest/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nucliadb/ingest/serialize.py,sha256=l2cIIHgo0rgkbaYvAY5slzjr8keVo_3Vb3B6DI120IY,15272
+nucliadb/ingest/serialize.py,sha256=03q9TBC9kbqbVq59SSL4ok1e3ThU0zeuYGdqY-B1V2M,15889
 nucliadb/ingest/settings.py,sha256=0B-wQNa8FLqtNcQgRzh-fuIuGptM816XHcbH1NQKfmE,3050
 nucliadb/ingest/utils.py,sha256=l1myURu3r8oA11dx3GpHw-gNTUc1AFX8xdPm9Lgl2rA,2275
 nucliadb/ingest/consumer/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
@@ -128,7 +128,7 @@ nucliadb/ingest/consumer/service.py,sha256=EZM1sABW_7bj6j2UgKUHUuK-EGIEYnLdtPAn8
 nucliadb/ingest/consumer/shard_creator.py,sha256=19wf-Bu_9hb_muCDVblamWuvLr09e5dMu9Id5I4-rGw,4324
 nucliadb/ingest/consumer/utils.py,sha256=jpX8D4lKzuPCpArQLZeX_Zczq3pfen_zAf8sPJfOEZU,2642
 nucliadb/ingest/fields/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/ingest/fields/base.py,sha256=GuyZaumtaaAzoy86-mewBMhNX9DaSDBIK4pCqrxZlDc,19836
+nucliadb/ingest/fields/base.py,sha256=b6QpVPsCiDirDiYG3-yOCMaSNznJSHmQB0z6J_eDIyw,20657
 nucliadb/ingest/fields/conversation.py,sha256=OcQOHvi72Pm0OyNGwxLo9gONo8f1NhwASq0_gS-E64A,7021
 nucliadb/ingest/fields/exceptions.py,sha256=LBZ-lw11f42Pk-ck-NSN9mSJ2kOw-NeRwb-UE31ILTQ,1171
 nucliadb/ingest/fields/file.py,sha256=1v4jLg3balUua2VmSV8hHkAwPFShTUCOzufZvIUQcQw,4740
@@ -136,13 +136,13 @@ nucliadb/ingest/fields/generic.py,sha256=elgtqv15aJUq3zY7X_g0bli_2BpcwPArVvzhe54
 nucliadb/ingest/fields/link.py,sha256=kN_gjRUEEj5cy8K_BwPijYg3TiWhedc24apXYlTbRJs,4172
 nucliadb/ingest/fields/text.py,sha256=tFvSQJAe0W7ePpp2_WDfLiE2yglR1OTU0Zht9acvOFw,1594
 nucliadb/ingest/orm/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/ingest/orm/brain.py,sha256=Hzq-3aarKaUCiUoa8H83unRUfduRE9TsQH1dEq0mvZY,28841
-nucliadb/ingest/orm/broker_message.py,sha256=JYYUJIZEL_EqovQuw6u-FmEkjyoYlxIXJq9hFekOiks,6441
+nucliadb/ingest/orm/brain.py,sha256=UND5EsNUdd7XdjScYqRqg4r_xCx3l-My8alGw5M9CWg,28398
+nucliadb/ingest/orm/broker_message.py,sha256=ZEMueoGuuRKO4tHgzc0P0AM1Ls1TTYey_4UvRQf0BpY,6915
 nucliadb/ingest/orm/entities.py,sha256=2PslT1FZ6yCvJtjR0UpKTSzxJrtS-C_gZx4ZTWHunTc,15759
 nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
-nucliadb/ingest/orm/knowledgebox.py,sha256=UpWJrVaVfCtk8R4qfSR6h6vzwOKXa8Teuwkna5QSljE,24508
+nucliadb/ingest/orm/knowledgebox.py,sha256=jWRBGic3KE1NRJzvUMpsRRLL6GHu9t28WsTb2DKtNhk,24901
 nucliadb/ingest/orm/metrics.py,sha256=OkwMSPKLZcKba0ZTwtTiIxwBgaLMX5ydhGieKvi2y7E,1096
-nucliadb/ingest/orm/resource.py,sha256=jvaKLsTlHtmIWUjjWCu8XBF7qQl5hoUihAa8sHDpLV8,59540
+nucliadb/ingest/orm/resource.py,sha256=KhucZzQzUbTBUm8_9gaCqxH68Fy1Q2u804IfTcjAIIk,43970
 nucliadb/ingest/orm/utils.py,sha256=vCe_9UxHu26JDFGLwQ0wH-XyzJIpQCTK-Ow9dtZR5Vg,2716
 nucliadb/ingest/orm/processor/__init__.py,sha256=Aqd9wCNTvggkMkCY3WvoI8spdr94Jnqk-0iq9XpLs18,922
 nucliadb/ingest/orm/processor/auditing.py,sha256=TeYhXGJRyQ7ROytbb2u8R0fIh_FYi3HgTu3S1ribY3U,4623
@@ -152,7 +152,7 @@ nucliadb/ingest/orm/processor/processor.py,sha256=2FxAetUvtHvg6l-24xYrmBdsyqc0RU
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=uqEphtI1Ir_yk9jRl2gPf7BlzzXWovbARY5MNZSBI_8,1704
 nucliadb/ingest/service/__init__.py,sha256=MME_G_ERxzJR6JW_hfE2qcfXpmpH1kdG-S0a-M0qRm8,2043
 nucliadb/ingest/service/exceptions.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/ingest/service/writer.py,sha256=4xXwZ8PFMTnHWxRV18Ic57bjI4qwybwHID9cc1Kxm94,22729
+nucliadb/ingest/service/writer.py,sha256=aBLLpPUJLlIf-VjAczBCUrcb-zMxRZOFHXkA0QE1pgw,22952
 nucliadb/middleware/__init__.py,sha256=A8NBlBuEkunCFMKpR9gnfNELsVn0Plc55BIQMbWDM8Q,2202
 nucliadb/migrator/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/migrator/command.py,sha256=dKbJ1tAmP6X4lMVRSSlz351euaqs2wBPpOczLjATUes,2089
@@ -165,7 +165,7 @@ nucliadb/migrator/settings.py,sha256=jOUX0ZMunCXN8HpF9xXN0aunJYRhu4Vdr_ffjRIqwtw
 nucliadb/migrator/utils.py,sha256=NgUreUvON8_nWEzTxELBMWlfV7E6-6qi-g0DMEbVEz4,2885
 nucliadb/models/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/models/responses.py,sha256=qnuOoc7TrVSUnpikfTwHLKez47_DE4mSFzpxrwtqijA,1599
-nucliadb/purge/__init__.py,sha256=tcXwO99714cqflLVJyZzOv6_64H9pt7r6V0UogDd4oA,10389
+nucliadb/purge/__init__.py,sha256=ijcigiWz38ohXmVVwDU87aCki1BkmAIQRjDoNQ3LPRM,11647
 nucliadb/purge/orphan_shards.py,sha256=fA5yqRRN-M50OIk8dkAi1_ShFVjwDYEYqzMA9dYP0eU,9227
 nucliadb/reader/__init__.py,sha256=C5Efic7WlGm2U2C5WOyquMFbIj2Pojwe_8mwzVYnOzE,1304
 nucliadb/reader/app.py,sha256=Se-BFTE6d1v1msLzQn4q5XIhjnSxa2ckDSHdvm7NRf8,3096
@@ -274,8 +274,9 @@ nucliadb/train/app.py,sha256=TiRttTvekLuZdIvi46E4HyuumDTkR4G4Luqq3fEdjes,2824
 nucliadb/train/generator.py,sha256=0_zqWsLUHmJZl0lXhGorO5CWSkl42-k78dqb1slZ5h0,3904
 nucliadb/train/lifecycle.py,sha256=aCNaRURu0ZOUJaWLTZuEjwTstnB9MuLtzxOMztQoGxc,1773
 nucliadb/train/models.py,sha256=BmgmMjDsu_1Ih5JDAqo6whhume90q0ASJcDP9dkMQm8,1198
-nucliadb/train/nodes.py,sha256=ha0AsGupmyvxUpoc1THQ6-eN7ziPkjM_gkKgKYT0SCg,5782
+nucliadb/train/nodes.py,sha256=_89ZIpBb0HnR2jejvuO6aPsgHVSGbasPWz0lkGmVnvU,5925
 nucliadb/train/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nucliadb/train/resource.py,sha256=3qQ_9Zdt5JAbtD-wpmt7OeDGRNKS-fQdKAuIQfznZm0,16219
 nucliadb/train/run.py,sha256=evz6CKVfJOzkbHMoaYz2mTMlKjJnNOb1O8zBBWMpeBw,1400
 nucliadb/train/servicer.py,sha256=scbmq8FriKsJGkOcoZB2Fg_IyIExn9Ux4W30mGDlkJQ,5728
 nucliadb/train/settings.py,sha256=rrLtgdBmuthtIObLuZUaeuo4VBGU2PJRazquQbtPBeI,1383
@@ -338,9 +339,9 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.2.1.post2864.dist-info/METADATA,sha256=-W5qHhu9x0clURMbD6zCYn6SsU9etVobB7qbn9jrebo,4689
-nucliadb-6.2.1.post2864.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-nucliadb-6.2.1.post2864.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.2.1.post2864.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.2.1.post2864.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-nucliadb-6.2.1.post2864.dist-info/RECORD,,
+nucliadb-6.2.1.post2869.dist-info/METADATA,sha256=yxkdjP13oz_FLAljFOB2S9kRxsEfkv6H27RhZS-7Dls,4689
+nucliadb-6.2.1.post2869.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+nucliadb-6.2.1.post2869.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.2.1.post2869.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.2.1.post2869.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+nucliadb-6.2.1.post2869.dist-info/RECORD,,