nucliadb 6.4.0.post4210__py3-none-any.whl → 6.4.0.post4224__py3-none-any.whl

nucliadb/ingest/app.py

@@ -103,7 +103,7 @@ async def initialize_pull_workers() -> list[Callable[[], Awaitable[None]]]:
     finalizers = await initialize_grpc()
     pull_workers = await consumer_service.start_pull_workers(SERVICE_NAME)
 
-    return [pull_workers] + finalizers
+    return pull_workers + finalizers
 
 
 async def main_consumer():  # pragma: no cover
@@ -117,7 +117,7 @@ async def main_consumer():  # pragma: no cover
     ingest_consumers = await consumer_service.start_ingest_consumers(SERVICE_NAME)
 
     await run_until_exit(
-        [grpc_health_finalizer, pull_workers, ingest_consumers, metrics_server.shutdown] + finalizers
+        [grpc_health_finalizer, ingest_consumers, metrics_server.shutdown] + pull_workers + finalizers
     )
 
 

nucliadb/ingest/consumer/pull.py

@@ -19,11 +19,14 @@
 #
 import asyncio
 import base64
+from datetime import datetime, timezone
 from typing import Optional
 
 from aiohttp.client_exceptions import ClientConnectorError
 
 from nucliadb.common import datamanagers
+from nucliadb.common.back_pressure.materializer import BackPressureMaterializer
+from nucliadb.common.back_pressure.utils import BackPressureException
 from nucliadb.common.http_clients.processing import ProcessingHTTPClient, get_nua_api_id
 from nucliadb.common.maindb.driver import Driver
 from nucliadb.ingest import logger, logger_activity
@@ -57,6 +60,7 @@ class PullWorker:
         local_subscriber: bool = False,
         pull_time_empty_backoff: float = 5.0,
         pull_api_timeout: int = 60,
+        back_pressure: Optional[BackPressureMaterializer] = None,
     ):
         self.partition = partition
         self.pull_time_error_backoff = pull_time_error_backoff
@@ -65,6 +69,7 @@ class PullWorker:
         self.local_subscriber = local_subscriber
 
         self.processor = Processor(driver, storage, pubsub, partition)
+        self.back_pressure = back_pressure
 
     def __str__(self) -> str:
         return f"PullWorker(partition={self.partition})"
@@ -112,11 +117,29 @@ class PullWorker:
                 transaction_check=False,
             )
 
+    async def back_pressure_check(self) -> None:
+        if self.back_pressure is None:
+            return
+        while True:
+            try:
+                self.back_pressure.check_indexing()
+                self.back_pressure.check_ingest()
+                break
+            except BackPressureException as exc:
+                sleep_time = (datetime.now(timezone.utc) - exc.data.try_after).total_seconds()
+                logger.warning(f"Back pressure active! Sleeping for {sleep_time} seconds", exc_info=True)
+                await asyncio.sleep(sleep_time)
+            except Exception as e:
+                errors.capture_exception(e)
+                logger.exception("Error while checking back pressure. Moving on")
+                break
+
     async def loop(self):
         """
         Run this forever
         """
         while True:
+            await self.back_pressure_check()
             try:
                 await self._loop()
             except ReallyStopPulling:

nucliadb/ingest/consumer/service.py

@@ -22,18 +22,22 @@ import sys
 from functools import partial
 from typing import Awaitable, Callable, Optional
 
+from nucliadb.common.back_pressure.materializer import BackPressureMaterializer
+from nucliadb.common.back_pressure.settings import settings as back_pressure_settings
+from nucliadb.common.back_pressure.utils import is_back_pressure_enabled
 from nucliadb.common.maindb.utils import setup_driver
 from nucliadb.ingest import SERVICE_NAME, logger
 from nucliadb.ingest.consumer.consumer import IngestConsumer, IngestProcessedConsumer
 from nucliadb.ingest.consumer.pull import PullWorker
 from nucliadb.ingest.settings import settings
 from nucliadb_utils.exceptions import ConfigurationError
-from nucliadb_utils.settings import transaction_settings
+from nucliadb_utils.settings import indexing_settings, transaction_settings
 from nucliadb_utils.utilities import (
     get_audit,
     get_nats_manager,
     get_pubsub,
     get_storage,
+    start_nats_manager,
 )
 
 from .auditing import IndexAuditHandler, ResourceWritesAuditHandler
@@ -54,12 +58,39 @@ async def _exit_tasks(tasks: list[asyncio.Task]) -> None:
     await asyncio.gather(*tasks, return_exceptions=True)
 
 
+async def start_back_pressure() -> BackPressureMaterializer:
+    logger.info("Starting back pressure materializer")
+    nats_manager = await start_nats_manager(
+        SERVICE_NAME,
+        indexing_settings.index_jetstream_servers,
+        indexing_settings.index_jetstream_auth,
+    )
+    back_pressure = BackPressureMaterializer(
+        nats_manager,
+        indexing_check_interval=back_pressure_settings.indexing_check_interval,
+        ingest_check_interval=back_pressure_settings.ingest_check_interval,
+    )
+    await back_pressure.start()
+    return back_pressure
+
+
+async def stop_back_pressure(materializer: BackPressureMaterializer) -> None:
+    await materializer.stop()
+    await materializer.nats_manager.finalize()
+
+
 async def start_pull_workers(
     service_name: Optional[str] = None,
-) -> Callable[[], Awaitable[None]]:
+) -> list[Callable[[], Awaitable[None]]]:
+    finalizers: list[Callable[[], Awaitable[None]]] = []
+
     driver = await setup_driver()
     pubsub = await get_pubsub()
     storage = await get_storage(service_name=service_name or SERVICE_NAME)
+    back_pressure = None
+    if is_back_pressure_enabled():
+        back_pressure = await start_back_pressure()
+        finalizers.append(partial(stop_back_pressure, back_pressure))
     tasks = []
     for partition in settings.partitions:
         worker = PullWorker(
@@ -70,12 +101,14 @@ async def start_pull_workers(
             pubsub=pubsub,
             local_subscriber=transaction_settings.transaction_local,
             pull_api_timeout=settings.pull_api_timeout,
+            back_pressure=back_pressure,
         )
         task = asyncio.create_task(worker.loop())
         task.add_done_callback(_handle_task_result)
         tasks.append(task)
-
-    return partial(_exit_tasks, tasks)
+    if len(tasks):
+        finalizers.append(partial(_exit_tasks, tasks))
+    return finalizers
 
 
 async def start_ingest_consumers(

nucliadb/ingest/orm/brain_v2.py

@@ -70,7 +70,7 @@ class ParagraphClassifications:
     denied: dict[str, list[str]]
 
 
-class ResourceBrainV2:
+class ResourceBrain:
     def __init__(self, rid: str):
         self.rid = rid
         self.brain: PBBrainResource = PBBrainResource(resource=ResourceID(uuid=rid))

nucliadb/ingest/orm/index_message.py

@@ -27,14 +27,12 @@ from nidx_protos.noderesources_pb2 import Resource as IndexMessage
 from nucliadb.common import datamanagers
 from nucliadb.ingest.fields.exceptions import FieldAuthorNotFound
 from nucliadb.ingest.fields.file import File
-from nucliadb.ingest.orm.brain_v2 import ResourceBrainV2 as ResourceBrain
+from nucliadb.ingest.orm.brain_v2 import ResourceBrain
 from nucliadb.ingest.orm.metrics import index_message_observer as observer
 from nucliadb.ingest.orm.resource import Resource, get_file_page_positions
 from nucliadb_protos.knowledgebox_pb2 import VectorSetConfig
 from nucliadb_protos.resources_pb2 import Basic, FieldID, FieldType
 from nucliadb_protos.writer_pb2 import BrokerMessage
-from nucliadb_utils import const
-from nucliadb_utils.utilities import has_feature
 
 
 class IndexMessageBuilder:
@@ -403,14 +401,5 @@ async def get_resource_index_message(
     """
     Get the full index message for a resource.
     """
-    if has_feature(
-        const.Features.INDEX_MESSAGE_GENERATION_V2,
-        context={
-            "kbid": resource.kb.kbid,
-        },
-    ):
-        im_builder = IndexMessageBuilder(resource)
-        return await im_builder.full(reindex=reindex)
-    else:
-        # TODO: remove this code when we remove the old index message generation
-        return (await resource.generate_index_message(reindex=reindex)).brain
+    im_builder = IndexMessageBuilder(resource)
+    return await im_builder.full(reindex=reindex)

nucliadb/ingest/orm/processor/processor.py

@@ -461,8 +461,8 @@ class Processor:
                 source=source,
             )
 
-    @processor_observer.wrap({"type": "generate_index_message_v2"})
-    async def generate_index_message_v2(
+    @processor_observer.wrap({"type": "generate_index_message"})
+    async def generate_index_message(
         self,
         resource: Resource,
         messages: list[writer_pb2.BrokerMessage],
@@ -477,40 +477,6 @@ class Processor:
         else:  # pragma: no cover
             raise InvalidBrokerMessage(f"Unknown broker message source: {message_source}")
 
-    @processor_observer.wrap({"type": "generate_index_message_v1"})
-    async def generate_index_message_v1(
-        self,
-        resource: Resource,
-        messages: list[writer_pb2.BrokerMessage],
-    ) -> PBBrainResource:
-        if any(needs_reindex(m) for m in messages):
-            # when reindexing, let's just generate full new index message
-            # TODO - This should be improved in the future as it's not optimal for very large resources:
-            # As of now, there are some API operations that require fully reindexing all the fields of a resource.
-            # An example of this is classification label changes - we need to reindex all the fields of a resource to
-            # propagate the label changes to the index.
-            resource.replace_indexer(await resource.generate_index_message(reindex=True))
-        else:
-            # TODO - Ideally we should only update the fields that have been changed in the current transaction.
-            await resource.compute_global_text()
-            await resource.compute_global_tags(resource.indexer)
-            await resource.compute_security(resource.indexer)
-        return resource.indexer.brain
-
-    async def generate_index_message(
-        self,
-        resource: Resource,
-        messages: list[writer_pb2.BrokerMessage],
-        resource_created: bool = False,
-    ) -> PBBrainResource:
-        if has_feature(
-            const.Features.INDEX_MESSAGE_GENERATION_V2,
-            context={"kbid": resource.kb.kbid},
-        ):
-            return await self.generate_index_message_v2(resource, messages, resource_created)
-        else:
-            return await self.generate_index_message_v1(resource, messages)
-
     async def external_index_delete_resource(
         self, external_index_manager: ExternalIndexManager, resource_uuid: str
     ):

nucliadb/ingest/orm/resource.py

@@ -19,11 +19,9 @@
 #
 from __future__ import annotations
 
-import asyncio
 import logging
 from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor
-from functools import partial
 from typing import TYPE_CHECKING, Any, Optional, Sequence, Type
 
 from nucliadb.common import datamanagers
@@ -32,12 +30,11 @@ from nucliadb.common.ids import FIELD_TYPE_PB_TO_STR, FieldId
 from nucliadb.common.maindb.driver import Transaction
 from nucliadb.ingest.fields.base import Field
 from nucliadb.ingest.fields.conversation import Conversation
-from nucliadb.ingest.fields.exceptions import FieldAuthorNotFound
 from nucliadb.ingest.fields.file import File
 from nucliadb.ingest.fields.generic import VALID_GENERIC_FIELDS, Generic
 from nucliadb.ingest.fields.link import Link
 from nucliadb.ingest.fields.text import Text
-from nucliadb.ingest.orm.brain import FilePagePositions, ResourceBrain
+from nucliadb.ingest.orm.brain_v2 import FilePagePositions
 from nucliadb.ingest.orm.metrics import processor_observer
 from nucliadb_models import content_types
 from nucliadb_models.common import CloudLink
@@ -69,9 +66,7 @@ from nucliadb_protos.resources_pb2 import Metadata as PBMetadata
 from nucliadb_protos.resources_pb2 import Origin as PBOrigin
 from nucliadb_protos.resources_pb2 import Relations as PBRelations
 from nucliadb_protos.writer_pb2 import BrokerMessage
-from nucliadb_utils import const
 from nucliadb_utils.storages.storage import Storage
-from nucliadb_utils.utilities import has_feature
 
 if TYPE_CHECKING:  # pragma: no cover
     from nucliadb.ingest.orm.knowledgebox import KnowledgeBox
@@ -121,7 +116,6 @@ class Resource:
         self.extra: Optional[PBExtra] = None
         self.security: Optional[utils_pb2.Security] = None
         self.modified: bool = False
-        self._indexer: Optional[ResourceBrain] = None
         self._modified_extracted_text: list[FieldID] = []
 
         self.txn = txn
@@ -133,15 +127,6 @@ class Resource:
         self._previous_status: Optional[Metadata.Status.ValueType] = None
         self.user_relations: Optional[PBRelations] = None
 
-    @property
-    def indexer(self) -> ResourceBrain:
-        if self._indexer is None:
-            self._indexer = ResourceBrain(rid=self.uuid)
-        return self._indexer
-
-    def replace_indexer(self, indexer: ResourceBrain) -> None:
-        self._indexer = indexer
-
     async def set_slug(self):
         basic = await self.get_basic()
         new_key = KB_RESOURCE_SLUG.format(kbid=self.kb.kbid, slug=basic.slug)
@@ -159,14 +144,6 @@ class Resource:
         if basic_in_payload.HasField("metadata") and basic_in_payload.metadata.useful:
             current_basic.metadata.status = basic_in_payload.metadata.status
 
-    def has_index_message_v2_feature(self) -> bool:
-        return has_feature(
-            const.Features.INDEX_MESSAGE_GENERATION_V2,
-            context={
-                "kbid": self.kb.kbid,
-            },
-        )
-
     @processor_observer.wrap({"type": "set_basic"})
     async def set_basic(
         self,
@@ -219,30 +196,6 @@ class Resource:
                 del self.basic.fieldmetadata[:]
                 self.basic.fieldmetadata.extend(updated)
 
-                if not self.has_index_message_v2_feature():
-                    # TODO: Remove this when we remove the old indexer is removed
-                    # All modified field metadata should be indexed
-                    # TODO: could be improved to only index the diff
-                    for user_field_metadata in self.basic.fieldmetadata:
-                        field_id = self.generate_field_id(fieldmetadata.field)
-                        field_obj = await self.get_field(
-                            fieldmetadata.field.field, fieldmetadata.field.field_type
-                        )
-                        field_metadata = await field_obj.get_field_metadata()
-                        if field_metadata is not None:
-                            page_positions: Optional[FilePagePositions] = None
-                            if isinstance(field_obj, File):
-                                page_positions = await get_file_page_positions(field_obj)
-
-                            self.indexer.apply_field_metadata(
-                                field_id,
-                                field_metadata,
-                                page_positions=page_positions,
-                                extracted_text=await field_obj.get_extracted_text(),
-                                basic_user_field_metadata=user_field_metadata,
-                                replace_field=True,
-                            )
-
         # Some basic fields are computed off field metadata.
         # This means we need to recompute upon field deletions.
         if deleted_fields is not None and len(deleted_fields) > 0:
@@ -313,66 +266,6 @@ class Resource:
         self.modified = True
         self.user_relations = payload
 
-    @processor_observer.wrap({"type": "generate_index_message_old"})
-    async def generate_index_message(self, reindex: bool = False) -> ResourceBrain:
-        brain = ResourceBrain(rid=self.uuid)
-        basic = await self.get_basic()
-        await self.compute_security(brain)
-        await self.compute_global_tags(brain)
-        fields = await self.get_fields(force=True)
-        for (type_id, field_id), field in fields.items():
-            fieldid = FieldID(field_type=type_id, field=field_id)
-            await self.compute_global_text_field(fieldid, brain)
-
-            field_metadata = await field.get_field_metadata()
-            field_key = self.generate_field_id(fieldid)
-            if field_metadata is not None:
-                page_positions: Optional[FilePagePositions] = None
-                if type_id == FieldType.FILE and isinstance(field, File):
-                    page_positions = await get_file_page_positions(field)
-
-                user_field_metadata = None
-                if basic is not None:
-                    user_field_metadata = next(
-                        (
-                            fm
-                            for fm in basic.fieldmetadata
-                            if fm.field.field == field_id and fm.field.field_type == type_id
-                        ),
-                        None,
-                    )
-                brain.apply_field_metadata(
-                    field_key,
-                    field_metadata,
-                    page_positions=page_positions,
-                    extracted_text=await field.get_extracted_text(),
-                    basic_user_field_metadata=user_field_metadata,
-                    replace_field=reindex,
-                )
-
-            if self.disable_vectors is False:
-                vectorset_configs = []
-                async for vectorset_id, vectorset_config in datamanagers.vectorsets.iter(
-                    self.txn, kbid=self.kb.kbid
-                ):
-                    vectorset_configs.append(vectorset_config)
-
-                for vectorset_config in vectorset_configs:
-                    vo = await field.get_vectors(
-                        vectorset=vectorset_config.vectorset_id,
-                        storage_key_kind=vectorset_config.storage_key_kind,
-                    )
-                    if vo is not None:
-                        dimension = vectorset_config.vectorset_index_config.vector_dimension
-                        brain.apply_field_vectors(
-                            field_key,
-                            vo,
-                            vectorset=vectorset_config.vectorset_id,
-                            vector_dimension=dimension,
-                            replace_field=reindex,
-                        )
-        return brain
-
     # Fields
     async def get_fields(self, force: bool = False) -> dict[tuple[FieldType.ValueType, str], Field]:
         # Get all fields
@@ -445,11 +338,6 @@ class Resource:
             if field in self.all_fields_keys:
                 self.all_fields_keys.remove(field)
 
-        # TODO: Remove this when we remove the old indexer
-        if not self.has_index_message_v2_feature():
-            field_key = self.generate_field_id(FieldID(field_type=type, field=key))
-            self.indexer.delete_field(field_key=field_key)
-
         await field_obj.delete()
 
     def has_field(self, type: FieldType.ValueType, field: str) -> bool:
@@ -668,7 +556,6 @@ class Resource:
         update_basic_languages(self.basic, extracted_languages)
 
         # Upload to binary storage
-        # Vector indexing
         if self.disable_vectors is False:
             await self._apply_extracted_vectors(message.field_vectors)
 
@@ -828,38 +715,7 @@ class Resource:
             field_metadata.field.field_type,
             load=False,
         )
-        metadata = await field_obj.set_field_metadata(field_metadata)
-
-        # TODO: Remove this when we remove the old indexer
-        if not self.has_index_message_v2_feature():
-            field_key = self.generate_field_id(field_metadata.field)
-
-            page_positions: Optional[FilePagePositions] = None
-            if field_metadata.field.field_type == FieldType.FILE and isinstance(field_obj, File):
-                page_positions = await get_file_page_positions(field_obj)
-
-            user_field_metadata = next(
-                (
-                    fm
-                    for fm in self.basic.fieldmetadata
-                    if fm.field.field == field_metadata.field.field
-                    and fm.field.field_type == field_metadata.field.field_type
-                ),
-                None,
-            )
-
-            extracted_text = await field_obj.get_extracted_text()
-            apply_field_metadata = partial(
-                self.indexer.apply_field_metadata,
-                field_key,
-                metadata,
-                page_positions=page_positions,
-                extracted_text=extracted_text,
-                basic_user_field_metadata=user_field_metadata,
-                replace_field=True,
-            )
-            loop = asyncio.get_running_loop()
-            await loop.run_in_executor(_executor, apply_field_metadata)
+        await field_obj.set_field_metadata(field_metadata)
 
         maybe_update_basic_thumbnail(self.basic, field_metadata.metadata.metadata.thumbnail)
 
@@ -913,27 +769,6 @@ class Resource:
             if vo is None:
                 raise AttributeError("Vector object not found on set_vectors")
 
-            if self.has_index_message_v2_feature():
-                continue
-
-            # TODO: Remove this when we remove the old indexer
-            # Prepare vectors to be indexed
-            field_key = self.generate_field_id(field_vectors.field)
-            dimension = vectorset.vectorset_index_config.vector_dimension
-            if not dimension:
-                raise ValueError(f"Vector dimension not set for vectorset '{vectorset.vectorset_id}'")
-
-            apply_field_vectors_partial = partial(
-                self.indexer.apply_field_vectors,
-                field_key,
-                vo,
-                vectorset=vectorset.vectorset_id,
-                replace_field=True,
-                vector_dimension=dimension,
-            )
-            loop = asyncio.get_running_loop()
-            await loop.run_in_executor(_executor, apply_field_vectors_partial)
-
     async def _apply_field_large_metadata(self, field_large_metadata: LargeComputedMetadataWrapper):
         field_obj = await self.get_field(
             field_large_metadata.field.field,
@@ -946,67 +781,7 @@ class Resource:
     def generate_field_id(self, field: FieldID) -> str:
         return f"{FIELD_TYPE_PB_TO_STR[field.field_type]}/{field.field}"
 
-    async def compute_security(self, brain: ResourceBrain):
-        security = await self.get_security()
-        if security is None:
-            return
-        brain.set_security(security)
-
-    @processor_observer.wrap({"type": "compute_global_tags"})
-    async def compute_global_tags(self, brain: ResourceBrain):
-        origin = await self.get_origin()
-        basic = await self.get_basic()
-        user_relations = await self.get_user_relations()
-        if basic is None:
-            raise KeyError("Resource not found")
-
-        brain.set_processing_status(basic=basic, previous_status=self._previous_status)
-        brain.set_resource_metadata(basic=basic, origin=origin, user_relations=user_relations)
-        for type, field in await self.get_fields_ids(force=True):
-            fieldobj = await self.get_field(field, type, load=False)
-            fieldid = FieldID(field_type=type, field=field)
-            fieldkey = self.generate_field_id(fieldid)
-            extracted_metadata = await fieldobj.get_field_metadata()
-            valid_user_field_metadata = None
-            for user_field_metadata in basic.fieldmetadata:
-                if (
-                    user_field_metadata.field.field == field
-                    and user_field_metadata.field.field_type == type
-                ):
-                    valid_user_field_metadata = user_field_metadata
-                    break
-            try:
-                generated_by = await fieldobj.generated_by()
-            except FieldAuthorNotFound:
-                generated_by = None
-            brain.apply_field_labels(
-                fieldkey,
-                extracted_metadata,
-                self.uuid,
-                generated_by,
-                basic.usermetadata,
-                valid_user_field_metadata,
-            )
-
-    @processor_observer.wrap({"type": "compute_global_text"})
-    async def compute_global_text(self):
-        for type, field in await self.get_fields_ids(force=True):
-            fieldid = FieldID(field_type=type, field=field)
-            await self.compute_global_text_field(fieldid, self.indexer)
-
-    async def compute_global_text_field(self, fieldid: FieldID, brain: ResourceBrain):
-        fieldobj = await self.get_field(fieldid.field, fieldid.field_type, load=False)
-        fieldkey = self.generate_field_id(fieldid)
-        extracted_text = await fieldobj.get_extracted_text()
-        if extracted_text is None:
-            return
-        field_text = extracted_text.text
-        for _, split in extracted_text.split_text.items():
-            field_text += f" {split} "
-        brain.apply_field_text(fieldkey, field_text, replace_field=True)
-
     def clean(self):
-        self._indexer = None
         self.txn = None
 
 

{nucliadb-6.4.0.post4210.dist-info → nucliadb-6.4.0.post4224.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.4.0.post4210
+Version: 6.4.0.post4224
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.4.0.post4210
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.4.0.post4210
-Requires-Dist: nucliadb-protos>=6.4.0.post4210
-Requires-Dist: nucliadb-models>=6.4.0.post4210
-Requires-Dist: nidx-protos>=6.4.0.post4210
+Requires-Dist: nucliadb-telemetry[all]>=6.4.0.post4224
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.4.0.post4224
+Requires-Dist: nucliadb-protos>=6.4.0.post4224
+Requires-Dist: nucliadb-models>=6.4.0.post4224
+Requires-Dist: nidx-protos>=6.4.0.post4224
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn[standard]

{nucliadb-6.4.0.post4210.dist-info → nucliadb-6.4.0.post4224.dist-info}/RECORD

@@ -122,7 +122,7 @@ nucliadb/export_import/models.py,sha256=dbjScNkiMRv4X3Ktudy1JRliD25bfoDTy3JmEZgQ
 nucliadb/export_import/tasks.py,sha256=DWbdqY97ffoyfipelGXz3Jqz1iam6JCjQSh367Fc3NA,2947
 nucliadb/export_import/utils.py,sha256=8XOVMYXXw8b4ikojG7RjQ4tKN3Xu7nfu2yCUOqD50sk,23216
 nucliadb/ingest/__init__.py,sha256=fsw3C38VP50km3R-nHL775LNGPpJ4JxqXJ2Ib1f5SqE,1011
-nucliadb/ingest/app.py,sha256=KCptzFq1Msq4eHFxvEol4TFwSLdmkG2v1EfQ3C8PhyY,7547
+nucliadb/ingest/app.py,sha256=BKmjpdBEskHcRIHwOnI_jG4gFGs6dV0KKVH9MLJeA48,7546
 nucliadb/ingest/partitions.py,sha256=2NIhMYbNT0TNBL6bX1UMSi7vxFGICstCKEqsB0TXHOE,2410
 nucliadb/ingest/processing.py,sha256=QmkHq-BU4vub7JRWe9VHvQ2DcAmT6-CzgFXuZxXhcBU,20953
 nucliadb/ingest/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -134,8 +134,8 @@ nucliadb/ingest/consumer/auditing.py,sha256=xK21DIa_ZAiOJVVbnkmT4jgCRGshNGyPyxsq
 nucliadb/ingest/consumer/consumer.py,sha256=OgS1fr5Yo55u-XbC6zypTH1aJ562Y1vZHnPDlJJpCXQ,13703
 nucliadb/ingest/consumer/materializer.py,sha256=tgD_rDI2twQzcz8kKNiW_L4YIth16IGh9mUfD5wiSD4,3858
 nucliadb/ingest/consumer/metrics.py,sha256=ji1l_4cKiHJthQd8YNem1ft4iMbw9KThmVvJmLcv3Xg,1075
-nucliadb/ingest/consumer/pull.py,sha256=EYT0ImngMQgatStG68p2GSrPQBbJxeuq8nFm8DdAbwk,9280
-nucliadb/ingest/consumer/service.py,sha256=BLM_dmKZkFBsYl3sj4MZZp5M3kkxHLuO7sE18PqIatw,6538
+nucliadb/ingest/consumer/pull.py,sha256=vv1AyN0EhVgbgnZyT0D_1_IB4hWy7jPd4lAWPAOHGNc,10374
+nucliadb/ingest/consumer/service.py,sha256=GhuqlK-9Lvhzd8kBox8wOlKlJgM3W_gssKoWSfVVdoI,7897
 nucliadb/ingest/consumer/shard_creator.py,sha256=w0smEu01FU_2cjZnsfBRNqT_Ntho11X17zTMST-vKbc,4359
 nucliadb/ingest/consumer/utils.py,sha256=jpX8D4lKzuPCpArQLZeX_Zczq3pfen_zAf8sPJfOEZU,2642
 nucliadb/ingest/fields/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
@@ -147,21 +147,20 @@ nucliadb/ingest/fields/generic.py,sha256=elgtqv15aJUq3zY7X_g0bli_2BpcwPArVvzhe54
 nucliadb/ingest/fields/link.py,sha256=kN_gjRUEEj5cy8K_BwPijYg3TiWhedc24apXYlTbRJs,4172
 nucliadb/ingest/fields/text.py,sha256=2grxo8twWbpXEd_iwUMBw9q0dWorVmlPONmY5d1ThwQ,1684
 nucliadb/ingest/orm/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/ingest/orm/brain.py,sha256=q8iW7TWgABcGG5gIrR4SN5Flv_hOy6uuSv3VL8aWgAk,29098
-nucliadb/ingest/orm/brain_v2.py,sha256=0OYqH9srWghajGh0l1oqTFPBh1Jtlw3ui3Qpww6IC7A,33573
+nucliadb/ingest/orm/brain_v2.py,sha256=qX81wvU-KCcEZ-hNgkQOskMOlZmdbJqDyAfe7eXbGLw,33571
 nucliadb/ingest/orm/broker_message.py,sha256=XWaiZgDOz94NPOPT-hqbRr5ZkpVimUw6PjUJNftfoVw,7514
 nucliadb/ingest/orm/entities.py,sha256=kXyeF6XOpFKhEsGLcY-GLIk21Exp0cJst4XQQ9jJoug,14791
 nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
-nucliadb/ingest/orm/index_message.py,sha256=7Pl2qtqoI3b3NAjWfgoiLQktayngdsJ_NfDH0wpTJBw,16041
+nucliadb/ingest/orm/index_message.py,sha256=hI85nSNVChNLLdEFuEJvOt61Tsir-Gq-2_WZoayAdvk,15617
 nucliadb/ingest/orm/knowledgebox.py,sha256=_rkeTMIXMhR64gbYtZpFHoUHghV2DTJ2lUBqZsoqC_4,23898
 nucliadb/ingest/orm/metrics.py,sha256=OiuggTh-n3kZHA2G73NEUdIlh8c3yFrbusI88DK-Mko,1273
-nucliadb/ingest/orm/resource.py,sha256=nk-aT9mEPenmZ4blyqTbTOXCpJaSSi1AE-wTJ01V2vA,47007
+nucliadb/ingest/orm/resource.py,sha256=hGELQgnzK2wIWgD478bR5OiVDyAxHn6WrFSq2YuHANU,36896
 nucliadb/ingest/orm/utils.py,sha256=fCQRuyecgqhaY7mcBG93oaXMkzkKb9BFjOcy4-ZiSNw,2693
 nucliadb/ingest/orm/processor/__init__.py,sha256=Aqd9wCNTvggkMkCY3WvoI8spdr94Jnqk-0iq9XpLs18,922
 nucliadb/ingest/orm/processor/auditing.py,sha256=TeYhXGJRyQ7ROytbb2u8R0fIh_FYi3HgTu3S1ribY3U,4623
 nucliadb/ingest/orm/processor/data_augmentation.py,sha256=v-pj4GbBWSuO8dQyahs5UDr5ghsyfhCZDS0ftKd6ZYc,5179
 nucliadb/ingest/orm/processor/pgcatalog.py,sha256=ht9_I5WlPc6sSFTY8PsxHlpjN-EsaBaChwqsLlMXwUk,3100
-nucliadb/ingest/orm/processor/processor.py,sha256=jLGLyfj6TIXoLubmNvQvkj5K32MxDDSoktY6M8z5pVk,33149
+nucliadb/ingest/orm/processor/processor.py,sha256=jaEBwbv--WyoC8zcdxWAyF0dAzVA5crVDJl56Bqv1eI,31444
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=uqEphtI1Ir_yk9jRl2gPf7BlzzXWovbARY5MNZSBI_8,1704
 nucliadb/ingest/service/__init__.py,sha256=LHQFUkdmNBOWqBG0Md9sMMI7g5TQZ-hLAnhw6ZblrJg,2002
 nucliadb/ingest/service/exceptions.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
@@ -369,8 +368,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.4.0.post4210.dist-info/METADATA,sha256=SB9gIMgWxoWNtUEexRLH85E0PL-MnroGhJ6aOambTT4,4223
-nucliadb-6.4.0.post4210.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-nucliadb-6.4.0.post4210.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.4.0.post4210.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.4.0.post4210.dist-info/RECORD,,
+nucliadb-6.4.0.post4224.dist-info/METADATA,sha256=G9L1810f7GDMjI54RDmZj-ZcpBD3_duqsGRR2q3c6yY,4223
+nucliadb-6.4.0.post4224.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+nucliadb-6.4.0.post4224.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.4.0.post4224.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.4.0.post4224.dist-info/RECORD,,