nucliadb 6.9.1.post5187__py3-none-any.whl → 6.9.1.post5207__py3-none-any.whl

nucliadb/common/cluster/rollover.py

@@ -34,8 +34,10 @@ from nucliadb.common.external_index_providers.base import ExternalIndexManager
 from nucliadb.common.external_index_providers.manager import (
     get_external_index_manager,
 )
+from nucliadb.common.maindb.utils import get_driver
 from nucliadb.common.nidx import get_nidx_api_client
 from nucliadb.common.vector_index_config import nucliadb_index_config_to_nidx
+from nucliadb.ingest.orm.knowledgebox import KnowledgeBox
 from nucliadb.migrator.settings import settings
 from nucliadb_protos import utils_pb2, writer_pb2
 from nucliadb_telemetry import errors
@@ -45,6 +47,7 @@ from .utils import (
     get_resource,
     get_rollover_resource_index_message,
     index_resource_to_shard,
+    wait_for_nidx,
 )
 
 logger = logging.getLogger(__name__)
@@ -254,6 +257,7 @@ async def index_to_rollover_index(
             for rid in resource_ids
         ]
         await asyncio.gather(*batch)
+        await wait_for_indexing_to_catch_up(app_context)
 
     async with datamanagers.with_transaction() as txn:
         state.resources_indexed = True
@@ -262,6 +266,22 @@ async def index_to_rollover_index(
         await txn.commit()
 
 
+async def wait_for_indexing_to_catch_up(app_context: ApplicationContext):
+    try:
+        app_context.nats_manager
+    except AssertionError:
+        logger.warning("Nats manager not initialized. Cannot wait for indexing to catch up")
+        return
+    max_pending = 1000
+    while True:
+        try:
+            await wait_for_nidx(app_context.nats_manager, max_wait_seconds=60, max_pending=max_pending)
+            return
+        except asyncio.TimeoutError:
+            logger.warning(f"Nidx is behind more than {max_pending} messages. Throttling rollover.")
+            await asyncio.sleep(30)
+
+
 async def _index_resource_to_rollover_index(
     app_context: ApplicationContext,
     rollover_shards: writer_pb2.Shards,
@@ -415,6 +435,15 @@ async def cutover_shards(app_context: ApplicationContext, kbid: str) -> None:
 
         await txn.commit()
 
+    # For KBs with pre-warm enabled, we must configure the new shards. There may
+    # be some small delay between this call and the shards being actually
+    # prewarmed, but rollovers are quite unusual and we prefer this rather than
+    # prewarming old and new shards at the same time
+    kb_config = await datamanagers.atomic.kb.get_config(kbid=kbid)
+    if kb_config is not None and kb_config.prewarm_enabled:
+        driver = get_driver()
+        await KnowledgeBox.configure_shards(driver, kbid, prewarm=True)
+
 
 async def validate_indexed_data(
     app_context: ApplicationContext, kbid: str, external: Optional[ExternalIndexManager] = None

nucliadb/common/cluster/utils.py

@@ -32,6 +32,7 @@ from nucliadb.common.cluster.settings import settings
 from nucliadb.ingest.orm import index_message
 from nucliadb.ingest.orm.resource import Resource
 from nucliadb_protos import writer_pb2
+from nucliadb_utils.nats import NatsConnectionManager
 from nucliadb_utils.utilities import Utility, clean_utility, get_utility, set_utility
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -125,3 +126,28 @@ async def delete_resource_from_shard(
     partition = partitioning.generate_partition(kbid, resource_id)
 
     await sm.delete_resource(shard, resource_id, 0, str(partition), kbid)
+
+
+async def get_nats_consumer_pending_messages(
+    nats_manager: NatsConnectionManager, *, stream: str, consumer: str
+) -> int:
+    # get raw js client
+    js = nats_manager.js
+    consumer_info = await js.consumer_info(stream, consumer)
+    return consumer_info.num_pending
+
+
+async def wait_for_nidx(
+    nats_manager: NatsConnectionManager,
+    max_pending: int,
+    poll_interval_seconds: int = 5,
+    max_wait_seconds: int = 60,
+):
+    async with asyncio.timeout(max_wait_seconds):  # type: ignore
+        while True:
+            pending = await get_nats_consumer_pending_messages(
+                nats_manager, stream="nidx", consumer="nidx"
+            )
+            if pending < max_pending:
+                return
+            await asyncio.sleep(poll_interval_seconds)

nucliadb/common/datamanagers/atomic.py

@@ -42,6 +42,7 @@ from typing_extensions import Concatenate, ParamSpec
 
 from nucliadb.common.maindb.driver import Transaction
 
+from . import cluster as cluster_dm
 from . import kb as kb_dm
 from . import labels as labels_dm
 from . import resources as resources_dm
@@ -73,6 +74,10 @@ def rw_txn_wrap(fun: Callable[Concatenate[Transaction, P], Awaitable[T]]) -> Cal
     return wrapper
 
 
+class cluster:
+    get_kb_shards = ro_txn_wrap(cluster_dm.get_kb_shards)
+
+
 class kb:
     exists_kb = ro_txn_wrap(kb_dm.exists_kb)
     get_config = ro_txn_wrap(kb_dm.get_config)

nucliadb/ingest/consumer/service.py

@@ -140,9 +140,8 @@ async def start_shard_creator() -> Callable[[], Awaitable[None]]:
     driver = await setup_driver()
     pubsub = await get_pubsub()
     assert pubsub is not None, "Pubsub is not configured"
-    storage = await get_storage(service_name=SERVICE_NAME)
 
-    shard_creator = ShardCreatorHandler(driver=driver, storage=storage, pubsub=pubsub)
+    shard_creator = ShardCreatorHandler(driver=driver, pubsub=pubsub)
     await shard_creator.initialize()
 
     return shard_creator.finalize

nucliadb/ingest/consumer/shard_creator.py

@@ -25,14 +25,14 @@ from typing import Any
 
 from nidx_protos import nodereader_pb2, noderesources_pb2
 
-from nucliadb.common import locking
+from nucliadb.common import datamanagers, locking
+from nucliadb.common.cluster.settings import settings
 from nucliadb.common.cluster.utils import get_shard_manager
 from nucliadb.common.maindb.driver import Driver
 from nucliadb.common.nidx import get_nidx_api_client
 from nucliadb_protos import writer_pb2
 from nucliadb_utils import const
 from nucliadb_utils.cache.pubsub import PubSubDriver
-from nucliadb_utils.storages.storage import Storage
 
 from . import metrics
 from .utils import DelayedTaskHandler
@@ -52,12 +52,10 @@ class ShardCreatorHandler:
         self,
         *,
         driver: Driver,
-        storage: Storage,
         pubsub: PubSubDriver,
         check_delay: float = 10.0,
     ):
         self.driver = driver
-        self.storage = storage
         self.pubsub = pubsub
         self.shard_manager = get_shard_manager()
         self.task_handler = DelayedTaskHandler(check_delay)
@@ -111,4 +109,17 @@ class ShardCreatorHandler:
                     shard_id=noderesources_pb2.ShardId(id=current_shard.nidx_shard_id)
                 )  # type: ignore
             )
-            await self.shard_manager.maybe_create_new_shard(kbid, shard.paragraphs)
+
+            if not should_create_new_shard(shard.paragraphs):
+                return
+
+            logger.info({"message": "Adding shard", "kbid": kbid})
+            async with datamanagers.with_rw_transaction() as txn:
+                kb_config = await datamanagers.kb.get_config(txn, kbid=kbid)
+                prewarm = kb_config is not None and kb_config.prewarm_enabled
+                await self.shard_manager.create_shard_by_kbid(txn, kbid, prewarm_enabled=prewarm)
+                await txn.commit()
+
+
+def should_create_new_shard(num_paragraphs: int) -> bool:
+    return num_paragraphs > settings.max_shard_paragraphs

nucliadb/ingest/orm/knowledgebox.py

@@ -24,7 +24,7 @@ from uuid import uuid4
 
 from grpc import StatusCode
 from grpc.aio import AioRpcError
-from nidx_protos import noderesources_pb2
+from nidx_protos import nidx_pb2, noderesources_pb2
 
 from nucliadb.common import datamanagers
 from nucliadb.common.cluster.exceptions import ShardNotFound
@@ -108,6 +108,7 @@ class KnowledgeBox:
         external_index_provider: CreateExternalIndexProviderMetadata = CreateExternalIndexProviderMetadata(),
         hidden_resources_enabled: bool = False,
         hidden_resources_hide_on_creation: bool = False,
+        prewarm_enabled: bool = False,
     ) -> tuple[str, str]:
         """Creates a new knowledge box and return its id and slug."""
 
@@ -194,6 +195,7 @@ class KnowledgeBox:
                     migration_version=get_latest_version(),
                     hidden_resources_enabled=hidden_resources_enabled,
                     hidden_resources_hide_on_creation=hidden_resources_hide_on_creation,
+                    prewarm_enabled=prewarm_enabled,
                 )
                 config.external_index_provider.CopyFrom(stored_external_index_provider)
                 await datamanagers.kb.set_config(txn, kbid=kbid, config=config)
@@ -220,7 +222,7 @@ class KnowledgeBox:
                 shard_manager = get_shard_manager()
                 # XXX creating a shard is a slow IO operation that requires a write
                 # txn to be open!
-                await shard_manager.create_shard_by_kbid(txn, kbid)
+                await shard_manager.create_shard_by_kbid(txn, kbid, prewarm_enabled=prewarm_enabled)
                 # shards don't need a rollback as they will be eventually purged
 
                 await txn.commit()
@@ -243,39 +245,86 @@ class KnowledgeBox:
     @classmethod
     async def update(
         cls,
-        txn: Transaction,
-        uuid: str,
+        driver: Driver,
+        kbid: str,
+        *,
         slug: Optional[str] = None,
-        config: Optional[KnowledgeBoxConfig] = None,
+        title: Optional[str] = None,
+        description: Optional[str] = None,
+        migration_version: Optional[int] = None,
+        external_index_provider: Optional[StoredExternalIndexProviderMetadata] = None,
+        hidden_resources_enabled: Optional[bool] = None,
+        hidden_resources_hide_on_creation: Optional[bool] = None,
+        prewarm_enabled: Optional[bool] = None,
     ) -> str:
-        exist = await datamanagers.kb.get_config(txn, kbid=uuid, for_update=True)
-        if not exist:
-            raise datamanagers.exceptions.KnowledgeBoxNotFound()
-
-        if slug:
-            await txn.delete(datamanagers.kb.KB_SLUGS.format(slug=exist.slug))
-            await txn.set(
-                datamanagers.kb.KB_SLUGS.format(slug=slug),
-                uuid.encode(),
-            )
-            if config:
-                config.slug = slug
-            else:
-                exist.slug = slug
+        async with driver.rw_transaction() as txn:
+            stored = await datamanagers.kb.get_config(txn, kbid=kbid, for_update=True)
+            if not stored:
+                raise datamanagers.exceptions.KnowledgeBoxNotFound()
+
+            if slug:
+                await txn.delete(datamanagers.kb.KB_SLUGS.format(slug=stored.slug))
+                await txn.set(
+                    datamanagers.kb.KB_SLUGS.format(slug=slug),
+                    kbid.encode(),
+                )
+                stored.slug = slug
 
-        if config and exist != config:
-            exist.MergeFrom(config)
-            exist.hidden_resources_enabled = config.hidden_resources_enabled
-            exist.hidden_resources_hide_on_creation = config.hidden_resources_hide_on_creation
+            if title is not None:
+                stored.title = title
+            if description is not None:
+                stored.description = description
 
-        if exist.hidden_resources_hide_on_creation and not exist.hidden_resources_enabled:
-            raise KnowledgeBoxCreationError(
-                "Cannot hide new resources if the hidden resources feature is disabled"
-            )
+            if migration_version is not None:
+                stored.migration_version = migration_version
+
+            if external_index_provider is not None:
+                stored.external_index_provider.MergeFrom(external_index_provider)
 
-        await datamanagers.kb.set_config(txn, kbid=uuid, config=exist)
+            if hidden_resources_enabled is not None:
+                stored.hidden_resources_enabled = hidden_resources_enabled
+            if hidden_resources_hide_on_creation is not None:
+                stored.hidden_resources_hide_on_creation = hidden_resources_hide_on_creation
+
+            update_nidx_prewarm = None
+            if prewarm_enabled is not None:
+                if stored.prewarm_enabled != prewarm_enabled:
+                    update_nidx_prewarm = prewarm_enabled
+                stored.prewarm_enabled = prewarm_enabled
+
+            if stored.hidden_resources_hide_on_creation and not stored.hidden_resources_enabled:
+                raise KnowledgeBoxCreationError(
+                    "Cannot hide new resources if the hidden resources feature is disabled"
+                )
 
-        return uuid
+            await datamanagers.kb.set_config(txn, kbid=kbid, config=stored)
+
+            await txn.commit()
+
+        if update_nidx_prewarm is not None:
+            await cls.configure_shards(driver, kbid, prewarm=update_nidx_prewarm)
+
+        return kbid
+
+    @classmethod
+    async def configure_shards(cls, driver: Driver, kbid: str, *, prewarm: bool):
+        shards_obj = await datamanagers.atomic.cluster.get_kb_shards(kbid=kbid)
+        if shards_obj is None:
+            logger.warning(f"Shards not found for KB while updating pre-warm flag", extra={"kbid": kbid})
+            return
+
+        nidx_shard_ids = [shard.nidx_shard_id for shard in shards_obj.shards]
+
+        nidx_api = get_nidx_api_client()
+        if nidx_api is not None and len(nidx_shard_ids) > 0:
+            configs = [
+                nidx_pb2.ShardConfig(
+                    shard_id=shard_id,
+                    prewarm_enabled=prewarm,
+                )
+                for shard_id in nidx_shard_ids
+            ]
+            await nidx_api.ConfigureShards(nidx_pb2.ShardsConfig(configs=configs))
 
     @classmethod
     async def delete(cls, driver: Driver, kbid: str):

nucliadb/ingest/orm/processor/processor.py

@@ -446,22 +446,27 @@ class Processor:
             # a resource was move to another shard while it was being indexed
             shard_id = await datamanagers.resources.get_resource_shard_id(txn, kbid=kbid, rid=uuid)
 
-        shard = None
-        if shard_id is not None:
-            # Resource already has a shard assigned
-            shard = await kb.get_resource_shard(shard_id)
-            if shard is None:
-                raise AttributeError("Shard not available")
-        else:
-            # It's a new resource, get KB's current active shard to place new resource on
-            shard = await self.index_node_shard_manager.get_current_active_shard(txn, kbid)
-            if shard is None:
-                # No current shard available, create a new one
-                shard = await self.index_node_shard_manager.create_shard_by_kbid(txn, kbid)
-            await datamanagers.resources.set_resource_shard_id(
-                txn, kbid=kbid, rid=uuid, shard=shard.shard
-            )
-        return shard
+            shard = None
+            if shard_id is not None:
+                # Resource already has a shard assigned
+                shard = await kb.get_resource_shard(shard_id)
+                if shard is None:
+                    raise AttributeError("Shard not available")
+            else:
+                # It's a new resource, get KB's current active shard to place new resource on
+                shard = await self.index_node_shard_manager.get_current_active_shard(txn, kbid)
+                if shard is None:
+                    # No current shard available, create a new one
+                    async with locking.distributed_lock(locking.NEW_SHARD_LOCK.format(kbid=kbid)):
+                        kb_config = await datamanagers.kb.get_config(txn, kbid=kbid)
+                        prewarm = kb_config is not None and kb_config.prewarm_enabled
+                        shard = await self.index_node_shard_manager.create_shard_by_kbid(
+                            txn, kbid, prewarm_enabled=prewarm
+                        )
+                await datamanagers.resources.set_resource_shard_id(
+                    txn, kbid=kbid, rid=uuid, shard=shard.shard
+                )
+            return shard
 
     @processor_observer.wrap({"type": "index_resource"})
     async def index_resource(

nucliadb/ingest/service/writer.py

@@ -126,6 +126,7 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
                 external_index_provider=request.external_index_provider,
                 hidden_resources_enabled=request.hidden_resources_enabled,
                 hidden_resources_hide_on_creation=request.hidden_resources_hide_on_creation,
+                prewarm_enabled=request.prewarm_enabled,
             )
 
         except KnowledgeBoxConflict:
@@ -167,11 +168,17 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
             )
 
         try:
-            async with self.driver.rw_transaction() as txn:
-                kbid = await KnowledgeBoxORM.update(
-                    txn, uuid=request.uuid, slug=request.slug, config=request.config
-                )
-                await txn.commit()
+            kbid = await KnowledgeBoxORM.update(
+                self.driver,
+                kbid=request.uuid,
+                slug=request.slug,
+                title=request.config.title or None,
+                description=request.config.description or None,
+                external_index_provider=request.config.external_index_provider or None,
+                hidden_resources_enabled=request.config.hidden_resources_enabled,
+                hidden_resources_hide_on_creation=request.config.hidden_resources_hide_on_creation,
+                prewarm_enabled=request.config.prewarm_enabled,
+            )
         except KnowledgeBoxNotFound:
             return UpdateKnowledgeBoxResponse(status=KnowledgeBoxResponseStatus.NOTFOUND)
         except Exception:

nucliadb/migrator/datamanager.py

@@ -77,13 +77,7 @@ class MigrationsDataManager:
         return KnowledgeBoxInfo(current_version=kb_config.migration_version)
 
     async def update_kb_info(self, *, kbid: str, current_version: int) -> None:
-        async with self.driver.rw_transaction() as txn:
-            kb_config = await datamanagers.kb.get_config(txn, kbid=kbid, for_update=True)
-            if kb_config is None:
-                raise Exception(f"KB {kbid} does not exist")
-            kb_config.migration_version = current_version
-            await KnowledgeBoxORM.update(txn, kbid, config=kb_config)
-            await txn.commit()
+        await KnowledgeBoxORM.update(self.driver, kbid, migration_version=current_version)
 
     async def get_global_info(self) -> GlobalInfo:
         async with self.driver.ro_transaction() as txn:

nucliadb/purge/__init__.py

@@ -19,6 +19,7 @@
 #
 import asyncio
 import importlib.metadata
+from itertools import batched  # type: ignore
 from typing import AsyncGenerator
 
 from nucliadb.common import datamanagers
@@ -233,7 +234,7 @@ async def purge_kb_vectorsets(driver: Driver, storage: Storage):
                     fields.extend((await resource.get_fields(force=True)).values())
 
             logger.info(f"Purging {len(fields)} fields for vectorset {vectorset}", extra={"kbid": kbid})
-            for fields_batch in batchify(fields, 20):
+            for fields_batch in batched(fields, n=20):
                 tasks = []
                 for field in fields_batch:
                     if purge_payload.storage_key_kind == VectorSetConfig.StorageKeyKind.UNSET:
@@ -317,9 +318,3 @@ def run() -> int:  # pragma: no cover
     setup_logging()
     errors.setup_error_handling(importlib.metadata.distribution("nucliadb").version)
     return asyncio.run(main())
-
-
-def batchify(iterable, n=1):
-    """Yield successive n-sized chunks from iterable."""
-    for i in range(0, len(iterable), n):
-        yield iterable[i : i + n]

nucliadb/writer/api/v1/knowledgebox.py

@@ -147,8 +147,6 @@ async def create_kb(item: KnowledgeBoxConfig) -> tuple[str, str]:
 @requires(NucliaDBRoles.MANAGER)
 @version(1)
 async def update_kb(request: Request, kbid: str, item: KnowledgeBoxConfig) -> KnowledgeBoxObjID:
-    driver = get_driver()
-    config = None
     if (
         item.slug
         or item.title
@@ -156,29 +154,24 @@ async def update_kb(request: Request, kbid: str, item: KnowledgeBoxConfig) -> Kn
         or item.hidden_resources_enabled
         or item.hidden_resources_hide_on_creation
     ):
-        config = knowledgebox_pb2.KnowledgeBoxConfig(
-            slug=item.slug or "",
-            title=item.title or "",
-            description=item.description or "",
-            hidden_resources_enabled=item.hidden_resources_enabled,
-            hidden_resources_hide_on_creation=item.hidden_resources_hide_on_creation,
-        )
-    try:
-        async with driver.rw_transaction() as txn:
+        try:
+            driver = get_driver()
             await KnowledgeBox.update(
-                txn,
-                uuid=kbid,
+                driver,
+                kbid=kbid,
                 slug=item.slug,
-                config=config,
+                title=item.title,
+                description=item.description,
+                hidden_resources_enabled=item.hidden_resources_enabled,
+                hidden_resources_hide_on_creation=item.hidden_resources_hide_on_creation,
             )
-            await txn.commit()
-    except datamanagers.exceptions.KnowledgeBoxNotFound:
-        raise HTTPException(status_code=404, detail="Knowledge box does not exist")
-    except Exception as exc:
-        logger.exception("Could not update KB", exc_info=exc, extra={"kbid": kbid})
-        raise HTTPException(status_code=500, detail="Error updating knowledge box")
-    else:
-        return KnowledgeBoxObjID(uuid=kbid)
+        except datamanagers.exceptions.KnowledgeBoxNotFound:
+            raise HTTPException(status_code=404, detail="Knowledge box does not exist")
+        except Exception as exc:
+            logger.exception("Could not update KB", exc_info=exc, extra={"kbid": kbid})
+            raise HTTPException(status_code=500, detail="Error updating knowledge box")
+
+    return KnowledgeBoxObjID(uuid=kbid)
 
 
 @only_for_onprem

{nucliadb-6.9.1.post5187.dist-info → nucliadb-6.9.1.post5207.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.9.1.post5187
+Version: 6.9.1.post5207
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -19,11 +19,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.9.1.post5187
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.9.1.post5187
-Requires-Dist: nucliadb-protos>=6.9.1.post5187
-Requires-Dist: nucliadb-models>=6.9.1.post5187
-Requires-Dist: nidx-protos>=6.9.1.post5187
+Requires-Dist: nucliadb-telemetry[all]>=6.9.1.post5207
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.9.1.post5207
+Requires-Dist: nucliadb-protos>=6.9.1.post5207
+Requires-Dist: nucliadb-models>=6.9.1.post5207
+Requires-Dist: nidx-protos>=6.9.1.post5207
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.50.0
 Requires-Dist: uvicorn[standard]

{nucliadb-6.9.1.post5187.dist-info → nucliadb-6.9.1.post5207.dist-info}/RECORD

@@ -85,17 +85,17 @@ nucliadb/common/catalog/utils.py,sha256=lQLTe3rN_ra0CiNjTQIiIaU4lhx5beRkx1Va10ZB
 nucliadb/common/cluster/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/cluster/exceptions.py,sha256=t7v_l93t44l2tQpdQXgO_w-c4YZRcaayOz1A2i0w4RQ,1258
 nucliadb/common/cluster/grpc_node_dummy.py,sha256=JkufazWzMA4KFEU8EBkMbiiDW4C8lLcRhiiCxP7aCQY,2949
-nucliadb/common/cluster/manager.py,sha256=QOHA7trtZVwV5G9IIxM1Y-0P2KE5mEhsXUMPs8vxIg0,12816
-nucliadb/common/cluster/rebalance.py,sha256=238JxqdSl1_oeJNpKuGBgfVn8WXVmyX5yl2PyHomYX8,8715
-nucliadb/common/cluster/rollover.py,sha256=swuYNXJ8u5p11PvYaHxVtomiYIPVWVGVsicwTgm24-Q,25862
+nucliadb/common/cluster/manager.py,sha256=p-haaGEnCa-20t-I2XEo4QJ5ZC1QgJ6p2jzXFYVB6nQ,12346
+nucliadb/common/cluster/rebalance.py,sha256=s0YJ07Y358T9x22QnHHEvhmHqf1CnQWQD6oJznOo2Xc,22239
+nucliadb/common/cluster/rollover.py,sha256=kmVCdyjJ1dilnSodHMqf0EUkTjPC5H0aA4JqW9KsEa4,27168
 nucliadb/common/cluster/settings.py,sha256=f6Y5K0PGahkedwe5wtkWMnbqwDFJgOOwX_MOIGwH9Dg,2271
-nucliadb/common/cluster/utils.py,sha256=IfW5PA7Ab26xWUYNOc3yBoksWV1GK4BGhTRo1vnHNKg,4662
+nucliadb/common/cluster/utils.py,sha256=E4GqidwTKczJX_lTnncBCof2fL4CFVVF1eLiz9NWjlc,5494
 nucliadb/common/cluster/standalone/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
 nucliadb/common/cluster/standalone/utils.py,sha256=af3r-x_GF7A6dwIAhZLR-r-SZQEVxsFrDKeMfUTA6G0,1908
 nucliadb/common/context/__init__.py,sha256=IKAHuiCjbOEsqfLozWwJ6mRFzFncsZMyxNC5E_XZ5EM,6016
 nucliadb/common/context/fastapi.py,sha256=mH_8n5t7quNSPivNM2JS5EQf2sTVJsdzXW6LaY7EHAA,1629
 nucliadb/common/datamanagers/__init__.py,sha256=xKc6ZMqKUs20R90jJT4xkQ8TFMNwQnhhuWnBBqVnKdM,2084
-nucliadb/common/datamanagers/atomic.py,sha256=WihdtBWQIAuElZQjh1xQ--q5dJowwlkovqsW-OB_t2k,3230
+nucliadb/common/datamanagers/atomic.py,sha256=-3T2--A0_qFWmy573t-DS29EHFcMTyjGvcvky6J9tAM,3341
 nucliadb/common/datamanagers/cluster.py,sha256=iU0b7AESm1Yi8Wp3pIKgqixZGNMjeBrxSpvEKsaZKgY,1831
 nucliadb/common/datamanagers/entities.py,sha256=gI-0mbMlqrr9FiyhexEh6czhgYcMxE2s9m4o866EK9o,5340
 nucliadb/common/datamanagers/exceptions.py,sha256=Atz_PP_GGq4jgJaWcAkcRbHBoBaGcC9yJvFteylKtTE,883
@@ -150,8 +150,8 @@ nucliadb/ingest/consumer/consumer.py,sha256=1OetpJXp6glaAe4kKqUA_L46BS-ZyEccTkwt
 nucliadb/ingest/consumer/materializer.py,sha256=tgD_rDI2twQzcz8kKNiW_L4YIth16IGh9mUfD5wiSD4,3858
 nucliadb/ingest/consumer/metrics.py,sha256=ji1l_4cKiHJthQd8YNem1ft4iMbw9KThmVvJmLcv3Xg,1075
 nucliadb/ingest/consumer/pull.py,sha256=Ki_aHi72W83yD03lPt6Yz2l_uCeu62fd4upEMcOZcm4,9201
-nucliadb/ingest/consumer/service.py,sha256=mjiEJFO9t8nsQBdbWfGyyGajaW012-6mAwMARBkedgU,5600
-nucliadb/ingest/consumer/shard_creator.py,sha256=UKIk0yaS_jC_nGQqymn9NGJWzwZEqhIr0gznJYorlAE,4348
+nucliadb/ingest/consumer/service.py,sha256=_yj3gUKaCUA6Wp_73Bm8tO2Xk1vR8-aJV2xvPZpjAfM,5524
+nucliadb/ingest/consumer/shard_creator.py,sha256=qUEpxZLE1etw1nL8L3O9HvZBx5NNql7S1vKDyqelN2M,4849
 nucliadb/ingest/consumer/utils.py,sha256=jpX8D4lKzuPCpArQLZeX_Zczq3pfen_zAf8sPJfOEZU,2642
 nucliadb/ingest/fields/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/ingest/fields/base.py,sha256=D8NzawonF7hivDW9zvQBbV938TKA6e2OCGqV4kS96RU,24405
@@ -167,23 +167,23 @@ nucliadb/ingest/orm/broker_message.py,sha256=XWaiZgDOz94NPOPT-hqbRr5ZkpVimUw6PjU
 nucliadb/ingest/orm/entities.py,sha256=kXyeF6XOpFKhEsGLcY-GLIk21Exp0cJst4XQQ9jJoug,14791
 nucliadb/ingest/orm/exceptions.py,sha256=gsp7TtVNQPiIEh-zf_UEJClwuFU0iu-5vzj0OrKMScg,1550
 nucliadb/ingest/orm/index_message.py,sha256=mWlpQ0-KChSVIbHewVE8sXCe-7LiPIIh0cBqr3axU8o,16554
-nucliadb/ingest/orm/knowledgebox.py,sha256=n72wuqordGbKhUcFd1jEbf3c7E3IQPnyBXq9rIEDtvM,23428
+nucliadb/ingest/orm/knowledgebox.py,sha256=rjZKD8dIhr-2kvm77K0OBjQKv8VU1FoWi0YQDswZ9fo,25630
 nucliadb/ingest/orm/metrics.py,sha256=OiuggTh-n3kZHA2G73NEUdIlh8c3yFrbusI88DK-Mko,1273
 nucliadb/ingest/orm/resource.py,sha256=zQeZyZ-tCxr-DhonLobfZRkz_iEew0Y-cGfXeNNIHG0,40432
 nucliadb/ingest/orm/utils.py,sha256=fCQRuyecgqhaY7mcBG93oaXMkzkKb9BFjOcy4-ZiSNw,2693
 nucliadb/ingest/orm/processor/__init__.py,sha256=xhDNKCxY0XNOlIVKEtM8QT75vDUkJIt7K-_VgGbbOQU,904
 nucliadb/ingest/orm/processor/auditing.py,sha256=gxn5v30KVaH0TnIjo715mWjzKGJ-DMviElEXJG9BNN4,4612
 nucliadb/ingest/orm/processor/data_augmentation.py,sha256=v-pj4GbBWSuO8dQyahs5UDr5ghsyfhCZDS0ftKd6ZYc,5179
-nucliadb/ingest/orm/processor/processor.py,sha256=3wlGiCXXaVpqLyCJ3AHWHfpKTwtywU78_LkNtpqOiuQ,31391
+nucliadb/ingest/orm/processor/processor.py,sha256=S7myNucnK8v_5y4gZxS_5nL-glGx_fZe2l8gSzDy2CU,31808
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=kUH0bCuM6NqpA0xSwfyb9igig3Btu57pc8VYnKggqx4,1693
 nucliadb/ingest/service/__init__.py,sha256=LHQFUkdmNBOWqBG0Md9sMMI7g5TQZ-hLAnhw6ZblrJg,2002
 nucliadb/ingest/service/exceptions.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/ingest/service/writer.py,sha256=6LvBcwdu5MbL8SWI4HhQwULFsTnnbeHpg4xIzj5Hw_M,21570
+nucliadb/ingest/service/writer.py,sha256=ilHBV3cW4D-vO0hWKZP0hA9j5xNnEgsXrrw7r0VOt9M,21988
 nucliadb/middleware/__init__.py,sha256=A8NBlBuEkunCFMKpR9gnfNELsVn0Plc55BIQMbWDM8Q,2202
 nucliadb/migrator/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/migrator/command.py,sha256=dKbJ1tAmP6X4lMVRSSlz351euaqs2wBPpOczLjATUes,2089
 nucliadb/migrator/context.py,sha256=3h6M_AG2HFjTYtQC2XDmKps7wywdUduT1RM3C30S_S0,1334
-nucliadb/migrator/datamanager.py,sha256=WLOKVbkU-G4dtWscKMc4WhQHpncWq_hH3Kf4qHfPzXI,5237
+nucliadb/migrator/datamanager.py,sha256=PttueVXSqwXHNvrT8p2626Td-NHb4eGBbfBY_F4vVdw,4928
 nucliadb/migrator/exceptions.py,sha256=jTj3YhKmFwUyjjgoKUNoCAiGrpEbB64X1Um212nSNQ8,889
 nucliadb/migrator/migrator.py,sha256=NUOhaCyEoAldXVexYEqClppIN0TMXrF37Ulz0y0TWCY,10840
 nucliadb/migrator/models.py,sha256=3PJkL2PGvKgIG0KIBv4H5XCsOVmwWMlRV3m0ntDj10Q,1145
@@ -193,7 +193,7 @@ nucliadb/models/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,8
 nucliadb/models/responses.py,sha256=qnuOoc7TrVSUnpikfTwHLKez47_DE4mSFzpxrwtqijA,1599
 nucliadb/models/internal/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/models/internal/processing.py,sha256=F_WpDItHsBWuMNeLMxXOKqpZoRYdjPuB2l0s29PVcfI,4213
-nucliadb/purge/__init__.py,sha256=FWqY2ln2ecGgVcZaMhfs8nTt4HzpprubkCsurSrIrC8,13070
+nucliadb/purge/__init__.py,sha256=RFqMj3JRML2sGrQtmRm2fzRg350f0PhkBpZh-KRWnlc,12954
 nucliadb/purge/orphan_shards.py,sha256=5QvTBoYJ2h14TUvJmZxphHJPf_WB-VVC9453n-zjSrU,7747
 nucliadb/reader/__init__.py,sha256=C5Efic7WlGm2U2C5WOyquMFbIj2Pojwe_8mwzVYnOzE,1304
 nucliadb/reader/app.py,sha256=Se-BFTE6d1v1msLzQn4q5XIhjnSxa2ckDSHdvm7NRf8,3096
@@ -362,7 +362,7 @@ nucliadb/writer/api/utils.py,sha256=wIQHlU8RQiIGVLI72suvyVIKlCU44Unh0Ae0IiN6Qwo,
 nucliadb/writer/api/v1/__init__.py,sha256=akI9A_jloNLb0dU4T5zjfdyvmSAiDeIdjAlzNx74FlU,1128
 nucliadb/writer/api/v1/export_import.py,sha256=v0sU55TtRSqDzwkDgcwv2uSaqKCuQTtGcMpYoHQYBQA,8192
 nucliadb/writer/api/v1/field.py,sha256=nO3IEV6v5hokdIo5HoaecdwDqvr1PzCJlh5DafzcNTw,19130
-nucliadb/writer/api/v1/knowledgebox.py,sha256=oSeM0HkOhvDIvjwrBeqbd9ubeN91x2-PiuhJgNuBn-w,8084
+nucliadb/writer/api/v1/knowledgebox.py,sha256=hx3J91fziaISU1aYdfI0eIQNuJj8BrBzGb6B4Rt2j-I,7881
 nucliadb/writer/api/v1/learning_config.py,sha256=DTLEzKJ3dHvi8pbZscjElUqCH_ZvLc6WZgvalFqHo10,4450
 nucliadb/writer/api/v1/resource.py,sha256=IfcT6HXnR5sC5wSnQSuKmFzEWcLTh7OzZEAV4hYmXnA,20442
 nucliadb/writer/api/v1/router.py,sha256=RjuoWLpZer6Kl2BW_wznpNo6XL3BOpdTGqXZCn3QrrQ,1034
@@ -385,8 +385,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vu1BGg4VqJ_x2P1u2BxqPKlSfw5orT_a3R-Ln5oPUpU,8483
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.9.1.post5187.dist-info/METADATA,sha256=dhIhNOcYtZA3MpFYQVS5YCtNOA7kQI09_R6FPPzR3TQ,4158
-nucliadb-6.9.1.post5187.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb-6.9.1.post5187.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.9.1.post5187.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.9.1.post5187.dist-info/RECORD,,
+nucliadb-6.9.1.post5207.dist-info/METADATA,sha256=icEK-KGKOcUE2rL5qX_iNdbLDueZLT3XNaPSe6P5n1E,4158
+nucliadb-6.9.1.post5207.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb-6.9.1.post5207.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.9.1.post5207.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.9.1.post5207.dist-info/RECORD,,