nucliadb 6.9.1.post5192__py3-none-any.whl → 6.10.0.post5705__py3-none-any.whl

migrations/0023_backfill_pg_catalog.py

@@ -27,11 +27,11 @@ Backfill the data into the PG catalog
 import logging
 from typing import cast
 
-from nucliadb.common import datamanagers
 from nucliadb.common.catalog import catalog_update, get_catalog
 from nucliadb.common.catalog.pg import PGCatalog
 from nucliadb.common.maindb.pg import PGDriver, PGTransaction
 from nucliadb.ingest.orm.index_message import get_resource_index_message
+from nucliadb.ingest.orm.resource import Resource
 from nucliadb.migrator.context import ExecutionContext
 
 logger = logging.getLogger(__name__)
@@ -73,7 +73,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
                 # Index each resource
                 for rid in resources_to_index:
                     rid = str(rid).replace("-", "")
-                    resource = await datamanagers.resources.get_resource(txn, kbid=kbid, rid=rid)
+                    resource = await Resource.get(txn, kbid=kbid, rid=rid)
                     if resource is None:
                         logger.warning(f"Could not load resource {rid} for kbid {kbid}")
                         continue

migrations/0029_backfill_field_status.py

@@ -24,7 +24,6 @@ Backfill field status (from error)
 """
 
 import logging
-from typing import Optional
 
 from nucliadb.migrator.context import ExecutionContext
 from nucliadb_protos import resources_pb2, writer_pb2
@@ -33,7 +32,7 @@ logger = logging.getLogger(__name__)
 
 
 async def migrate(context: ExecutionContext) -> None:
-    start: Optional[str] = ""
+    start: str | None = ""
     while True:
         if start is None:
             break
@@ -43,7 +42,7 @@ async def migrate(context: ExecutionContext) -> None:
 async def migrate_kb(context: ExecutionContext, kbid: str) -> None: ...
 
 
-async def do_batch(context: ExecutionContext, start: str) -> Optional[str]:
+async def do_batch(context: ExecutionContext, start: str) -> str | None:
     logger.info(f"Running batch from {start}")
     async with context.kv_driver.rw_transaction() as txn:
         async with txn.connection.cursor() as cur:  # type: ignore
@@ -64,7 +63,7 @@ async def do_batch(context: ExecutionContext, start: str) -> Optional[str]:
             field_keys = [r[0] for r in records]
 
             # Retrieve resources basic (to check status)
-            resource_keys = set(["/".join(f.split("/")[:5]) for f in field_keys])
+            resource_keys = {"/".join(f.split("/")[:5]) for f in field_keys}
             await cur.execute(
                 """
                 SELECT key, value FROM resources

migrations/0032_remove_old_relations.py

@@ -26,7 +26,6 @@ is stored in object storage.
 """
 
 import logging
-from typing import Optional
 
 from nucliadb.migrator.context import ExecutionContext
 
@@ -34,7 +33,7 @@ logger = logging.getLogger(__name__)
 
 
 async def migrate(context: ExecutionContext) -> None:
-    start: Optional[str] = ""
+    start: str | None = ""
     while True:
         if start is None:
             break
@@ -45,7 +44,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
     pass
 
 
-async def do_batch(context: ExecutionContext, start: str) -> Optional[str]:
+async def do_batch(context: ExecutionContext, start: str) -> str | None:
     logger.info(f"Running batch from {start}")
     async with context.kv_driver.rw_transaction() as txn:
         async with txn.connection.cursor() as cur:  # type: ignore

migrations/0038_backfill_catalog_field_labels.py

@@ -27,11 +27,11 @@ Backfill the catalog with labels from fields metadata
 import logging
 from typing import cast
 
-from nucliadb.common import datamanagers
 from nucliadb.common.catalog import catalog_update, get_catalog
 from nucliadb.common.catalog.pg import PGCatalog
 from nucliadb.common.maindb.pg import PGDriver, PGTransaction
 from nucliadb.ingest.orm.index_message import get_resource_index_message
+from nucliadb.ingest.orm.resource import Resource
 from nucliadb.migrator.context import ExecutionContext
 from nucliadb_protos import resources_pb2
 
@@ -82,7 +82,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
                 # Index each resource
                 for key in to_index:
                     rid = key.split("/")[4]
-                    resource = await datamanagers.resources.get_resource(txn, kbid=kbid, rid=rid)
+                    resource = await Resource.get(txn, kbid=kbid, rid=rid)
                     if resource is None:
                         logger.warning(f"Could not load resource {rid} for kbid {kbid}")
                         continue

migrations/0039_backfill_converation_splits_metadata.py

@@ -36,7 +36,7 @@ from nucliadb.ingest.fields.conversation import (
 from nucliadb.ingest.orm.knowledgebox import KnowledgeBox as KnowledgeBoxORM
 from nucliadb.migrator.context import ExecutionContext
 from nucliadb_protos import resources_pb2
-from nucliadb_protos.resources_pb2 import SplitMetadata, SplitsMetadata
+from nucliadb_protos.resources_pb2 import SplitsMetadata
 from nucliadb_utils.storages.storage import Storage
 
 logger = logging.getLogger(__name__)
@@ -102,5 +102,5 @@ async def build_splits_metadata(
         if page is None:
             continue
         for message in page.messages:
-            splits_metadata.metadata.setdefault(message.ident, SplitMetadata())
+            splits_metadata.metadata.get_or_create(message.ident)
     return splits_metadata

migrations/0041_reindex_conversations.py

@@ -0,0 +1,137 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import logging
+import uuid
+from collections.abc import AsyncIterator
+from typing import cast
+
+from nucliadb.common import datamanagers
+from nucliadb.common.maindb.pg import PGTransaction
+from nucliadb.ingest.orm.index_message import get_resource_index_message
+from nucliadb.ingest.orm.resource import Resource
+from nucliadb.migrator.context import ExecutionContext
+from nucliadb_protos.writer_pb2 import ShardObject, Shards
+
+logger = logging.getLogger(__name__)
+
+
+async def migrate(context: ExecutionContext) -> None: ...
+
+
+async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
+    """
+    Reindex resources that have conversation fields
+    """
+    kb_shards = await datamanagers.atomic.cluster.get_kb_shards(kbid=kbid, for_update=False)
+    if kb_shards is not None:
+        async for rid in iter_affected_resource_ids(context, kbid):
+            await reindex_resource(context, kbid, rid, kb_shards)
+    else:
+        logger.warning(
+            "Migration 41: KB shards not found, skipping reindexing",
+            extra={"kbid": kbid},
+        )
+
+
+async def reindex_resource(
+    context: ExecutionContext,
+    kbid: str,
+    rid: str,
+    kb_shards: Shards,
+) -> None:
+    """
+    Reindex a single resource
+    """
+    async with datamanagers.with_ro_transaction() as rs_txn:
+        # Fetch the resource
+        resource = await Resource.get(rs_txn, kbid=kbid, rid=rid)
+        if resource is None:
+            logger.warning(
+                "Migration 41: Resource not found, skipping reindexing",
+                extra={"kbid": kbid, "rid": rid},
+            )
+            return
+
+        # Get the shard for the resource
+        shard: ShardObject | None = None
+        shard_id = await datamanagers.resources.get_resource_shard_id(
+            rs_txn, kbid=kbid, rid=rid, for_update=False
+        )
+        if shard_id is not None:
+            shard = next((shard for shard in kb_shards.shards if shard.shard == shard_id), None)
+        if shard is None:
+            logger.warning(
+                "Migration 41: Shard not found for resource, skipping reindexing",
+                extra={"kbid": kbid, "rid": rid, "shard_id": shard_id},
+            )
+            return
+
+        # Create the index message and reindex the resource
+        index_message = await get_resource_index_message(resource, reindex=True)
+        await context.shard_manager.add_resource(
+            shard,
+            index_message,
+            0,
+            partition="0",
+            kb=kbid,
+            reindex_id=uuid.uuid4().hex,
+        )
+        logger.info(
+            "Migration 41: Resource reindexed",
+            extra={"kbid": kbid, "rid": rid},
+        )
+
+
+async def iter_affected_resource_ids(context: ExecutionContext, kbid: str) -> AsyncIterator[str]:
+    start = ""
+    while True:
+        keys_batch = await get_batch(context, kbid, start)
+        if keys_batch is None:
+            break
+        start = keys_batch[-1]
+        for key in keys_batch:
+            # The keys have the format /kbs/{kbid}/r/{rid}/f/c/{field_id}
+            rid = key.split("/")[4]
+            yield rid
+
+
+async def get_batch(context: ExecutionContext, kbid: str, start: str) -> list[str] | None:
+    """
+    Get a batch of resource keys that hold conversation fields for the given KB.
+    Starting after the given start key.
+    Returns None if no more keys are found.
+    """
+    batch_size = 100
+    async with context.kv_driver.rw_transaction() as txn:
+        txn = cast(PGTransaction, txn)
+        async with txn.connection.cursor() as cur:
+            await cur.execute(
+                """
+                SELECT key FROM resources
+                WHERE key ~ ('^/kbs/' || %s || '/r/[^/]*/f/c/[^/]*$')
+                AND key > %s
+                ORDER BY key
+                LIMIT %s""",
+                (kbid, start, batch_size),
+            )
+            rows = await cur.fetchall()
+            if len(rows) == 0:
+                return None
+            return [row[0] for row in rows]

migrations/pg/0010_shards_index.py

@@ -0,0 +1,34 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from nucliadb.common.maindb.pg import PGTransaction
+
+
+async def migrate(txn: PGTransaction) -> None:
+    # Concurrent index must be created outside of a transaction but psycopg automatically
+    # creates transactions. We temporarily disable this for building indexes.
+    await txn.connection.commit()
+    try:
+        await txn.connection.set_autocommit(True)
+        await txn.connection.execute(
+            "CREATE INDEX CONCURRENTLY ON resources (key, value) WHERE key ~ '/kbs/[^/]*/r/[^/]*/shard$';"
+        )
+    finally:
+        await txn.connection.set_autocommit(False)

nucliadb/search/api/v1/resource/utils.py → migrations/pg/0011_catalog_statistics.py

@@ -18,11 +18,10 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-from typing import Optional
+from nucliadb.common.maindb.pg import PGTransaction
 
-from nucliadb.common import datamanagers
 
-
-async def get_resource_uuid_by_slug(kbid: str, slug: str) -> Optional[str]:
-    async with datamanagers.with_ro_transaction() as txn:
-        return await datamanagers.resources.get_resource_uuid_from_slug(txn, kbid=kbid, slug=slug)
+async def migrate(txn: PGTransaction) -> None:
+    async with txn.connection.cursor() as cur:
+        await cur.execute("CREATE STATISTICS catalog_kbid_labels ON kbid, labels FROM catalog;")
+        await cur.execute("ANALYZE catalog;")

migrations/pg/0012_catalog_statistics_undo.py

@@ -0,0 +1,26 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from nucliadb.common.maindb.pg import PGTransaction
+
+
+async def migrate(txn: PGTransaction) -> None:
+    async with txn.connection.cursor() as cur:
+        await cur.execute("DROP STATISTICS catalog_kbid_labels;")

nucliadb/backups/create.py

@@ -21,8 +21,8 @@ import asyncio
 import json
 import logging
 import tarfile
+from collections.abc import AsyncIterator
 from datetime import datetime, timezone
-from typing import AsyncIterator, Optional
 
 from nucliadb.backups.const import (
     BackupFinishedStream,
@@ -37,7 +37,6 @@ from nucliadb.export_import.utils import (
     download_binary,
     get_broker_message,
     get_cloud_files,
-    get_entities,
     get_labels,
     get_search_configurations,
     get_synonyms,
@@ -76,7 +75,6 @@ async def backup_kb(context: ApplicationContext, kbid: str, backup_id: str):
     """
     await backup_resources(context, kbid, backup_id)
     await backup_labels(context, kbid, backup_id)
-    await backup_entities(context, kbid, backup_id)
     await backup_synonyms(context, kbid, backup_id)
     await backup_search_configurations(context, kbid, backup_id)
     await notify_backup_completed(context, kbid, backup_id)
@@ -235,15 +233,6 @@ async def backup_labels(context: ApplicationContext, kbid: str, backup_id: str):
     )
 
 
-async def backup_entities(context: ApplicationContext, kbid: str, backup_id: str):
-    entities = await get_entities(context, kbid)
-    await context.blob_storage.upload_object(
-        bucket=settings.backups_bucket,
-        key=StorageKeys.ENTITIES.format(backup_id=backup_id),
-        data=entities.SerializeToString(),
-    )
-
-
 async def backup_synonyms(context: ApplicationContext, kbid: str, backup_id: str):
     synonyms = await get_synonyms(context, kbid)
     await context.blob_storage.upload_object(
@@ -266,9 +255,7 @@ async def backup_search_configurations(context: ApplicationContext, kbid: str, b
     )
 
 
-async def get_metadata(
-    context: ApplicationContext, kbid: str, backup_id: str
-) -> Optional[BackupMetadata]:
+async def get_metadata(context: ApplicationContext, kbid: str, backup_id: str) -> BackupMetadata | None:
     async with context.kv_driver.ro_transaction() as txn:
         metadata_raw = await txn.get(MaindbKeys.METADATA.format(kbid=kbid, backup_id=backup_id))
         if metadata_raw is None:

nucliadb/backups/restore.py

@@ -24,7 +24,8 @@ import functools
 import json
 import logging
 import tarfile
-from typing import Any, AsyncIterator, Callable, Optional, Union
+from collections.abc import AsyncIterator, Callable
+from typing import Any
 
 from pydantic import TypeAdapter
 
@@ -35,7 +36,6 @@ from nucliadb.common.context import ApplicationContext
 from nucliadb.export_import.utils import (
     import_binary,
     restore_broker_message,
-    set_entities_groups,
     set_labels,
     set_search_configurations,
     set_synonyms,
@@ -74,7 +74,6 @@ async def restore_kb(context: ApplicationContext, kbid: str, backup_id: str):
     """
     await restore_resources(context, kbid, backup_id)
     await restore_labels(context, kbid, backup_id)
-    await restore_entities(context, kbid, backup_id)
     await restore_synonyms(context, kbid, backup_id)
     await restore_search_configurations(context, kbid, backup_id)
     await delete_last_restored(context, kbid, backup_id)
@@ -101,7 +100,7 @@ async def restore_resources(context: ApplicationContext, kbid: str, backup_id: s
         await set_last_restored(context, kbid, backup_id, key)
 
 
-async def get_last_restored(context: ApplicationContext, kbid: str, backup_id: str) -> Optional[str]:
+async def get_last_restored(context: ApplicationContext, kbid: str, backup_id: str) -> str | None:
     key = MaindbKeys.LAST_RESTORED.format(kbid=kbid, backup_id=backup_id)
     async with context.kv_driver.ro_transaction() as txn:
         raw = await txn.get(key)
@@ -191,7 +190,7 @@ class ResourceBackupReader:
         data = await self.read(tarinfo_size + padding_bytes)
         return data[:tarinfo_size]
 
-    async def read_item(self) -> Union[BrokerMessage, CloudFile, CloudFileBinary]:
+    async def read_item(self) -> BrokerMessage | CloudFile | CloudFileBinary:
         tarinfo = await self.read_tarinfo()
         if tarinfo.name.startswith("broker-message"):
             raw_bm = await self.read_data(tarinfo)
@@ -257,16 +256,6 @@ async def restore_labels(context: ApplicationContext, kbid: str, backup_id: str)
     await set_labels(context, kbid, labels)
 
 
-async def restore_entities(context: ApplicationContext, kbid: str, backup_id: str):
-    raw = await context.blob_storage.downloadbytes(
-        bucket=settings.backups_bucket,
-        key=StorageKeys.ENTITIES.format(backup_id=backup_id),
-    )
-    entities = kb_pb2.EntitiesGroups()
-    entities.ParseFromString(raw.getvalue())
-    await set_entities_groups(context, kbid, entities)
-
-
 async def restore_synonyms(context: ApplicationContext, kbid: str, backup_id: str):
     raw = await context.blob_storage.downloadbytes(
         bucket=settings.backups_bucket,

nucliadb/backups/tasks.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from typing import Awaitable, Callable
+from collections.abc import Awaitable, Callable
 
 from nucliadb.backups.const import BackupsNatsConfig
 from nucliadb.backups.create import backup_kb_task
@@ -38,6 +38,7 @@ def creator_consumer() -> NatsTaskConsumer[CreateBackupRequest]:
         callback=backup_kb_task,
         msg_type=CreateBackupRequest,
         max_concurrent_messages=10,
+        max_retries=100,
     )
     return consumer
 
@@ -64,6 +65,7 @@ def restorer_consumer() -> NatsTaskConsumer[RestoreBackupRequest]:
         callback=restore_kb_task,
         msg_type=RestoreBackupRequest,
         max_concurrent_messages=10,
+        max_retries=100,
     )
     return consumer
 
@@ -90,6 +92,7 @@ def deleter_consumer() -> NatsTaskConsumer[DeleteBackupRequest]:
         callback=delete_backup_task,
         msg_type=DeleteBackupRequest,
         max_concurrent_messages=2,
+        max_retries=100,
     )
     return consumer
 

nucliadb/common/back_pressure/cache.py

@@ -21,7 +21,6 @@ import contextlib
 import logging
 import threading
 from datetime import datetime, timezone
-from typing import Optional
 
 from cachetools import TTLCache
 
@@ -47,7 +46,7 @@ class BackPressureCache:
         self._cache = TTLCache(maxsize=1024, ttl=5 * 60)
         self._lock = threading.Lock()
 
-    def get(self, key: str) -> Optional[BackPressureData]:
+    def get(self, key: str) -> BackPressureData | None:
         with self._lock:
             data = self._cache.get(key, None)
             if data is None:
@@ -72,7 +71,7 @@ def cached_back_pressure(cache_key: str):
     Context manager that handles the caching of the try again in time so that
     we don't recompute try again times if we have already applied back pressure.
     """
-    data: Optional[BackPressureData] = _cache.get(cache_key)
+    data: BackPressureData | None = _cache.get(cache_key)
     if data is not None:
         back_pressure_type = data.type
         RATE_LIMITED_REQUESTS_COUNTER.inc({"type": back_pressure_type, "cached": "true"})

nucliadb/common/back_pressure/materializer.py

@@ -20,7 +20,6 @@
 import asyncio
 import logging
 import threading
-from typing import Optional
 
 from cachetools import TTLCache
 from fastapi import HTTPException
@@ -118,12 +117,6 @@ class BackPressureMaterializer:
                     extra={"kbid": kbid},
                 )
                 return 0
-
-            if pending > 0:
-                logger.info(
-                    f"Processing returned {pending} pending messages for KB",
-                    extra={"kbid": kbid},
-                )
             self.processing_pending_cache[kbid] = pending
             return pending
 
@@ -184,7 +177,7 @@ class BackPressureMaterializer:
                 pending=pending,
                 max_wait=settings.max_wait_time,
             )
-            data = BackPressureData(type="indexing", try_after=try_after)
+            data = BackPressureData(type="indexing", try_after=try_after, pending=pending)
             raise BackPressureException(data)
 
     def check_ingest(self):
@@ -199,7 +192,7 @@ class BackPressureMaterializer:
                 pending=ingest_pending,
                 max_wait=settings.max_wait_time,
             )
-            data = BackPressureData(type="ingest", try_after=try_after)
+            data = BackPressureData(type="ingest", try_after=try_after, pending=ingest_pending)
             raise BackPressureException(data)
 
     async def check_processing(self, kbid: str):
@@ -215,11 +208,11 @@ class BackPressureMaterializer:
                 pending=kb_pending,
                 max_wait=settings.max_wait_time,
             )
-            data = BackPressureData(type="processing", try_after=try_after)
+            data = BackPressureData(type="processing", try_after=try_after, pending=kb_pending)
             raise BackPressureException(data)
 
 
-MATERIALIZER: Optional[BackPressureMaterializer] = None
+MATERIALIZER: BackPressureMaterializer | None = None
 materializer_lock = threading.Lock()
 
 
@@ -268,7 +261,7 @@ def get_materializer() -> BackPressureMaterializer:
     return MATERIALIZER
 
 
-async def maybe_back_pressure(kbid: str, resource_uuid: Optional[str] = None) -> None:
+async def maybe_back_pressure(kbid: str, resource_uuid: str | None = None) -> None:
     """
     This function does system checks to see if we need to put back pressure on writes.
     In that case, a HTTP 429 will be raised with the estimated time to try again.
@@ -278,7 +271,7 @@ async def maybe_back_pressure(kbid: str, resource_uuid: Optional[str] = None) ->
     await back_pressure_checks(kbid, resource_uuid)
 
 
-async def back_pressure_checks(kbid: str, resource_uuid: Optional[str] = None):
+async def back_pressure_checks(kbid: str, resource_uuid: str | None = None):
     """
     Will raise a 429 if back pressure is needed:
     - If the processing engine is behind.
@@ -299,6 +292,7 @@ async def back_pressure_checks(kbid: str, resource_uuid: Optional[str] = None):
                 "resource_uuid": resource_uuid,
                 "try_after": exc.data.try_after,
                 "back_pressure_type": exc.data.type,
+                "pending": exc.data.pending,
             },
         )
         raise HTTPException(

nucliadb/common/back_pressure/settings.py

@@ -29,30 +29,30 @@ class BackPressureSettings(BaseSettings):
     )
     indexing_rate: float = Field(
         default=10,
-        description="Estimation of the indexing rate in messages per second. This is used to calculate the try again in time",  # noqa
+        description="Estimation of the indexing rate in messages per second. This is used to calculate the try again in time",
     )
     ingest_rate: float = Field(
         default=4,
-        description="Estimation of the ingest processed consumer rate in messages per second. This is used to calculate the try again in time",  # noqa
+        description="Estimation of the ingest processed consumer rate in messages per second. This is used to calculate the try again in time",
     )
     processing_rate: float = Field(
         default=1,
-        description="Estimation of the processing rate in messages per second. This is used to calculate the try again in time",  # noqa
+        description="Estimation of the processing rate in messages per second. This is used to calculate the try again in time",
     )
     max_indexing_pending: int = Field(
         default=1000,
-        description="Max number of messages pending to index in a node queue before rate limiting writes. Set to 0 to disable indexing back pressure checks",  # noqa
+        description="Max number of messages pending to index in a node queue before rate limiting writes. Set to 0 to disable indexing back pressure checks",
         alias="back_pressure_max_indexing_pending",
     )
     max_ingest_pending: int = Field(
         # Disabled by default
         default=0,
-        description="Max number of messages pending to be ingested by processed consumers before rate limiting writes. Set to 0 to disable ingest back pressure checks",  # noqa
+        description="Max number of messages pending to be ingested by processed consumers before rate limiting writes. Set to 0 to disable ingest back pressure checks",
         alias="back_pressure_max_ingest_pending",
     )
     max_processing_pending: int = Field(
         default=1000,
-        description="Max number of messages pending to process per Knowledge Box before rate limiting writes. Set to 0 to disable processing back pressure checks",  # noqa
+        description="Max number of messages pending to process per Knowledge Box before rate limiting writes. Set to 0 to disable processing back pressure checks",
         alias="back_pressure_max_processing_pending",
     )
     indexing_check_interval: int = Field(

nucliadb/common/back_pressure/utils.py

@@ -28,6 +28,7 @@ from nucliadb_utils.nats import NatsConnectionManager
 class BackPressureData:
     type: str
     try_after: datetime
+    pending: int = 0
 
 
 class BackPressureException(Exception):