nucliadb 6.4.0.post4279__py3-none-any.whl → 6.4.0.post4283__py3-none-any.whl

nucliadb/ingest/consumer/pull.py

@@ -19,6 +19,7 @@
 #
 import asyncio
 import base64
+import time
 from contextlib import contextmanager
 from datetime import datetime, timezone
 from typing import Optional
@@ -41,6 +42,7 @@ from nucliadb.common.http_clients.processing import (
 )
 from nucliadb.common.maindb.driver import Driver
 from nucliadb.ingest import SERVICE_NAME, logger, logger_activity
+from nucliadb.ingest.consumer.consumer import consumer_observer
 from nucliadb.ingest.orm.exceptions import ReallyStopPulling
 from nucliadb.ingest.orm.processor import Processor
 from nucliadb_protos.writer_pb2 import BrokerMessage, BrokerMessageBlobReference
@@ -52,7 +54,7 @@ from nucliadb_utils.cache.pubsub import PubSubDriver
 from nucliadb_utils.settings import nuclia_settings
 from nucliadb_utils.storages.storage import Storage
 from nucliadb_utils.transaction import MaxTransactionSizeExceededError
-from nucliadb_utils.utilities import get_storage, get_transaction_utility
+from nucliadb_utils.utilities import get_storage, get_transaction_utility, pull_subscriber_utilization
 
 processing_pending_messages = Gauge("nucliadb_processing_pending_messages")
 
@@ -309,11 +311,13 @@ class PullV2Worker:
 
         logger.debug(f"Resource: {pb.uuid} KB: {pb.kbid} ProcessingID: {pb.processing_id}")
 
-        await self.processor.process(
-            pb,
-            seq,
-            transaction_check=False,
-        )
+        source = "writer" if pb.source == pb.MessageSource.WRITER else "processor"
+        with consumer_observer({"source": source, "partition": "-1"}):
+            await self.processor.process(
+                pb,
+                seq,
+                transaction_check=False,
+            )
 
     async def loop(self):
         """
@@ -331,6 +335,7 @@ class PullV2Worker:
                 await asyncio.sleep(10)
 
     async def _loop(self):
+        usage_metric = pull_subscriber_utilization
         headers = {}
         data = None
         if nuclia_settings.nuclia_service_account is not None:
@@ -346,6 +351,8 @@ class PullV2Worker:
         async with ProcessingHTTPClient() as processing_http_client:
             while True:
                 try:
+                    start_time = time.monotonic()
+
                     # The code is only really prepared to pull 1 message at a time. If changing this, review MessageProgressUpdate usage
                     pull = await processing_http_client.pull_v2(ack_tokens=ack_tokens, limit=1)
                     ack_tokens.clear()
@@ -353,10 +360,14 @@ class PullV2Worker:
                         processing_pending_messages.set(0)
                         logger_activity.debug(f"No messages waiting in processing pull")
                         await asyncio.sleep(self.pull_time_empty_backoff)
+                        usage_metric.inc({"status": "waiting"}, time.monotonic() - start_time)
                         continue
 
-                    logger.info("Message received from proxy", extra={"seq": [pull.messages[0].seq]})
+                    received_time = time.monotonic()
+                    usage_metric.inc({"status": "waiting"}, received_time - start_time)
                     processing_pending_messages.set(pull.pending)
+
+                    logger.info("Message received from proxy", extra={"seq": [pull.messages[0].seq]})
                     try:
                         for message in pull.messages:
                             async with ProcessingPullMessageProgressUpdater(
@@ -365,6 +376,8 @@ class PullV2Worker:
                                 with run_in_span(message.headers):
                                     await self.handle_message(message.seq, message.payload)
                                     ack_tokens.append(message.ack_token)
+
+                        usage_metric.inc({"status": "processing"}, time.monotonic() - received_time)
                     except Exception as e:
                         errors.capture_exception(e)
                         logger.exception("Error while pulling and processing message/s")

{nucliadb-6.4.0.post4279.dist-info → nucliadb-6.4.0.post4283.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.4.0.post4279
+Version: 6.4.0.post4283
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.4.0.post4279
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.4.0.post4279
-Requires-Dist: nucliadb-protos>=6.4.0.post4279
-Requires-Dist: nucliadb-models>=6.4.0.post4279
-Requires-Dist: nidx-protos>=6.4.0.post4279
+Requires-Dist: nucliadb-telemetry[all]>=6.4.0.post4283
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.4.0.post4283
+Requires-Dist: nucliadb-protos>=6.4.0.post4283
+Requires-Dist: nucliadb-models>=6.4.0.post4283
+Requires-Dist: nidx-protos>=6.4.0.post4283
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn[standard]

{nucliadb-6.4.0.post4279.dist-info → nucliadb-6.4.0.post4283.dist-info}/RECORD

@@ -134,7 +134,7 @@ nucliadb/ingest/consumer/auditing.py,sha256=xK21DIa_ZAiOJVVbnkmT4jgCRGshNGyPyxsq
 nucliadb/ingest/consumer/consumer.py,sha256=GfdlrNlnt7PWYyk75xtyzn2SHZse7475U4U9q_9jKr0,13711
 nucliadb/ingest/consumer/materializer.py,sha256=tgD_rDI2twQzcz8kKNiW_L4YIth16IGh9mUfD5wiSD4,3858
 nucliadb/ingest/consumer/metrics.py,sha256=ji1l_4cKiHJthQd8YNem1ft4iMbw9KThmVvJmLcv3Xg,1075
-nucliadb/ingest/consumer/pull.py,sha256=gfdyQ8IMFA_bpGnEpmRB9qmOJywBEwxn7pGYaueZizU,16874
+nucliadb/ingest/consumer/pull.py,sha256=x39G6AcNXSnw_GRPxJfafmD5pehZzMBd6v_f_yrNbUI,17594
 nucliadb/ingest/consumer/service.py,sha256=WXBN8dY7MlmYWxqQHIbIO7w_SdVJRY1RuHAWlQUXf8o,8852
 nucliadb/ingest/consumer/shard_creator.py,sha256=w0smEu01FU_2cjZnsfBRNqT_Ntho11X17zTMST-vKbc,4359
 nucliadb/ingest/consumer/utils.py,sha256=jpX8D4lKzuPCpArQLZeX_Zczq3pfen_zAf8sPJfOEZU,2642
@@ -368,8 +368,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.4.0.post4279.dist-info/METADATA,sha256=ISm2mlidMfyHGlEXRXBJcbnPe52rP58sBjj5NLRFf68,4223
-nucliadb-6.4.0.post4279.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-nucliadb-6.4.0.post4279.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.4.0.post4279.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.4.0.post4279.dist-info/RECORD,,
+nucliadb-6.4.0.post4283.dist-info/METADATA,sha256=qNXC_Jl6XRiEVh5KfaC09owsizR7ozLE3OJ2FG_Ygsg,4223
+nucliadb-6.4.0.post4283.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+nucliadb-6.4.0.post4283.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.4.0.post4283.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.4.0.post4283.dist-info/RECORD,,