nucliadb-utils 6.4.0.post4146__py3-none-any.whl → 6.4.0.post4161__py3-none-any.whl

nucliadb_utils/nats.py

@@ -34,6 +34,7 @@ from nats.js.client import JetStreamContext
 
 from nucliadb_telemetry.errors import capture_exception
 from nucliadb_telemetry.jetstream import JetStreamContextTelemetry
+from nucliadb_telemetry.metrics import Counter
 from nucliadb_telemetry.utils import get_telemetry
 
 logger = logging.getLogger(__name__)
@@ -119,6 +120,7 @@ class NatsConnectionManager:
         service_name: str,
         nats_servers: list[str],
         nats_creds: Optional[str] = None,
+        pull_utilization_metrics: Optional[Counter] = None,
     ):
         self._service_name = service_name
         self._nats_servers = nats_servers
@@ -132,6 +134,7 @@ class NatsConnectionManager:
         self._reconnect_task: Optional[asyncio.Task] = None
         self._expected_subscriptions: set[str] = set()
         self._initialized = False
+        self.pull_utilization_metrics = pull_utilization_metrics
 
     def healthy(self) -> bool:
         if not self._healthy:
@@ -364,15 +367,29 @@ class NatsConnectionManager:
                 if cancelled.is_set():
                     break
                 try:
+                    if self.pull_utilization_metrics:
+                        start_wait = time.monotonic()
+
                     messages = await psub.fetch(batch=1)
+
+                    if self.pull_utilization_metrics:
+                        received = time.monotonic()
+                        self.pull_utilization_metrics.inc({"status": "waiting"}, received - start_wait)
+
                     for message in messages:
                         await cb(message)
+
+                    if self.pull_utilization_metrics:
+                        processed = time.monotonic()
+                        self.pull_utilization_metrics.inc({"status": "processing"}, processed - received)
                 except asyncio.CancelledError:
                     # Handle task cancellation
                     logger.info("Pull subscription consume task cancelled", extra={"subject": subject})
                     break
                 except TimeoutError:
-                    pass
+                    if self.pull_utilization_metrics:
+                        received = time.monotonic()
+                        self.pull_utilization_metrics.inc({"status": "waiting"}, received - start_wait)
                 except Exception:
                     logger.exception("Error in pull_subscribe task", extra={"subject": subject})
 

nucliadb_utils/utilities.py

@@ -27,6 +27,7 @@ from enum import Enum
 from typing import TYPE_CHECKING, Any, List, Optional, Union, cast
 
 from nucliadb_protos.writer_pb2_grpc import WriterStub
+from nucliadb_telemetry.metrics import Counter
 from nucliadb_utils import featureflagging
 from nucliadb_utils.aiopynecone.client import PineconeSession
 from nucliadb_utils.audit.audit import AuditStorage
@@ -58,6 +59,10 @@ if TYPE_CHECKING:  # pragma: no cover
 
 logger = logging.getLogger(__name__)
 
+pull_subscriber_utilization = Counter(
+    "nucliadb_pull_subscriber_utilization_seconds", labels={"status": ""}
+)
+
 
 class Utility(str, Enum):
     INGEST = "ingest"
@@ -359,6 +364,7 @@ async def start_nats_manager(
         service_name=service_name,
         nats_servers=nats_servers,
         nats_creds=nats_creds,
+        pull_utilization_metrics=pull_subscriber_utilization,
     )
     await nats_manager.initialize()
     set_utility(Utility.NATS_MANAGER, nats_manager)

{nucliadb_utils-6.4.0.post4146.dist-info → nucliadb_utils-6.4.0.post4161.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb_utils
-Version: 6.4.0.post4146
+Version: 6.4.0.post4161
 Summary: NucliaDB util library
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -27,8 +27,8 @@ Requires-Dist: nats-py[nkeys]>=2.6.0
 Requires-Dist: PyNaCl
 Requires-Dist: pyjwt>=2.4.0
 Requires-Dist: mrflagly>=0.2.9
-Requires-Dist: nucliadb-protos>=6.4.0.post4146
-Requires-Dist: nucliadb-telemetry>=6.4.0.post4146
+Requires-Dist: nucliadb-protos>=6.4.0.post4161
+Requires-Dist: nucliadb-telemetry>=6.4.0.post4161
 Provides-Extra: cache
 Requires-Dist: redis>=4.3.4; extra == "cache"
 Requires-Dist: orjson>=3.6.7; extra == "cache"

{nucliadb_utils-6.4.0.post4146.dist-info → nucliadb_utils-6.4.0.post4161.dist-info}/RECORD

@@ -7,7 +7,7 @@ nucliadb_utils/exceptions.py,sha256=y_3wk77WLVUtdo-5FtbBsdSkCtK_DsJkdWb5BoPn3qo,
 nucliadb_utils/featureflagging.py,sha256=SYqr31e0_NeOMx6dZgSiDO9VJiV6xxYhM_gWkk8ITTQ,2386
 nucliadb_utils/grpc.py,sha256=apu0uePnkGHCAT7GRQ9YZfRYyFj26kJ440i8jitbM3U,3314
 nucliadb_utils/helpers.py,sha256=nPw8yod3hP-pxq80VF8QC36s7ygSg0dBUdfI-LatvCs,1600
-nucliadb_utils/nats.py,sha256=-ehKk0u5ZhiJEyg67yV26YysBGEGp4TDSQKq3aydJn0,15275
+nucliadb_utils/nats.py,sha256=LrUNBPffjK3e_EuF_s2MGmmE7ExOVD9cdi0yZjHNSv4,16167
 nucliadb_utils/partition.py,sha256=jBgy4Hu5Iwn4gjbPPcthSykwf-qNx-GcLAIwbzPd1d0,1157
 nucliadb_utils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb_utils/run.py,sha256=Es0_Bu5Yc-LWczvwL6gzWqSwC85RjDCk-0oFQAJi9g4,1827
@@ -15,7 +15,7 @@ nucliadb_utils/settings.py,sha256=RnGhEUvwv6faNqALiqDCivvzNOyyXVBflYh_37uNkow,81
 nucliadb_utils/signals.py,sha256=lo_Mk12NIX5Au--3H3WObvDOXq_OMurql2qiC2TnAao,2676
 nucliadb_utils/store.py,sha256=kQ35HemE0v4_Qg6xVqNIJi8vSFAYQtwI3rDtMsNy62Y,890
 nucliadb_utils/transaction.py,sha256=z_VeiTIta48rosS2SXMqx86XaavprSMRWf6s6zWIeEs,7920
-nucliadb_utils/utilities.py,sha256=Vc4zLpDf-FQh9bs0ZyAfGAjzlbbMTMWf3VWt2Ao5V3k,15379
+nucliadb_utils/utilities.py,sha256=w4nsdyiFgZVEq5aBhjzVg176UDJNxfhIEfDbEi8bJdA,15604
 nucliadb_utils/aiopynecone/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb_utils/aiopynecone/client.py,sha256=MPyHnDXwhukJr7U3CJh7BpsekfSuOkyM4g5b9LLtzc8,22941
 nucliadb_utils/aiopynecone/exceptions.py,sha256=fUErx3ceKQK1MUbOnYcZhIzpNe8UVAptZE9JIRDLXDE,4000
@@ -57,7 +57,7 @@ nucliadb_utils/tests/gcs.py,sha256=MBMzn_UHU5SU6iILuCsB5zU4umhNcaCw_MKrxZhwvOc,4
 nucliadb_utils/tests/local.py,sha256=cxIfPrKuqs5Ef0nbrVYQQAH2mwc4E0iD9bC2sWegS-c,1934
 nucliadb_utils/tests/nats.py,sha256=RWHjwqq5esuO7OFbP24yYX1cXnpPLcWJwDUdmwCpH28,1897
 nucliadb_utils/tests/s3.py,sha256=DACUh3HvgH3BchKFZ9R7RFUzsrg3v9A-cxTcXx4nmvA,3734
-nucliadb_utils-6.4.0.post4146.dist-info/METADATA,sha256=LQOCHAfy9qQfnuGtGdKmX9iQ8VRLiNCXkN4TtgoySE0,2205
-nucliadb_utils-6.4.0.post4146.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-nucliadb_utils-6.4.0.post4146.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
-nucliadb_utils-6.4.0.post4146.dist-info/RECORD,,
+nucliadb_utils-6.4.0.post4161.dist-info/METADATA,sha256=9cW6zAHR0b3P2YQHwcwyL7kUdoR3QllRFUznTCFI3YE,2205
+nucliadb_utils-6.4.0.post4161.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
+nucliadb_utils-6.4.0.post4161.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
+nucliadb_utils-6.4.0.post4161.dist-info/RECORD,,

{nucliadb_utils-6.4.0.post4146.dist-info → nucliadb_utils-6.4.0.post4161.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (79.0.1)
+Generator: setuptools (80.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any