abstract-block-dumper 0.0.8__py3-none-any.whl → 0.0.9__py3-none-any.whl

abstract_block_dumper/_internal/services/backfill_scheduler.py

@@ -18,8 +18,10 @@ import abstract_block_dumper._internal.services.utils as abd_utils
 from abstract_block_dumper._internal.services.block_processor import BlockProcessor, block_processor_factory
 from abstract_block_dumper._internal.services.metrics import (
     BlockProcessingTimer,
+    increment_archive_network_usage,
     increment_blocks_processed,
     set_backfill_progress,
+    set_block_lag,
     set_current_block,
 )
 from abstract_block_dumper._internal.services.utils import serialize_args
@@ -275,6 +277,10 @@ class BackfillScheduler:
                     set_backfill_progress(self.from_block, self.to_block, block_number)
                     increment_blocks_processed("backfill")
 
+                    # Track block lag (distance from chain head)
+                    if self._current_head_cache:
+                        set_block_lag("backfill", self._current_head_cache - block_number)
+
                     # Log progress periodically
                     if processed_count % PROGRESS_LOG_INTERVAL == 0:
                         progress_pct = (processed_count / total_blocks) * 100
@@ -380,6 +386,8 @@ class BackfillScheduler:
             try:
                 if registry_item.match_condition(block_number, **args):
                     use_archive = self._requires_archive_network(block_number)
+                    if use_archive:
+                        increment_archive_network_usage()
                     logger.debug(
                         "Backfilling block",
                         function_name=registry_item.function.__name__,

abstract_block_dumper/_internal/services/executor.py

@@ -4,6 +4,7 @@ import structlog
 
 import abstract_block_dumper._internal.dal.django_dal as abd_dal
 from abstract_block_dumper._internal.dal.memory_registry import RegistryItem
+from abstract_block_dumper._internal.services.metrics import increment_tasks_submitted
 from abstract_block_dumper.models import TaskAttempt
 
 logger = structlog.get_logger(__name__)
@@ -61,4 +62,8 @@ class CeleryExecutor:
         # The task ID can be retrieved from the task_attempt if needed
         registry_item.function.apply_async(**apply_async_kwargs)
 
+        # Track task submission metric
+        task_name = registry_item.executable_path.split(".")[-1]
+        increment_tasks_submitted(task_name)
+
         logger.debug("Celery task scheduled", task_id=task_attempt.id)

abstract_block_dumper/_internal/services/metrics.py

@@ -33,6 +33,15 @@ BACKFILL_PROGRESS = None
 BACKFILL_FROM_BLOCK = None
 BACKFILL_TO_BLOCK = None
 BLOCK_PROCESSING_TIME = None
+# Task-level metrics
+TASK_EXECUTIONS = None
+TASK_EXECUTION_TIME = None
+TASK_RETRIES = None
+# Business/observability metrics
+BLOCK_LAG = None  # How far behind the chain head
+PENDING_TASKS = None  # Current pending tasks count
+REGISTERED_TASKS = None  # Number of registered task types
+ARCHIVE_NETWORK_USAGE = None  # Counter for archive network fallback
 
 if PROMETHEUS_AVAILABLE:
     BLOCKS_PROCESSED = Counter(  # type: ignore
@@ -67,6 +76,41 @@ if PROMETHEUS_AVAILABLE:
         "Time spent processing each block",
         ["mode"],
     )
+    # Task-level metrics
+    TASK_EXECUTIONS = Counter(  # type: ignore
+        "block_dumper_task_executions_total",
+        "Total task executions by status",
+        ["task_name", "status"],  # status: 'success', 'failed'
+    )
+    TASK_EXECUTION_TIME = Histogram(  # type: ignore
+        "block_dumper_task_execution_seconds",
+        "Time spent executing each task",
+        ["task_name"],
+        buckets=(0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0),
+    )
+    TASK_RETRIES = Counter(  # type: ignore
+        "block_dumper_task_retries_total",
+        "Total task retry attempts",
+        ["task_name"],
+    )
+    # Business/observability metrics
+    BLOCK_LAG = Gauge(  # type: ignore
+        "block_dumper_block_lag",
+        "Number of blocks behind the chain head",
+        ["mode"],  # 'realtime' or 'backfill'
+    )
+    PENDING_TASKS = Gauge(  # type: ignore
+        "block_dumper_pending_tasks",
+        "Current number of pending tasks in queue",
+    )
+    REGISTERED_TASKS = Gauge(  # type: ignore
+        "block_dumper_registered_tasks",
+        "Number of registered task types",
+    )
+    ARCHIVE_NETWORK_USAGE = Counter(  # type: ignore
+        "block_dumper_archive_network_requests_total",
+        "Total requests using archive network",
+    )
 
 
 def increment_blocks_processed(mode: str) -> None:
@@ -105,6 +149,71 @@ def set_backfill_progress(from_block: int, to_block: int, current_block: int) ->
             BACKFILL_PROGRESS.set(progress)
 
 
+def set_block_lag(mode: str, lag: int) -> None:
+    """Set the current block lag (distance from chain head)."""
+    if PROMETHEUS_AVAILABLE and BLOCK_LAG is not None:
+        BLOCK_LAG.labels(mode=mode).set(lag)
+
+
+def set_pending_tasks(count: int) -> None:
+    """Set the current number of pending tasks."""
+    if PROMETHEUS_AVAILABLE and PENDING_TASKS is not None:
+        PENDING_TASKS.set(count)
+
+
+def set_registered_tasks(count: int) -> None:
+    """Set the number of registered task types."""
+    if PROMETHEUS_AVAILABLE and REGISTERED_TASKS is not None:
+        REGISTERED_TASKS.set(count)
+
+
+def increment_archive_network_usage() -> None:
+    """Increment the archive network usage counter."""
+    if PROMETHEUS_AVAILABLE and ARCHIVE_NETWORK_USAGE is not None:
+        ARCHIVE_NETWORK_USAGE.inc()
+
+
+def record_task_execution(task_name: str, status: str) -> None:
+    """Record a task execution with status (success/failed)."""
+    if PROMETHEUS_AVAILABLE and TASK_EXECUTIONS is not None:
+        TASK_EXECUTIONS.labels(task_name=task_name, status=status).inc()
+
+
+def record_task_retry(task_name: str) -> None:
+    """Record a task retry attempt."""
+    if PROMETHEUS_AVAILABLE and TASK_RETRIES is not None:
+        TASK_RETRIES.labels(task_name=task_name).inc()
+
+
+def observe_task_execution_time(task_name: str, duration: float) -> None:
+    """Record task execution duration in seconds."""
+    if PROMETHEUS_AVAILABLE and TASK_EXECUTION_TIME is not None:
+        TASK_EXECUTION_TIME.labels(task_name=task_name).observe(duration)
+
+
+class TaskExecutionTimer:
+    """Context manager for timing task execution."""
+
+    def __init__(self, task_name: str) -> None:
+        self.task_name = task_name
+        self._timer: Any = None
+
+    def __enter__(self) -> Self:
+        if PROMETHEUS_AVAILABLE and TASK_EXECUTION_TIME is not None:
+            self._timer = TASK_EXECUTION_TIME.labels(task_name=self.task_name).time()
+            self._timer.__enter__()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        if self._timer is not None:
+            self._timer.__exit__(exc_type, exc_val, exc_tb)
+
+
 class BlockProcessingTimer:
     """Context manager for timing block processing."""
 

abstract_block_dumper/_internal/services/scheduler.py

@@ -10,7 +10,9 @@ from abstract_block_dumper._internal.services.block_processor import BlockProces
 from abstract_block_dumper._internal.services.metrics import (
     BlockProcessingTimer,
     increment_blocks_processed,
+    set_block_lag,
     set_current_block,
+    set_registered_tasks,
 )
 
 logger = structlog.get_logger(__name__)
@@ -94,10 +96,13 @@ class TaskScheduler:
 
         self.initialize_last_block()
 
+        registered_tasks_count = len(self.block_processor.registry.get_functions())
+        set_registered_tasks(registered_tasks_count)
+
         logger.info(
             "TaskScheduler started",
             last_processed_block=self.last_processed_block,
-            registry_functions=len(self.block_processor.registry.get_functions()),
+            registry_functions=registered_tasks_count,
         )
 
         while self.is_running:
@@ -117,6 +122,7 @@ class TaskScheduler:
 
                         set_current_block("realtime", current_block)
                         increment_blocks_processed("realtime")
+                        set_block_lag("realtime", 0)  # Head-only mode has no lag
                         self.last_processed_block = current_block
 
                     time.sleep(self.poll_interval)
@@ -129,6 +135,7 @@ class TaskScheduler:
                         # Update metrics
                         set_current_block("realtime", block_number)
                         increment_blocks_processed("realtime")
+                        set_block_lag("realtime", current_block - block_number)
 
                         time.sleep(self.poll_interval)
                         self.last_processed_block = block_number

abstract_block_dumper/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.0.8'
-__version_tuple__ = version_tuple = (0, 0, 8)
+__version__ = version = '0.0.9'
+__version_tuple__ = version_tuple = (0, 0, 9)
 
 __commit_id__ = commit_id = None

abstract_block_dumper/v1/decorators.py

@@ -1,4 +1,5 @@
 import inspect
+import time
 from collections.abc import Callable
 from typing import Any, cast
 
@@ -10,6 +11,11 @@ import abstract_block_dumper._internal.dal.django_dal as abd_dal
 import abstract_block_dumper._internal.services.utils as abd_utils
 from abstract_block_dumper._internal.dal.memory_registry import RegistryItem, task_registry
 from abstract_block_dumper._internal.exceptions import CeleryTaskLockedError
+from abstract_block_dumper._internal.services.metrics import (
+    observe_task_execution_time,
+    record_task_execution,
+    record_task_retry,
+)
 from abstract_block_dumper.models import TaskAttempt
 
 logger = structlog.get_logger(__name__)
@@ -62,6 +68,10 @@ def schedule_retry(task_attempt: TaskAttempt) -> None:
         eta=task_attempt.next_retry_at,
     )
 
+    # Record retry metric
+    task_name = task_attempt.executable_path.split(".")[-1]
+    record_task_retry(task_name)
+
 
 def _celery_task_wrapper(
     func: Callable[..., Any], block_number: int, **kwargs: dict[str, Any]
@@ -102,10 +112,20 @@ def _celery_task_wrapper(
         abd_dal.task_mark_as_started(task_attempt, abd_utils.get_current_celery_task_id())
 
         # Start task execution
+        # Pass _use_archive_network only if the function accepts **kwargs
+        # Otherwise, strip it to avoid TypeError
+        execution_kwargs = {"block_number": block_number, **kwargs}
+
+        # Check if function accepts **kwargs before adding _use_archive_network
+        sig = inspect.signature(func)
+        has_var_keyword = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values())
+        if has_var_keyword:
+            execution_kwargs["_use_archive_network"] = use_archive_network
+
+        task_name = executable_path.split(".")[-1]  # Get short task name
+        start_time = time.perf_counter()
+
         try:
-            # Pass _use_archive_network only if the function accepts **kwargs
-            # Otherwise, strip it to avoid TypeError
-            execution_kwargs = {"block_number": block_number, **kwargs}
             logger.info(
                 "Starting task execution",
                 task_id=task_attempt.id,
@@ -115,27 +135,32 @@ def _celery_task_wrapper(
                 use_archive_network=use_archive_network,
             )
 
-            # Check if function accepts **kwargs before adding _use_archive_network
-            sig = inspect.signature(func)
-            has_var_keyword = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values())
-            if has_var_keyword:
-                execution_kwargs["_use_archive_network"] = use_archive_network
-
             result = func(**execution_kwargs)
+            execution_duration = time.perf_counter() - start_time
 
             abd_dal.task_mark_as_success(task_attempt, result)
 
-            logger.info("Task completed successfully", task_id=task_attempt.id)
+            # Record success metrics
+            record_task_execution(task_name, "success")
+            observe_task_execution_time(task_name, execution_duration)
+
+            logger.info("Task completed successfully", task_id=task_attempt.id, duration=execution_duration)
             return {"result": result}
         except Exception as e:
+            execution_duration = time.perf_counter() - start_time
             logger.exception(
                 "Task execution failed",
                 task_id=task_attempt.id,
                 error_type=type(e).__name__,
                 error_message=str(e),
+                duration=execution_duration,
             )
             abd_dal.task_mark_as_failed(task_attempt)
 
+            # Record failure metrics
+            record_task_execution(task_name, "failed")
+            observe_task_execution_time(task_name, execution_duration)
+
     # Schedule retry after transaction commits:
     if abd_dal.task_can_retry(task_attempt):
         try:

{abstract_block_dumper-0.0.8.dist-info → abstract_block_dumper-0.0.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract-block-dumper
-Version: 0.0.8
+Version: 0.0.9
 Project-URL: Source, https://github.com/bactensor/abstract-block-dumper
 Project-URL: Issue Tracker, https://github.com/bactensor/abstract-block-dumper/issues
 Author-email: Reef Technologies <opensource@reef.pl>

{abstract_block_dumper-0.0.8.dist-info → abstract_block_dumper-0.0.9.dist-info}/RECORD

@@ -1,5 +1,5 @@
 abstract_block_dumper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_version.py,sha256=j-ar4gJGiWIawqKXhvv9hGJWLfwu0tISl-0GV97B7a0,704
+abstract_block_dumper/_version.py,sha256=X2FLFwBoUmgJPsOV-1l-SxIXNSTX1TQ036Kf2j9Mc68,704
 abstract_block_dumper/admin.py,sha256=3J3I_QOKFgfMNpTXW-rTQGO_q5Ls6uNuL0FkPVdIsYg,1654
 abstract_block_dumper/apps.py,sha256=DXATdrjsL3T2IletTbKeD6unr8ScLaxg7wz0nAHTAns,215
 abstract_block_dumper/models.py,sha256=MO9824dmHB6xF3PrFE_RERh7whVjQtS4tt6QA0wSbg0,2022
@@ -11,11 +11,11 @@ abstract_block_dumper/_internal/dal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQ
 abstract_block_dumper/_internal/dal/django_dal.py,sha256=i9jocanfptjXw5lfE2xBYvx5mo1g98IoMjlS-WjGP88,5623
 abstract_block_dumper/_internal/dal/memory_registry.py,sha256=m9Yms-cuemi9_5q_Kn_zsJnxDPEiuAUkESIAltD60QY,2943
 abstract_block_dumper/_internal/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_internal/services/backfill_scheduler.py,sha256=PTBs3tN4LLvFYuXKVsHVpme66RAsnPt166RWlRNA4xQ,15955
+abstract_block_dumper/_internal/services/backfill_scheduler.py,sha256=XgsVYXaz6pR4PBA9giesjhR74x1qLX2281-eQgM5qhs,16311
 abstract_block_dumper/_internal/services/block_processor.py,sha256=NC7p1oD38FpaZb6EbykBolP32uY069abumOvXrjOBV0,6644
-abstract_block_dumper/_internal/services/executor.py,sha256=vuAALKuP7KTQuQyG-P8JfrU22Sr-90HzXsIdfYbXZy4,2080
-abstract_block_dumper/_internal/services/metrics.py,sha256=FLhpq63WVccc0N6K1rN2VnV90jywFfAiG2ZxoDPALv0,3929
-abstract_block_dumper/_internal/services/scheduler.py,sha256=1Ls9dn_siEKl6yXJTS_B4NKhA3ZpZxrMJM_whRtEdxk,7453
+abstract_block_dumper/_internal/services/executor.py,sha256=WhpHhOAi4cI-qdEE8-DSt9xZwooOpSc9_uDMQBBoHUM,2317
+abstract_block_dumper/_internal/services/metrics.py,sha256=Gg-PQYZ98caaS52wm1EqhtPURXlfrVjk2t3-8nccqfo,7821
+abstract_block_dumper/_internal/services/scheduler.py,sha256=B62ZoOxSv8i5mverBWv67BzoyBd14f0wWnNQ329pJds,7770
 abstract_block_dumper/_internal/services/utils.py,sha256=QSs2hBHWOPgNgKPf_ZmADXuqEiqK5mWZp7JblvQgxZQ,1140
 abstract_block_dumper/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 abstract_block_dumper/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -25,8 +25,8 @@ abstract_block_dumper/migrations/0001_initial.py,sha256=ImPHC3G6kPkq4Xn_4YVAm4La
 abstract_block_dumper/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 abstract_block_dumper/v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 abstract_block_dumper/v1/celery.py,sha256=X4IqVs5i6ZpyY7fy1SqMZgsZy4SXP-jK2qG-FYnjU38,1722
-abstract_block_dumper/v1/decorators.py,sha256=Lua91tR-d0juif9VGVxnC8bzhcgyn8_1bSR33hw7IK0,8920
+abstract_block_dumper/v1/decorators.py,sha256=4FXsBScT_5Wadl8FadRcZZtyLytZUzSTX4V5DI0IuRs,9820
 abstract_block_dumper/v1/tasks.py,sha256=u9iMYdDUqzYT3yPrNwZecHnlweZ3yFipV9BcIWHCbus,2647
-abstract_block_dumper-0.0.8.dist-info/METADATA,sha256=O5E4ChDm8gX0XSs_zvImjBgl9Q_DJkNu_rW5KSx-fMk,12993
-abstract_block_dumper-0.0.8.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-abstract_block_dumper-0.0.8.dist-info/RECORD,,
+abstract_block_dumper-0.0.9.dist-info/METADATA,sha256=xWskTf2HCMNuFGA5M2KSyyejjBLKgQ6OBv2Q5VuL5_I,12993
+abstract_block_dumper-0.0.9.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+abstract_block_dumper-0.0.9.dist-info/RECORD,,