abstract-block-dumper 0.0.6__py3-none-any.whl → 0.0.8__py3-none-any.whl

abstract_block_dumper/_internal/dal/django_dal.py

@@ -21,13 +21,19 @@ def get_ready_to_retry_attempts() -> QuerySet[abd_models.TaskAttempt]:
 
 
 def executed_block_numbers(executable_path: str, args_json: str, from_block: int, to_block: int) -> set[int]:
-    block_numbers = abd_models.TaskAttempt.objects.filter(
-        executable_path=executable_path,
-        args_json=args_json,
-        block_number__gte=from_block,
-        block_number__lt=to_block,
-        status=abd_models.TaskAttempt.Status.SUCCESS,
-    ).values_list("block_number", flat=True)
+    # Use iterator() to avoid Django's QuerySet caching which causes memory leaks
+    # during long-running backfill operations
+    block_numbers = (
+        abd_models.TaskAttempt.objects.filter(
+            executable_path=executable_path,
+            args_json=args_json,
+            block_number__gte=from_block,
+            block_number__lt=to_block,
+            status=abd_models.TaskAttempt.Status.SUCCESS,
+        )
+        .values_list("block_number", flat=True)
+        .iterator()
+    )
     return set(block_numbers)
 
 

abstract_block_dumper/_internal/dal/memory_registry.py

@@ -23,15 +23,14 @@ class RegistryItem:
         """Check if condition matches for given block and arguments."""
         try:
             return self.condition(block_number, **kwargs)
-        except Exception as e:
-            logger.error(
+        except Exception as exc:
+            logger.exception(
                 "Condition evaluation failed",
                 condition=self.function.__name__,
                 block_number=block_number,
-                exc_info=True,
             )
-            msg = f"Failed to evaluate condition: {e}"
-            raise ConditionEvaluationError(msg) from e
+            msg = "Failed to evaluate condition"
+            raise ConditionEvaluationError(msg) from exc
 
     def get_execution_args(self) -> list[dict[str, Any]]:
         """Get list of argument sets for execution."""

abstract_block_dumper/_internal/services/backfill_scheduler.py

@@ -0,0 +1,438 @@
+"""
+Backfill scheduler for historical block processing.
+
+This module provides a dedicated scheduler for backfilling historical blocks
+with rate limiting and automatic archive network switching.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+import structlog
+
+import abstract_block_dumper._internal.dal.django_dal as abd_dal
+import abstract_block_dumper._internal.services.utils as abd_utils
+from abstract_block_dumper._internal.services.block_processor import BlockProcessor, block_processor_factory
+from abstract_block_dumper._internal.services.metrics import (
+    BlockProcessingTimer,
+    increment_blocks_processed,
+    set_backfill_progress,
+    set_current_block,
+)
+from abstract_block_dumper._internal.services.utils import serialize_args
+
+if TYPE_CHECKING:
+    import bittensor as bt
+
+    from abstract_block_dumper._internal.dal.memory_registry import RegistryItem
+
+logger = structlog.get_logger(__name__)
+
+# Blocks older than this threshold from current head require archive network
+ARCHIVE_BLOCK_THRESHOLD = 300
+
+# Progress logging interval
+PROGRESS_LOG_INTERVAL = 100
+ARCHIVE_NETWORK = "archive"
+
+# Memory cleanup interval (every N blocks)
+MEMORY_CLEANUP_INTERVAL = 1000
+
+
+@dataclass
+class DryRunStats:
+    """Statistics for dry-run mode."""
+
+    total_blocks: int = 0
+    already_processed: int = 0
+    blocks_needing_tasks: int = 0
+    estimated_tasks: int = 0
+
+
+class BackfillScheduler:
+    """Scheduler for backfilling historical blocks with rate limiting."""
+
+    def __init__(
+        self,
+        block_processor: BlockProcessor,
+        network: str,
+        from_block: int,
+        to_block: int,
+        rate_limit: float = 1.0,
+        dry_run: bool = False,
+    ) -> None:
+        """
+        Initialize the backfill scheduler.
+
+        Args:
+            block_processor: The block processor to use for task execution.
+            network: The bittensor network name (e.g., 'finney').
+            from_block: Starting block number (inclusive).
+            to_block: Ending block number (inclusive).
+            rate_limit: Seconds to sleep between processing each block.
+            dry_run: If True, preview what would be processed without executing.
+
+        """
+        self.block_processor = block_processor
+        self.network = network
+        self.from_block = from_block
+        self.to_block = to_block
+        self.rate_limit = rate_limit
+        self.dry_run = dry_run
+        self.is_running = False
+        self._subtensor: bt.Subtensor | None = None
+        self._archive_subtensor: bt.Subtensor | None = None
+        self._current_head_cache: int | None = None
+
+    @property
+    def subtensor(self) -> bt.Subtensor:
+        """Get the regular subtensor connection, creating it if needed."""
+        if self._subtensor is None:
+            self._subtensor = abd_utils.get_bittensor_client(self.network)
+        return self._subtensor
+
+    @property
+    def archive_subtensor(self) -> bt.Subtensor:
+        """Get the archive subtensor connection, creating it if needed."""
+        if self._archive_subtensor is None:
+            self._archive_subtensor = abd_utils.get_bittensor_client("archive")
+        return self._archive_subtensor
+
+    def get_subtensor_for_block(self, block_number: int) -> bt.Subtensor:
+        """
+        Get the appropriate subtensor for the given block number.
+
+        Uses archive network for blocks older than ARCHIVE_BLOCK_THRESHOLD
+        from the current head.
+        """
+        if self._current_head_cache is None:
+            self._current_head_cache = self.subtensor.get_current_block()
+
+        blocks_behind = self._current_head_cache - block_number
+
+        if blocks_behind > ARCHIVE_BLOCK_THRESHOLD:
+            logger.debug(
+                "Using archive network for old block",
+                block_number=block_number,
+                blocks_behind=blocks_behind,
+            )
+            return self.archive_subtensor
+        return self.subtensor
+
+    def _get_network_type_for_block(self, block_number: int) -> str:
+        """Get the network type string for a block (for display purposes)."""
+        if self._current_head_cache is None:
+            self._current_head_cache = self.subtensor.get_current_block()
+
+        blocks_behind = self._current_head_cache - block_number
+        return ARCHIVE_NETWORK if blocks_behind > ARCHIVE_BLOCK_THRESHOLD else self.network
+
+    def start(self) -> DryRunStats | None:
+        """
+        Start processing blocks from from_block to to_block.
+
+        Returns:
+            DryRunStats if dry_run is True, None otherwise.
+
+        """
+        self.is_running = True
+
+        # Refresh current head for accurate archive network decisions
+        self._current_head_cache = self.subtensor.get_current_block()
+
+        total_blocks = self.to_block - self.from_block + 1
+        network_type = self._get_network_type_for_block(self.from_block)
+
+        logger.info(
+            "BackfillScheduler starting",
+            from_block=self.from_block,
+            to_block=self.to_block,
+            total_blocks=total_blocks,
+            rate_limit=self.rate_limit,
+            dry_run=self.dry_run,
+            network_type=network_type,
+            current_head=self._current_head_cache,
+        )
+
+        if self.dry_run:
+            return self._run_dry_run()
+
+        self._run_backfill()
+        return None
+
+    def _run_dry_run(self) -> DryRunStats:
+        """
+        Run in dry-run mode to preview what would be processed.
+
+        Optimized to fetch all executed blocks in one query per registry item,
+        instead of querying for each block individually.
+        """
+        stats = DryRunStats(total_blocks=self.to_block - self.from_block + 1)
+
+        registry_items = self.block_processor.registry.get_functions()
+
+        # Pre-fetch all executed blocks for each registry item + args combination
+        # This reduces N queries (one per block) to M queries (one per registry item + args)
+        executed_blocks_cache: dict[tuple[str, str], set[int]] = {}
+
+        logger.info(
+            "Dry run: pre-fetching executed blocks",
+            from_block=self.from_block,
+            to_block=self.to_block,
+            registry_items_count=len(registry_items),
+        )
+
+        for registry_item in registry_items:
+            for args in registry_item.get_execution_args():
+                args_json = serialize_args(args)
+                cache_key = (registry_item.executable_path, args_json)
+
+                # Fetch all executed blocks in the range with a single query
+                executed_blocks_cache[cache_key] = abd_dal.executed_block_numbers(
+                    registry_item.executable_path,
+                    args_json,
+                    self.from_block,
+                    self.to_block + 1,
+                )
+
+        logger.info(
+            "Dry run: analyzing blocks",
+            cache_entries=len(executed_blocks_cache),
+        )
+
+        # Track which blocks have at least one task
+        blocks_with_tasks: set[int] = set()
+
+        for registry_item in registry_items:
+            for args in registry_item.get_execution_args():
+                args_json = serialize_args(args)
+                cache_key = (registry_item.executable_path, args_json)
+                executed_blocks = executed_blocks_cache[cache_key]
+
+                for block_number in range(self.from_block, self.to_block + 1):
+                    if not self.is_running:
+                        break
+
+                    if block_number in executed_blocks:
+                        continue
+
+                    # Check if condition matches
+                    try:
+                        if registry_item.match_condition(block_number, **args):
+                            stats.estimated_tasks += 1
+                            blocks_with_tasks.add(block_number)
+                    except Exception as exc:
+                        logger.debug(
+                            "Error evaluating match condition during dry run",
+                            function_name=registry_item.function.__name__,
+                            block_number=block_number,
+                            args=args,
+                            error=str(exc),
+                        )
+
+        stats.blocks_needing_tasks = len(blocks_with_tasks)
+        stats.already_processed = stats.total_blocks - stats.blocks_needing_tasks
+
+        return stats
+
+    def _run_backfill(self) -> None:
+        """Run the actual backfill process."""
+        processed_count = 0
+        total_blocks = self.to_block - self.from_block + 1
+
+        # Set initial metrics
+        set_backfill_progress(self.from_block, self.to_block, self.from_block)
+
+        # Pre-fetch all executed blocks to avoid per-block DB queries
+        logger.info(
+            "Pre-fetching executed blocks",
+            from_block=self.from_block,
+            to_block=self.to_block,
+        )
+        executed_blocks_cache = self._prefetch_executed_blocks()
+        logger.info(
+            "Pre-fetch complete",
+            cache_entries=len(executed_blocks_cache),
+        )
+
+        try:
+            for block_number in range(self.from_block, self.to_block + 1):
+                if not self.is_running:
+                    logger.info("BackfillScheduler stopping early", processed_count=processed_count)
+                    break
+
+                try:
+                    with BlockProcessingTimer(mode="backfill"):
+                        self._process_block(block_number, executed_blocks_cache)
+
+                    processed_count += 1
+
+                    # Update metrics
+                    set_current_block("backfill", block_number)
+                    set_backfill_progress(self.from_block, self.to_block, block_number)
+                    increment_blocks_processed("backfill")
+
+                    # Log progress periodically
+                    if processed_count % PROGRESS_LOG_INTERVAL == 0:
+                        progress_pct = (processed_count / total_blocks) * 100
+                        logger.info(
+                            "Backfill progress",
+                            processed=processed_count,
+                            total=total_blocks,
+                            progress_percent=f"{progress_pct:.1f}%",
+                            current_block=block_number,
+                        )
+
+                    # Rate limiting between block submissions
+                    if block_number < self.to_block and self.rate_limit > 0:
+                        time.sleep(self.rate_limit)
+
+                except KeyboardInterrupt:
+                    raise
+                except Exception:
+                    logger.exception(
+                        "Error processing block during backfill",
+                        block_number=block_number,
+                    )
+                    # Continue with next block
+                    time.sleep(self.rate_limit)
+
+        except KeyboardInterrupt:
+            logger.info(
+                "BackfillScheduler interrupted",
+                processed_count=processed_count,
+                last_block=self.from_block + processed_count - 1 if processed_count > 0 else self.from_block,
+            )
+            self.stop()
+
+        logger.info(
+            "BackfillScheduler completed",
+            processed_count=processed_count,
+            total_blocks=total_blocks,
+        )
+
+    def _prefetch_executed_blocks(self) -> dict[tuple[str, str], set[int]]:
+        """Pre-fetch all executed blocks for all registry items in the range."""
+        cache: dict[tuple[str, str], set[int]] = {}
+
+        for registry_item in self.block_processor.registry.get_functions():
+            for args in registry_item.get_execution_args():
+                args_json = serialize_args(args)
+                cache_key = (registry_item.executable_path, args_json)
+
+                cache[cache_key] = abd_dal.executed_block_numbers(
+                    registry_item.executable_path,
+                    args_json,
+                    self.from_block,
+                    self.to_block + 1,
+                )
+
+        return cache
+
+    def _process_block(
+        self,
+        block_number: int,
+        executed_blocks_cache: dict[tuple[str, str], set[int]],
+    ) -> None:
+        """Process a single block during backfill."""
+        for registry_item in self.block_processor.registry.get_functions():
+            try:
+                self._process_registry_item_for_backfill(
+                    registry_item,
+                    block_number,
+                    executed_blocks_cache,
+                )
+            except Exception:
+                logger.exception(
+                    "Error processing registry item during backfill",
+                    function_name=registry_item.function.__name__,
+                    block_number=block_number,
+                )
+
+    def _requires_archive_network(self, block_number: int) -> bool:
+        """Check if a block requires archive network based on age."""
+        if self._current_head_cache is None:
+            return False
+        blocks_behind = self._current_head_cache - block_number
+        return blocks_behind > ARCHIVE_BLOCK_THRESHOLD
+
+    def _process_registry_item_for_backfill(
+        self,
+        registry_item: RegistryItem,
+        block_number: int,
+        executed_blocks_cache: dict[tuple[str, str], set[int]],
+    ) -> None:
+        """Process a registry item for backfill - only submits if not already executed."""
+        for args in registry_item.get_execution_args():
+            args_json = serialize_args(args)
+            cache_key = (registry_item.executable_path, args_json)
+
+            # Check if already executed using pre-fetched cache
+            executed_blocks = executed_blocks_cache.get(cache_key, set())
+
+            if block_number in executed_blocks:
+                continue
+
+            # Check condition and execute
+            try:
+                if registry_item.match_condition(block_number, **args):
+                    use_archive = self._requires_archive_network(block_number)
+                    logger.debug(
+                        "Backfilling block",
+                        function_name=registry_item.function.__name__,
+                        block_number=block_number,
+                        args=args,
+                        use_archive=use_archive,
+                    )
+                    self.block_processor.executor.execute(
+                        registry_item,
+                        block_number,
+                        args,
+                        use_archive=use_archive,
+                    )
+            except Exception:
+                logger.exception(
+                    "Error during backfill task execution",
+                    function_name=registry_item.function.__name__,
+                    block_number=block_number,
+                    args=args,
+                )
+
+    def stop(self) -> None:
+        """Stop the backfill scheduler."""
+        self.is_running = False
+        logger.info("BackfillScheduler stopped")
+
+
+def backfill_scheduler_factory(
+    from_block: int,
+    to_block: int,
+    network: str = "finney",
+    rate_limit: float = 1.0,
+    dry_run: bool = False,
+) -> BackfillScheduler:
+    """
+    Factory for BackfillScheduler.
+
+    Args:
+        from_block: Starting block number (inclusive).
+        to_block: Ending block number (inclusive).
+        network: Bittensor network name. Defaults to "finney".
+        rate_limit: Seconds to sleep between blocks. Defaults to 1.0.
+        dry_run: If True, preview without executing. Defaults to False.
+
+    Returns:
+        Configured BackfillScheduler instance.
+
+    """
+    return BackfillScheduler(
+        block_processor=block_processor_factory(),
+        network=network,
+        from_block=from_block,
+        to_block=to_block,
+        rate_limit=rate_limit,
+        dry_run=dry_run,
+    )

abstract_block_dumper/_internal/services/block_processor.py

@@ -1,3 +1,5 @@
+import time
+
 import structlog
 from django.db import transaction
 
@@ -5,7 +7,6 @@ import abstract_block_dumper._internal.dal.django_dal as abd_dal
 from abstract_block_dumper._internal.dal.memory_registry import BaseRegistry, RegistryItem, task_registry
 from abstract_block_dumper._internal.exceptions import ConditionEvaluationError
 from abstract_block_dumper._internal.services.executor import CeleryExecutor
-from abstract_block_dumper._internal.services.utils import serialize_args
 from abstract_block_dumper.models import TaskAttempt
 
 logger = structlog.get_logger(__name__)
@@ -18,16 +19,15 @@ class BlockProcessor:
         self._cleanup_phantom_tasks()
 
     def process_block(self, block_number: int) -> None:
+        """Process a single block - executes registered tasks for this block only."""
         for registry_item in self.registry.get_functions():
             try:
-                self.process_backfill(registry_item, block_number)
                 self.process_registry_item(registry_item, block_number)
             except Exception:
-                logger.error(
+                logger.exception(
                     "Error processing registry item",
                     function_name=registry_item.function.__name__,
                     block_number=block_number,
-                    exc_info=True,
                 )
 
     def process_registry_item(self, registry_item: RegistryItem, block_number: int) -> None:
@@ -43,64 +43,28 @@ class BlockProcessor:
                 )
                 # Continue with other tasks
             except Exception:
-                logger.error("Unexpected error processing task", exc_info=True)
-
-    def process_backfill(self, registry_item: RegistryItem, current_block: int) -> None:
-        if not registry_item.backfilling_lookback:
-            return None
-
-        start_block = max(0, current_block - registry_item.backfilling_lookback)
-
-        logger.info(
-            "Processing backfill",
-            function_name=registry_item.function.__name__,
-            start_block=start_block,
-            current_block=current_block,
-            lookback=registry_item.backfilling_lookback,
-        )
-
-        execution_args_list = registry_item.get_execution_args()
-
-        for args in execution_args_list:
-            args_json = serialize_args(args)
-
-            executed_blocks = abd_dal.executed_block_numbers(
-                registry_item.executable_path,
-                args_json,
-                start_block,
-                current_block,
-            )
-
-            for block_number in range(start_block, current_block):
-                if block_number in executed_blocks:
-                    continue
+                logger.exception("Unexpected error processing task")
 
-                try:
-                    if registry_item.match_condition(block_number, **args):
-                        logger.debug(
-                            "Backfilling block",
-                            function_name=registry_item.function.__name__,
-                            block_number=block_number,
-                            args=args,
-                        )
-                        self.executor.execute(registry_item, block_number, args)
-                except Exception:
-                    logger.error(
-                        "Error during backfill",
-                        function_name=registry_item.function.__name__,
-                        block_number=block_number,
-                        args=args,
-                        exc_info=True,
-                    )
-
-    def recover_failed_retries(self) -> None:
+    def recover_failed_retries(self, poll_interval: int, batch_size: int | None = None) -> None:
         """
         Recover failed tasks that are ready to be retried.
 
         This handles tasks that may have been lost due to scheduler restarts.
+
+        Args:
+            poll_interval: Seconds to sleep between processing each retry.
+            batch_size: Maximum number of retries to process. If None, process all.
+
         """
         retry_count = 0
-        for retry_attempt in abd_dal.get_ready_to_retry_attempts():
+        retry_attempts = abd_dal.get_ready_to_retry_attempts()
+
+        # Apply batch size limit if specified
+        if batch_size is not None:
+            retry_attempts = retry_attempts[:batch_size]
+
+        for retry_attempt in retry_attempts:
+            time.sleep(poll_interval)
             try:
                 # Find the registry item to get celery_kwargs
                 registry_item = self.registry.get_by_executable_path(retry_attempt.executable_path)
@@ -148,10 +112,9 @@ class BlockProcessor:
                     attempt_count=task_attempt.attempt_count,
                 )
             except Exception:
-                logger.error(
+                logger.exception(
                     "Failed to recover retry",
                     task_id=retry_attempt.id,
-                    exc_info=True,
                 )
                 # Reload task to see current state after potential execution failure
                 try:

abstract_block_dumper/_internal/services/executor.py

@@ -10,7 +10,14 @@ logger = structlog.get_logger(__name__)
 
 
 class CeleryExecutor:
-    def execute(self, registry_item: RegistryItem, block_number: int, args: dict[str, Any]) -> None:
+    def execute(
+        self,
+        registry_item: RegistryItem,
+        block_number: int,
+        args: dict[str, Any],
+        *,
+        use_archive: bool = False,
+    ) -> None:
         task_attempt, created = abd_dal.task_create_or_get_pending(
             block_number=block_number,
             executable_path=registry_item.executable_path,
@@ -26,6 +33,7 @@ class CeleryExecutor:
 
         task_kwargs = {
             "block_number": block_number,
+            "_use_archive_network": use_archive,  # Runtime hint, not stored in DB
             **args,
         }
 
@@ -49,6 +57,8 @@ class CeleryExecutor:
             celery_kwargs=apply_async_kwargs,
         )
 
-        celery_task = registry_item.function.apply_async(**apply_async_kwargs)
+        # Don't store AsyncResult to avoid memory accumulation during long runs
+        # The task ID can be retrieved from the task_attempt if needed
+        registry_item.function.apply_async(**apply_async_kwargs)
 
-        logger.debug("Celery task scheduled", task_id=task_attempt.id, celery_task_id=celery_task.id)
+        logger.debug("Celery task scheduled", task_id=task_attempt.id)

abstract_block_dumper/_internal/services/metrics.py

@@ -0,0 +1,128 @@
+"""
+Optional Prometheus metrics for block dumper.
+
+Metrics are only available if prometheus_client is installed.
+Install with: pip install abstract-block-dumper[prometheus]
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Self
+
+import structlog
+
+if TYPE_CHECKING:
+    from types import TracebackType
+
+logger = structlog.get_logger(__name__)
+
+# Conditional import - metrics only work if prometheus_client is installed
+try:
+    from prometheus_client import Counter, Gauge, Histogram
+
+    PROMETHEUS_AVAILABLE = True
+except ImportError:
+    PROMETHEUS_AVAILABLE = False
+
+
+# Define no-op placeholders when prometheus is not available
+BLOCKS_PROCESSED = None
+TASKS_SUBMITTED = None
+CURRENT_BLOCK = None
+BACKFILL_PROGRESS = None
+BACKFILL_FROM_BLOCK = None
+BACKFILL_TO_BLOCK = None
+BLOCK_PROCESSING_TIME = None
+
+if PROMETHEUS_AVAILABLE:
+    BLOCKS_PROCESSED = Counter(  # type: ignore
+        "block_dumper_blocks_processed_total",
+        "Total blocks processed",
+        ["mode"],  # 'realtime' or 'backfill'
+    )
+    TASKS_SUBMITTED = Counter(  # type: ignore
+        "block_dumper_tasks_submitted_total",
+        "Total tasks submitted to Celery",
+        ["task_name"],
+    )
+    CURRENT_BLOCK = Gauge(  # type: ignore
+        "block_dumper_current_block",
+        "Current block being processed",
+        ["mode"],
+    )
+    BACKFILL_PROGRESS = Gauge(  # type: ignore
+        "block_dumper_backfill_progress_percent",
+        "Backfill progress percentage",
+    )
+    BACKFILL_FROM_BLOCK = Gauge(  # type: ignore
+        "block_dumper_backfill_from_block",
+        "Backfill starting block number",
+    )
+    BACKFILL_TO_BLOCK = Gauge(  # type: ignore
+        "block_dumper_backfill_to_block",
+        "Backfill target block number",
+    )
+    BLOCK_PROCESSING_TIME = Histogram(  # type: ignore
+        "block_dumper_block_processing_seconds",
+        "Time spent processing each block",
+        ["mode"],
+    )
+
+
+def increment_blocks_processed(mode: str) -> None:
+    """Increment the blocks processed counter."""
+    if PROMETHEUS_AVAILABLE and BLOCKS_PROCESSED is not None:
+        BLOCKS_PROCESSED.labels(mode=mode).inc()
+
+
+def increment_tasks_submitted(task_name: str) -> None:
+    """Increment the tasks submitted counter."""
+    if PROMETHEUS_AVAILABLE and TASKS_SUBMITTED is not None:
+        TASKS_SUBMITTED.labels(task_name=task_name).inc()
+
+
+def set_current_block(mode: str, block_number: int) -> None:
+    """Set the current block being processed."""
+    if PROMETHEUS_AVAILABLE and CURRENT_BLOCK is not None:
+        CURRENT_BLOCK.labels(mode=mode).set(block_number)
+
+
+def set_backfill_progress(from_block: int, to_block: int, current_block: int) -> None:
+    """Set backfill progress metrics."""
+    if not PROMETHEUS_AVAILABLE:
+        return
+
+    if BACKFILL_FROM_BLOCK is not None:
+        BACKFILL_FROM_BLOCK.set(from_block)
+    if BACKFILL_TO_BLOCK is not None:
+        BACKFILL_TO_BLOCK.set(to_block)
+
+    if BACKFILL_PROGRESS is not None:
+        total_blocks = to_block - from_block
+        if total_blocks > 0:
+            processed = current_block - from_block
+            progress = (processed / total_blocks) * 100
+            BACKFILL_PROGRESS.set(progress)
+
+
+class BlockProcessingTimer:
+    """Context manager for timing block processing."""
+
+    def __init__(self, mode: str) -> None:
+        self.mode = mode
+        self._timer: Any = None
+
+    def __enter__(self) -> Self:
+        if PROMETHEUS_AVAILABLE and BLOCK_PROCESSING_TIME is not None:
+            self._timer = BLOCK_PROCESSING_TIME.labels(mode=self.mode).time()
+            self._timer.__enter__()  # Start the timer
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        if self._timer is not None:
+            self._timer.__exit__(exc_type, exc_val, exc_tb)

abstract_block_dumper/_internal/services/scheduler.py

@@ -7,22 +7,87 @@ from django.conf import settings
 import abstract_block_dumper._internal.dal.django_dal as abd_dal
 import abstract_block_dumper._internal.services.utils as abd_utils
 from abstract_block_dumper._internal.services.block_processor import BlockProcessor, block_processor_factory
+from abstract_block_dumper._internal.services.metrics import (
+    BlockProcessingTimer,
+    increment_blocks_processed,
+    set_current_block,
+)
 
 logger = structlog.get_logger(__name__)
 
+# Blocks older than this threshold from current head require archive network
+ARCHIVE_BLOCK_THRESHOLD = 300
+
 
 class TaskScheduler:
     def __init__(
         self,
         block_processor: BlockProcessor,
-        subtensor: bt.Subtensor,
+        network: str,
         poll_interval: int,
+        realtime_head_only: bool = False,
     ) -> None:
         self.block_processor = block_processor
-        self.subtensor = subtensor
+        self.network = network
         self.poll_interval = poll_interval
+        self.realtime_head_only = realtime_head_only
         self.last_processed_block = -1
         self.is_running = False
+        self._subtensor: bt.Subtensor | None = None
+        self._archive_subtensor: bt.Subtensor | None = None
+        self._current_block_cache: int | None = None
+
+    @property
+    def subtensor(self) -> bt.Subtensor:
+        """Get the regular subtensor connection, creating it if needed."""
+        if self._subtensor is None:
+            self._subtensor = abd_utils.get_bittensor_client(self.network)
+        return self._subtensor
+
+    @subtensor.setter
+    def subtensor(self, value: bt.Subtensor | None) -> None:
+        """Set or reset the subtensor connection."""
+        self._subtensor = value
+
+    @property
+    def archive_subtensor(self) -> bt.Subtensor:
+        """Get the archive subtensor connection, creating it if needed."""
+        if self._archive_subtensor is None:
+            self._archive_subtensor = abd_utils.get_bittensor_client("archive")
+        return self._archive_subtensor
+
+    @archive_subtensor.setter
+    def archive_subtensor(self, value: bt.Subtensor | None) -> None:
+        """Set or reset the archive subtensor connection."""
+        self._archive_subtensor = value
+
+    def get_subtensor_for_block(self, block_number: int) -> bt.Subtensor:
+        """
+        Get the appropriate subtensor for the given block number.
+
+        Uses archive network for blocks older than ARCHIVE_BLOCK_THRESHOLD
+        from the current head.
+        """
+        if self._current_block_cache is None:
+            self._current_block_cache = self.subtensor.get_current_block()
+
+        blocks_behind = self._current_block_cache - block_number
+
+        if blocks_behind > ARCHIVE_BLOCK_THRESHOLD:
+            logger.debug(
+                "Using archive network for old block",
+                block_number=block_number,
+                blocks_behind=blocks_behind,
+            )
+            return self.archive_subtensor
+        return self.subtensor
+
+    def refresh_connections(self) -> None:
+        """Reset all subtensor connections to force re-establishment."""
+        self._subtensor = None
+        self._archive_subtensor = None
+        self._current_block_cache = None
+        logger.info("Subtensor connections reset")
 
     def start(self) -> None:
         self.is_running = True
@@ -37,16 +102,37 @@ class TaskScheduler:
 
         while self.is_running:
             try:
-                # Process lost retries first
-                self.block_processor.recover_failed_retries()
+                if self._current_block_cache is not None:
+                    self.subtensor = self.get_subtensor_for_block(self._current_block_cache)
 
-                current_block = self.subtensor.get_current_block()
+                # Update current block cache for archive network decision
+                self._current_block_cache = self.subtensor.get_current_block()
+                current_block = self._current_block_cache
 
-                for block_number in range(self.last_processed_block + 1, current_block + 1):
-                    self.block_processor.process_block(block_number)
-                    self.last_processed_block = block_number
+                if self.realtime_head_only:
+                    # Only process the current head block, skip if already processed
+                    if current_block != self.last_processed_block:
+                        with BlockProcessingTimer(mode="realtime"):
+                            self.block_processor.process_block(current_block)
+
+                        set_current_block("realtime", current_block)
+                        increment_blocks_processed("realtime")
+                        self.last_processed_block = current_block
+
+                    time.sleep(self.poll_interval)
+                else:
+                    # Original behavior: process all blocks from last_processed to current
+                    for block_number in range(self.last_processed_block + 1, current_block + 1):
+                        with BlockProcessingTimer(mode="realtime"):
+                            self.block_processor.process_block(block_number)
+
+                        # Update metrics
+                        set_current_block("realtime", block_number)
+                        increment_blocks_processed("realtime")
+
+                        time.sleep(self.poll_interval)
+                        self.last_processed_block = block_number
 
-                time.sleep(self.poll_interval)
             except KeyboardInterrupt:
                 logger.info("TaskScheduler stopping due to KeyboardInterrupt.")
                 self.stop()
@@ -86,9 +172,17 @@ class TaskScheduler:
             )
 
 
-def task_scheduler_factory() -> TaskScheduler:
+def task_scheduler_factory(network: str = "finney") -> TaskScheduler:
+    """
+    Factory for TaskScheduler.
+
+    Args:
+        network (str): Bittensor network name. Defaults to "finney"
+
+    """
     return TaskScheduler(
         block_processor=block_processor_factory(),
-        subtensor=abd_utils.get_bittensor_client(),
+        network=network,
         poll_interval=getattr(settings, "BLOCK_DUMPER_POLL_INTERVAL", 1),
+        realtime_head_only=getattr(settings, "BLOCK_DUMPER_REALTIME_HEAD_ONLY", True),
     )

abstract_block_dumper/_internal/services/utils.py

@@ -1,6 +1,5 @@
 import json
 from collections.abc import Callable
-from functools import cache
 
 import bittensor as bt
 import structlog
@@ -10,16 +9,13 @@ from django.conf import settings
 logger = structlog.get_logger(__name__)
 
 
-@cache
-def get_bittensor_client() -> bt.Subtensor:
+def get_bittensor_client(network: str = "finney") -> bt.Subtensor:
     """
     Get a cached bittensor client.
 
     The client is cached indefinitely since network configuration
     doesn't change during runtime.
     """
-    DEFAULT_BITTENSOR_NETWORK = "finney"
-    network = getattr(settings, "BITTENSOR_NETWORK", DEFAULT_BITTENSOR_NETWORK)
     logger.info("Creating new bittensor client for network", network=network)
     return bt.subtensor(network=network)
 

abstract_block_dumper/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.0.6'
-__version_tuple__ = version_tuple = (0, 0, 6)
+__version__ = version = '0.0.8'
+__version_tuple__ = version_tuple = (0, 0, 8)
 
 __commit_id__ = commit_id = None

abstract_block_dumper/management/commands/backfill_blocks_v1.py

@@ -0,0 +1,162 @@
+"""Management command for backfilling historical blocks."""
+
+from django.core.management.base import BaseCommand
+
+from abstract_block_dumper._internal.dal.memory_registry import task_registry
+from abstract_block_dumper._internal.discovery import ensure_modules_loaded
+from abstract_block_dumper._internal.services.backfill_scheduler import (
+    ARCHIVE_BLOCK_THRESHOLD,
+    BackfillScheduler,
+    backfill_scheduler_factory,
+)
+
+
+class Command(BaseCommand):
+    help = "Backfill historical blocks with rate limiting."
+
+    def add_arguments(self, parser) -> None:
+        parser.add_argument(
+            "--from-block",
+            type=int,
+            required=True,
+            help="Starting block number (inclusive)",
+        )
+        parser.add_argument(
+            "--to-block",
+            type=int,
+            required=True,
+            help="Ending block number (inclusive)",
+        )
+        parser.add_argument(
+            "--rate-limit",
+            type=float,
+            default=1.0,
+            help="Seconds to sleep between processing each block (default: 1.0)",
+        )
+        parser.add_argument(
+            "--network",
+            type=str,
+            default="finney",
+            help="Bittensor network name (default: finney)",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Preview blocks to backfill without executing tasks",
+        )
+
+    def handle(self, *args, **options) -> None:
+        from_block = options["from_block"]
+        to_block = options["to_block"]
+        rate_limit = options["rate_limit"]
+        network = options["network"]
+        dry_run = options["dry_run"]
+
+        # Validate arguments
+        if from_block > to_block:
+            self.stderr.write(self.style.ERROR(f"--from-block ({from_block}) must be <= --to-block ({to_block})"))
+            return
+
+        if rate_limit < 0:
+            self.stderr.write(self.style.ERROR("--rate-limit must be >= 0"))
+            return
+
+        # Load registered functions
+        self.stdout.write("Syncing decorated functions...")
+        ensure_modules_loaded()
+        functions_counter = len(task_registry.get_functions())
+        self.stdout.write(self.style.SUCCESS(f"Synced {functions_counter} functions"))
+
+        if functions_counter == 0:
+            self.stderr.write(self.style.WARNING("No functions registered. Nothing to backfill."))
+            return
+
+        # Create scheduler
+        scheduler = backfill_scheduler_factory(
+            from_block=from_block,
+            to_block=to_block,
+            network=network,
+            rate_limit=rate_limit,
+            dry_run=dry_run,
+        )
+
+        total_blocks = to_block - from_block + 1
+
+        if dry_run:
+            self._handle_dry_run(scheduler, from_block, to_block, total_blocks, rate_limit)
+        else:
+            self._handle_backfill(scheduler, from_block, to_block, total_blocks, rate_limit)
+
+    def _handle_dry_run(
+        self, scheduler: BackfillScheduler, from_block: int, to_block: int, total_blocks: int, rate_limit: float
+    ) -> None:
+        """Handle dry-run mode output."""
+        self.stdout.write("")
+        self.stdout.write(self.style.WARNING("Dry-run mode: previewing blocks to backfill (no tasks will be executed)"))
+        self.stdout.write("")
+
+        # Get network type
+        scheduler._current_head_cache = scheduler.subtensor.get_current_block()
+        network_type = scheduler._get_network_type_for_block(from_block)
+
+        self.stdout.write(f"Block range: {from_block} -> {to_block} ({total_blocks} blocks)")
+        operator = ">" if network_type == "archive" else "<="
+        self.stdout.write(f"Network: {network_type} (blocks {operator}{ARCHIVE_BLOCK_THRESHOLD} behind head)")
+        self.stdout.write(f"Current head: {scheduler._current_head_cache}")
+        self.stdout.write("")
+
+        # Show registry items
+        self.stdout.write("Registry items:")
+        for registry_item in scheduler.block_processor.registry.get_functions():
+            self.stdout.write(f"  - {registry_item.executable_path}")
+        self.stdout.write("")
+
+        # Run dry-run
+        self.stdout.write("Analyzing blocks...")
+        stats = scheduler.start()
+
+        if stats is None:
+            self.stderr.write(self.style.ERROR("Dry-run failed"))
+            return
+
+        # Output summary
+        self.stdout.write("")
+        self.stdout.write(self.style.SUCCESS("Summary:"))
+        self.stdout.write(f"  Total blocks in range: {stats.total_blocks}")
+        self.stdout.write(f"  Already processed (all tasks done): {stats.already_processed}")
+        self.stdout.write(f"  Blocks needing tasks: {stats.blocks_needing_tasks}")
+        self.stdout.write(f"  Estimated tasks to submit: {stats.estimated_tasks}")
+
+        if rate_limit > 0 and stats.blocks_needing_tasks > 0:
+            estimated_seconds = stats.blocks_needing_tasks * rate_limit
+            if estimated_seconds < 60:
+                time_str = f"~{estimated_seconds:.0f} seconds"
+            elif estimated_seconds < 3600:
+                time_str = f"~{estimated_seconds / 60:.1f} minutes"
+            else:
+                time_str = f"~{estimated_seconds / 3600:.1f} hours"
+            self.stdout.write(f"  Estimated time at {rate_limit}s rate limit: {time_str}")
+
+    def _handle_backfill(self, scheduler, from_block: int, to_block: int, total_blocks: int, rate_limit: float) -> None:
+        """Handle actual backfill execution."""
+        self.stdout.write("")
+        self.stdout.write(f"Starting backfill: {from_block} -> {to_block} ({total_blocks} blocks)")
+        self.stdout.write(f"Rate limit: {rate_limit} seconds between blocks")
+
+        if rate_limit > 0:
+            estimated_seconds = total_blocks * rate_limit
+            if estimated_seconds < 60:
+                time_str = f"~{estimated_seconds:.0f} seconds"
+            elif estimated_seconds < 3600:
+                time_str = f"~{estimated_seconds / 60:.1f} minutes"
+            else:
+                time_str = f"~{estimated_seconds / 3600:.1f} hours"
+            self.stdout.write(f"Estimated max time: {time_str}")
+
+        self.stdout.write("")
+        self.stdout.write("Press Ctrl+C to stop gracefully...")
+        self.stdout.write("")
+
+        scheduler.start()
+
+        self.stdout.write(self.style.SUCCESS("Backfill completed"))

abstract_block_dumper/v1/decorators.py

@@ -1,3 +1,4 @@
+import inspect
 from collections.abc import Callable
 from typing import Any, cast
 
@@ -67,12 +68,18 @@ def _celery_task_wrapper(
 ) -> dict[str, Any] | None:
     executable_path = abd_utils.get_executable_path(func)
 
+    # Extract runtime hints that shouldn't be stored in DB
+    use_archive_network = kwargs.pop("_use_archive_network", False)
+
+    # Create db_kwargs without runtime hints for DB lookup
+    db_kwargs = {k: v for k, v in kwargs.items() if not k.startswith("_")}
+
     with transaction.atomic():
         try:
             task_attempt = TaskAttempt.objects.select_for_update(nowait=True).get(
                 block_number=block_number,
                 executable_path=executable_path,
-                args_json=abd_utils.serialize_args(kwargs),
+                args_json=abd_utils.serialize_args(db_kwargs),
             )
         except TaskAttempt.DoesNotExist as exc:
             msg = "TaskAttempt not found - task may have been canceled directly"
@@ -96,6 +103,8 @@ def _celery_task_wrapper(
 
         # Start task execution
         try:
+            # Pass _use_archive_network only if the function accepts **kwargs
+            # Otherwise, strip it to avoid TypeError
             execution_kwargs = {"block_number": block_number, **kwargs}
             logger.info(
                 "Starting task execution",
@@ -103,9 +112,15 @@ def _celery_task_wrapper(
                 block_number=block_number,
                 executable_path=executable_path,
                 celery_task_id=task_attempt.celery_task_id,
-                execution_kwargs=execution_kwargs,
+                use_archive_network=use_archive_network,
             )
 
+            # Check if function accepts **kwargs before adding _use_archive_network
+            sig = inspect.signature(func)
+            has_var_keyword = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values())
+            if has_var_keyword:
+                execution_kwargs["_use_archive_network"] = use_archive_network
+
             result = func(**execution_kwargs)
 
             abd_dal.task_mark_as_success(task_attempt, result)

{abstract_block_dumper-0.0.6.dist-info → abstract_block_dumper-0.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract-block-dumper
-Version: 0.0.6
+Version: 0.0.8
 Project-URL: Source, https://github.com/bactensor/abstract-block-dumper
 Project-URL: Issue Tracker, https://github.com/bactensor/abstract-block-dumper/issues
 Author-email: Reef Technologies <opensource@reef.pl>
@@ -18,6 +18,8 @@ Requires-Dist: bittensor>=9.10.1
 Requires-Dist: celery>=5.3
 Requires-Dist: django<6.0,>=3.2
 Requires-Dist: structlog>=25.4.0
+Provides-Extra: prometheus
+Requires-Dist: prometheus-client>=0.17.0; extra == 'prometheus'
 Description-Content-Type: text/markdown
 
 # Abstract Block Dumper

{abstract_block_dumper-0.0.6.dist-info → abstract_block_dumper-0.0.8.dist-info}/RECORD

@@ -1,5 +1,5 @@
 abstract_block_dumper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_version.py,sha256=7MyqQ3iPP2mJruPfRYGCNCq1z7_Nk7c-eyYecYITxsY,704
+abstract_block_dumper/_version.py,sha256=j-ar4gJGiWIawqKXhvv9hGJWLfwu0tISl-0GV97B7a0,704
 abstract_block_dumper/admin.py,sha256=3J3I_QOKFgfMNpTXW-rTQGO_q5Ls6uNuL0FkPVdIsYg,1654
 abstract_block_dumper/apps.py,sha256=DXATdrjsL3T2IletTbKeD6unr8ScLaxg7wz0nAHTAns,215
 abstract_block_dumper/models.py,sha256=MO9824dmHB6xF3PrFE_RERh7whVjQtS4tt6QA0wSbg0,2022
@@ -8,22 +8,25 @@ abstract_block_dumper/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 abstract_block_dumper/_internal/discovery.py,sha256=sISOL8vq6rC0pOndrCfWKDZjyYwzzZIChG-BH9mteq0,745
 abstract_block_dumper/_internal/exceptions.py,sha256=jVXQ8b3gneno2XYvO0XisJPMlkAWb6H5u10egIpPJ4k,335
 abstract_block_dumper/_internal/dal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_internal/dal/django_dal.py,sha256=pBGEFeo_U0ac2Za-dwzJvf04Ng8lP51aR60c_DUrGIw,5426
-abstract_block_dumper/_internal/dal/memory_registry.py,sha256=yMNF7jrvWGF-S1pqyR2zOCNLWwrdsImcvV6cGqu1wYE,2972
+abstract_block_dumper/_internal/dal/django_dal.py,sha256=i9jocanfptjXw5lfE2xBYvx5mo1g98IoMjlS-WjGP88,5623
+abstract_block_dumper/_internal/dal/memory_registry.py,sha256=m9Yms-cuemi9_5q_Kn_zsJnxDPEiuAUkESIAltD60QY,2943
 abstract_block_dumper/_internal/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_internal/services/block_processor.py,sha256=wB-zeft3Ys8zmqCdF_v12rXd6umNWvGfy2Ts6XSGkL8,8132
-abstract_block_dumper/_internal/services/executor.py,sha256=ZZmQ9TzoNEoAE4amiU8lHRsTfP7YusUkWXasrArfo2g,1806
-abstract_block_dumper/_internal/services/scheduler.py,sha256=NrT3t0oVR-osf50tWWqcxojkVkxhd2PHsk0PuXD5RMc,3593
-abstract_block_dumper/_internal/services/utils.py,sha256=Y8b8KdKn53mcuWchw6b5EJq9ipO4p1FFf6g_Fpbg7cQ,1273
+abstract_block_dumper/_internal/services/backfill_scheduler.py,sha256=PTBs3tN4LLvFYuXKVsHVpme66RAsnPt166RWlRNA4xQ,15955
+abstract_block_dumper/_internal/services/block_processor.py,sha256=NC7p1oD38FpaZb6EbykBolP32uY069abumOvXrjOBV0,6644
+abstract_block_dumper/_internal/services/executor.py,sha256=vuAALKuP7KTQuQyG-P8JfrU22Sr-90HzXsIdfYbXZy4,2080
+abstract_block_dumper/_internal/services/metrics.py,sha256=FLhpq63WVccc0N6K1rN2VnV90jywFfAiG2ZxoDPALv0,3929
+abstract_block_dumper/_internal/services/scheduler.py,sha256=1Ls9dn_siEKl6yXJTS_B4NKhA3ZpZxrMJM_whRtEdxk,7453
+abstract_block_dumper/_internal/services/utils.py,sha256=QSs2hBHWOPgNgKPf_ZmADXuqEiqK5mWZp7JblvQgxZQ,1140
 abstract_block_dumper/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 abstract_block_dumper/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+abstract_block_dumper/management/commands/backfill_blocks_v1.py,sha256=EmNUozAZn8uThjCvusZe7poNrw9RYy-MafMg2wu3XeQ,6392
 abstract_block_dumper/management/commands/block_tasks_v1.py,sha256=jSi04ahIKYwlm_dNKCUGL_cmALv1iP-ZjfXrmz0pn-4,880
 abstract_block_dumper/migrations/0001_initial.py,sha256=ImPHC3G6kPkq4Xn_4YVAm4Labh1Xi7PkCRszYRGpTiI,2298
 abstract_block_dumper/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 abstract_block_dumper/v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 abstract_block_dumper/v1/celery.py,sha256=X4IqVs5i6ZpyY7fy1SqMZgsZy4SXP-jK2qG-FYnjU38,1722
-abstract_block_dumper/v1/decorators.py,sha256=SBl8XP9qhKyTdsKaRREW870BZGidEe0C_nmxnwh76lo,8156
+abstract_block_dumper/v1/decorators.py,sha256=Lua91tR-d0juif9VGVxnC8bzhcgyn8_1bSR33hw7IK0,8920
 abstract_block_dumper/v1/tasks.py,sha256=u9iMYdDUqzYT3yPrNwZecHnlweZ3yFipV9BcIWHCbus,2647
-abstract_block_dumper-0.0.6.dist-info/METADATA,sha256=yXyU72VareEjcSb8A4Tur5hWkgk5k6lBLy9cnqR--kY,12902
-abstract_block_dumper-0.0.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-abstract_block_dumper-0.0.6.dist-info/RECORD,,
+abstract_block_dumper-0.0.8.dist-info/METADATA,sha256=O5E4ChDm8gX0XSs_zvImjBgl9Q_DJkNu_rW5KSx-fMk,12993
+abstract_block_dumper-0.0.8.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+abstract_block_dumper-0.0.8.dist-info/RECORD,,

{abstract_block_dumper-0.0.6.dist-info → abstract_block_dumper-0.0.8.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any