abstract-block-dumper 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

abstract_block_dumper/_internal/dal/django_dal.py

@@ -1,8 +1,10 @@
+from collections.abc import Iterator
 from datetime import timedelta
 from typing import Any
 
 from django.conf import settings
 from django.db import transaction
+from django.db.models import Max
 from django.db.models.query import QuerySet
 from django.utils import timezone
 
@@ -10,19 +12,21 @@ import abstract_block_dumper._internal.services.utils as abd_utils
 import abstract_block_dumper.models as abd_models
 
 
-def get_ready_to_retry_attempts() -> QuerySet[abd_models.TaskAttempt]:
-    return abd_models.TaskAttempt.objects.filter(
-        next_retry_at__isnull=False,
-        next_retry_at__lte=timezone.now(),
-        attempt_count__lt=abd_utils.get_max_attempt_limit(),
-    ).exclude(
-        status=abd_models.TaskAttempt.Status.SUCCESS,
+def get_ready_to_retry_attempts() -> Iterator[abd_models.TaskAttempt]:
+    return (
+        abd_models.TaskAttempt.objects.filter(
+            next_retry_at__isnull=False,
+            next_retry_at__lte=timezone.now(),
+            attempt_count__lt=abd_utils.get_max_attempt_limit(),
+        )
+        .exclude(
+            status=abd_models.TaskAttempt.Status.SUCCESS,
+        )
+        .iterator()
     )
 
 
 def executed_block_numbers(executable_path: str, args_json: str, from_block: int, to_block: int) -> set[int]:
-    # Use iterator() to avoid Django's QuerySet caching which causes memory leaks
-    # during long-running backfill operations
     block_numbers = (
         abd_models.TaskAttempt.objects.filter(
             executable_path=executable_path,
@@ -151,7 +155,5 @@ def task_create_or_get_pending(
 
 
 def get_the_latest_executed_block_number() -> int | None:
-    qs = abd_models.TaskAttempt.objects.order_by("-block_number").first()
-    if qs:
-        return qs.block_number
-    return None
+    result = abd_models.TaskAttempt.objects.aggregate(max_block=Max("block_number"))
+    return result["max_block"]

abstract_block_dumper/_internal/providers/bittensor_client.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import bittensor as bt
+import structlog
+
+import abstract_block_dumper._internal.services.utils as abd_utils
+
+if TYPE_CHECKING:
+    import types
+
+logger = structlog.get_logger(__name__)
+
+
+# Blocks older than this threshold from current head require archive network
+ARCHIVE_BLOCK_THRESHOLD = 300
+
+
+class BittensorConnectionClient:
+    """
+    Manages connections to regular and archive Bittensor subtensor networks.
+
+    Supports context manager protocol for safe connection cleanup:
+        with BittensorConnectionClient(network="finney") as client:
+            block = client.subtensor.get_current_block()
+    """
+
+    def __init__(self, network: str) -> None:
+        self.network = network
+        self._subtensor: bt.Subtensor | None = None
+        self._archive_subtensor: bt.Subtensor | None = None
+        self._current_block_cache: int | None = None
+
+    def __enter__(self) -> BittensorConnectionClient:
+        """Context manager entry."""
+        return self
+
+    def __exit__(
+        self,
+        _exc_type: type[BaseException] | None,
+        _exc_val: BaseException | None,
+        _exc_tb: types.TracebackType | None,
+    ) -> None:
+        """Context manager exit - ensures connections are closed."""
+        self.close()
+
+    def close(self) -> None:
+        """Close all subtensor connections to prevent memory leaks."""
+        if self._subtensor is not None:
+            try:
+                self._subtensor.close()
+            except Exception:
+                logger.warning("Error closing subtensor connection", exc_info=True)
+            self._subtensor = None
+
+        if self._archive_subtensor is not None:
+            try:
+                self._archive_subtensor.close()
+            except Exception:
+                logger.warning("Error closing archive subtensor connection", exc_info=True)
+            self._archive_subtensor = None
+
+        self._current_block_cache = None
+        logger.debug("Subtensor connections closed")
+
+    def get_for_block(self, block_number: int) -> bt.Subtensor:
+        """Get the appropriate subtensor client for the given block number."""
+        raise NotImplementedError
+
+    @property
+    def subtensor(self) -> bt.Subtensor:
+        """Get the regular subtensor connection, creating it if needed."""
+        if self._subtensor is None:
+            self._subtensor = abd_utils.get_bittensor_client(self.network)
+        return self._subtensor
+
+    @subtensor.setter
+    def subtensor(self, value: bt.Subtensor | None) -> None:
+        """Set or reset the subtensor connection."""
+        self._subtensor = value
+
+    @property
+    def archive_subtensor(self) -> bt.Subtensor:
+        """Get the archive subtensor connection, creating it if needed."""
+        if self._archive_subtensor is None:
+            self._archive_subtensor = abd_utils.get_bittensor_client("archive")
+        return self._archive_subtensor
+
+    @archive_subtensor.setter
+    def archive_subtensor(self, value: bt.Subtensor | None) -> None:
+        """Set or reset the archive subtensor connection."""
+        self._archive_subtensor = value
+
+    def get_subtensor_for_block(self, block_number: int) -> bt.Subtensor:
+        """
+        Get the appropriate subtensor for the given block number.
+
+        Uses archive network for blocks older than ARCHIVE_BLOCK_THRESHOLD
+        from the current head.
+        """
+        if self._current_block_cache is None:
+            self._current_block_cache = self.subtensor.get_current_block()
+
+        blocks_behind = self._current_block_cache - block_number
+
+        if blocks_behind > ARCHIVE_BLOCK_THRESHOLD:
+            logger.debug(
+                "Using archive network for old block",
+                block_number=block_number,
+                blocks_behind=blocks_behind,
+            )
+            return self.archive_subtensor
+        return self.subtensor
+
+    def refresh_connections(self) -> None:
+        """Close and reset all subtensor connections to force re-establishment."""
+        self.close()
+        logger.info("Subtensor connections refreshed")

abstract_block_dumper/_internal/services/backfill_scheduler.py

@@ -15,7 +15,7 @@ import structlog
 
 import abstract_block_dumper._internal.dal.django_dal as abd_dal
 import abstract_block_dumper._internal.services.utils as abd_utils
-from abstract_block_dumper._internal.services.block_processor import BlockProcessor, block_processor_factory
+from abstract_block_dumper._internal.services.block_processor import BaseBlockProcessor, block_processor_factory
 from abstract_block_dumper._internal.services.metrics import (
     BlockProcessingTimer,
     increment_archive_network_usage,
@@ -36,8 +36,6 @@ logger = structlog.get_logger(__name__)
 # Blocks older than this threshold from current head require archive network
 ARCHIVE_BLOCK_THRESHOLD = 300
 
-# Progress logging interval
-PROGRESS_LOG_INTERVAL = 100
 ARCHIVE_NETWORK = "archive"
 
 # Memory cleanup interval (every N blocks)
@@ -59,7 +57,7 @@ class BackfillScheduler:
 
     def __init__(
         self,
-        block_processor: BlockProcessor,
+        block_processor: BaseBlockProcessor,
         network: str,
         from_block: int,
         to_block: int,
@@ -281,16 +279,13 @@ class BackfillScheduler:
                     if self._current_head_cache:
                         set_block_lag("backfill", self._current_head_cache - block_number)
 
-                    # Log progress periodically
-                    if processed_count % PROGRESS_LOG_INTERVAL == 0:
-                        progress_pct = (processed_count / total_blocks) * 100
-                        logger.info(
-                            "Backfill progress",
-                            processed=processed_count,
-                            total=total_blocks,
-                            progress_percent=f"{progress_pct:.1f}%",
-                            current_block=block_number,
-                        )
+                    # Log each block being processed
+                    progress_pct = (processed_count / total_blocks) * 100
+                    logger.info(
+                        "Backfilling block",
+                        block=block_number,
+                        progress=f"{processed_count}/{total_blocks} ({progress_pct:.1f}%)",
+                    )
 
                     # Rate limiting between block submissions
                     if block_number < self.to_block and self.rate_limit > 0:

abstract_block_dumper/_internal/services/block_processor.py

@@ -1,4 +1,6 @@
+import itertools
 import time
+from typing import Protocol
 
 import structlog
 from django.db import transaction
@@ -12,6 +14,25 @@ from abstract_block_dumper.models import TaskAttempt
 logger = structlog.get_logger(__name__)
 
 
+class BaseBlockProcessor(Protocol):
+    """Protocol defining the interface for block processors."""
+
+    executor: CeleryExecutor
+    registry: BaseRegistry
+
+    def process_block(self, block_number: int) -> None:
+        """Process a single block - executes registered tasks for this block only."""
+        ...
+
+    def process_registry_item(self, registry_item: RegistryItem, block_number: int) -> None:
+        """Process a single registry item for a given block."""
+        ...
+
+    def recover_failed_retries(self, poll_interval: int, batch_size: int | None = None) -> None:
+        """Recover failed tasks that are ready to be retried."""
+        ...
+
+
 class BlockProcessor:
     def __init__(self, executor: CeleryExecutor, registry: BaseRegistry) -> None:
         self.executor = executor
@@ -59,9 +80,9 @@ class BlockProcessor:
         retry_count = 0
         retry_attempts = abd_dal.get_ready_to_retry_attempts()
 
-        # Apply batch size limit if specified
+        # Apply batch size limit if specified (use islice for iterator compatibility)
         if batch_size is not None:
-            retry_attempts = retry_attempts[:batch_size]
+            retry_attempts = itertools.islice(retry_attempts, batch_size)
 
         for retry_attempt in retry_attempts:
             time.sleep(poll_interval)
@@ -147,7 +168,7 @@ class BlockProcessor:
 def block_processor_factory(
     executor: CeleryExecutor | None = None,
     registry: BaseRegistry | None = None,
-) -> BlockProcessor:
+) -> BaseBlockProcessor:
     return BlockProcessor(
         executor=executor or CeleryExecutor(),
         registry=registry or task_registry,

abstract_block_dumper/_internal/services/scheduler.py

@@ -1,12 +1,12 @@
 import time
+from typing import Protocol
 
-import bittensor as bt
 import structlog
 from django.conf import settings
 
 import abstract_block_dumper._internal.dal.django_dal as abd_dal
-import abstract_block_dumper._internal.services.utils as abd_utils
-from abstract_block_dumper._internal.services.block_processor import BlockProcessor, block_processor_factory
+from abstract_block_dumper._internal.providers.bittensor_client import BittensorConnectionClient
+from abstract_block_dumper._internal.services.block_processor import BaseBlockProcessor, block_processor_factory
 from abstract_block_dumper._internal.services.metrics import (
     BlockProcessingTimer,
     increment_blocks_processed,
@@ -17,83 +17,49 @@ from abstract_block_dumper._internal.services.metrics import (
 
 logger = structlog.get_logger(__name__)
 
-# Blocks older than this threshold from current head require archive network
-ARCHIVE_BLOCK_THRESHOLD = 300
+
+class BlockStateResolver(Protocol):
+    """Protocol defining the interface for block state resolvers."""
+
+    def get_starting_block(self) -> int:
+        """Determine which block to start processing from."""
+        ...
+
+
+class DefaultBlockStateResolver:
+    """Default implementation that reads from settings and database."""
+
+    def __init__(self, bittensor_client: BittensorConnectionClient) -> None:
+        self.bittensor_client = bittensor_client
+
+    def get_starting_block(self) -> int:
+        start_setting = getattr(settings, "BLOCK_DUMPER_START_FROM_BLOCK", None)
+        if start_setting == "current":
+            return self.bittensor_client.subtensor.get_current_block()
+        if isinstance(start_setting, int):
+            return start_setting
+
+        # Default: resume from DB or current
+        return abd_dal.get_the_latest_executed_block_number() or self.bittensor_client.subtensor.get_current_block()
 
 
 class TaskScheduler:
     def __init__(
         self,
-        block_processor: BlockProcessor,
-        network: str,
+        block_processor: BaseBlockProcessor,
+        bittensor_client: BittensorConnectionClient,
+        state_resolver: BlockStateResolver,
         poll_interval: int,
     ) -> None:
         self.block_processor = block_processor
-        self.network = network
         self.poll_interval = poll_interval
-        self.last_processed_block = -1
+        self.bittensor_client = bittensor_client
+        self.last_processed_block = state_resolver.get_starting_block()
         self.is_running = False
-        self._subtensor: bt.Subtensor | None = None
-        self._archive_subtensor: bt.Subtensor | None = None
-        self._current_block_cache: int | None = None
-
-    @property
-    def subtensor(self) -> bt.Subtensor:
-        """Get the regular subtensor connection, creating it if needed."""
-        if self._subtensor is None:
-            self._subtensor = abd_utils.get_bittensor_client(self.network)
-        return self._subtensor
-
-    @subtensor.setter
-    def subtensor(self, value: bt.Subtensor | None) -> None:
-        """Set or reset the subtensor connection."""
-        self._subtensor = value
-
-    @property
-    def archive_subtensor(self) -> bt.Subtensor:
-        """Get the archive subtensor connection, creating it if needed."""
-        if self._archive_subtensor is None:
-            self._archive_subtensor = abd_utils.get_bittensor_client("archive")
-        return self._archive_subtensor
-
-    @archive_subtensor.setter
-    def archive_subtensor(self, value: bt.Subtensor | None) -> None:
-        """Set or reset the archive subtensor connection."""
-        self._archive_subtensor = value
-
-    def get_subtensor_for_block(self, block_number: int) -> bt.Subtensor:
-        """
-        Get the appropriate subtensor for the given block number.
-
-        Uses archive network for blocks older than ARCHIVE_BLOCK_THRESHOLD
-        from the current head.
-        """
-        if self._current_block_cache is None:
-            self._current_block_cache = self.subtensor.get_current_block()
-
-        blocks_behind = self._current_block_cache - block_number
-
-        if blocks_behind > ARCHIVE_BLOCK_THRESHOLD:
-            logger.debug(
-                "Using archive network for old block",
-                block_number=block_number,
-                blocks_behind=blocks_behind,
-            )
-            return self.archive_subtensor
-        return self.subtensor
-
-    def refresh_connections(self) -> None:
-        """Reset all subtensor connections to force re-establishment."""
-        self._subtensor = None
-        self._archive_subtensor = None
-        self._current_block_cache = None
-        logger.info("Subtensor connections reset")
 
     def start(self) -> None:
         self.is_running = True
 
-        self.initialize_last_block()
-
         registered_tasks_count = len(self.block_processor.registry.get_functions())
         set_registered_tasks(registered_tasks_count)
 
@@ -105,12 +71,7 @@ class TaskScheduler:
 
         while self.is_running:
             try:
-                if self._current_block_cache is not None:
-                    self.subtensor = self.get_subtensor_for_block(self._current_block_cache)
-
-                # Update current block cache for archive network decision
-                self._current_block_cache = self.subtensor.get_current_block()
-                current_block = self._current_block_cache
+                current_block = self.bittensor_client.subtensor.get_current_block()
 
                 # Only process the current head block, skip if already processed
                 if current_block != self.last_processed_block:
@@ -129,39 +90,14 @@ class TaskScheduler:
                 self.stop()
                 break
             except Exception:
-                logger.error("Fatal scheduler error", exc_info=True)
-                # resume the loop even if task failed
+                logger.exception("Error in TaskScheduler loop")
                 time.sleep(self.poll_interval)
 
     def stop(self) -> None:
         self.is_running = False
+        self.bittensor_client.close()
         logger.info("TaskScheduler stopped.")
 
-    def initialize_last_block(self) -> None:
-        # Safe getattr in case setting is not defined
-        start_from_block_setting = getattr(settings, "BLOCK_DUMPER_START_FROM_BLOCK", None)
-
-        if start_from_block_setting is not None:
-            if start_from_block_setting == "current":
-                self.last_processed_block = self.subtensor.get_current_block()
-                logger.info("Starting from current blockchain block", block_number=self.last_processed_block)
-
-            elif isinstance(start_from_block_setting, int):
-                self.last_processed_block = start_from_block_setting
-                logger.info("Starting from configured block", block_number=self.last_processed_block)
-            else:
-                error_msg = f"Invalid BLOCK_DUMPER_START_FROM_BLOCK value: {start_from_block_setting}"
-                raise ValueError(error_msg)
-        else:
-            # Default behavior - resume from database
-            last_block_number = abd_dal.get_the_latest_executed_block_number()
-
-            self.last_processed_block = last_block_number or self.subtensor.get_current_block()
-            logger.info(
-                "Resume from the last database block or start from the current block",
-                last_processed_block=self.last_processed_block,
-            )
-
 
 def task_scheduler_factory(network: str = "finney") -> TaskScheduler:
     """
@@ -171,8 +107,11 @@ def task_scheduler_factory(network: str = "finney") -> TaskScheduler:
         network (str): Bittensor network name. Defaults to "finney"
 
     """
+    bittensor_client = BittensorConnectionClient(network=network)
+    state_resolver = DefaultBlockStateResolver(bittensor_client=bittensor_client)
     return TaskScheduler(
         block_processor=block_processor_factory(),
-        network=network,
-        poll_interval=getattr(settings, "BLOCK_DUMPER_POLL_INTERVAL", 1),
+        poll_interval=getattr(settings, "BLOCK_DUMPER_POLL_INTERVAL", 5),
+        bittensor_client=bittensor_client,
+        state_resolver=state_resolver,
     )

abstract_block_dumper/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.0'
-__version_tuple__ = version_tuple = (0, 1, 0)
+__version__ = version = '0.1.2'
+__version_tuple__ = version_tuple = (0, 1, 2)
 
 __commit_id__ = commit_id = None

{abstract_block_dumper-0.1.0.dist-info → abstract_block_dumper-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract-block-dumper
-Version: 0.1.0
+Version: 0.1.2
 Project-URL: Source, https://github.com/bactensor/abstract-block-dumper
 Project-URL: Issue Tracker, https://github.com/bactensor/abstract-block-dumper/issues
 Author-email: Reef Technologies <opensource@reef.pl>

{abstract_block_dumper-0.1.0.dist-info → abstract_block_dumper-0.1.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 abstract_block_dumper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_version.py,sha256=5jwwVncvCiTnhOedfkzzxmxsggwmTBORdFL_4wq0ZeY,704
+abstract_block_dumper/_version.py,sha256=Ok5oAXdWgR9aghaFXTafTeDW6sYO3uVe6d2Nket57R4,704
 abstract_block_dumper/admin.py,sha256=3J3I_QOKFgfMNpTXW-rTQGO_q5Ls6uNuL0FkPVdIsYg,1654
 abstract_block_dumper/apps.py,sha256=DXATdrjsL3T2IletTbKeD6unr8ScLaxg7wz0nAHTAns,215
 abstract_block_dumper/models.py,sha256=MO9824dmHB6xF3PrFE_RERh7whVjQtS4tt6QA0wSbg0,2022
@@ -8,14 +8,16 @@ abstract_block_dumper/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 abstract_block_dumper/_internal/discovery.py,sha256=sISOL8vq6rC0pOndrCfWKDZjyYwzzZIChG-BH9mteq0,745
 abstract_block_dumper/_internal/exceptions.py,sha256=jVXQ8b3gneno2XYvO0XisJPMlkAWb6H5u10egIpPJ4k,335
 abstract_block_dumper/_internal/dal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_internal/dal/django_dal.py,sha256=i9jocanfptjXw5lfE2xBYvx5mo1g98IoMjlS-WjGP88,5623
+abstract_block_dumper/_internal/dal/django_dal.py,sha256=QbDsikUthIAhVC_FwSynUUdQL3OWlCo3_Cg65M91Cb4,5618
 abstract_block_dumper/_internal/dal/memory_registry.py,sha256=m9Yms-cuemi9_5q_Kn_zsJnxDPEiuAUkESIAltD60QY,2943
+abstract_block_dumper/_internal/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+abstract_block_dumper/_internal/providers/bittensor_client.py,sha256=wlKjFrGN4Q2DfQyD_Fx-eH83ZMB6AbzLs5keYq6FGUw,4124
 abstract_block_dumper/_internal/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_internal/services/backfill_scheduler.py,sha256=XgsVYXaz6pR4PBA9giesjhR74x1qLX2281-eQgM5qhs,16311
-abstract_block_dumper/_internal/services/block_processor.py,sha256=NC7p1oD38FpaZb6EbykBolP32uY069abumOvXrjOBV0,6644
+abstract_block_dumper/_internal/services/backfill_scheduler.py,sha256=3X9NRXCamnb1jUI47sUXqmsrQ1nBzK0BLKe5CeLt00E,16091
+abstract_block_dumper/_internal/services/block_processor.py,sha256=P8_LZR4ZSyNKbtnqFbAUkpT1XmEh9yX5Pgj5J__IwdA,7409
 abstract_block_dumper/_internal/services/executor.py,sha256=WhpHhOAi4cI-qdEE8-DSt9xZwooOpSc9_uDMQBBoHUM,2317
 abstract_block_dumper/_internal/services/metrics.py,sha256=Gg-PQYZ98caaS52wm1EqhtPURXlfrVjk2t3-8nccqfo,7821
-abstract_block_dumper/_internal/services/scheduler.py,sha256=pFVNV1YBHujNIRW9kq_1bxaa_W1Bn_pr7DrgwKaUItw,6779
+abstract_block_dumper/_internal/services/scheduler.py,sha256=BIQ7c-HYSebW3CKq5ynsMZjULEO9c5YP0qWFN1aqg24,4164
 abstract_block_dumper/_internal/services/utils.py,sha256=QZxdQyWIcUnezyVmegS4g3x3BoB3-oijYJ_i9nLQWHo,1140
 abstract_block_dumper/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 abstract_block_dumper/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -27,6 +29,6 @@ abstract_block_dumper/v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 abstract_block_dumper/v1/celery.py,sha256=X4IqVs5i6ZpyY7fy1SqMZgsZy4SXP-jK2qG-FYnjU38,1722
 abstract_block_dumper/v1/decorators.py,sha256=yQglsy1dU1u7ShwaTqZLahDcybHmetibTIOi53o_ZOM,9829
 abstract_block_dumper/v1/tasks.py,sha256=u9iMYdDUqzYT3yPrNwZecHnlweZ3yFipV9BcIWHCbus,2647
-abstract_block_dumper-0.1.0.dist-info/METADATA,sha256=UypVClIjNRQlbzkGmmR2Indrw3c-2gbrlbjHaXin_ys,12993
-abstract_block_dumper-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-abstract_block_dumper-0.1.0.dist-info/RECORD,,
+abstract_block_dumper-0.1.2.dist-info/METADATA,sha256=6y5tq_8Wp3JNHuYATBIx_XQ2I0mXqBU5tgyf97rHahc,12993
+abstract_block_dumper-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+abstract_block_dumper-0.1.2.dist-info/RECORD,,