abstract-block-dumper 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

abstract_block_dumper/_internal/dal/django_dal.py

@@ -1,8 +1,10 @@
+from collections.abc import Iterator
 from datetime import timedelta
 from typing import Any
 
 from django.conf import settings
 from django.db import transaction
+from django.db.models import Max
 from django.db.models.query import QuerySet
 from django.utils import timezone
 
@@ -10,19 +12,21 @@ import abstract_block_dumper._internal.services.utils as abd_utils
 import abstract_block_dumper.models as abd_models
 
 
-def get_ready_to_retry_attempts() -> QuerySet[abd_models.TaskAttempt]:
-    return abd_models.TaskAttempt.objects.filter(
-        next_retry_at__isnull=False,
-        next_retry_at__lte=timezone.now(),
-        attempt_count__lt=abd_utils.get_max_attempt_limit(),
-    ).exclude(
-        status=abd_models.TaskAttempt.Status.SUCCESS,
+def get_ready_to_retry_attempts() -> Iterator[abd_models.TaskAttempt]:
+    return (
+        abd_models.TaskAttempt.objects.filter(
+            next_retry_at__isnull=False,
+            next_retry_at__lte=timezone.now(),
+            attempt_count__lt=abd_utils.get_max_attempt_limit(),
+        )
+        .exclude(
+            status=abd_models.TaskAttempt.Status.SUCCESS,
+        )
+        .iterator()
     )
 
 
 def executed_block_numbers(executable_path: str, args_json: str, from_block: int, to_block: int) -> set[int]:
-    # Use iterator() to avoid Django's QuerySet caching which causes memory leaks
-    # during long-running backfill operations
     block_numbers = (
         abd_models.TaskAttempt.objects.filter(
             executable_path=executable_path,
@@ -151,7 +155,5 @@ def task_create_or_get_pending(
 
 
 def get_the_latest_executed_block_number() -> int | None:
-    qs = abd_models.TaskAttempt.objects.order_by("-block_number").first()
-    if qs:
-        return qs.block_number
-    return None
+    result = abd_models.TaskAttempt.objects.aggregate(max_block=Max("block_number"))
+    return result["max_block"]

abstract_block_dumper/_internal/providers/bittensor_client.py

@@ -0,0 +1,78 @@
+import bittensor as bt
+import structlog
+
+import abstract_block_dumper._internal.services.utils as abd_utils
+
+logger = structlog.get_logger(__name__)
+
+
+# Blocks older than this threshold from current head require archive network
+ARCHIVE_BLOCK_THRESHOLD = 300
+
+
+class BittensorConnectionClient:
+    """
+    Manages connections to regular and archive Bittensor subtensor networks.
+    """
+
+    def __init__(self, network: str) -> None:
+        self.network = network
+        self._subtensor: bt.Subtensor | None = None
+        self._archive_subtensor: bt.Subtensor | None = None
+        self._current_block_cache: int | None = None
+
+    def get_for_block(self, block_number: int) -> bt.Subtensor:
+        """Get the appropriate subtensor client for the given block number."""
+        raise NotImplementedError
+
+    @property
+    def subtensor(self) -> bt.Subtensor:
+        """Get the regular subtensor connection, creating it if needed."""
+        if self._subtensor is None:
+            self._subtensor = abd_utils.get_bittensor_client(self.network)
+        return self._subtensor
+
+    @subtensor.setter
+    def subtensor(self, value: bt.Subtensor | None) -> None:
+        """Set or reset the subtensor connection."""
+        self._subtensor = value
+
+    @property
+    def archive_subtensor(self) -> bt.Subtensor:
+        """Get the archive subtensor connection, creating it if needed."""
+        if self._archive_subtensor is None:
+            self._archive_subtensor = abd_utils.get_bittensor_client("archive")
+        return self._archive_subtensor
+
+    @archive_subtensor.setter
+    def archive_subtensor(self, value: bt.Subtensor | None) -> None:
+        """Set or reset the archive subtensor connection."""
+        self._archive_subtensor = value
+
+    def get_subtensor_for_block(self, block_number: int) -> bt.Subtensor:
+        """
+        Get the appropriate subtensor for the given block number.
+
+        Uses archive network for blocks older than ARCHIVE_BLOCK_THRESHOLD
+        from the current head.
+        """
+        if self._current_block_cache is None:
+            self._current_block_cache = self.subtensor.get_current_block()
+
+        blocks_behind = self._current_block_cache - block_number
+
+        if blocks_behind > ARCHIVE_BLOCK_THRESHOLD:
+            logger.debug(
+                "Using archive network for old block",
+                block_number=block_number,
+                blocks_behind=blocks_behind,
+            )
+            return self.archive_subtensor
+        return self.subtensor
+
+    def refresh_connections(self) -> None:
+        """Reset all subtensor connections to force re-establishment."""
+        self._subtensor = None
+        self._archive_subtensor = None
+        self._current_block_cache = None
+        logger.info("Subtensor connections reset")

abstract_block_dumper/_internal/services/backfill_scheduler.py

@@ -15,7 +15,7 @@ import structlog
 
 import abstract_block_dumper._internal.dal.django_dal as abd_dal
 import abstract_block_dumper._internal.services.utils as abd_utils
-from abstract_block_dumper._internal.services.block_processor import BlockProcessor, block_processor_factory
+from abstract_block_dumper._internal.services.block_processor import BaseBlockProcessor, block_processor_factory
 from abstract_block_dumper._internal.services.metrics import (
     BlockProcessingTimer,
     increment_archive_network_usage,
@@ -59,7 +59,7 @@ class BackfillScheduler:
 
     def __init__(
         self,
-        block_processor: BlockProcessor,
+        block_processor: BaseBlockProcessor,
         network: str,
         from_block: int,
         to_block: int,

abstract_block_dumper/_internal/services/block_processor.py

@@ -1,4 +1,6 @@
+import itertools
 import time
+from typing import Protocol
 
 import structlog
 from django.db import transaction
@@ -12,6 +14,25 @@ from abstract_block_dumper.models import TaskAttempt
 logger = structlog.get_logger(__name__)
 
 
+class BaseBlockProcessor(Protocol):
+    """Protocol defining the interface for block processors."""
+
+    executor: CeleryExecutor
+    registry: BaseRegistry
+
+    def process_block(self, block_number: int) -> None:
+        """Process a single block - executes registered tasks for this block only."""
+        ...
+
+    def process_registry_item(self, registry_item: RegistryItem, block_number: int) -> None:
+        """Process a single registry item for a given block."""
+        ...
+
+    def recover_failed_retries(self, poll_interval: int, batch_size: int | None = None) -> None:
+        """Recover failed tasks that are ready to be retried."""
+        ...
+
+
 class BlockProcessor:
     def __init__(self, executor: CeleryExecutor, registry: BaseRegistry) -> None:
         self.executor = executor
@@ -59,9 +80,9 @@ class BlockProcessor:
         retry_count = 0
         retry_attempts = abd_dal.get_ready_to_retry_attempts()
 
-        # Apply batch size limit if specified
+        # Apply batch size limit if specified (use islice for iterator compatibility)
         if batch_size is not None:
-            retry_attempts = retry_attempts[:batch_size]
+            retry_attempts = itertools.islice(retry_attempts, batch_size)
 
         for retry_attempt in retry_attempts:
             time.sleep(poll_interval)
@@ -147,7 +168,7 @@ class BlockProcessor:
 def block_processor_factory(
     executor: CeleryExecutor | None = None,
     registry: BaseRegistry | None = None,
-) -> BlockProcessor:
+) -> BaseBlockProcessor:
     return BlockProcessor(
         executor=executor or CeleryExecutor(),
         registry=registry or task_registry,

abstract_block_dumper/_internal/services/scheduler.py

@@ -1,12 +1,13 @@
 import time
+from typing import Protocol
 
-import bittensor as bt
 import structlog
+from django import db
 from django.conf import settings
 
 import abstract_block_dumper._internal.dal.django_dal as abd_dal
-import abstract_block_dumper._internal.services.utils as abd_utils
-from abstract_block_dumper._internal.services.block_processor import BlockProcessor, block_processor_factory
+from abstract_block_dumper._internal.providers.bittensor_client import BittensorConnectionClient
+from abstract_block_dumper._internal.services.block_processor import BaseBlockProcessor, block_processor_factory
 from abstract_block_dumper._internal.services.metrics import (
     BlockProcessingTimer,
     increment_blocks_processed,
@@ -15,85 +16,54 @@ from abstract_block_dumper._internal.services.metrics import (
     set_registered_tasks,
 )
 
+# Refresh bittensor connections every N blocks to prevent memory leaks from internal caches
+CONNECTION_REFRESH_INTERVAL = 1000
+
 logger = structlog.get_logger(__name__)
 
-# Blocks older than this threshold from current head require archive network
-ARCHIVE_BLOCK_THRESHOLD = 300
+
+class BlockStateResolver(Protocol):
+    """Protocol defining the interface for block state resolvers."""
+
+    def get_starting_block(self) -> int:
+        """Determine which block to start processing from."""
+        ...
+
+
+class DefaultBlockStateResolver:
+    """Default implementation that reads from settings and database."""
+
+    def __init__(self, bittensor_client: BittensorConnectionClient) -> None:
+        self.bittensor_client = bittensor_client
+
+    def get_starting_block(self) -> int:
+        start_setting = getattr(settings, "BLOCK_DUMPER_START_FROM_BLOCK", None)
+        if start_setting == "current":
+            return self.bittensor_client.subtensor.get_current_block()
+        if isinstance(start_setting, int):
+            return start_setting
+        # Default: resume from DB or current
+        return abd_dal.get_the_latest_executed_block_number() or self.bittensor_client.subtensor.get_current_block()
 
 
 class TaskScheduler:
     def __init__(
         self,
-        block_processor: BlockProcessor,
-        network: str,
+        block_processor: BaseBlockProcessor,
+        bittensor_client: BittensorConnectionClient,
+        state_resolver: BlockStateResolver,
         poll_interval: int,
     ) -> None:
         self.block_processor = block_processor
-        self.network = network
         self.poll_interval = poll_interval
-        self.last_processed_block = -1
+        self.bittensor_client = bittensor_client
+        self.last_processed_block = state_resolver.get_starting_block()
         self.is_running = False
-        self._subtensor: bt.Subtensor | None = None
-        self._archive_subtensor: bt.Subtensor | None = None
-        self._current_block_cache: int | None = None
-
-    @property
-    def subtensor(self) -> bt.Subtensor:
-        """Get the regular subtensor connection, creating it if needed."""
-        if self._subtensor is None:
-            self._subtensor = abd_utils.get_bittensor_client(self.network)
-        return self._subtensor
-
-    @subtensor.setter
-    def subtensor(self, value: bt.Subtensor | None) -> None:
-        """Set or reset the subtensor connection."""
-        self._subtensor = value
-
-    @property
-    def archive_subtensor(self) -> bt.Subtensor:
-        """Get the archive subtensor connection, creating it if needed."""
-        if self._archive_subtensor is None:
-            self._archive_subtensor = abd_utils.get_bittensor_client("archive")
-        return self._archive_subtensor
-
-    @archive_subtensor.setter
-    def archive_subtensor(self, value: bt.Subtensor | None) -> None:
-        """Set or reset the archive subtensor connection."""
-        self._archive_subtensor = value
-
-    def get_subtensor_for_block(self, block_number: int) -> bt.Subtensor:
-        """
-        Get the appropriate subtensor for the given block number.
-
-        Uses archive network for blocks older than ARCHIVE_BLOCK_THRESHOLD
-        from the current head.
-        """
-        if self._current_block_cache is None:
-            self._current_block_cache = self.subtensor.get_current_block()
-
-        blocks_behind = self._current_block_cache - block_number
-
-        if blocks_behind > ARCHIVE_BLOCK_THRESHOLD:
-            logger.debug(
-                "Using archive network for old block",
-                block_number=block_number,
-                blocks_behind=blocks_behind,
-            )
-            return self.archive_subtensor
-        return self.subtensor
-
-    def refresh_connections(self) -> None:
-        """Reset all subtensor connections to force re-establishment."""
-        self._subtensor = None
-        self._archive_subtensor = None
-        self._current_block_cache = None
-        logger.info("Subtensor connections reset")
+        self._blocks_since_refresh = 0
 
     def start(self) -> None:
         self.is_running = True
 
-        self.initialize_last_block()
-
         registered_tasks_count = len(self.block_processor.registry.get_functions())
         set_registered_tasks(registered_tasks_count)
 
@@ -105,12 +75,7 @@ class TaskScheduler:
 
         while self.is_running:
             try:
-                if self._current_block_cache is not None:
-                    self.subtensor = self.get_subtensor_for_block(self._current_block_cache)
-
-                # Update current block cache for archive network decision
-                self._current_block_cache = self.subtensor.get_current_block()
-                current_block = self._current_block_cache
+                current_block = self.bittensor_client.subtensor.get_current_block()
 
                 # Only process the current head block, skip if already processed
                 if current_block != self.last_processed_block:
@@ -121,6 +86,11 @@ class TaskScheduler:
                     increment_blocks_processed("realtime")
                     set_block_lag("realtime", 0)  # Head-only mode has no lag
                     self.last_processed_block = current_block
+                    self._blocks_since_refresh += 1
+
+                    # Periodic memory cleanup
+                    if self._blocks_since_refresh >= CONNECTION_REFRESH_INTERVAL:
+                        self._perform_cleanup()
 
                 time.sleep(self.poll_interval)
 
@@ -129,38 +99,23 @@ class TaskScheduler:
                 self.stop()
                 break
             except Exception:
-                logger.error("Fatal scheduler error", exc_info=True)
-                # resume the loop even if task failed
+                logger.exception("Error in TaskScheduler loop")
                 time.sleep(self.poll_interval)
 
     def stop(self) -> None:
         self.is_running = False
         logger.info("TaskScheduler stopped.")
 
-    def initialize_last_block(self) -> None:
-        # Safe getattr in case setting is not defined
-        start_from_block_setting = getattr(settings, "BLOCK_DUMPER_START_FROM_BLOCK", None)
-
-        if start_from_block_setting is not None:
-            if start_from_block_setting == "current":
-                self.last_processed_block = self.subtensor.get_current_block()
-                logger.info("Starting from current blockchain block", block_number=self.last_processed_block)
+    def _perform_cleanup(self) -> None:
+        """Perform periodic memory cleanup to prevent leaks in long-running processes."""
+        # Reset bittensor connections to clear internal caches
+        self.bittensor_client.refresh_connections()
 
-            elif isinstance(start_from_block_setting, int):
-                self.last_processed_block = start_from_block_setting
-                logger.info("Starting from configured block", block_number=self.last_processed_block)
-            else:
-                error_msg = f"Invalid BLOCK_DUMPER_START_FROM_BLOCK value: {start_from_block_setting}"
-                raise ValueError(error_msg)
-        else:
-            # Default behavior - resume from database
-            last_block_number = abd_dal.get_the_latest_executed_block_number()
+        # Clear Django's query log (only accumulates if DEBUG=True)
+        db.reset_queries()
 
-            self.last_processed_block = last_block_number or self.subtensor.get_current_block()
-            logger.info(
-                "Resume from the last database block or start from the current block",
-                last_processed_block=self.last_processed_block,
-            )
+        self._blocks_since_refresh = 0
+        logger.debug("Memory cleanup performed", blocks_processed=CONNECTION_REFRESH_INTERVAL)
 
 
 def task_scheduler_factory(network: str = "finney") -> TaskScheduler:
@@ -171,8 +126,11 @@ def task_scheduler_factory(network: str = "finney") -> TaskScheduler:
         network (str): Bittensor network name. Defaults to "finney"
 
     """
+    bittensor_client = BittensorConnectionClient(network=network)
+    state_resolver = DefaultBlockStateResolver(bittensor_client=bittensor_client)
     return TaskScheduler(
         block_processor=block_processor_factory(),
-        network=network,
-        poll_interval=getattr(settings, "BLOCK_DUMPER_POLL_INTERVAL", 1),
+        poll_interval=getattr(settings, "BLOCK_DUMPER_POLL_INTERVAL", 5),
+        bittensor_client=bittensor_client,
+        state_resolver=state_resolver,
     )

abstract_block_dumper/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.0'
-__version_tuple__ = version_tuple = (0, 1, 0)
+__version__ = version = '0.1.1'
+__version_tuple__ = version_tuple = (0, 1, 1)
 
 __commit_id__ = commit_id = None

{abstract_block_dumper-0.1.0.dist-info → abstract_block_dumper-0.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract-block-dumper
-Version: 0.1.0
+Version: 0.1.1
 Project-URL: Source, https://github.com/bactensor/abstract-block-dumper
 Project-URL: Issue Tracker, https://github.com/bactensor/abstract-block-dumper/issues
 Author-email: Reef Technologies <opensource@reef.pl>

{abstract_block_dumper-0.1.0.dist-info → abstract_block_dumper-0.1.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 abstract_block_dumper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_version.py,sha256=5jwwVncvCiTnhOedfkzzxmxsggwmTBORdFL_4wq0ZeY,704
+abstract_block_dumper/_version.py,sha256=m8HxkqoKGw_wAJtc4ZokpJKNLXqp4zwnNhbnfDtro7w,704
 abstract_block_dumper/admin.py,sha256=3J3I_QOKFgfMNpTXW-rTQGO_q5Ls6uNuL0FkPVdIsYg,1654
 abstract_block_dumper/apps.py,sha256=DXATdrjsL3T2IletTbKeD6unr8ScLaxg7wz0nAHTAns,215
 abstract_block_dumper/models.py,sha256=MO9824dmHB6xF3PrFE_RERh7whVjQtS4tt6QA0wSbg0,2022
@@ -8,14 +8,16 @@ abstract_block_dumper/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 abstract_block_dumper/_internal/discovery.py,sha256=sISOL8vq6rC0pOndrCfWKDZjyYwzzZIChG-BH9mteq0,745
 abstract_block_dumper/_internal/exceptions.py,sha256=jVXQ8b3gneno2XYvO0XisJPMlkAWb6H5u10egIpPJ4k,335
 abstract_block_dumper/_internal/dal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_internal/dal/django_dal.py,sha256=i9jocanfptjXw5lfE2xBYvx5mo1g98IoMjlS-WjGP88,5623
+abstract_block_dumper/_internal/dal/django_dal.py,sha256=QbDsikUthIAhVC_FwSynUUdQL3OWlCo3_Cg65M91Cb4,5618
 abstract_block_dumper/_internal/dal/memory_registry.py,sha256=m9Yms-cuemi9_5q_Kn_zsJnxDPEiuAUkESIAltD60QY,2943
+abstract_block_dumper/_internal/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+abstract_block_dumper/_internal/providers/bittensor_client.py,sha256=hkUdkhz8wOZInRloyySo3DDN4JSMEiPwLv0YV5IDZ7o,2798
 abstract_block_dumper/_internal/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_internal/services/backfill_scheduler.py,sha256=XgsVYXaz6pR4PBA9giesjhR74x1qLX2281-eQgM5qhs,16311
-abstract_block_dumper/_internal/services/block_processor.py,sha256=NC7p1oD38FpaZb6EbykBolP32uY069abumOvXrjOBV0,6644
+abstract_block_dumper/_internal/services/backfill_scheduler.py,sha256=uZqM4bSoMgNRKwBy-W3a7ydh2Z37Sum8cDRGo4T4KhA,16319
+abstract_block_dumper/_internal/services/block_processor.py,sha256=P8_LZR4ZSyNKbtnqFbAUkpT1XmEh9yX5Pgj5J__IwdA,7409
 abstract_block_dumper/_internal/services/executor.py,sha256=WhpHhOAi4cI-qdEE8-DSt9xZwooOpSc9_uDMQBBoHUM,2317
 abstract_block_dumper/_internal/services/metrics.py,sha256=Gg-PQYZ98caaS52wm1EqhtPURXlfrVjk2t3-8nccqfo,7821
-abstract_block_dumper/_internal/services/scheduler.py,sha256=pFVNV1YBHujNIRW9kq_1bxaa_W1Bn_pr7DrgwKaUItw,6779
+abstract_block_dumper/_internal/services/scheduler.py,sha256=pkcOlh0XsR-3sStEtTbiSpHFJhMG9tGIgSUFeH-FITk,5020
 abstract_block_dumper/_internal/services/utils.py,sha256=QZxdQyWIcUnezyVmegS4g3x3BoB3-oijYJ_i9nLQWHo,1140
 abstract_block_dumper/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 abstract_block_dumper/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -27,6 +29,6 @@ abstract_block_dumper/v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 abstract_block_dumper/v1/celery.py,sha256=X4IqVs5i6ZpyY7fy1SqMZgsZy4SXP-jK2qG-FYnjU38,1722
 abstract_block_dumper/v1/decorators.py,sha256=yQglsy1dU1u7ShwaTqZLahDcybHmetibTIOi53o_ZOM,9829
 abstract_block_dumper/v1/tasks.py,sha256=u9iMYdDUqzYT3yPrNwZecHnlweZ3yFipV9BcIWHCbus,2647
-abstract_block_dumper-0.1.0.dist-info/METADATA,sha256=UypVClIjNRQlbzkGmmR2Indrw3c-2gbrlbjHaXin_ys,12993
-abstract_block_dumper-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-abstract_block_dumper-0.1.0.dist-info/RECORD,,
+abstract_block_dumper-0.1.1.dist-info/METADATA,sha256=CVrs2LOv57Y4JOGxJgKsr1Xnnhl0i0wa534T-0hTU58,12993
+abstract_block_dumper-0.1.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+abstract_block_dumper-0.1.1.dist-info/RECORD,,