abstract-block-dumper 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl

abstract_block_dumper/_internal/providers/bittensor_client.py

@@ -1,8 +1,15 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 import bittensor as bt
 import structlog
 
 import abstract_block_dumper._internal.services.utils as abd_utils
 
+if TYPE_CHECKING:
+    import types
+
 logger = structlog.get_logger(__name__)
 
 
@@ -13,6 +20,10 @@ ARCHIVE_BLOCK_THRESHOLD = 300
 class BittensorConnectionClient:
     """
     Manages connections to regular and archive Bittensor subtensor networks.
+
+    Supports context manager protocol for safe connection cleanup:
+        with BittensorConnectionClient(network="finney") as client:
+            block = client.subtensor.get_current_block()
     """
 
     def __init__(self, network: str) -> None:
@@ -21,6 +32,38 @@ class BittensorConnectionClient:
         self._archive_subtensor: bt.Subtensor | None = None
         self._current_block_cache: int | None = None
 
+    def __enter__(self) -> BittensorConnectionClient:
+        """Context manager entry."""
+        return self
+
+    def __exit__(
+        self,
+        _exc_type: type[BaseException] | None,
+        _exc_val: BaseException | None,
+        _exc_tb: types.TracebackType | None,
+    ) -> None:
+        """Context manager exit - ensures connections are closed."""
+        self.close()
+
+    def close(self) -> None:
+        """Close all subtensor connections to prevent memory leaks."""
+        if self._subtensor is not None:
+            try:
+                self._subtensor.close()
+            except Exception:
+                logger.warning("Error closing subtensor connection", exc_info=True)
+            self._subtensor = None
+
+        if self._archive_subtensor is not None:
+            try:
+                self._archive_subtensor.close()
+            except Exception:
+                logger.warning("Error closing archive subtensor connection", exc_info=True)
+            self._archive_subtensor = None
+
+        self._current_block_cache = None
+        logger.debug("Subtensor connections closed")
+
     def get_for_block(self, block_number: int) -> bt.Subtensor:
         """Get the appropriate subtensor client for the given block number."""
         raise NotImplementedError
@@ -71,8 +114,6 @@ class BittensorConnectionClient:
         return self.subtensor
 
     def refresh_connections(self) -> None:
-        """Reset all subtensor connections to force re-establishment."""
-        self._subtensor = None
-        self._archive_subtensor = None
-        self._current_block_cache = None
-        logger.info("Subtensor connections reset")
+        """Close and reset all subtensor connections to force re-establishment."""
+        self.close()
+        logger.info("Subtensor connections refreshed")

abstract_block_dumper/_internal/services/backfill_scheduler.py

@@ -36,8 +36,6 @@ logger = structlog.get_logger(__name__)
 # Blocks older than this threshold from current head require archive network
 ARCHIVE_BLOCK_THRESHOLD = 300
 
-# Progress logging interval
-PROGRESS_LOG_INTERVAL = 100
 ARCHIVE_NETWORK = "archive"
 
 # Memory cleanup interval (every N blocks)
@@ -281,16 +279,13 @@ class BackfillScheduler:
                     if self._current_head_cache:
                         set_block_lag("backfill", self._current_head_cache - block_number)
 
-                    # Log progress periodically
-                    if processed_count % PROGRESS_LOG_INTERVAL == 0:
-                        progress_pct = (processed_count / total_blocks) * 100
-                        logger.info(
-                            "Backfill progress",
-                            processed=processed_count,
-                            total=total_blocks,
-                            progress_percent=f"{progress_pct:.1f}%",
-                            current_block=block_number,
-                        )
+                    # Log each block being processed
+                    progress_pct = (processed_count / total_blocks) * 100
+                    logger.info(
+                        "Backfilling block",
+                        block=block_number,
+                        progress=f"{processed_count}/{total_blocks} ({progress_pct:.1f}%)",
+                    )
 
                     # Rate limiting between block submissions
                     if block_number < self.to_block and self.rate_limit > 0:

abstract_block_dumper/_internal/services/scheduler.py

@@ -2,7 +2,6 @@ import time
 from typing import Protocol
 
 import structlog
-from django import db
 from django.conf import settings
 
 import abstract_block_dumper._internal.dal.django_dal as abd_dal
@@ -16,9 +15,6 @@ from abstract_block_dumper._internal.services.metrics import (
     set_registered_tasks,
 )
 
-# Refresh bittensor connections every N blocks to prevent memory leaks from internal caches
-CONNECTION_REFRESH_INTERVAL = 1000
-
 logger = structlog.get_logger(__name__)
 
 
@@ -42,6 +38,7 @@ class DefaultBlockStateResolver:
             return self.bittensor_client.subtensor.get_current_block()
         if isinstance(start_setting, int):
             return start_setting
+
         # Default: resume from DB or current
         return abd_dal.get_the_latest_executed_block_number() or self.bittensor_client.subtensor.get_current_block()
 
@@ -59,7 +56,6 @@ class TaskScheduler:
         self.bittensor_client = bittensor_client
         self.last_processed_block = state_resolver.get_starting_block()
         self.is_running = False
-        self._blocks_since_refresh = 0
 
     def start(self) -> None:
         self.is_running = True
@@ -86,11 +82,6 @@ class TaskScheduler:
                     increment_blocks_processed("realtime")
                     set_block_lag("realtime", 0)  # Head-only mode has no lag
                     self.last_processed_block = current_block
-                    self._blocks_since_refresh += 1
-
-                    # Periodic memory cleanup
-                    if self._blocks_since_refresh >= CONNECTION_REFRESH_INTERVAL:
-                        self._perform_cleanup()
 
                 time.sleep(self.poll_interval)
 
@@ -104,19 +95,9 @@ class TaskScheduler:
 
     def stop(self) -> None:
         self.is_running = False
+        self.bittensor_client.close()
         logger.info("TaskScheduler stopped.")
 
-    def _perform_cleanup(self) -> None:
-        """Perform periodic memory cleanup to prevent leaks in long-running processes."""
-        # Reset bittensor connections to clear internal caches
-        self.bittensor_client.refresh_connections()
-
-        # Clear Django's query log (only accumulates if DEBUG=True)
-        db.reset_queries()
-
-        self._blocks_since_refresh = 0
-        logger.debug("Memory cleanup performed", blocks_processed=CONNECTION_REFRESH_INTERVAL)
-
 
 def task_scheduler_factory(network: str = "finney") -> TaskScheduler:
     """

abstract_block_dumper/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.1'
-__version_tuple__ = version_tuple = (0, 1, 1)
+__version__ = version = '0.1.2'
+__version_tuple__ = version_tuple = (0, 1, 2)
 
 __commit_id__ = commit_id = None

{abstract_block_dumper-0.1.1.dist-info → abstract_block_dumper-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract-block-dumper
-Version: 0.1.1
+Version: 0.1.2
 Project-URL: Source, https://github.com/bactensor/abstract-block-dumper
 Project-URL: Issue Tracker, https://github.com/bactensor/abstract-block-dumper/issues
 Author-email: Reef Technologies <opensource@reef.pl>

{abstract_block_dumper-0.1.1.dist-info → abstract_block_dumper-0.1.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 abstract_block_dumper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_version.py,sha256=m8HxkqoKGw_wAJtc4ZokpJKNLXqp4zwnNhbnfDtro7w,704
+abstract_block_dumper/_version.py,sha256=Ok5oAXdWgR9aghaFXTafTeDW6sYO3uVe6d2Nket57R4,704
 abstract_block_dumper/admin.py,sha256=3J3I_QOKFgfMNpTXW-rTQGO_q5Ls6uNuL0FkPVdIsYg,1654
 abstract_block_dumper/apps.py,sha256=DXATdrjsL3T2IletTbKeD6unr8ScLaxg7wz0nAHTAns,215
 abstract_block_dumper/models.py,sha256=MO9824dmHB6xF3PrFE_RERh7whVjQtS4tt6QA0wSbg0,2022
@@ -11,13 +11,13 @@ abstract_block_dumper/_internal/dal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQ
 abstract_block_dumper/_internal/dal/django_dal.py,sha256=QbDsikUthIAhVC_FwSynUUdQL3OWlCo3_Cg65M91Cb4,5618
 abstract_block_dumper/_internal/dal/memory_registry.py,sha256=m9Yms-cuemi9_5q_Kn_zsJnxDPEiuAUkESIAltD60QY,2943
 abstract_block_dumper/_internal/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_internal/providers/bittensor_client.py,sha256=hkUdkhz8wOZInRloyySo3DDN4JSMEiPwLv0YV5IDZ7o,2798
+abstract_block_dumper/_internal/providers/bittensor_client.py,sha256=wlKjFrGN4Q2DfQyD_Fx-eH83ZMB6AbzLs5keYq6FGUw,4124
 abstract_block_dumper/_internal/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_internal/services/backfill_scheduler.py,sha256=uZqM4bSoMgNRKwBy-W3a7ydh2Z37Sum8cDRGo4T4KhA,16319
+abstract_block_dumper/_internal/services/backfill_scheduler.py,sha256=3X9NRXCamnb1jUI47sUXqmsrQ1nBzK0BLKe5CeLt00E,16091
 abstract_block_dumper/_internal/services/block_processor.py,sha256=P8_LZR4ZSyNKbtnqFbAUkpT1XmEh9yX5Pgj5J__IwdA,7409
 abstract_block_dumper/_internal/services/executor.py,sha256=WhpHhOAi4cI-qdEE8-DSt9xZwooOpSc9_uDMQBBoHUM,2317
 abstract_block_dumper/_internal/services/metrics.py,sha256=Gg-PQYZ98caaS52wm1EqhtPURXlfrVjk2t3-8nccqfo,7821
-abstract_block_dumper/_internal/services/scheduler.py,sha256=pkcOlh0XsR-3sStEtTbiSpHFJhMG9tGIgSUFeH-FITk,5020
+abstract_block_dumper/_internal/services/scheduler.py,sha256=BIQ7c-HYSebW3CKq5ynsMZjULEO9c5YP0qWFN1aqg24,4164
 abstract_block_dumper/_internal/services/utils.py,sha256=QZxdQyWIcUnezyVmegS4g3x3BoB3-oijYJ_i9nLQWHo,1140
 abstract_block_dumper/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 abstract_block_dumper/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -29,6 +29,6 @@ abstract_block_dumper/v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 abstract_block_dumper/v1/celery.py,sha256=X4IqVs5i6ZpyY7fy1SqMZgsZy4SXP-jK2qG-FYnjU38,1722
 abstract_block_dumper/v1/decorators.py,sha256=yQglsy1dU1u7ShwaTqZLahDcybHmetibTIOi53o_ZOM,9829
 abstract_block_dumper/v1/tasks.py,sha256=u9iMYdDUqzYT3yPrNwZecHnlweZ3yFipV9BcIWHCbus,2647
-abstract_block_dumper-0.1.1.dist-info/METADATA,sha256=CVrs2LOv57Y4JOGxJgKsr1Xnnhl0i0wa534T-0hTU58,12993
-abstract_block_dumper-0.1.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-abstract_block_dumper-0.1.1.dist-info/RECORD,,
+abstract_block_dumper-0.1.2.dist-info/METADATA,sha256=6y5tq_8Wp3JNHuYATBIx_XQ2I0mXqBU5tgyf97rHahc,12993
+abstract_block_dumper-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+abstract_block_dumper-0.1.2.dist-info/RECORD,,