abstract-block-dumper 0.0.9__tar.gz → 0.1.1__tar.gz

{abstract_block_dumper-0.0.9 → abstract_block_dumper-0.1.1}/CHANGELOG.md

@@ -9,6 +9,16 @@ upcoming release can be found in [changelog.d](changelog.d).
 
 <!-- towncrier release notes start -->
 
+## [0.1.1](https://github.com/bactensor/abstract-block-dumper/releases/tag/v0.1.1) - 2025-12-18
+
+No significant changes.
+
+
+## [0.1.0](https://github.com/bactensor/abstract-block-dumper/releases/tag/v0.1.0) - 2025-12-10
+
+No significant changes.
+
+
 ## [0.0.9](https://github.com/bactensor/abstract-block-dumper/releases/tag/v0.0.9) - 2025-12-04
 
 No significant changes.

{abstract_block_dumper-0.0.9 → abstract_block_dumper-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract-block-dumper
-Version: 0.0.9
+Version: 0.1.1
 Project-URL: Source, https://github.com/bactensor/abstract-block-dumper
 Project-URL: Issue Tracker, https://github.com/bactensor/abstract-block-dumper/issues
 Author-email: Reef Technologies <opensource@reef.pl>

{abstract_block_dumper-0.0.9 → abstract_block_dumper-0.1.1}/pyproject.toml

@@ -144,6 +144,9 @@ directory = "infrastructure"
 name = "Infrastructure"
 showcontent = true
 
+[tool.pyright]
+typeCheckingMode = "off"
+
 [tool.mypy]
 plugins = ["mypy_django_plugin.main"]
 strict_optional = true

{abstract_block_dumper-0.0.9 → abstract_block_dumper-0.1.1}/src/abstract_block_dumper/_internal/dal/django_dal.py

@@ -1,8 +1,10 @@
+from collections.abc import Iterator
 from datetime import timedelta
 from typing import Any
 
 from django.conf import settings
 from django.db import transaction
+from django.db.models import Max
 from django.db.models.query import QuerySet
 from django.utils import timezone
 
@@ -10,19 +12,21 @@ import abstract_block_dumper._internal.services.utils as abd_utils
 import abstract_block_dumper.models as abd_models
 
 
-def get_ready_to_retry_attempts() -> QuerySet[abd_models.TaskAttempt]:
-    return abd_models.TaskAttempt.objects.filter(
-        next_retry_at__isnull=False,
-        next_retry_at__lte=timezone.now(),
-        attempt_count__lt=abd_utils.get_max_attempt_limit(),
-    ).exclude(
-        status=abd_models.TaskAttempt.Status.SUCCESS,
+def get_ready_to_retry_attempts() -> Iterator[abd_models.TaskAttempt]:
+    return (
+        abd_models.TaskAttempt.objects.filter(
+            next_retry_at__isnull=False,
+            next_retry_at__lte=timezone.now(),
+            attempt_count__lt=abd_utils.get_max_attempt_limit(),
+        )
+        .exclude(
+            status=abd_models.TaskAttempt.Status.SUCCESS,
+        )
+        .iterator()
     )
 
 
 def executed_block_numbers(executable_path: str, args_json: str, from_block: int, to_block: int) -> set[int]:
-    # Use iterator() to avoid Django's QuerySet caching which causes memory leaks
-    # during long-running backfill operations
     block_numbers = (
         abd_models.TaskAttempt.objects.filter(
             executable_path=executable_path,
@@ -151,7 +155,5 @@ def task_create_or_get_pending(
 
 
 def get_the_latest_executed_block_number() -> int | None:
-    qs = abd_models.TaskAttempt.objects.order_by("-block_number").first()
-    if qs:
-        return qs.block_number
-    return None
+    result = abd_models.TaskAttempt.objects.aggregate(max_block=Max("block_number"))
+    return result["max_block"]

abstract_block_dumper-0.1.1/src/abstract_block_dumper/_internal/providers/bittensor_client.py

@@ -0,0 +1,78 @@
+import bittensor as bt
+import structlog
+
+import abstract_block_dumper._internal.services.utils as abd_utils
+
+logger = structlog.get_logger(__name__)
+
+
+# Blocks older than this threshold from current head require archive network
+ARCHIVE_BLOCK_THRESHOLD = 300
+
+
+class BittensorConnectionClient:
+    """
+    Manages connections to regular and archive Bittensor subtensor networks.
+    """
+
+    def __init__(self, network: str) -> None:
+        self.network = network
+        self._subtensor: bt.Subtensor | None = None
+        self._archive_subtensor: bt.Subtensor | None = None
+        self._current_block_cache: int | None = None
+
+    def get_for_block(self, block_number: int) -> bt.Subtensor:
+        """Get the appropriate subtensor client for the given block number."""
+        raise NotImplementedError
+
+    @property
+    def subtensor(self) -> bt.Subtensor:
+        """Get the regular subtensor connection, creating it if needed."""
+        if self._subtensor is None:
+            self._subtensor = abd_utils.get_bittensor_client(self.network)
+        return self._subtensor
+
+    @subtensor.setter
+    def subtensor(self, value: bt.Subtensor | None) -> None:
+        """Set or reset the subtensor connection."""
+        self._subtensor = value
+
+    @property
+    def archive_subtensor(self) -> bt.Subtensor:
+        """Get the archive subtensor connection, creating it if needed."""
+        if self._archive_subtensor is None:
+            self._archive_subtensor = abd_utils.get_bittensor_client("archive")
+        return self._archive_subtensor
+
+    @archive_subtensor.setter
+    def archive_subtensor(self, value: bt.Subtensor | None) -> None:
+        """Set or reset the archive subtensor connection."""
+        self._archive_subtensor = value
+
+    def get_subtensor_for_block(self, block_number: int) -> bt.Subtensor:
+        """
+        Get the appropriate subtensor for the given block number.
+
+        Uses archive network for blocks older than ARCHIVE_BLOCK_THRESHOLD
+        from the current head.
+        """
+        if self._current_block_cache is None:
+            self._current_block_cache = self.subtensor.get_current_block()
+
+        blocks_behind = self._current_block_cache - block_number
+
+        if blocks_behind > ARCHIVE_BLOCK_THRESHOLD:
+            logger.debug(
+                "Using archive network for old block",
+                block_number=block_number,
+                blocks_behind=blocks_behind,
+            )
+            return self.archive_subtensor
+        return self.subtensor
+
+    def refresh_connections(self) -> None:
+        """Reset all subtensor connections to force re-establishment."""
+        self._subtensor = None
+        self._archive_subtensor = None
+        self._current_block_cache = None
+        logger.info("Subtensor connections reset")

{abstract_block_dumper-0.0.9 → abstract_block_dumper-0.1.1}/src/abstract_block_dumper/_internal/services/backfill_scheduler.py

@@ -15,7 +15,7 @@ import structlog
 
 import abstract_block_dumper._internal.dal.django_dal as abd_dal
 import abstract_block_dumper._internal.services.utils as abd_utils
-from abstract_block_dumper._internal.services.block_processor import BlockProcessor, block_processor_factory
+from abstract_block_dumper._internal.services.block_processor import BaseBlockProcessor, block_processor_factory
 from abstract_block_dumper._internal.services.metrics import (
     BlockProcessingTimer,
     increment_archive_network_usage,
@@ -59,7 +59,7 @@ class BackfillScheduler:
 
     def __init__(
         self,
-        block_processor: BlockProcessor,
+        block_processor: BaseBlockProcessor,
         network: str,
         from_block: int,
         to_block: int,

{abstract_block_dumper-0.0.9 → abstract_block_dumper-0.1.1}/src/abstract_block_dumper/_internal/services/block_processor.py

@@ -1,4 +1,6 @@
+import itertools
 import time
+from typing import Protocol
 
 import structlog
 from django.db import transaction
@@ -12,6 +14,25 @@ from abstract_block_dumper.models import TaskAttempt
 logger = structlog.get_logger(__name__)
 
 
+class BaseBlockProcessor(Protocol):
+    """Protocol defining the interface for block processors."""
+
+    executor: CeleryExecutor
+    registry: BaseRegistry
+
+    def process_block(self, block_number: int) -> None:
+        """Process a single block - executes registered tasks for this block only."""
+        ...
+
+    def process_registry_item(self, registry_item: RegistryItem, block_number: int) -> None:
+        """Process a single registry item for a given block."""
+        ...
+
+    def recover_failed_retries(self, poll_interval: int, batch_size: int | None = None) -> None:
+        """Recover failed tasks that are ready to be retried."""
+        ...
+
+
 class BlockProcessor:
     def __init__(self, executor: CeleryExecutor, registry: BaseRegistry) -> None:
         self.executor = executor
@@ -59,9 +80,9 @@ class BlockProcessor:
         retry_count = 0
         retry_attempts = abd_dal.get_ready_to_retry_attempts()
 
-        # Apply batch size limit if specified
+        # Apply batch size limit if specified (use islice for iterator compatibility)
         if batch_size is not None:
-            retry_attempts = retry_attempts[:batch_size]
+            retry_attempts = itertools.islice(retry_attempts, batch_size)
 
         for retry_attempt in retry_attempts:
             time.sleep(poll_interval)
@@ -147,7 +168,7 @@ class BlockProcessor:
 def block_processor_factory(
     executor: CeleryExecutor | None = None,
     registry: BaseRegistry | None = None,
-) -> BlockProcessor:
+) -> BaseBlockProcessor:
     return BlockProcessor(
         executor=executor or CeleryExecutor(),
         registry=registry or task_registry,

abstract_block_dumper-0.1.1/src/abstract_block_dumper/_internal/services/scheduler.py

@@ -0,0 +1,136 @@
+import time
+from typing import Protocol
+
+import structlog
+from django import db
+from django.conf import settings
+
+import abstract_block_dumper._internal.dal.django_dal as abd_dal
+from abstract_block_dumper._internal.providers.bittensor_client import BittensorConnectionClient
+from abstract_block_dumper._internal.services.block_processor import BaseBlockProcessor, block_processor_factory
+from abstract_block_dumper._internal.services.metrics import (
+    BlockProcessingTimer,
+    increment_blocks_processed,
+    set_block_lag,
+    set_current_block,
+    set_registered_tasks,
+)
+
+# Refresh bittensor connections every N blocks to prevent memory leaks from internal caches
+CONNECTION_REFRESH_INTERVAL = 1000
+
+logger = structlog.get_logger(__name__)
+
+
+class BlockStateResolver(Protocol):
+    """Protocol defining the interface for block state resolvers."""
+
+    def get_starting_block(self) -> int:
+        """Determine which block to start processing from."""
+        ...
+
+
+class DefaultBlockStateResolver:
+    """Default implementation that reads from settings and database."""
+
+    def __init__(self, bittensor_client: BittensorConnectionClient) -> None:
+        self.bittensor_client = bittensor_client
+
+    def get_starting_block(self) -> int:
+        start_setting = getattr(settings, "BLOCK_DUMPER_START_FROM_BLOCK", None)
+        if start_setting == "current":
+            return self.bittensor_client.subtensor.get_current_block()
+        if isinstance(start_setting, int):
+            return start_setting
+        # Default: resume from DB or current
+        return abd_dal.get_the_latest_executed_block_number() or self.bittensor_client.subtensor.get_current_block()
+
+
+class TaskScheduler:
+    def __init__(
+        self,
+        block_processor: BaseBlockProcessor,
+        bittensor_client: BittensorConnectionClient,
+        state_resolver: BlockStateResolver,
+        poll_interval: int,
+    ) -> None:
+        self.block_processor = block_processor
+        self.poll_interval = poll_interval
+        self.bittensor_client = bittensor_client
+        self.last_processed_block = state_resolver.get_starting_block()
+        self.is_running = False
+        self._blocks_since_refresh = 0
+
+    def start(self) -> None:
+        self.is_running = True
+
+        registered_tasks_count = len(self.block_processor.registry.get_functions())
+        set_registered_tasks(registered_tasks_count)
+
+        logger.info(
+            "TaskScheduler started",
+            last_processed_block=self.last_processed_block,
+            registry_functions=registered_tasks_count,
+        )
+
+        while self.is_running:
+            try:
+                current_block = self.bittensor_client.subtensor.get_current_block()
+
+                # Only process the current head block, skip if already processed
+                if current_block != self.last_processed_block:
+                    with BlockProcessingTimer(mode="realtime"):
+                        self.block_processor.process_block(current_block)
+
+                    set_current_block("realtime", current_block)
+                    increment_blocks_processed("realtime")
+                    set_block_lag("realtime", 0)  # Head-only mode has no lag
+                    self.last_processed_block = current_block
+                    self._blocks_since_refresh += 1
+
+                    # Periodic memory cleanup
+                    if self._blocks_since_refresh >= CONNECTION_REFRESH_INTERVAL:
+                        self._perform_cleanup()
+
+                time.sleep(self.poll_interval)
+
+            except KeyboardInterrupt:
+                logger.info("TaskScheduler stopping due to KeyboardInterrupt.")
+                self.stop()
+                break
+            except Exception:
+                logger.exception("Error in TaskScheduler loop")
+                time.sleep(self.poll_interval)
+
+    def stop(self) -> None:
+        self.is_running = False
+        logger.info("TaskScheduler stopped.")
+
+    def _perform_cleanup(self) -> None:
+        """Perform periodic memory cleanup to prevent leaks in long-running processes."""
+        # Reset bittensor connections to clear internal caches
+        self.bittensor_client.refresh_connections()
+
+        # Clear Django's query log (only accumulates if DEBUG=True)
+        db.reset_queries()
+
+        self._blocks_since_refresh = 0
+        logger.debug("Memory cleanup performed", blocks_processed=CONNECTION_REFRESH_INTERVAL)
+
+
+def task_scheduler_factory(network: str = "finney") -> TaskScheduler:
+    """
+    Factory for TaskScheduler.
+
+    Args:
+        network (str): Bittensor network name. Defaults to "finney"
+
+    """
+    bittensor_client = BittensorConnectionClient(network=network)
+    state_resolver = DefaultBlockStateResolver(bittensor_client=bittensor_client)
+    return TaskScheduler(
+        block_processor=block_processor_factory(),
+        poll_interval=getattr(settings, "BLOCK_DUMPER_POLL_INTERVAL", 5),
+        bittensor_client=bittensor_client,
+        state_resolver=state_resolver,
+    )

{abstract_block_dumper-0.0.9 → abstract_block_dumper-0.1.1}/src/abstract_block_dumper/_internal/services/utils.py

@@ -17,7 +17,7 @@ def get_bittensor_client(network: str = "finney") -> bt.Subtensor:
     doesn't change during runtime.
     """
     logger.info("Creating new bittensor client for network", network=network)
-    return bt.subtensor(network=network)
+    return bt.Subtensor(network=network)
 
 
 def get_current_celery_task_id() -> str:

{abstract_block_dumper-0.0.9 → abstract_block_dumper-0.1.1}/src/abstract_block_dumper/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.0.9'
-__version_tuple__ = version_tuple = (0, 0, 9)
+__version__ = version = '0.1.1'
+__version_tuple__ = version_tuple = (0, 1, 1)
 
 __commit_id__ = commit_id = None

{abstract_block_dumper-0.0.9 → abstract_block_dumper-0.1.1}/src/abstract_block_dumper/v1/decorators.py

@@ -74,7 +74,9 @@ def schedule_retry(task_attempt: TaskAttempt) -> None:
 
 
 def _celery_task_wrapper(
-    func: Callable[..., Any], block_number: int, **kwargs: dict[str, Any]
+    func: Callable[..., Any],
+    block_number: int,
+    **kwargs: dict[str, Any],
 ) -> dict[str, Any] | None:
     executable_path = abd_utils.get_executable_path(func)
 

abstract_block_dumper-0.1.1/tests/conftest.py

@@ -0,0 +1,115 @@
+from collections.abc import Generator
+from typing import Any, NoReturn
+
+import django
+import pytest
+from celery import Celery
+from django.conf import settings
+
+from abstract_block_dumper._internal.dal.memory_registry import RegistryItem, task_registry
+from abstract_block_dumper.v1.decorators import block_task
+
+from .django_fixtures import *  # noqa: F401, F403
+
+# Ensure Django is set up
+if not settings.configured:
+    django.setup()
+
+
+class MockedBlockProcessor:
+    """Mock implementation of BaseBlockProcessor for testing."""
+
+    def __init__(self, executor: Any = None, registry: Any = None) -> None:
+        self.executor = executor
+        self.registry = registry or task_registry
+        self.processed_blocks: list[int] = []
+        self.processed_registry_items: list[tuple[RegistryItem, int]] = []
+        self.recover_failed_retries_calls: list[tuple[int, int | None]] = []
+
+    def process_block(self, block_number: int) -> None:
+        self.processed_blocks.append(block_number)
+
+    def process_registry_item(self, registry_item: RegistryItem, block_number: int) -> None:
+        self.processed_registry_items.append((registry_item, block_number))
+
+    def recover_failed_retries(self, poll_interval: int, batch_size: int | None = None) -> None:
+        self.recover_failed_retries_calls.append((poll_interval, batch_size))
+
+
+class MockedSubtensor:
+    """Mock implementation of bt.Subtensor for testing."""
+
+    def __init__(self, current_block: int = 1000) -> None:
+        self._current_block = current_block
+
+    def get_current_block(self) -> int:
+        return self._current_block
+
+    def set_current_block(self, block: int) -> None:
+        self._current_block = block
+
+
+@pytest.fixture(autouse=True)
+def celery_test_app() -> Generator[Celery, Any, None]:
+    """Configure Celery for testing with eager mode."""
+    app = Celery("test_app")
+    app.config_from_object(settings, namespace="CELERY")
+    return app
+
+
+def every_block_task_func(block_number: int):
+    """
+    Test function for every block execution.
+    """
+    return f"Processed block {block_number}"
+
+
+def modulo_task_func(block_number: int, netuid: int):
+    """
+    Test function for modulo condition execution.
+    """
+    return f"Modulo task processed block {block_number} for netuid {netuid}"
+
+
+def failing_task_func(block_number: int) -> NoReturn:
+    """
+    Test function that always fails.
+    """
+    raise ValueError("Test error")
+
+
+@pytest.fixture
+def setup_test_tasks():
+    # Register test tasks using decorators
+
+    # every block
+    block_task(condition=lambda bn: True)(every_block_task_func)
+
+    # every 5 blocks
+    block_task(condition=lambda bn, netuid: bn % 5 == 0, args=[{"netuid": 1}, {"netuid": 2}])(modulo_task_func)
+
+    yield
+
+
+@pytest.fixture(autouse=True)
+def cleanup_memory_registry():
+    task_registry.clear()
+
+    yield
+
+    task_registry.clear()
+
+
+@pytest.fixture
+def mock_block_processor() -> MockedBlockProcessor:
+    return MockedBlockProcessor()
+
+
+@pytest.fixture
+def mock_subtensor() -> MockedSubtensor:
+    return MockedSubtensor(current_block=1000)
+
+
+@pytest.fixture
+def mock_archive_subtensor() -> MockedSubtensor:
+    return MockedSubtensor(current_block=1000)

{abstract_block_dumper-0.0.9 → abstract_block_dumper-0.1.1}/tests/integration/test_scheduler.py

@@ -7,6 +7,7 @@ from django.utils import timezone
 import abstract_block_dumper._internal.dal.django_dal as abd_dal
 import abstract_block_dumper._internal.services.utils as abd_utils
 from abstract_block_dumper._internal.dal.memory_registry import task_registry
+from abstract_block_dumper._internal.providers.bittensor_client import BittensorConnectionClient
 from abstract_block_dumper._internal.services.block_processor import block_processor_factory
 from abstract_block_dumper.models import TaskAttempt
 from abstract_block_dumper.v1.decorators import block_task
@@ -151,3 +152,36 @@ def test_retry_recover_mechanism():
         status=TaskAttempt.Status.SUCCESS,
     )
     assert qs.count() == len(recover_ids)
+
+
+def test_bittensor_client_uses_archive_for_old_blocks(
+    mock_subtensor,
+    mock_archive_subtensor,
+) -> None:
+    """Test that BittensorConnectionClient uses archive_subtensor for blocks older than 300 from current head."""
+    current_block = 1000
+
+    client = BittensorConnectionClient(network="testnet")
+    client._subtensor = mock_subtensor
+    client._archive_subtensor = mock_archive_subtensor
+    client._current_block_cache = current_block
+
+    # For old block (400 behind), should return archive_subtensor
+    old_block = 600  # 400 blocks behind (> 300 threshold)
+    result = client.get_subtensor_for_block(old_block)
+    assert result is mock_archive_subtensor
+
+    # For recent block (within 300), should return regular subtensor
+    recent_block = 800  # 200 blocks behind (< 300 threshold)
+    result = client.get_subtensor_for_block(recent_block)
+    assert result is mock_subtensor
+
+    # For block exactly at threshold (300 behind), should return regular subtensor
+    threshold_block = 700  # exactly 300 blocks behind
+    result = client.get_subtensor_for_block(threshold_block)
+    assert result is mock_subtensor
+
+    # For block just over threshold (301 behind), should return archive_subtensor
+    just_over_threshold_block = 699  # 301 blocks behind
+    result = client.get_subtensor_for_block(just_over_threshold_block)
+    assert result is mock_archive_subtensor

abstract_block_dumper-0.0.9/src/abstract_block_dumper/_internal/services/scheduler.py

@@ -1,195 +0,0 @@
-import time
-
-import bittensor as bt
-import structlog
-from django.conf import settings
-
-import abstract_block_dumper._internal.dal.django_dal as abd_dal
-import abstract_block_dumper._internal.services.utils as abd_utils
-from abstract_block_dumper._internal.services.block_processor import BlockProcessor, block_processor_factory
-from abstract_block_dumper._internal.services.metrics import (
-    BlockProcessingTimer,
-    increment_blocks_processed,
-    set_block_lag,
-    set_current_block,
-    set_registered_tasks,
-)
-
-logger = structlog.get_logger(__name__)
-
-# Blocks older than this threshold from current head require archive network
-ARCHIVE_BLOCK_THRESHOLD = 300
-
-
-class TaskScheduler:
-    def __init__(
-        self,
-        block_processor: BlockProcessor,
-        network: str,
-        poll_interval: int,
-        realtime_head_only: bool = False,
-    ) -> None:
-        self.block_processor = block_processor
-        self.network = network
-        self.poll_interval = poll_interval
-        self.realtime_head_only = realtime_head_only
-        self.last_processed_block = -1
-        self.is_running = False
-        self._subtensor: bt.Subtensor | None = None
-        self._archive_subtensor: bt.Subtensor | None = None
-        self._current_block_cache: int | None = None
-
-    @property
-    def subtensor(self) -> bt.Subtensor:
-        """Get the regular subtensor connection, creating it if needed."""
-        if self._subtensor is None:
-            self._subtensor = abd_utils.get_bittensor_client(self.network)
-        return self._subtensor
-
-    @subtensor.setter
-    def subtensor(self, value: bt.Subtensor | None) -> None:
-        """Set or reset the subtensor connection."""
-        self._subtensor = value
-
-    @property
-    def archive_subtensor(self) -> bt.Subtensor:
-        """Get the archive subtensor connection, creating it if needed."""
-        if self._archive_subtensor is None:
-            self._archive_subtensor = abd_utils.get_bittensor_client("archive")
-        return self._archive_subtensor
-
-    @archive_subtensor.setter
-    def archive_subtensor(self, value: bt.Subtensor | None) -> None:
-        """Set or reset the archive subtensor connection."""
-        self._archive_subtensor = value
-
-    def get_subtensor_for_block(self, block_number: int) -> bt.Subtensor:
-        """
-        Get the appropriate subtensor for the given block number.
-
-        Uses archive network for blocks older than ARCHIVE_BLOCK_THRESHOLD
-        from the current head.
-        """
-        if self._current_block_cache is None:
-            self._current_block_cache = self.subtensor.get_current_block()
-
-        blocks_behind = self._current_block_cache - block_number
-
-        if blocks_behind > ARCHIVE_BLOCK_THRESHOLD:
-            logger.debug(
-                "Using archive network for old block",
-                block_number=block_number,
-                blocks_behind=blocks_behind,
-            )
-            return self.archive_subtensor
-        return self.subtensor
-
-    def refresh_connections(self) -> None:
-        """Reset all subtensor connections to force re-establishment."""
-        self._subtensor = None
-        self._archive_subtensor = None
-        self._current_block_cache = None
-        logger.info("Subtensor connections reset")
-
-    def start(self) -> None:
-        self.is_running = True
-
-        self.initialize_last_block()
-
-        registered_tasks_count = len(self.block_processor.registry.get_functions())
-        set_registered_tasks(registered_tasks_count)
-
-        logger.info(
-            "TaskScheduler started",
-            last_processed_block=self.last_processed_block,
-            registry_functions=registered_tasks_count,
-        )
-
-        while self.is_running:
-            try:
-                if self._current_block_cache is not None:
-                    self.subtensor = self.get_subtensor_for_block(self._current_block_cache)
-
-                # Update current block cache for archive network decision
-                self._current_block_cache = self.subtensor.get_current_block()
-                current_block = self._current_block_cache
-
-                if self.realtime_head_only:
-                    # Only process the current head block, skip if already processed
-                    if current_block != self.last_processed_block:
-                        with BlockProcessingTimer(mode="realtime"):
-                            self.block_processor.process_block(current_block)
-
-                        set_current_block("realtime", current_block)
-                        increment_blocks_processed("realtime")
-                        set_block_lag("realtime", 0)  # Head-only mode has no lag
-                        self.last_processed_block = current_block
-
-                    time.sleep(self.poll_interval)
-                else:
-                    # Original behavior: process all blocks from last_processed to current
-                    for block_number in range(self.last_processed_block + 1, current_block + 1):
-                        with BlockProcessingTimer(mode="realtime"):
-                            self.block_processor.process_block(block_number)
-
-                        # Update metrics
-                        set_current_block("realtime", block_number)
-                        increment_blocks_processed("realtime")
-                        set_block_lag("realtime", current_block - block_number)
-
-                        time.sleep(self.poll_interval)
-                        self.last_processed_block = block_number
-
-            except KeyboardInterrupt:
-                logger.info("TaskScheduler stopping due to KeyboardInterrupt.")
-                self.stop()
-                break
-            except Exception:
-                logger.error("Fatal scheduler error", exc_info=True)
-                # resume the loop even if task failed
-                time.sleep(self.poll_interval)
-
-    def stop(self) -> None:
-        self.is_running = False
-        logger.info("TaskScheduler stopped.")
-
-    def initialize_last_block(self) -> None:
-        # Safe getattr in case setting is not defined
-        start_from_block_setting = getattr(settings, "BLOCK_DUMPER_START_FROM_BLOCK", None)
-
-        if start_from_block_setting is not None:
-            if start_from_block_setting == "current":
-                self.last_processed_block = self.subtensor.get_current_block()
-                logger.info("Starting from current blockchain block", block_number=self.last_processed_block)
-
-            elif isinstance(start_from_block_setting, int):
-                self.last_processed_block = start_from_block_setting
-                logger.info("Starting from configured block", block_number=self.last_processed_block)
-            else:
-                error_msg = f"Invalid BLOCK_DUMPER_START_FROM_BLOCK value: {start_from_block_setting}"
-                raise ValueError(error_msg)
-        else:
-            # Default behavior - resume from database
-            last_block_number = abd_dal.get_the_latest_executed_block_number()
-
-            self.last_processed_block = last_block_number or self.subtensor.get_current_block()
-            logger.info(
-                "Resume from the last database block or start from the current block",
-                last_processed_block=self.last_processed_block,
-            )
-
-
-def task_scheduler_factory(network: str = "finney") -> TaskScheduler:
-    """
-    Factory for TaskScheduler.
-
-    Args:
-        network (str): Bittensor network name. Defaults to "finney"
-
-    """
-    return TaskScheduler(
-        block_processor=block_processor_factory(),
-        network=network,
-        poll_interval=getattr(settings, "BLOCK_DUMPER_POLL_INTERVAL", 1),
-        realtime_head_only=getattr(settings, "BLOCK_DUMPER_REALTIME_HEAD_ONLY", True),
-    )

abstract_block_dumper-0.0.9/tests/conftest.py

@@ -1,65 +0,0 @@
-import django
-import pytest
-from celery import Celery
-from django.conf import settings
-
-from abstract_block_dumper._internal.dal.memory_registry import task_registry
-
-from .django_fixtures import *  # noqa: F401, F403
-
-# Ensure Django is set up
-if not settings.configured:
-    django.setup()
-
-
-@pytest.fixture(autouse=True)
-def celery_test_app():
-    """Configure Celery for testing with eager mode."""
-    app = Celery("test_app")
-    app.config_from_object(settings, namespace="CELERY")
-
-    yield app
-
-
-def every_block_task_func(block_number: int):
-    """
-    Test function for every block execution.
-    """
-    return f"Processed block {block_number}"
-
-
-def modulo_task_func(block_number: int, netuid: int):
-    """
-    Test function for modulo condition execution.
-    """
-    return f"Modulo task processed block {block_number} for netuid {netuid}"
-
-
-def failing_task_func(block_number: int):
-    """
-    Test function that always fails.
-    """
-    raise ValueError("Test error")
-
-
-@pytest.fixture
-def setup_test_tasks():
-    # Register test tasks using decorators
-    from abstract_block_dumper.v1.decorators import block_task
-
-    # every block
-    block_task(condition=lambda bn: True)(every_block_task_func)
-
-    # every 5 blocks
-    block_task(condition=lambda bn, netuid: bn % 5 == 0, args=[{"netuid": 1}, {"netuid": 2}])(modulo_task_func)
-
-    yield
-
-
-@pytest.fixture(autouse=True)
-def cleanup_memory_registry():
-    task_registry.clear()
-
-    yield
-
-    task_registry.clear()