abstract-block-dumper 0.1.2__py3-none-any.whl → 0.1.5__py3-none-any.whl

abstract_block_dumper/_internal/dal/django_dal.py

@@ -4,7 +4,7 @@ from typing import Any
 
 from django.conf import settings
 from django.db import transaction
-from django.db.models import Max
+from django.db.models import Max, Min
 from django.db.models.query import QuerySet
 from django.utils import timezone
 
@@ -157,3 +157,27 @@ def task_create_or_get_pending(
 def get_the_latest_executed_block_number() -> int | None:
     result = abd_models.TaskAttempt.objects.aggregate(max_block=Max("block_number"))
     return result["max_block"]
+
+
+def get_block_range() -> tuple[int | None, int | None]:
+    """Get the min and max block numbers from all task attempts."""
+    result = abd_models.TaskAttempt.objects.aggregate(
+        min_block=Min("block_number"),
+        max_block=Max("block_number"),
+    )
+    return result["min_block"], result["max_block"]
+
+
+def get_successful_block_numbers(from_block: int, to_block: int) -> set[int]:
+    """Get all block numbers with at least one successful task in the range."""
+    block_numbers = (
+        abd_models.TaskAttempt.objects.filter(
+            block_number__gte=from_block,
+            block_number__lte=to_block,
+            status=abd_models.TaskAttempt.Status.SUCCESS,
+        )
+        .values_list("block_number", flat=True)
+        .distinct()
+        .iterator()
+    )
+    return set(block_numbers)

abstract_block_dumper/_internal/discovery.py

@@ -20,5 +20,5 @@ def ensure_modules_loaded() -> None:
             except ModuleNotFoundError:
                 continue
             except ImportError as e:
-                logger.warning(f"Failed to import {app_config.name}.{module_suffix}: {e}")
+                logger.warning("Failed to import %s.%s: %s", app_config.name, module_suffix, e)
                 continue

abstract_block_dumper/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.2'
-__version_tuple__ = version_tuple = (0, 1, 2)
+__version__ = version = '0.1.5'
+__version_tuple__ = version_tuple = (0, 1, 5)
 
 __commit_id__ = commit_id = None

abstract_block_dumper/management/commands/backfill_blocks_v1.py

@@ -1,7 +1,11 @@
-"""Management command for backfilling historical blocks."""
+import argparse
+import time
+from dataclasses import dataclass
+from typing import TypedDict
 
 from django.core.management.base import BaseCommand
 
+import abstract_block_dumper._internal.dal.django_dal as abd_dal
 from abstract_block_dumper._internal.dal.memory_registry import task_registry
 from abstract_block_dumper._internal.discovery import ensure_modules_loaded
 from abstract_block_dumper._internal.services.backfill_scheduler import (
@@ -10,22 +14,74 @@ from abstract_block_dumper._internal.services.backfill_scheduler import (
     backfill_scheduler_factory,
 )
 
+SECONDS_PER_HOUR = 3600
+SECONDS_PER_MINUTE = 60
+
+
+class BackfillOptions(TypedDict):
+    """Type definition for command options."""
+
+    from_block: int | None
+    to_block: int | None
+    rate_limit: float
+    network: str
+    dry_run: bool
+    no_gap_detection: bool
+
+
+@dataclass
+class Gap:
+    """Represents a gap of missing blocks."""
+
+    start: int
+    end: int
+
+    @property
+    def size(self) -> int:
+        """
+        Get the size of the gap (number of missing blocks).
+        """
+        return self.end - self.start + 1
+
+
+def find_gaps(from_block: int, to_block: int, processed_blocks: set[int]) -> list[Gap]:
+    """Find gaps (missing blocks) in the given range."""
+    gaps = []
+    gap_start = None
+
+    for block_num in range(from_block, to_block + 1):
+        if block_num not in processed_blocks:
+            if gap_start is None:
+                gap_start = block_num
+        elif gap_start is not None:
+            gaps.append(Gap(start=gap_start, end=block_num - 1))
+            gap_start = None
+
+    # Handle gap at the end
+    if gap_start is not None:
+        gaps.append(Gap(start=gap_start, end=to_block))
+
+    return gaps
+
 
 class Command(BaseCommand):
-    help = "Backfill historical blocks with rate limiting."
+    help = "Backfill historical blocks with rate limiting. Discovers gaps by default."
 
-    def add_arguments(self, parser) -> None:
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        """
+        Add command-line arguments to the parser.
+        """
         parser.add_argument(
             "--from-block",
             type=int,
-            required=True,
-            help="Starting block number (inclusive)",
+            required=False,
+            help="Starting block number (inclusive). If not provided, uses min block from database.",
         )
         parser.add_argument(
             "--to-block",
             type=int,
-            required=True,
-            help="Ending block number (inclusive)",
+            required=False,
+            help="Ending block number (inclusive). If not provided, uses max block from database.",
         )
         parser.add_argument(
             "--rate-limit",
@@ -44,13 +100,46 @@ class Command(BaseCommand):
             action="store_true",
             help="Preview blocks to backfill without executing tasks",
         )
+        parser.add_argument(
+            "--no-gap-detection",
+            action="store_true",
+            help="Process all blocks in range instead of only gaps (original behavior)",
+        )
 
-    def handle(self, *args, **options) -> None:
+    def handle(self, **options: BackfillOptions) -> None:  # type: ignore[override]
+        """
+        Main command handler.
+        """
         from_block = options["from_block"]
         to_block = options["to_block"]
         rate_limit = options["rate_limit"]
         network = options["network"]
         dry_run = options["dry_run"]
+        no_gap_detection = options["no_gap_detection"]
+
+        # Load registered functions
+        self.stdout.write("Syncing decorated functions...")
+        ensure_modules_loaded()
+        functions_counter = len(task_registry.get_functions())
+        self.stdout.write(self.style.SUCCESS(f"Synced {functions_counter} functions"))
+
+        if functions_counter == 0:
+            self.stderr.write(self.style.WARNING("No functions registered. Nothing to backfill."))
+            return
+
+        # Determine block range
+        if from_block is None or to_block is None:
+            min_block, max_block = abd_dal.get_block_range()
+            if min_block is None or max_block is None:
+                self.stderr.write(
+                    self.style.ERROR("No blocks found in database. Provide --from-block and --to-block."),
+                )
+                return
+
+            from_block = from_block if from_block is not None else min_block
+            to_block = to_block if to_block is not None else max_block
+
+            self.stdout.write(f"Using block range from database: {from_block} -> {to_block}")
 
         # Validate arguments
         if from_block > to_block:
@@ -61,17 +150,133 @@ class Command(BaseCommand):
             self.stderr.write(self.style.ERROR("--rate-limit must be >= 0"))
             return
 
-        # Load registered functions
-        self.stdout.write("Syncing decorated functions...")
-        ensure_modules_loaded()
-        functions_counter = len(task_registry.get_functions())
-        self.stdout.write(self.style.SUCCESS(f"Synced {functions_counter} functions"))
+        # Use gap detection by default
+        if no_gap_detection:
+            self._handle_range_backfill(from_block, to_block, rate_limit, network, dry_run=dry_run)
+        else:
+            self._handle_gap_backfill(from_block, to_block, rate_limit, network, dry_run=dry_run)
 
-        if functions_counter == 0:
-            self.stderr.write(self.style.WARNING("No functions registered. Nothing to backfill."))
+    def _handle_gap_backfill(
+        self,
+        from_block: int,
+        to_block: int,
+        rate_limit: float,
+        network: str,
+        *,
+        dry_run: bool,
+    ) -> None:
+        """Discover and process gaps in the block range."""
+        self.stdout.write("")
+        self.stdout.write("Discovering gaps in block range...")
+
+        # Get all successfully processed blocks in the range
+        processed_blocks = abd_dal.get_successful_block_numbers(from_block, to_block)
+        total_blocks = to_block - from_block + 1
+
+        self.stdout.write(f"Block range: {from_block} -> {to_block} ({total_blocks} blocks)")
+        self.stdout.write(f"Successfully processed blocks: {len(processed_blocks)}")
+
+        # Find gaps
+        gaps = find_gaps(from_block, to_block, processed_blocks)
+
+        if not gaps:
+            self.stdout.write(self.style.SUCCESS("No gaps found! All blocks have been processed."))
+            return
+
+        total_missing = sum(gap.size for gap in gaps)
+        self.stdout.write(f"Found {len(gaps)} gap(s) with {total_missing} missing blocks:")
+        self.stdout.write("")
+
+        # Display gaps
+        for i, gap in enumerate(gaps, 1):
+            if gap.size == 1:
+                self.stdout.write(f"  Gap {i}: block {gap.start}")
+            else:
+                self.stdout.write(f"  Gap {i}: blocks {gap.start} -> {gap.end} ({gap.size} blocks)")
+
+        self.stdout.write("")
+
+        if dry_run:
+            self._handle_gap_dry_run(gaps, total_missing, rate_limit, network)
+        else:
+            self._handle_gap_execution(gaps, total_missing, rate_limit, network)
+
+    def _handle_gap_dry_run(self, gaps: list[Gap], total_missing: int, rate_limit: float, network: str) -> None:
+        """Handle dry-run mode for gap backfill."""
+        self.stdout.write(self.style.WARNING("Dry-run mode: previewing gaps to backfill"))
+        self.stdout.write("")
+
+        estimated_tasks = 0
+        for gap in gaps:
+            scheduler = backfill_scheduler_factory(
+                from_block=gap.start,
+                to_block=gap.end,
+                network=network,
+                rate_limit=rate_limit,
+                dry_run=True,
+            )
+            stats = scheduler.start()
+            if stats:
+                estimated_tasks += stats.estimated_tasks
+
+        self.stdout.write(self.style.SUCCESS("Summary:"))
+        self.stdout.write(f"  Total gaps: {len(gaps)}")
+        self.stdout.write(f"  Total missing blocks: {total_missing}")
+        self.stdout.write(f"  Estimated tasks to submit: {estimated_tasks}")
+
+        if rate_limit > 0 and total_missing > 0:
+            estimated_seconds = total_missing * rate_limit
+            self._print_time_estimate(estimated_seconds, rate_limit)
+
+    def _handle_gap_execution(self, gaps: list[Gap], total_missing: int, rate_limit: float, network: str) -> None:
+        """Execute backfill for all gaps."""
+        self.stdout.write(f"Starting backfill of {len(gaps)} gap(s) with {total_missing} missing blocks")
+        self.stdout.write(f"Rate limit: {rate_limit} seconds between blocks")
+
+        if rate_limit > 0:
+            estimated_seconds = total_missing * rate_limit
+            self._print_time_estimate(estimated_seconds, rate_limit)
+
+        self.stdout.write("")
+        self.stdout.write("Press Ctrl+C to stop gracefully...")
+        self.stdout.write("")
+
+        blocks_processed = 0
+        try:
+            for i, gap in enumerate(gaps, 1):
+                self.stdout.write(f"Processing gap {i}/{len(gaps)}: {gap.start} -> {gap.end} ({gap.size} blocks)")
+
+                scheduler = backfill_scheduler_factory(
+                    from_block=gap.start,
+                    to_block=gap.end,
+                    network=network,
+                    rate_limit=rate_limit,
+                    dry_run=False,
+                )
+                scheduler.start()
+                blocks_processed += gap.size
+
+                # Small pause between gaps
+                if i < len(gaps) and rate_limit > 0:
+                    time.sleep(rate_limit)
+
+        except KeyboardInterrupt:
+            self.stdout.write("")
+            self.stdout.write(self.style.WARNING(f"Interrupted. Processed {blocks_processed}/{total_missing} blocks."))
             return
 
-        # Create scheduler
+        self.stdout.write(self.style.SUCCESS(f"Gap backfill completed. Processed {blocks_processed} blocks."))
+
+    def _handle_range_backfill(
+        self,
+        from_block: int,
+        to_block: int,
+        rate_limit: float,
+        network: str,
+        *,
+        dry_run: bool,
+    ) -> None:
+        """Handle backfill for the entire range (original behavior)."""
         scheduler = backfill_scheduler_factory(
             from_block=from_block,
             to_block=to_block,
@@ -88,7 +293,12 @@ class Command(BaseCommand):
             self._handle_backfill(scheduler, from_block, to_block, total_blocks, rate_limit)
 
     def _handle_dry_run(
-        self, scheduler: BackfillScheduler, from_block: int, to_block: int, total_blocks: int, rate_limit: float
+        self,
+        scheduler: BackfillScheduler,
+        from_block: int,
+        to_block: int,
+        total_blocks: int,
+        rate_limit: float,
     ) -> None:
         """Handle dry-run mode output."""
         self.stdout.write("")
@@ -96,13 +306,13 @@ class Command(BaseCommand):
         self.stdout.write("")
 
         # Get network type
-        scheduler._current_head_cache = scheduler.subtensor.get_current_block()
-        network_type = scheduler._get_network_type_for_block(from_block)
+        scheduler._current_head_cache = scheduler.subtensor.get_current_block()  # noqa: SLF001
+        network_type = scheduler._get_network_type_for_block(from_block)  # noqa: SLF001
 
         self.stdout.write(f"Block range: {from_block} -> {to_block} ({total_blocks} blocks)")
         operator = ">" if network_type == "archive" else "<="
         self.stdout.write(f"Network: {network_type} (blocks {operator}{ARCHIVE_BLOCK_THRESHOLD} behind head)")
-        self.stdout.write(f"Current head: {scheduler._current_head_cache}")
+        self.stdout.write(f"Current head: {scheduler._current_head_cache}")  # noqa: SLF001
         self.stdout.write("")
 
         # Show registry items
@@ -129,15 +339,16 @@ class Command(BaseCommand):
 
         if rate_limit > 0 and stats.blocks_needing_tasks > 0:
             estimated_seconds = stats.blocks_needing_tasks * rate_limit
-            if estimated_seconds < 60:
-                time_str = f"~{estimated_seconds:.0f} seconds"
-            elif estimated_seconds < 3600:
-                time_str = f"~{estimated_seconds / 60:.1f} minutes"
-            else:
-                time_str = f"~{estimated_seconds / 3600:.1f} hours"
-            self.stdout.write(f"  Estimated time at {rate_limit}s rate limit: {time_str}")
+            self._print_time_estimate(estimated_seconds, rate_limit)
 
-    def _handle_backfill(self, scheduler, from_block: int, to_block: int, total_blocks: int, rate_limit: float) -> None:
+    def _handle_backfill(
+        self,
+        scheduler: BackfillScheduler,
+        from_block: int,
+        to_block: int,
+        total_blocks: int,
+        rate_limit: float,
+    ) -> None:
         """Handle actual backfill execution."""
         self.stdout.write("")
         self.stdout.write(f"Starting backfill: {from_block} -> {to_block} ({total_blocks} blocks)")
@@ -145,13 +356,7 @@ class Command(BaseCommand):
 
         if rate_limit > 0:
             estimated_seconds = total_blocks * rate_limit
-            if estimated_seconds < 60:
-                time_str = f"~{estimated_seconds:.0f} seconds"
-            elif estimated_seconds < 3600:
-                time_str = f"~{estimated_seconds / 60:.1f} minutes"
-            else:
-                time_str = f"~{estimated_seconds / 3600:.1f} hours"
-            self.stdout.write(f"Estimated max time: {time_str}")
+            self._print_time_estimate(estimated_seconds, rate_limit)
 
         self.stdout.write("")
         self.stdout.write("Press Ctrl+C to stop gracefully...")
@@ -160,3 +365,13 @@ class Command(BaseCommand):
         scheduler.start()
 
         self.stdout.write(self.style.SUCCESS("Backfill completed"))
+
+    def _print_time_estimate(self, estimated_seconds: float, rate_limit: float) -> None:
+        """Print estimated time."""
+        if estimated_seconds < SECONDS_PER_MINUTE:
+            time_str = f"~{estimated_seconds:.0f} seconds"
+        elif estimated_seconds < SECONDS_PER_HOUR:
+            time_str = f"~{estimated_seconds / SECONDS_PER_MINUTE:.1f} minutes"
+        else:
+            time_str = f"~{estimated_seconds / SECONDS_PER_HOUR:.1f} hours"
+        self.stdout.write(f"Estimated time at {rate_limit}s rate limit: {time_str}")

{abstract_block_dumper-0.1.2.dist-info → abstract_block_dumper-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: abstract-block-dumper
-Version: 0.1.2
+Version: 0.1.5
 Project-URL: Source, https://github.com/bactensor/abstract-block-dumper
 Project-URL: Issue Tracker, https://github.com/bactensor/abstract-block-dumper/issues
 Author-email: Reef Technologies <opensource@reef.pl>

{abstract_block_dumper-0.1.2.dist-info → abstract_block_dumper-0.1.5.dist-info}/RECORD

@@ -1,14 +1,14 @@
 abstract_block_dumper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_version.py,sha256=Ok5oAXdWgR9aghaFXTafTeDW6sYO3uVe6d2Nket57R4,704
+abstract_block_dumper/_version.py,sha256=rdxBMYpwzYxiWk08QbPLHSAxHoDfeKWwyaJIAM0lSic,704
 abstract_block_dumper/admin.py,sha256=3J3I_QOKFgfMNpTXW-rTQGO_q5Ls6uNuL0FkPVdIsYg,1654
 abstract_block_dumper/apps.py,sha256=DXATdrjsL3T2IletTbKeD6unr8ScLaxg7wz0nAHTAns,215
 abstract_block_dumper/models.py,sha256=MO9824dmHB6xF3PrFE_RERh7whVjQtS4tt6QA0wSbg0,2022
 abstract_block_dumper/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 abstract_block_dumper/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_internal/discovery.py,sha256=sISOL8vq6rC0pOndrCfWKDZjyYwzzZIChG-BH9mteq0,745
+abstract_block_dumper/_internal/discovery.py,sha256=w3FJ6mcndik__ohzbj0U4QnkFznao9MsHd5rfceDumg,750
 abstract_block_dumper/_internal/exceptions.py,sha256=jVXQ8b3gneno2XYvO0XisJPMlkAWb6H5u10egIpPJ4k,335
 abstract_block_dumper/_internal/dal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/_internal/dal/django_dal.py,sha256=QbDsikUthIAhVC_FwSynUUdQL3OWlCo3_Cg65M91Cb4,5618
+abstract_block_dumper/_internal/dal/django_dal.py,sha256=G6Uf9U6Z-ax6i8neqHyaZZCAD1ad5BY7yMpww8Gcyn0,6443
 abstract_block_dumper/_internal/dal/memory_registry.py,sha256=m9Yms-cuemi9_5q_Kn_zsJnxDPEiuAUkESIAltD60QY,2943
 abstract_block_dumper/_internal/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 abstract_block_dumper/_internal/providers/bittensor_client.py,sha256=wlKjFrGN4Q2DfQyD_Fx-eH83ZMB6AbzLs5keYq6FGUw,4124
@@ -21,7 +21,7 @@ abstract_block_dumper/_internal/services/scheduler.py,sha256=BIQ7c-HYSebW3CKq5yn
 abstract_block_dumper/_internal/services/utils.py,sha256=QZxdQyWIcUnezyVmegS4g3x3BoB3-oijYJ_i9nLQWHo,1140
 abstract_block_dumper/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 abstract_block_dumper/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-abstract_block_dumper/management/commands/backfill_blocks_v1.py,sha256=EmNUozAZn8uThjCvusZe7poNrw9RYy-MafMg2wu3XeQ,6392
+abstract_block_dumper/management/commands/backfill_blocks_v1.py,sha256=IgZgoYYnH88DnfXHo4fbXijWEMICEOhuq03z6345d1Y,13849
 abstract_block_dumper/management/commands/block_tasks_v1.py,sha256=jSi04ahIKYwlm_dNKCUGL_cmALv1iP-ZjfXrmz0pn-4,880
 abstract_block_dumper/migrations/0001_initial.py,sha256=ImPHC3G6kPkq4Xn_4YVAm4Labh1Xi7PkCRszYRGpTiI,2298
 abstract_block_dumper/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -29,6 +29,6 @@ abstract_block_dumper/v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 abstract_block_dumper/v1/celery.py,sha256=X4IqVs5i6ZpyY7fy1SqMZgsZy4SXP-jK2qG-FYnjU38,1722
 abstract_block_dumper/v1/decorators.py,sha256=yQglsy1dU1u7ShwaTqZLahDcybHmetibTIOi53o_ZOM,9829
 abstract_block_dumper/v1/tasks.py,sha256=u9iMYdDUqzYT3yPrNwZecHnlweZ3yFipV9BcIWHCbus,2647
-abstract_block_dumper-0.1.2.dist-info/METADATA,sha256=6y5tq_8Wp3JNHuYATBIx_XQ2I0mXqBU5tgyf97rHahc,12993
-abstract_block_dumper-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-abstract_block_dumper-0.1.2.dist-info/RECORD,,
+abstract_block_dumper-0.1.5.dist-info/METADATA,sha256=s9EW_Tiw1S8shZtD9FcF6xzcmeUB28HoRptkVMCCxBY,12993
+abstract_block_dumper-0.1.5.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+abstract_block_dumper-0.1.5.dist-info/RECORD,,