abstract-block-dumper 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- abstract_block_dumper/_internal/dal/django_dal.py +25 -1
- abstract_block_dumper/_internal/discovery.py +1 -1
- abstract_block_dumper/_internal/providers/bittensor_client.py +46 -5
- abstract_block_dumper/_internal/services/backfill_scheduler.py +7 -12
- abstract_block_dumper/_internal/services/scheduler.py +2 -21
- abstract_block_dumper/_version.py +2 -2
- abstract_block_dumper/management/commands/backfill_blocks_v1.py +239 -35
- {abstract_block_dumper-0.1.1.dist-info → abstract_block_dumper-0.1.3.dist-info}/METADATA +1 -1
- {abstract_block_dumper-0.1.1.dist-info → abstract_block_dumper-0.1.3.dist-info}/RECORD +10 -10
- {abstract_block_dumper-0.1.1.dist-info → abstract_block_dumper-0.1.3.dist-info}/WHEEL +0 -0
|
@@ -4,7 +4,7 @@ from typing import Any
|
|
|
4
4
|
|
|
5
5
|
from django.conf import settings
|
|
6
6
|
from django.db import transaction
|
|
7
|
-
from django.db.models import Max
|
|
7
|
+
from django.db.models import Max, Min
|
|
8
8
|
from django.db.models.query import QuerySet
|
|
9
9
|
from django.utils import timezone
|
|
10
10
|
|
|
@@ -157,3 +157,27 @@ def task_create_or_get_pending(
|
|
|
157
157
|
def get_the_latest_executed_block_number() -> int | None:
|
|
158
158
|
result = abd_models.TaskAttempt.objects.aggregate(max_block=Max("block_number"))
|
|
159
159
|
return result["max_block"]
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def get_block_range() -> tuple[int | None, int | None]:
|
|
163
|
+
"""Get the min and max block numbers from all task attempts."""
|
|
164
|
+
result = abd_models.TaskAttempt.objects.aggregate(
|
|
165
|
+
min_block=Min("block_number"),
|
|
166
|
+
max_block=Max("block_number"),
|
|
167
|
+
)
|
|
168
|
+
return result["min_block"], result["max_block"]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def get_successful_block_numbers(from_block: int, to_block: int) -> set[int]:
|
|
172
|
+
"""Get all block numbers with at least one successful task in the range."""
|
|
173
|
+
block_numbers = (
|
|
174
|
+
abd_models.TaskAttempt.objects.filter(
|
|
175
|
+
block_number__gte=from_block,
|
|
176
|
+
block_number__lte=to_block,
|
|
177
|
+
status=abd_models.TaskAttempt.Status.SUCCESS,
|
|
178
|
+
)
|
|
179
|
+
.values_list("block_number", flat=True)
|
|
180
|
+
.distinct()
|
|
181
|
+
.iterator()
|
|
182
|
+
)
|
|
183
|
+
return set(block_numbers)
|
|
@@ -20,5 +20,5 @@ def ensure_modules_loaded() -> None:
|
|
|
20
20
|
except ModuleNotFoundError:
|
|
21
21
|
continue
|
|
22
22
|
except ImportError as e:
|
|
23
|
-
logger.warning(
|
|
23
|
+
logger.warning("Failed to import %s.%s: %s", app_config.name, module_suffix, e)
|
|
24
24
|
continue
|
|
@@ -1,8 +1,15 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
1
5
|
import bittensor as bt
|
|
2
6
|
import structlog
|
|
3
7
|
|
|
4
8
|
import abstract_block_dumper._internal.services.utils as abd_utils
|
|
5
9
|
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
import types
|
|
12
|
+
|
|
6
13
|
logger = structlog.get_logger(__name__)
|
|
7
14
|
|
|
8
15
|
|
|
@@ -13,6 +20,10 @@ ARCHIVE_BLOCK_THRESHOLD = 300
|
|
|
13
20
|
class BittensorConnectionClient:
|
|
14
21
|
"""
|
|
15
22
|
Manages connections to regular and archive Bittensor subtensor networks.
|
|
23
|
+
|
|
24
|
+
Supports context manager protocol for safe connection cleanup:
|
|
25
|
+
with BittensorConnectionClient(network="finney") as client:
|
|
26
|
+
block = client.subtensor.get_current_block()
|
|
16
27
|
"""
|
|
17
28
|
|
|
18
29
|
def __init__(self, network: str) -> None:
|
|
@@ -21,6 +32,38 @@ class BittensorConnectionClient:
|
|
|
21
32
|
self._archive_subtensor: bt.Subtensor | None = None
|
|
22
33
|
self._current_block_cache: int | None = None
|
|
23
34
|
|
|
35
|
+
def __enter__(self) -> BittensorConnectionClient:
|
|
36
|
+
"""Context manager entry."""
|
|
37
|
+
return self
|
|
38
|
+
|
|
39
|
+
def __exit__(
|
|
40
|
+
self,
|
|
41
|
+
_exc_type: type[BaseException] | None,
|
|
42
|
+
_exc_val: BaseException | None,
|
|
43
|
+
_exc_tb: types.TracebackType | None,
|
|
44
|
+
) -> None:
|
|
45
|
+
"""Context manager exit - ensures connections are closed."""
|
|
46
|
+
self.close()
|
|
47
|
+
|
|
48
|
+
def close(self) -> None:
|
|
49
|
+
"""Close all subtensor connections to prevent memory leaks."""
|
|
50
|
+
if self._subtensor is not None:
|
|
51
|
+
try:
|
|
52
|
+
self._subtensor.close()
|
|
53
|
+
except Exception:
|
|
54
|
+
logger.warning("Error closing subtensor connection", exc_info=True)
|
|
55
|
+
self._subtensor = None
|
|
56
|
+
|
|
57
|
+
if self._archive_subtensor is not None:
|
|
58
|
+
try:
|
|
59
|
+
self._archive_subtensor.close()
|
|
60
|
+
except Exception:
|
|
61
|
+
logger.warning("Error closing archive subtensor connection", exc_info=True)
|
|
62
|
+
self._archive_subtensor = None
|
|
63
|
+
|
|
64
|
+
self._current_block_cache = None
|
|
65
|
+
logger.debug("Subtensor connections closed")
|
|
66
|
+
|
|
24
67
|
def get_for_block(self, block_number: int) -> bt.Subtensor:
|
|
25
68
|
"""Get the appropriate subtensor client for the given block number."""
|
|
26
69
|
raise NotImplementedError
|
|
@@ -71,8 +114,6 @@ class BittensorConnectionClient:
|
|
|
71
114
|
return self.subtensor
|
|
72
115
|
|
|
73
116
|
def refresh_connections(self) -> None:
|
|
74
|
-
"""
|
|
75
|
-
self.
|
|
76
|
-
|
|
77
|
-
self._current_block_cache = None
|
|
78
|
-
logger.info("Subtensor connections reset")
|
|
117
|
+
"""Close and reset all subtensor connections to force re-establishment."""
|
|
118
|
+
self.close()
|
|
119
|
+
logger.info("Subtensor connections refreshed")
|
|
@@ -36,8 +36,6 @@ logger = structlog.get_logger(__name__)
|
|
|
36
36
|
# Blocks older than this threshold from current head require archive network
|
|
37
37
|
ARCHIVE_BLOCK_THRESHOLD = 300
|
|
38
38
|
|
|
39
|
-
# Progress logging interval
|
|
40
|
-
PROGRESS_LOG_INTERVAL = 100
|
|
41
39
|
ARCHIVE_NETWORK = "archive"
|
|
42
40
|
|
|
43
41
|
# Memory cleanup interval (every N blocks)
|
|
@@ -281,16 +279,13 @@ class BackfillScheduler:
|
|
|
281
279
|
if self._current_head_cache:
|
|
282
280
|
set_block_lag("backfill", self._current_head_cache - block_number)
|
|
283
281
|
|
|
284
|
-
# Log
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
progress_percent=f"{progress_pct:.1f}%",
|
|
292
|
-
current_block=block_number,
|
|
293
|
-
)
|
|
282
|
+
# Log each block being processed
|
|
283
|
+
progress_pct = (processed_count / total_blocks) * 100
|
|
284
|
+
logger.info(
|
|
285
|
+
"Backfilling block",
|
|
286
|
+
block=block_number,
|
|
287
|
+
progress=f"{processed_count}/{total_blocks} ({progress_pct:.1f}%)",
|
|
288
|
+
)
|
|
294
289
|
|
|
295
290
|
# Rate limiting between block submissions
|
|
296
291
|
if block_number < self.to_block and self.rate_limit > 0:
|
|
@@ -2,7 +2,6 @@ import time
|
|
|
2
2
|
from typing import Protocol
|
|
3
3
|
|
|
4
4
|
import structlog
|
|
5
|
-
from django import db
|
|
6
5
|
from django.conf import settings
|
|
7
6
|
|
|
8
7
|
import abstract_block_dumper._internal.dal.django_dal as abd_dal
|
|
@@ -16,9 +15,6 @@ from abstract_block_dumper._internal.services.metrics import (
|
|
|
16
15
|
set_registered_tasks,
|
|
17
16
|
)
|
|
18
17
|
|
|
19
|
-
# Refresh bittensor connections every N blocks to prevent memory leaks from internal caches
|
|
20
|
-
CONNECTION_REFRESH_INTERVAL = 1000
|
|
21
|
-
|
|
22
18
|
logger = structlog.get_logger(__name__)
|
|
23
19
|
|
|
24
20
|
|
|
@@ -42,6 +38,7 @@ class DefaultBlockStateResolver:
|
|
|
42
38
|
return self.bittensor_client.subtensor.get_current_block()
|
|
43
39
|
if isinstance(start_setting, int):
|
|
44
40
|
return start_setting
|
|
41
|
+
|
|
45
42
|
# Default: resume from DB or current
|
|
46
43
|
return abd_dal.get_the_latest_executed_block_number() or self.bittensor_client.subtensor.get_current_block()
|
|
47
44
|
|
|
@@ -59,7 +56,6 @@ class TaskScheduler:
|
|
|
59
56
|
self.bittensor_client = bittensor_client
|
|
60
57
|
self.last_processed_block = state_resolver.get_starting_block()
|
|
61
58
|
self.is_running = False
|
|
62
|
-
self._blocks_since_refresh = 0
|
|
63
59
|
|
|
64
60
|
def start(self) -> None:
|
|
65
61
|
self.is_running = True
|
|
@@ -86,11 +82,6 @@ class TaskScheduler:
|
|
|
86
82
|
increment_blocks_processed("realtime")
|
|
87
83
|
set_block_lag("realtime", 0) # Head-only mode has no lag
|
|
88
84
|
self.last_processed_block = current_block
|
|
89
|
-
self._blocks_since_refresh += 1
|
|
90
|
-
|
|
91
|
-
# Periodic memory cleanup
|
|
92
|
-
if self._blocks_since_refresh >= CONNECTION_REFRESH_INTERVAL:
|
|
93
|
-
self._perform_cleanup()
|
|
94
85
|
|
|
95
86
|
time.sleep(self.poll_interval)
|
|
96
87
|
|
|
@@ -104,19 +95,9 @@ class TaskScheduler:
|
|
|
104
95
|
|
|
105
96
|
def stop(self) -> None:
|
|
106
97
|
self.is_running = False
|
|
98
|
+
self.bittensor_client.close()
|
|
107
99
|
logger.info("TaskScheduler stopped.")
|
|
108
100
|
|
|
109
|
-
def _perform_cleanup(self) -> None:
|
|
110
|
-
"""Perform periodic memory cleanup to prevent leaks in long-running processes."""
|
|
111
|
-
# Reset bittensor connections to clear internal caches
|
|
112
|
-
self.bittensor_client.refresh_connections()
|
|
113
|
-
|
|
114
|
-
# Clear Django's query log (only accumulates if DEBUG=True)
|
|
115
|
-
db.reset_queries()
|
|
116
|
-
|
|
117
|
-
self._blocks_since_refresh = 0
|
|
118
|
-
logger.debug("Memory cleanup performed", blocks_processed=CONNECTION_REFRESH_INTERVAL)
|
|
119
|
-
|
|
120
101
|
|
|
121
102
|
def task_scheduler_factory(network: str = "finney") -> TaskScheduler:
|
|
122
103
|
"""
|
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '0.1.
|
|
32
|
-
__version_tuple__ = version_tuple = (0, 1,
|
|
31
|
+
__version__ = version = '0.1.3'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 1, 3)
|
|
33
33
|
|
|
34
34
|
__commit_id__ = commit_id = None
|
|
@@ -1,7 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
import argparse
|
|
2
|
+
import time
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any
|
|
2
5
|
|
|
3
6
|
from django.core.management.base import BaseCommand
|
|
4
7
|
|
|
8
|
+
import abstract_block_dumper._internal.dal.django_dal as abd_dal
|
|
5
9
|
from abstract_block_dumper._internal.dal.memory_registry import task_registry
|
|
6
10
|
from abstract_block_dumper._internal.discovery import ensure_modules_loaded
|
|
7
11
|
from abstract_block_dumper._internal.services.backfill_scheduler import (
|
|
@@ -10,22 +14,63 @@ from abstract_block_dumper._internal.services.backfill_scheduler import (
|
|
|
10
14
|
backfill_scheduler_factory,
|
|
11
15
|
)
|
|
12
16
|
|
|
17
|
+
SECONDS_PER_HOUR = 3600
|
|
18
|
+
SECONDS_PER_MINUTE = 60
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class Gap:
|
|
23
|
+
"""Represents a gap of missing blocks."""
|
|
24
|
+
|
|
25
|
+
start: int
|
|
26
|
+
end: int
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def size(self) -> int:
|
|
30
|
+
"""
|
|
31
|
+
Get the size of the gap (number of missing blocks).
|
|
32
|
+
"""
|
|
33
|
+
return self.end - self.start + 1
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def find_gaps(from_block: int, to_block: int, processed_blocks: set[int]) -> list[Gap]:
|
|
37
|
+
"""Find gaps (missing blocks) in the given range."""
|
|
38
|
+
gaps = []
|
|
39
|
+
gap_start = None
|
|
40
|
+
|
|
41
|
+
for block_num in range(from_block, to_block + 1):
|
|
42
|
+
if block_num not in processed_blocks:
|
|
43
|
+
if gap_start is None:
|
|
44
|
+
gap_start = block_num
|
|
45
|
+
elif gap_start is not None:
|
|
46
|
+
gaps.append(Gap(start=gap_start, end=block_num - 1))
|
|
47
|
+
gap_start = None
|
|
48
|
+
|
|
49
|
+
# Handle gap at the end
|
|
50
|
+
if gap_start is not None:
|
|
51
|
+
gaps.append(Gap(start=gap_start, end=to_block))
|
|
52
|
+
|
|
53
|
+
return gaps
|
|
54
|
+
|
|
13
55
|
|
|
14
56
|
class Command(BaseCommand):
|
|
15
|
-
help = "Backfill historical blocks with rate limiting."
|
|
57
|
+
help = "Backfill historical blocks with rate limiting. Discovers gaps by default."
|
|
16
58
|
|
|
17
|
-
def add_arguments(self, parser) -> None:
|
|
59
|
+
def add_arguments(self, parser: argparse.ArgumentParser) -> None:
|
|
60
|
+
"""
|
|
61
|
+
Add command-line arguments to the parser.
|
|
62
|
+
"""
|
|
18
63
|
parser.add_argument(
|
|
19
64
|
"--from-block",
|
|
20
65
|
type=int,
|
|
21
|
-
required=
|
|
22
|
-
help="Starting block number (inclusive)",
|
|
66
|
+
required=False,
|
|
67
|
+
help="Starting block number (inclusive). If not provided, uses min block from database.",
|
|
23
68
|
)
|
|
24
69
|
parser.add_argument(
|
|
25
70
|
"--to-block",
|
|
26
71
|
type=int,
|
|
27
|
-
required=
|
|
28
|
-
help="Ending block number (inclusive)",
|
|
72
|
+
required=False,
|
|
73
|
+
help="Ending block number (inclusive). If not provided, uses max block from database.",
|
|
29
74
|
)
|
|
30
75
|
parser.add_argument(
|
|
31
76
|
"--rate-limit",
|
|
@@ -44,13 +89,46 @@ class Command(BaseCommand):
|
|
|
44
89
|
action="store_true",
|
|
45
90
|
help="Preview blocks to backfill without executing tasks",
|
|
46
91
|
)
|
|
92
|
+
parser.add_argument(
|
|
93
|
+
"--no-gap-detection",
|
|
94
|
+
action="store_true",
|
|
95
|
+
help="Process all blocks in range instead of only gaps (original behavior)",
|
|
96
|
+
)
|
|
47
97
|
|
|
48
|
-
def handle(self,
|
|
98
|
+
def handle(self, **options: dict[str, Any]) -> None:
|
|
99
|
+
"""
|
|
100
|
+
Main command handler.
|
|
101
|
+
"""
|
|
49
102
|
from_block = options["from_block"]
|
|
50
103
|
to_block = options["to_block"]
|
|
51
104
|
rate_limit = options["rate_limit"]
|
|
52
105
|
network = options["network"]
|
|
53
106
|
dry_run = options["dry_run"]
|
|
107
|
+
no_gap_detection = options["no_gap_detection"]
|
|
108
|
+
|
|
109
|
+
# Load registered functions
|
|
110
|
+
self.stdout.write("Syncing decorated functions...")
|
|
111
|
+
ensure_modules_loaded()
|
|
112
|
+
functions_counter = len(task_registry.get_functions())
|
|
113
|
+
self.stdout.write(self.style.SUCCESS(f"Synced {functions_counter} functions"))
|
|
114
|
+
|
|
115
|
+
if functions_counter == 0:
|
|
116
|
+
self.stderr.write(self.style.WARNING("No functions registered. Nothing to backfill."))
|
|
117
|
+
return
|
|
118
|
+
|
|
119
|
+
# Determine block range
|
|
120
|
+
if from_block is None or to_block is None:
|
|
121
|
+
min_block, max_block = abd_dal.get_block_range()
|
|
122
|
+
if min_block is None or max_block is None:
|
|
123
|
+
self.stderr.write(
|
|
124
|
+
self.style.ERROR("No blocks found in database. Provide --from-block and --to-block."),
|
|
125
|
+
)
|
|
126
|
+
return
|
|
127
|
+
|
|
128
|
+
from_block = from_block if from_block is not None else min_block
|
|
129
|
+
to_block = to_block if to_block is not None else max_block
|
|
130
|
+
|
|
131
|
+
self.stdout.write(f"Using block range from database: {from_block} -> {to_block}")
|
|
54
132
|
|
|
55
133
|
# Validate arguments
|
|
56
134
|
if from_block > to_block:
|
|
@@ -61,17 +139,133 @@ class Command(BaseCommand):
|
|
|
61
139
|
self.stderr.write(self.style.ERROR("--rate-limit must be >= 0"))
|
|
62
140
|
return
|
|
63
141
|
|
|
64
|
-
#
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
142
|
+
# Use gap detection by default
|
|
143
|
+
if no_gap_detection:
|
|
144
|
+
self._handle_range_backfill(from_block, to_block, rate_limit, network, dry_run)
|
|
145
|
+
else:
|
|
146
|
+
self._handle_gap_backfill(from_block, to_block, rate_limit, network, dry_run)
|
|
69
147
|
|
|
70
|
-
|
|
71
|
-
|
|
148
|
+
def _handle_gap_backfill(
|
|
149
|
+
self,
|
|
150
|
+
from_block: int,
|
|
151
|
+
to_block: int,
|
|
152
|
+
rate_limit: float,
|
|
153
|
+
network: str,
|
|
154
|
+
*,
|
|
155
|
+
dry_run: bool,
|
|
156
|
+
) -> None:
|
|
157
|
+
"""Discover and process gaps in the block range."""
|
|
158
|
+
self.stdout.write("")
|
|
159
|
+
self.stdout.write("Discovering gaps in block range...")
|
|
160
|
+
|
|
161
|
+
# Get all successfully processed blocks in the range
|
|
162
|
+
processed_blocks = abd_dal.get_successful_block_numbers(from_block, to_block)
|
|
163
|
+
total_blocks = to_block - from_block + 1
|
|
164
|
+
|
|
165
|
+
self.stdout.write(f"Block range: {from_block} -> {to_block} ({total_blocks} blocks)")
|
|
166
|
+
self.stdout.write(f"Successfully processed blocks: {len(processed_blocks)}")
|
|
167
|
+
|
|
168
|
+
# Find gaps
|
|
169
|
+
gaps = find_gaps(from_block, to_block, processed_blocks)
|
|
170
|
+
|
|
171
|
+
if not gaps:
|
|
172
|
+
self.stdout.write(self.style.SUCCESS("No gaps found! All blocks have been processed."))
|
|
173
|
+
return
|
|
174
|
+
|
|
175
|
+
total_missing = sum(gap.size for gap in gaps)
|
|
176
|
+
self.stdout.write(f"Found {len(gaps)} gap(s) with {total_missing} missing blocks:")
|
|
177
|
+
self.stdout.write("")
|
|
178
|
+
|
|
179
|
+
# Display gaps
|
|
180
|
+
for i, gap in enumerate(gaps, 1):
|
|
181
|
+
if gap.size == 1:
|
|
182
|
+
self.stdout.write(f" Gap {i}: block {gap.start}")
|
|
183
|
+
else:
|
|
184
|
+
self.stdout.write(f" Gap {i}: blocks {gap.start} -> {gap.end} ({gap.size} blocks)")
|
|
185
|
+
|
|
186
|
+
self.stdout.write("")
|
|
187
|
+
|
|
188
|
+
if dry_run:
|
|
189
|
+
self._handle_gap_dry_run(gaps, total_missing, rate_limit, network)
|
|
190
|
+
else:
|
|
191
|
+
self._handle_gap_execution(gaps, total_missing, rate_limit, network)
|
|
192
|
+
|
|
193
|
+
def _handle_gap_dry_run(self, gaps: list[Gap], total_missing: int, rate_limit: float, network: str) -> None:
|
|
194
|
+
"""Handle dry-run mode for gap backfill."""
|
|
195
|
+
self.stdout.write(self.style.WARNING("Dry-run mode: previewing gaps to backfill"))
|
|
196
|
+
self.stdout.write("")
|
|
197
|
+
|
|
198
|
+
estimated_tasks = 0
|
|
199
|
+
for gap in gaps:
|
|
200
|
+
scheduler = backfill_scheduler_factory(
|
|
201
|
+
from_block=gap.start,
|
|
202
|
+
to_block=gap.end,
|
|
203
|
+
network=network,
|
|
204
|
+
rate_limit=rate_limit,
|
|
205
|
+
dry_run=True,
|
|
206
|
+
)
|
|
207
|
+
stats = scheduler.start()
|
|
208
|
+
if stats:
|
|
209
|
+
estimated_tasks += stats.estimated_tasks
|
|
210
|
+
|
|
211
|
+
self.stdout.write(self.style.SUCCESS("Summary:"))
|
|
212
|
+
self.stdout.write(f" Total gaps: {len(gaps)}")
|
|
213
|
+
self.stdout.write(f" Total missing blocks: {total_missing}")
|
|
214
|
+
self.stdout.write(f" Estimated tasks to submit: {estimated_tasks}")
|
|
215
|
+
|
|
216
|
+
if rate_limit > 0 and total_missing > 0:
|
|
217
|
+
estimated_seconds = total_missing * rate_limit
|
|
218
|
+
self._print_time_estimate(estimated_seconds, rate_limit)
|
|
219
|
+
|
|
220
|
+
def _handle_gap_execution(self, gaps: list[Gap], total_missing: int, rate_limit: float, network: str) -> None:
|
|
221
|
+
"""Execute backfill for all gaps."""
|
|
222
|
+
self.stdout.write(f"Starting backfill of {len(gaps)} gap(s) with {total_missing} missing blocks")
|
|
223
|
+
self.stdout.write(f"Rate limit: {rate_limit} seconds between blocks")
|
|
224
|
+
|
|
225
|
+
if rate_limit > 0:
|
|
226
|
+
estimated_seconds = total_missing * rate_limit
|
|
227
|
+
self._print_time_estimate(estimated_seconds, rate_limit)
|
|
228
|
+
|
|
229
|
+
self.stdout.write("")
|
|
230
|
+
self.stdout.write("Press Ctrl+C to stop gracefully...")
|
|
231
|
+
self.stdout.write("")
|
|
232
|
+
|
|
233
|
+
blocks_processed = 0
|
|
234
|
+
try:
|
|
235
|
+
for i, gap in enumerate(gaps, 1):
|
|
236
|
+
self.stdout.write(f"Processing gap {i}/{len(gaps)}: {gap.start} -> {gap.end} ({gap.size} blocks)")
|
|
237
|
+
|
|
238
|
+
scheduler = backfill_scheduler_factory(
|
|
239
|
+
from_block=gap.start,
|
|
240
|
+
to_block=gap.end,
|
|
241
|
+
network=network,
|
|
242
|
+
rate_limit=rate_limit,
|
|
243
|
+
dry_run=False,
|
|
244
|
+
)
|
|
245
|
+
scheduler.start()
|
|
246
|
+
blocks_processed += gap.size
|
|
247
|
+
|
|
248
|
+
# Small pause between gaps
|
|
249
|
+
if i < len(gaps) and rate_limit > 0:
|
|
250
|
+
time.sleep(rate_limit)
|
|
251
|
+
|
|
252
|
+
except KeyboardInterrupt:
|
|
253
|
+
self.stdout.write("")
|
|
254
|
+
self.stdout.write(self.style.WARNING(f"Interrupted. Processed {blocks_processed}/{total_missing} blocks."))
|
|
72
255
|
return
|
|
73
256
|
|
|
74
|
-
|
|
257
|
+
self.stdout.write(self.style.SUCCESS(f"Gap backfill completed. Processed {blocks_processed} blocks."))
|
|
258
|
+
|
|
259
|
+
def _handle_range_backfill(
|
|
260
|
+
self,
|
|
261
|
+
from_block: int,
|
|
262
|
+
to_block: int,
|
|
263
|
+
rate_limit: float,
|
|
264
|
+
network: str,
|
|
265
|
+
*,
|
|
266
|
+
dry_run: bool,
|
|
267
|
+
) -> None:
|
|
268
|
+
"""Handle backfill for the entire range (original behavior)."""
|
|
75
269
|
scheduler = backfill_scheduler_factory(
|
|
76
270
|
from_block=from_block,
|
|
77
271
|
to_block=to_block,
|
|
@@ -88,7 +282,12 @@ class Command(BaseCommand):
|
|
|
88
282
|
self._handle_backfill(scheduler, from_block, to_block, total_blocks, rate_limit)
|
|
89
283
|
|
|
90
284
|
def _handle_dry_run(
|
|
91
|
-
self,
|
|
285
|
+
self,
|
|
286
|
+
scheduler: BackfillScheduler,
|
|
287
|
+
from_block: int,
|
|
288
|
+
to_block: int,
|
|
289
|
+
total_blocks: int,
|
|
290
|
+
rate_limit: float,
|
|
92
291
|
) -> None:
|
|
93
292
|
"""Handle dry-run mode output."""
|
|
94
293
|
self.stdout.write("")
|
|
@@ -96,13 +295,13 @@ class Command(BaseCommand):
|
|
|
96
295
|
self.stdout.write("")
|
|
97
296
|
|
|
98
297
|
# Get network type
|
|
99
|
-
scheduler._current_head_cache = scheduler.subtensor.get_current_block()
|
|
100
|
-
network_type = scheduler._get_network_type_for_block(from_block)
|
|
298
|
+
scheduler._current_head_cache = scheduler.subtensor.get_current_block() # noqa: SLF001
|
|
299
|
+
network_type = scheduler._get_network_type_for_block(from_block) # noqa: SLF001
|
|
101
300
|
|
|
102
301
|
self.stdout.write(f"Block range: {from_block} -> {to_block} ({total_blocks} blocks)")
|
|
103
302
|
operator = ">" if network_type == "archive" else "<="
|
|
104
303
|
self.stdout.write(f"Network: {network_type} (blocks {operator}{ARCHIVE_BLOCK_THRESHOLD} behind head)")
|
|
105
|
-
self.stdout.write(f"Current head: {scheduler._current_head_cache}")
|
|
304
|
+
self.stdout.write(f"Current head: {scheduler._current_head_cache}") # noqa: SLF001
|
|
106
305
|
self.stdout.write("")
|
|
107
306
|
|
|
108
307
|
# Show registry items
|
|
@@ -129,15 +328,16 @@ class Command(BaseCommand):
|
|
|
129
328
|
|
|
130
329
|
if rate_limit > 0 and stats.blocks_needing_tasks > 0:
|
|
131
330
|
estimated_seconds = stats.blocks_needing_tasks * rate_limit
|
|
132
|
-
|
|
133
|
-
time_str = f"~{estimated_seconds:.0f} seconds"
|
|
134
|
-
elif estimated_seconds < 3600:
|
|
135
|
-
time_str = f"~{estimated_seconds / 60:.1f} minutes"
|
|
136
|
-
else:
|
|
137
|
-
time_str = f"~{estimated_seconds / 3600:.1f} hours"
|
|
138
|
-
self.stdout.write(f" Estimated time at {rate_limit}s rate limit: {time_str}")
|
|
331
|
+
self._print_time_estimate(estimated_seconds, rate_limit)
|
|
139
332
|
|
|
140
|
-
def _handle_backfill(
|
|
333
|
+
def _handle_backfill(
|
|
334
|
+
self,
|
|
335
|
+
scheduler: BackfillScheduler,
|
|
336
|
+
from_block: int,
|
|
337
|
+
to_block: int,
|
|
338
|
+
total_blocks: int,
|
|
339
|
+
rate_limit: float,
|
|
340
|
+
) -> None:
|
|
141
341
|
"""Handle actual backfill execution."""
|
|
142
342
|
self.stdout.write("")
|
|
143
343
|
self.stdout.write(f"Starting backfill: {from_block} -> {to_block} ({total_blocks} blocks)")
|
|
@@ -145,13 +345,7 @@ class Command(BaseCommand):
|
|
|
145
345
|
|
|
146
346
|
if rate_limit > 0:
|
|
147
347
|
estimated_seconds = total_blocks * rate_limit
|
|
148
|
-
|
|
149
|
-
time_str = f"~{estimated_seconds:.0f} seconds"
|
|
150
|
-
elif estimated_seconds < 3600:
|
|
151
|
-
time_str = f"~{estimated_seconds / 60:.1f} minutes"
|
|
152
|
-
else:
|
|
153
|
-
time_str = f"~{estimated_seconds / 3600:.1f} hours"
|
|
154
|
-
self.stdout.write(f"Estimated max time: {time_str}")
|
|
348
|
+
self._print_time_estimate(estimated_seconds, rate_limit)
|
|
155
349
|
|
|
156
350
|
self.stdout.write("")
|
|
157
351
|
self.stdout.write("Press Ctrl+C to stop gracefully...")
|
|
@@ -160,3 +354,13 @@ class Command(BaseCommand):
|
|
|
160
354
|
scheduler.start()
|
|
161
355
|
|
|
162
356
|
self.stdout.write(self.style.SUCCESS("Backfill completed"))
|
|
357
|
+
|
|
358
|
+
def _print_time_estimate(self, estimated_seconds: float, rate_limit: float) -> None:
|
|
359
|
+
"""Print estimated time."""
|
|
360
|
+
if estimated_seconds < SECONDS_PER_MINUTE:
|
|
361
|
+
time_str = f"~{estimated_seconds:.0f} seconds"
|
|
362
|
+
elif estimated_seconds < SECONDS_PER_HOUR:
|
|
363
|
+
time_str = f"~{estimated_seconds / SECONDS_PER_MINUTE:.1f} minutes"
|
|
364
|
+
else:
|
|
365
|
+
time_str = f"~{estimated_seconds / SECONDS_PER_HOUR:.1f} hours"
|
|
366
|
+
self.stdout.write(f"Estimated time at {rate_limit}s rate limit: {time_str}")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: abstract-block-dumper
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Project-URL: Source, https://github.com/bactensor/abstract-block-dumper
|
|
5
5
|
Project-URL: Issue Tracker, https://github.com/bactensor/abstract-block-dumper/issues
|
|
6
6
|
Author-email: Reef Technologies <opensource@reef.pl>
|
|
@@ -1,27 +1,27 @@
|
|
|
1
1
|
abstract_block_dumper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
abstract_block_dumper/_version.py,sha256=
|
|
2
|
+
abstract_block_dumper/_version.py,sha256=q5nF98G8SoVeJqaknL0xdyxtv0egsqb0fK06_84Izu8,704
|
|
3
3
|
abstract_block_dumper/admin.py,sha256=3J3I_QOKFgfMNpTXW-rTQGO_q5Ls6uNuL0FkPVdIsYg,1654
|
|
4
4
|
abstract_block_dumper/apps.py,sha256=DXATdrjsL3T2IletTbKeD6unr8ScLaxg7wz0nAHTAns,215
|
|
5
5
|
abstract_block_dumper/models.py,sha256=MO9824dmHB6xF3PrFE_RERh7whVjQtS4tt6QA0wSbg0,2022
|
|
6
6
|
abstract_block_dumper/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
7
|
abstract_block_dumper/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
-
abstract_block_dumper/_internal/discovery.py,sha256=
|
|
8
|
+
abstract_block_dumper/_internal/discovery.py,sha256=w3FJ6mcndik__ohzbj0U4QnkFznao9MsHd5rfceDumg,750
|
|
9
9
|
abstract_block_dumper/_internal/exceptions.py,sha256=jVXQ8b3gneno2XYvO0XisJPMlkAWb6H5u10egIpPJ4k,335
|
|
10
10
|
abstract_block_dumper/_internal/dal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
-
abstract_block_dumper/_internal/dal/django_dal.py,sha256=
|
|
11
|
+
abstract_block_dumper/_internal/dal/django_dal.py,sha256=G6Uf9U6Z-ax6i8neqHyaZZCAD1ad5BY7yMpww8Gcyn0,6443
|
|
12
12
|
abstract_block_dumper/_internal/dal/memory_registry.py,sha256=m9Yms-cuemi9_5q_Kn_zsJnxDPEiuAUkESIAltD60QY,2943
|
|
13
13
|
abstract_block_dumper/_internal/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
-
abstract_block_dumper/_internal/providers/bittensor_client.py,sha256=
|
|
14
|
+
abstract_block_dumper/_internal/providers/bittensor_client.py,sha256=wlKjFrGN4Q2DfQyD_Fx-eH83ZMB6AbzLs5keYq6FGUw,4124
|
|
15
15
|
abstract_block_dumper/_internal/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
|
-
abstract_block_dumper/_internal/services/backfill_scheduler.py,sha256=
|
|
16
|
+
abstract_block_dumper/_internal/services/backfill_scheduler.py,sha256=3X9NRXCamnb1jUI47sUXqmsrQ1nBzK0BLKe5CeLt00E,16091
|
|
17
17
|
abstract_block_dumper/_internal/services/block_processor.py,sha256=P8_LZR4ZSyNKbtnqFbAUkpT1XmEh9yX5Pgj5J__IwdA,7409
|
|
18
18
|
abstract_block_dumper/_internal/services/executor.py,sha256=WhpHhOAi4cI-qdEE8-DSt9xZwooOpSc9_uDMQBBoHUM,2317
|
|
19
19
|
abstract_block_dumper/_internal/services/metrics.py,sha256=Gg-PQYZ98caaS52wm1EqhtPURXlfrVjk2t3-8nccqfo,7821
|
|
20
|
-
abstract_block_dumper/_internal/services/scheduler.py,sha256=
|
|
20
|
+
abstract_block_dumper/_internal/services/scheduler.py,sha256=BIQ7c-HYSebW3CKq5ynsMZjULEO9c5YP0qWFN1aqg24,4164
|
|
21
21
|
abstract_block_dumper/_internal/services/utils.py,sha256=QZxdQyWIcUnezyVmegS4g3x3BoB3-oijYJ_i9nLQWHo,1140
|
|
22
22
|
abstract_block_dumper/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
23
23
|
abstract_block_dumper/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
|
-
abstract_block_dumper/management/commands/backfill_blocks_v1.py,sha256=
|
|
24
|
+
abstract_block_dumper/management/commands/backfill_blocks_v1.py,sha256=TH58_4D-k5BPZtW8nSbUI1VGT8ruiyFJQSErLMfKkLs,13580
|
|
25
25
|
abstract_block_dumper/management/commands/block_tasks_v1.py,sha256=jSi04ahIKYwlm_dNKCUGL_cmALv1iP-ZjfXrmz0pn-4,880
|
|
26
26
|
abstract_block_dumper/migrations/0001_initial.py,sha256=ImPHC3G6kPkq4Xn_4YVAm4Labh1Xi7PkCRszYRGpTiI,2298
|
|
27
27
|
abstract_block_dumper/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -29,6 +29,6 @@ abstract_block_dumper/v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
|
|
|
29
29
|
abstract_block_dumper/v1/celery.py,sha256=X4IqVs5i6ZpyY7fy1SqMZgsZy4SXP-jK2qG-FYnjU38,1722
|
|
30
30
|
abstract_block_dumper/v1/decorators.py,sha256=yQglsy1dU1u7ShwaTqZLahDcybHmetibTIOi53o_ZOM,9829
|
|
31
31
|
abstract_block_dumper/v1/tasks.py,sha256=u9iMYdDUqzYT3yPrNwZecHnlweZ3yFipV9BcIWHCbus,2647
|
|
32
|
-
abstract_block_dumper-0.1.
|
|
33
|
-
abstract_block_dumper-0.1.
|
|
34
|
-
abstract_block_dumper-0.1.
|
|
32
|
+
abstract_block_dumper-0.1.3.dist-info/METADATA,sha256=DfV8q-CSoWkqJHJ3ON37v_PaT_t52XHMF5QK5tzfns0,12993
|
|
33
|
+
abstract_block_dumper-0.1.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
34
|
+
abstract_block_dumper-0.1.3.dist-info/RECORD,,
|
|
File without changes
|