binance-datatool 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- binance_datatool/__init__.py +3 -0
- binance_datatool/archive/__init__.py +53 -0
- binance_datatool/archive/checksum.py +55 -0
- binance_datatool/archive/client.py +391 -0
- binance_datatool/archive/downloader.py +212 -0
- binance_datatool/archive/filter.py +138 -0
- binance_datatool/archive/symbol_dir.py +203 -0
- binance_datatool/cli/__init__.py +41 -0
- binance_datatool/cli/archive.py +484 -0
- binance_datatool/common/__init__.py +47 -0
- binance_datatool/common/constants.py +61 -0
- binance_datatool/common/enums.py +80 -0
- binance_datatool/common/logging.py +32 -0
- binance_datatool/common/path.py +40 -0
- binance_datatool/common/progress.py +335 -0
- binance_datatool/common/symbols.py +195 -0
- binance_datatool/common/types.py +44 -0
- binance_datatool/py.typed +1 -0
- binance_datatool/workflow/__init__.py +33 -0
- binance_datatool/workflow/_shared.py +39 -0
- binance_datatool/workflow/download.py +193 -0
- binance_datatool/workflow/list_files.py +96 -0
- binance_datatool/workflow/list_symbols.py +125 -0
- binance_datatool/workflow/results.py +184 -0
- binance_datatool/workflow/verify.py +256 -0
- binance_datatool-0.1.0.dist-info/METADATA +172 -0
- binance_datatool-0.1.0.dist-info/RECORD +30 -0
- binance_datatool-0.1.0.dist-info/WHEEL +4 -0
- binance_datatool-0.1.0.dist-info/entry_points.txt +2 -0
- binance_datatool-0.1.0.dist-info/licenses/LICENSE +29 -0
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""Aria2-backed archive download helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import subprocess
|
|
8
|
+
import tempfile
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import TYPE_CHECKING
|
|
12
|
+
|
|
13
|
+
from loguru import logger
|
|
14
|
+
|
|
15
|
+
from binance_datatool.common.progress import ProgressEvent, make_reporter
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from collections.abc import Sequence
|
|
19
|
+
|
|
20
|
+
PROXY_ENV_VARS = (
|
|
21
|
+
"http_proxy",
|
|
22
|
+
"https_proxy",
|
|
23
|
+
"HTTP_PROXY",
|
|
24
|
+
"HTTPS_PROXY",
|
|
25
|
+
"all_proxy",
|
|
26
|
+
"ALL_PROXY",
|
|
27
|
+
"no_proxy",
|
|
28
|
+
"NO_PROXY",
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class Aria2NotFoundError(FileNotFoundError):
|
|
33
|
+
"""Raised when ``aria2c`` is not available in ``PATH``."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(slots=True, frozen=True)
|
|
37
|
+
class DownloadRequest:
|
|
38
|
+
"""Single direct-download request for aria2."""
|
|
39
|
+
|
|
40
|
+
url: str
|
|
41
|
+
local_path: Path
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(slots=True)
|
|
45
|
+
class Aria2DownloadResult:
|
|
46
|
+
"""Aggregated result for one aria2 download run."""
|
|
47
|
+
|
|
48
|
+
requested: int
|
|
49
|
+
failed_requests: list[DownloadRequest]
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def succeeded(self) -> int:
|
|
53
|
+
"""Return the number of successful requests."""
|
|
54
|
+
return self.requested - len(self.failed_requests)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _find_aria2c() -> str:
|
|
58
|
+
"""Return the ``aria2c`` executable path or raise a dedicated error."""
|
|
59
|
+
executable = shutil.which("aria2c")
|
|
60
|
+
if executable is None:
|
|
61
|
+
raise Aria2NotFoundError("aria2c executable not found in PATH.")
|
|
62
|
+
return executable
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _chunk_requests(
|
|
66
|
+
requests: Sequence[DownloadRequest],
|
|
67
|
+
batch_size: int,
|
|
68
|
+
) -> list[list[DownloadRequest]]:
|
|
69
|
+
"""Split requests into fixed-size batches."""
|
|
70
|
+
return [list(requests[i : i + batch_size]) for i in range(0, len(requests), batch_size)]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _build_env(*, inherit_proxy: bool) -> dict[str, str] | None:
|
|
74
|
+
"""Build the subprocess environment for aria2."""
|
|
75
|
+
if inherit_proxy:
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
env = os.environ.copy()
|
|
79
|
+
for variable in PROXY_ENV_VARS:
|
|
80
|
+
env.pop(variable, None)
|
|
81
|
+
return env
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _write_input_file(requests: Sequence[DownloadRequest]) -> Path:
|
|
85
|
+
"""Create an aria2 input file for one batch."""
|
|
86
|
+
with tempfile.NamedTemporaryFile(
|
|
87
|
+
mode="w",
|
|
88
|
+
encoding="utf-8",
|
|
89
|
+
delete=False,
|
|
90
|
+
prefix="archive_aria2_",
|
|
91
|
+
) as input_file:
|
|
92
|
+
for request in requests:
|
|
93
|
+
request.local_path.parent.mkdir(parents=True, exist_ok=True)
|
|
94
|
+
input_file.write(f"{request.url}\n")
|
|
95
|
+
input_file.write(f" dir={request.local_path.parent}\n")
|
|
96
|
+
return Path(input_file.name)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _run_batch(
|
|
100
|
+
executable: str,
|
|
101
|
+
requests: Sequence[DownloadRequest],
|
|
102
|
+
*,
|
|
103
|
+
inherit_proxy: bool,
|
|
104
|
+
) -> int:
|
|
105
|
+
"""Run aria2 for a single batch and return its exit code."""
|
|
106
|
+
input_file_path = _write_input_file(requests)
|
|
107
|
+
try:
|
|
108
|
+
cmd = [
|
|
109
|
+
executable,
|
|
110
|
+
"-i",
|
|
111
|
+
str(input_file_path),
|
|
112
|
+
"-j32",
|
|
113
|
+
"-x4",
|
|
114
|
+
"-q",
|
|
115
|
+
"--allow-overwrite=true",
|
|
116
|
+
"--auto-file-renaming=false",
|
|
117
|
+
]
|
|
118
|
+
completed = subprocess.run(
|
|
119
|
+
cmd,
|
|
120
|
+
check=False,
|
|
121
|
+
env=_build_env(inherit_proxy=inherit_proxy),
|
|
122
|
+
)
|
|
123
|
+
finally:
|
|
124
|
+
input_file_path.unlink(missing_ok=True)
|
|
125
|
+
|
|
126
|
+
return completed.returncode
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _is_download_complete(request: DownloadRequest) -> bool:
|
|
130
|
+
"""Check whether a single download request completed successfully.
|
|
131
|
+
|
|
132
|
+
A file is considered complete when it exists on disk and has no
|
|
133
|
+
leftover aria2 control file (which indicates a partial download).
|
|
134
|
+
"""
|
|
135
|
+
if not request.local_path.exists():
|
|
136
|
+
return False
|
|
137
|
+
control_file = request.local_path.with_name(request.local_path.name + ".aria2")
|
|
138
|
+
return not control_file.exists()
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _find_missing_requests(
|
|
142
|
+
requests: Sequence[DownloadRequest],
|
|
143
|
+
) -> list[DownloadRequest]:
|
|
144
|
+
"""Return only the requests whose files were not fully downloaded."""
|
|
145
|
+
return [r for r in requests if not _is_download_complete(r)]
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def download_archive_files(
|
|
149
|
+
requests: Sequence[DownloadRequest],
|
|
150
|
+
*,
|
|
151
|
+
inherit_proxy: bool,
|
|
152
|
+
batch_size: int = 4096,
|
|
153
|
+
max_tries: int = 3,
|
|
154
|
+
progress_bar: bool = False,
|
|
155
|
+
) -> Aria2DownloadResult:
|
|
156
|
+
"""Download requests with aria2 using per-file retry semantics.
|
|
157
|
+
|
|
158
|
+
Args:
|
|
159
|
+
requests: Direct download requests to execute.
|
|
160
|
+
inherit_proxy: Whether aria2 should inherit proxy-related env vars.
|
|
161
|
+
batch_size: Number of files per aria2 batch.
|
|
162
|
+
max_tries: Maximum retry rounds for failed batches.
|
|
163
|
+
progress_bar: Whether to render an interactive tqdm progress bar.
|
|
164
|
+
|
|
165
|
+
Returns:
|
|
166
|
+
Aggregated aria2 result.
|
|
167
|
+
"""
|
|
168
|
+
if not requests:
|
|
169
|
+
return Aria2DownloadResult(requested=0, failed_requests=[])
|
|
170
|
+
|
|
171
|
+
executable = _find_aria2c()
|
|
172
|
+
pending: list[DownloadRequest] = list(requests)
|
|
173
|
+
|
|
174
|
+
for attempt in range(1, max_tries + 1):
|
|
175
|
+
description = "Download" if attempt == 1 else f"Download retry {attempt - 1}"
|
|
176
|
+
still_missing: list[DownloadRequest] = []
|
|
177
|
+
batches = _chunk_requests(pending, batch_size=batch_size)
|
|
178
|
+
|
|
179
|
+
with make_reporter(
|
|
180
|
+
progress_bar,
|
|
181
|
+
total=len(pending),
|
|
182
|
+
description=description,
|
|
183
|
+
) as reporter:
|
|
184
|
+
for batch_index, batch in enumerate(batches, start=1):
|
|
185
|
+
batch_label = f"batch {batch_index}/{len(batches)}"
|
|
186
|
+
returncode = _run_batch(executable, batch, inherit_proxy=inherit_proxy)
|
|
187
|
+
|
|
188
|
+
if returncode == 0:
|
|
189
|
+
reporter.tick(ProgressEvent(name=batch_label, ok=True, count=len(batch)))
|
|
190
|
+
continue
|
|
191
|
+
|
|
192
|
+
# aria2 returned non-zero — check which files are actually missing.
|
|
193
|
+
batch_missing = _find_missing_requests(batch)
|
|
194
|
+
batch_ok_count = len(batch) - len(batch_missing)
|
|
195
|
+
|
|
196
|
+
if batch_ok_count > 0:
|
|
197
|
+
reporter.tick(ProgressEvent(name=batch_label, ok=True, count=batch_ok_count))
|
|
198
|
+
if batch_missing:
|
|
199
|
+
reporter.tick(
|
|
200
|
+
ProgressEvent(name=batch_label, ok=False, count=len(batch_missing))
|
|
201
|
+
)
|
|
202
|
+
still_missing.extend(batch_missing)
|
|
203
|
+
|
|
204
|
+
if not still_missing:
|
|
205
|
+
return Aria2DownloadResult(requested=len(requests), failed_requests=[])
|
|
206
|
+
|
|
207
|
+
pending = still_missing
|
|
208
|
+
|
|
209
|
+
return Aria2DownloadResult(
|
|
210
|
+
requested=len(requests),
|
|
211
|
+
failed_requests=pending,
|
|
212
|
+
)
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Symbol filters for archive listings."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
from binance_datatool.common import TradeType
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from collections.abc import Iterable
|
|
12
|
+
|
|
13
|
+
from binance_datatool.common import CmSymbolInfo, ContractType, SpotSymbolInfo, UmSymbolInfo
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(slots=True)
|
|
17
|
+
class SpotSymbolFilter:
|
|
18
|
+
"""Filter for spot symbols."""
|
|
19
|
+
|
|
20
|
+
quote_assets: frozenset[str] | None = None
|
|
21
|
+
exclude_leverage: bool = False
|
|
22
|
+
exclude_stable_pairs: bool = False
|
|
23
|
+
|
|
24
|
+
def matches(self, info: SpotSymbolInfo) -> bool:
|
|
25
|
+
"""Return whether the symbol matches the configured spot filters.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
info: Parsed spot symbol metadata.
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
``True`` when the symbol passes every enabled filter.
|
|
32
|
+
"""
|
|
33
|
+
if self.quote_assets is not None and info.quote_asset not in self.quote_assets:
|
|
34
|
+
return False
|
|
35
|
+
if self.exclude_leverage and info.is_leverage:
|
|
36
|
+
return False
|
|
37
|
+
return not (self.exclude_stable_pairs and info.is_stable_pair)
|
|
38
|
+
|
|
39
|
+
def __call__(self, infos: Iterable[SpotSymbolInfo]) -> list[SpotSymbolInfo]:
|
|
40
|
+
"""Return the matching spot symbols in input order."""
|
|
41
|
+
return [info for info in infos if self.matches(info)]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(slots=True)
|
|
45
|
+
class UmSymbolFilter:
|
|
46
|
+
"""Filter for USD-M futures symbols."""
|
|
47
|
+
|
|
48
|
+
quote_assets: frozenset[str] | None = None
|
|
49
|
+
contract_type: ContractType | None = None
|
|
50
|
+
exclude_stable_pairs: bool = False
|
|
51
|
+
|
|
52
|
+
def matches(self, info: UmSymbolInfo) -> bool:
|
|
53
|
+
"""Return whether the symbol matches the configured USD-M filters.
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
info: Parsed USD-M futures symbol metadata.
|
|
57
|
+
|
|
58
|
+
Returns:
|
|
59
|
+
``True`` when the symbol passes every enabled filter.
|
|
60
|
+
"""
|
|
61
|
+
if self.quote_assets is not None and info.quote_asset not in self.quote_assets:
|
|
62
|
+
return False
|
|
63
|
+
if self.contract_type is not None and info.contract_type is not self.contract_type:
|
|
64
|
+
return False
|
|
65
|
+
return not (self.exclude_stable_pairs and info.is_stable_pair)
|
|
66
|
+
|
|
67
|
+
def __call__(self, infos: Iterable[UmSymbolInfo]) -> list[UmSymbolInfo]:
|
|
68
|
+
"""Return the matching USD-M futures symbols in input order."""
|
|
69
|
+
return [info for info in infos if self.matches(info)]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(slots=True)
|
|
73
|
+
class CmSymbolFilter:
|
|
74
|
+
"""Filter for COIN-M futures symbols."""
|
|
75
|
+
|
|
76
|
+
contract_type: ContractType | None = None
|
|
77
|
+
|
|
78
|
+
def matches(self, info: CmSymbolInfo) -> bool:
|
|
79
|
+
"""Return whether the symbol matches the configured COIN-M filters.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
info: Parsed COIN-M futures symbol metadata.
|
|
83
|
+
|
|
84
|
+
Returns:
|
|
85
|
+
``True`` when the symbol passes every enabled filter.
|
|
86
|
+
"""
|
|
87
|
+
return not (self.contract_type is not None and info.contract_type is not self.contract_type)
|
|
88
|
+
|
|
89
|
+
def __call__(self, infos: Iterable[CmSymbolInfo]) -> list[CmSymbolInfo]:
|
|
90
|
+
"""Return the matching COIN-M futures symbols in input order."""
|
|
91
|
+
return [info for info in infos if self.matches(info)]
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
SymbolFilter = SpotSymbolFilter | UmSymbolFilter | CmSymbolFilter
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def build_symbol_filter(
|
|
98
|
+
trade_type: TradeType,
|
|
99
|
+
*,
|
|
100
|
+
quote_assets: frozenset[str] | None,
|
|
101
|
+
exclude_leverage: bool,
|
|
102
|
+
exclude_stable_pairs: bool,
|
|
103
|
+
contract_type: ContractType | None,
|
|
104
|
+
) -> SymbolFilter | None:
|
|
105
|
+
"""Build the market-specific symbol filter.
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
trade_type: Market segment for the symbol listing.
|
|
109
|
+
quote_assets: Optional normalized quote-asset allowlist.
|
|
110
|
+
exclude_leverage: Whether leveraged spot tokens should be excluded.
|
|
111
|
+
exclude_stable_pairs: Whether stablecoin pairs should be excluded.
|
|
112
|
+
contract_type: Optional futures contract-type constraint.
|
|
113
|
+
|
|
114
|
+
Returns:
|
|
115
|
+
A market-specific symbol filter, or ``None`` when no applicable
|
|
116
|
+
filters are enabled for the given market.
|
|
117
|
+
"""
|
|
118
|
+
match trade_type:
|
|
119
|
+
case TradeType.spot:
|
|
120
|
+
if quote_assets is None and not exclude_leverage and not exclude_stable_pairs:
|
|
121
|
+
return None
|
|
122
|
+
return SpotSymbolFilter(
|
|
123
|
+
quote_assets=quote_assets,
|
|
124
|
+
exclude_leverage=exclude_leverage,
|
|
125
|
+
exclude_stable_pairs=exclude_stable_pairs,
|
|
126
|
+
)
|
|
127
|
+
case TradeType.um:
|
|
128
|
+
if quote_assets is None and contract_type is None and not exclude_stable_pairs:
|
|
129
|
+
return None
|
|
130
|
+
return UmSymbolFilter(
|
|
131
|
+
quote_assets=quote_assets,
|
|
132
|
+
contract_type=contract_type,
|
|
133
|
+
exclude_stable_pairs=exclude_stable_pairs,
|
|
134
|
+
)
|
|
135
|
+
case TradeType.cm:
|
|
136
|
+
if contract_type is None:
|
|
137
|
+
return None
|
|
138
|
+
return CmSymbolFilter(contract_type=contract_type)
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""Local symbol archive directory helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
import time
|
|
7
|
+
from collections import defaultdict
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from collections.abc import Collection, Sequence
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from binance_datatool.common import DataFrequency, DataType, TradeType
|
|
16
|
+
|
|
17
|
+
__all__ = ["SymbolArchiveDir", "create_symbol_archive_dir"]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(slots=True)
|
|
21
|
+
class SymbolScanResult:
|
|
22
|
+
"""Per-symbol scan outcome used internally by archive workflows."""
|
|
23
|
+
|
|
24
|
+
to_verify: list[Path]
|
|
25
|
+
orphan_zips: list[Path]
|
|
26
|
+
orphan_checksums: list[Path]
|
|
27
|
+
skipped: int
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _match_marker_to_zip(marker_name: str, zip_names: set[str]) -> str | None:
|
|
31
|
+
"""Match a marker filename to the zip filename it belongs to."""
|
|
32
|
+
without_suffix = marker_name.removesuffix(".verified")
|
|
33
|
+
if without_suffix in zip_names:
|
|
34
|
+
return without_suffix
|
|
35
|
+
|
|
36
|
+
last_dot = without_suffix.rfind(".")
|
|
37
|
+
if last_dot != -1 and without_suffix[:last_dot] in zip_names:
|
|
38
|
+
return without_suffix[:last_dot]
|
|
39
|
+
|
|
40
|
+
return None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _parse_marker_timestamps(
|
|
44
|
+
verified_names: set[str],
|
|
45
|
+
zip_names: set[str],
|
|
46
|
+
) -> dict[str, list[int]]:
|
|
47
|
+
"""Parse timestamped verified markers and group them by zip name."""
|
|
48
|
+
markers: dict[str, list[int]] = defaultdict(list)
|
|
49
|
+
for name in verified_names:
|
|
50
|
+
without_suffix = name.removesuffix(".verified")
|
|
51
|
+
last_dot = without_suffix.rfind(".")
|
|
52
|
+
if last_dot == -1:
|
|
53
|
+
continue
|
|
54
|
+
|
|
55
|
+
ts_str = without_suffix[last_dot + 1 :]
|
|
56
|
+
zip_name = without_suffix[:last_dot]
|
|
57
|
+
try:
|
|
58
|
+
ts = int(ts_str)
|
|
59
|
+
except ValueError:
|
|
60
|
+
continue
|
|
61
|
+
|
|
62
|
+
if zip_name in zip_names:
|
|
63
|
+
markers[zip_name].append(ts)
|
|
64
|
+
|
|
65
|
+
return dict(markers)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def collect_markers_by_zip(zip_paths: Sequence[Path]) -> dict[Path, list[Path]]:
|
|
69
|
+
"""Collect all verification markers for a list of zip files."""
|
|
70
|
+
dirs_to_zip_names: dict[Path, set[str]] = defaultdict(set)
|
|
71
|
+
for zip_path in zip_paths:
|
|
72
|
+
dirs_to_zip_names[zip_path.parent].add(zip_path.name)
|
|
73
|
+
|
|
74
|
+
markers_by_zip: dict[Path, list[Path]] = defaultdict(list)
|
|
75
|
+
for dir_path, zip_names in dirs_to_zip_names.items():
|
|
76
|
+
for marker_path in dir_path.glob("*.verified"):
|
|
77
|
+
zip_name = _match_marker_to_zip(marker_path.name, zip_names)
|
|
78
|
+
if zip_name is not None:
|
|
79
|
+
markers_by_zip[dir_path / zip_name].append(marker_path)
|
|
80
|
+
|
|
81
|
+
return dict(markers_by_zip)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class SymbolArchiveDir:
|
|
85
|
+
"""One local symbol archive directory."""
|
|
86
|
+
|
|
87
|
+
def __init__(self, path: Path) -> None:
|
|
88
|
+
"""Initialize the symbol archive directory helper."""
|
|
89
|
+
self.path = path
|
|
90
|
+
|
|
91
|
+
def zip_path(self, zip_name: str) -> Path:
|
|
92
|
+
"""Return the local path for a zip file in this directory."""
|
|
93
|
+
return self.path / zip_name
|
|
94
|
+
|
|
95
|
+
def checksum_path(self, zip_name: str) -> Path:
|
|
96
|
+
"""Return the local checksum path for a zip file in this directory."""
|
|
97
|
+
return self.path / f"{zip_name}.CHECKSUM"
|
|
98
|
+
|
|
99
|
+
def clear_markers(self, zip_name: str) -> None:
|
|
100
|
+
"""Delete all verification markers for one zip file."""
|
|
101
|
+
self.clear_markers_many([zip_name])
|
|
102
|
+
|
|
103
|
+
def clear_markers_many(self, zip_names: Collection[str]) -> None:
|
|
104
|
+
"""Delete verification markers for one or more zip files."""
|
|
105
|
+
if not zip_names:
|
|
106
|
+
return
|
|
107
|
+
|
|
108
|
+
if len(zip_names) == 1:
|
|
109
|
+
zip_name = next(iter(zip_names))
|
|
110
|
+
(self.path / f"{zip_name}.verified").unlink(missing_ok=True)
|
|
111
|
+
for marker_path in self.path.glob(f"{zip_name}.*.verified"):
|
|
112
|
+
marker_path.unlink(missing_ok=True)
|
|
113
|
+
return
|
|
114
|
+
|
|
115
|
+
zip_name_set = set(zip_names)
|
|
116
|
+
for marker_path in self.path.glob("*.verified"):
|
|
117
|
+
if _match_marker_to_zip(marker_path.name, zip_name_set) is not None:
|
|
118
|
+
marker_path.unlink(missing_ok=True)
|
|
119
|
+
|
|
120
|
+
def max_source_mtime(self, zip_name: str) -> int:
|
|
121
|
+
"""Return the normalized freshness timestamp for a zip/checksum pair."""
|
|
122
|
+
zip_path = self.zip_path(zip_name)
|
|
123
|
+
checksum_path = self.checksum_path(zip_name)
|
|
124
|
+
return math.ceil(max(zip_path.stat().st_mtime, checksum_path.stat().st_mtime))
|
|
125
|
+
|
|
126
|
+
def is_marker_fresh(self, zip_name: str, timestamps: list[int]) -> bool:
|
|
127
|
+
"""Return True when any marker timestamp is fresh for this zip file."""
|
|
128
|
+
return bool(timestamps) and max(timestamps) >= self.max_source_mtime(zip_name)
|
|
129
|
+
|
|
130
|
+
def write_marker(self, zip_name: str) -> None:
|
|
131
|
+
"""Create a fresh timestamped marker for a verified zip file."""
|
|
132
|
+
timestamp = max(int(time.time()), self.max_source_mtime(zip_name))
|
|
133
|
+
(self.path / f"{zip_name}.{timestamp}.verified").touch()
|
|
134
|
+
|
|
135
|
+
def discard_failed(self, zip_name: str) -> None:
|
|
136
|
+
"""Delete a failed zip/checksum pair."""
|
|
137
|
+
self.zip_path(zip_name).unlink(missing_ok=True)
|
|
138
|
+
self.checksum_path(zip_name).unlink(missing_ok=True)
|
|
139
|
+
|
|
140
|
+
def remove_orphan_checksum(self, checksum_name: str) -> None:
|
|
141
|
+
"""Delete an orphan checksum file."""
|
|
142
|
+
(self.path / checksum_name).unlink(missing_ok=True)
|
|
143
|
+
|
|
144
|
+
def scan(self) -> SymbolScanResult:
|
|
145
|
+
"""Scan the symbol directory and classify local verify work."""
|
|
146
|
+
if not self.path.exists():
|
|
147
|
+
return SymbolScanResult(
|
|
148
|
+
to_verify=[],
|
|
149
|
+
orphan_zips=[],
|
|
150
|
+
orphan_checksums=[],
|
|
151
|
+
skipped=0,
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
zip_paths = sorted(self.path.glob("*.zip"))
|
|
155
|
+
zip_names = {path.name for path in zip_paths}
|
|
156
|
+
checksum_names = {path.name for path in self.path.glob("*.zip.CHECKSUM")}
|
|
157
|
+
verified_names = {path.name for path in self.path.glob("*.verified")}
|
|
158
|
+
|
|
159
|
+
marker_timestamps = _parse_marker_timestamps(verified_names, zip_names)
|
|
160
|
+
|
|
161
|
+
to_verify: list[Path] = []
|
|
162
|
+
orphan_zips: list[Path] = []
|
|
163
|
+
skipped = 0
|
|
164
|
+
|
|
165
|
+
for zip_path in zip_paths:
|
|
166
|
+
zip_name = zip_path.name
|
|
167
|
+
if f"{zip_name}.CHECKSUM" not in checksum_names:
|
|
168
|
+
orphan_zips.append(zip_path)
|
|
169
|
+
continue
|
|
170
|
+
|
|
171
|
+
if self.is_marker_fresh(zip_name, marker_timestamps.get(zip_name, [])):
|
|
172
|
+
skipped += 1
|
|
173
|
+
else:
|
|
174
|
+
to_verify.append(zip_path)
|
|
175
|
+
|
|
176
|
+
orphan_checksums = sorted(
|
|
177
|
+
self.path / checksum_name
|
|
178
|
+
for checksum_name in checksum_names
|
|
179
|
+
if checksum_name.removesuffix(".CHECKSUM") not in zip_names
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
return SymbolScanResult(
|
|
183
|
+
to_verify=to_verify,
|
|
184
|
+
orphan_zips=orphan_zips,
|
|
185
|
+
orphan_checksums=orphan_checksums,
|
|
186
|
+
skipped=skipped,
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def create_symbol_archive_dir(
|
|
191
|
+
archive_home: Path,
|
|
192
|
+
trade_type: TradeType,
|
|
193
|
+
data_freq: DataFrequency,
|
|
194
|
+
data_type: DataType,
|
|
195
|
+
symbol: str,
|
|
196
|
+
interval: str | None = None,
|
|
197
|
+
) -> SymbolArchiveDir:
|
|
198
|
+
"""Build the local directory helper that stores files for one symbol."""
|
|
199
|
+
path = archive_home / "data" / trade_type.s3_path / data_freq.value / data_type.value / symbol
|
|
200
|
+
if interval is not None:
|
|
201
|
+
path /= interval
|
|
202
|
+
|
|
203
|
+
return SymbolArchiveDir(path)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Typer application for binance-datatool."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Annotated
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from binance_datatool.common import configure_cli_logging
|
|
10
|
+
|
|
11
|
+
app = typer.Typer(name="binance-datatool", help="Binance DataTool CLI.", add_completion=False)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@app.callback()
|
|
15
|
+
def root_callback(
|
|
16
|
+
ctx: typer.Context,
|
|
17
|
+
verbose: Annotated[
|
|
18
|
+
int,
|
|
19
|
+
typer.Option(
|
|
20
|
+
"-v",
|
|
21
|
+
"--verbose",
|
|
22
|
+
count=True,
|
|
23
|
+
help="Increase log verbosity (-v=INFO, -vv=DEBUG).",
|
|
24
|
+
),
|
|
25
|
+
] = 0,
|
|
26
|
+
archive_home: Annotated[
|
|
27
|
+
str | None,
|
|
28
|
+
typer.Option(
|
|
29
|
+
"--archive-home",
|
|
30
|
+
help="Override the local archive data directory.",
|
|
31
|
+
),
|
|
32
|
+
] = None,
|
|
33
|
+
) -> None:
|
|
34
|
+
"""Configure shared CLI state before running a command."""
|
|
35
|
+
configure_cli_logging(verbose)
|
|
36
|
+
ctx.obj = {} if ctx.obj is None else dict(ctx.obj)
|
|
37
|
+
ctx.obj["archive_home_override"] = archive_home
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# Register root command modules (side-effect import).
|
|
41
|
+
from binance_datatool.cli import archive as _archive # noqa: F401,E402
|