binance-datatool 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,212 @@
1
+ """Aria2-backed archive download helpers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import shutil
7
+ import subprocess
8
+ import tempfile
9
+ from dataclasses import dataclass
10
+ from pathlib import Path
11
+ from typing import TYPE_CHECKING
12
+
13
+ from loguru import logger
14
+
15
+ from binance_datatool.common.progress import ProgressEvent, make_reporter
16
+
17
+ if TYPE_CHECKING:
18
+ from collections.abc import Sequence
19
+
20
+ PROXY_ENV_VARS = (
21
+ "http_proxy",
22
+ "https_proxy",
23
+ "HTTP_PROXY",
24
+ "HTTPS_PROXY",
25
+ "all_proxy",
26
+ "ALL_PROXY",
27
+ "no_proxy",
28
+ "NO_PROXY",
29
+ )
30
+
31
+
32
+ class Aria2NotFoundError(FileNotFoundError):
33
+ """Raised when ``aria2c`` is not available in ``PATH``."""
34
+
35
+
36
+ @dataclass(slots=True, frozen=True)
37
+ class DownloadRequest:
38
+ """Single direct-download request for aria2."""
39
+
40
+ url: str
41
+ local_path: Path
42
+
43
+
44
+ @dataclass(slots=True)
45
+ class Aria2DownloadResult:
46
+ """Aggregated result for one aria2 download run."""
47
+
48
+ requested: int
49
+ failed_requests: list[DownloadRequest]
50
+
51
+ @property
52
+ def succeeded(self) -> int:
53
+ """Return the number of successful requests."""
54
+ return self.requested - len(self.failed_requests)
55
+
56
+
57
+ def _find_aria2c() -> str:
58
+ """Return the ``aria2c`` executable path or raise a dedicated error."""
59
+ executable = shutil.which("aria2c")
60
+ if executable is None:
61
+ raise Aria2NotFoundError("aria2c executable not found in PATH.")
62
+ return executable
63
+
64
+
65
+ def _chunk_requests(
66
+ requests: Sequence[DownloadRequest],
67
+ batch_size: int,
68
+ ) -> list[list[DownloadRequest]]:
69
+ """Split requests into fixed-size batches."""
70
+ return [list(requests[i : i + batch_size]) for i in range(0, len(requests), batch_size)]
71
+
72
+
73
+ def _build_env(*, inherit_proxy: bool) -> dict[str, str] | None:
74
+ """Build the subprocess environment for aria2."""
75
+ if inherit_proxy:
76
+ return None
77
+
78
+ env = os.environ.copy()
79
+ for variable in PROXY_ENV_VARS:
80
+ env.pop(variable, None)
81
+ return env
82
+
83
+
84
+ def _write_input_file(requests: Sequence[DownloadRequest]) -> Path:
85
+ """Create an aria2 input file for one batch."""
86
+ with tempfile.NamedTemporaryFile(
87
+ mode="w",
88
+ encoding="utf-8",
89
+ delete=False,
90
+ prefix="archive_aria2_",
91
+ ) as input_file:
92
+ for request in requests:
93
+ request.local_path.parent.mkdir(parents=True, exist_ok=True)
94
+ input_file.write(f"{request.url}\n")
95
+ input_file.write(f" dir={request.local_path.parent}\n")
96
+ return Path(input_file.name)
97
+
98
+
99
+ def _run_batch(
100
+ executable: str,
101
+ requests: Sequence[DownloadRequest],
102
+ *,
103
+ inherit_proxy: bool,
104
+ ) -> int:
105
+ """Run aria2 for a single batch and return its exit code."""
106
+ input_file_path = _write_input_file(requests)
107
+ try:
108
+ cmd = [
109
+ executable,
110
+ "-i",
111
+ str(input_file_path),
112
+ "-j32",
113
+ "-x4",
114
+ "-q",
115
+ "--allow-overwrite=true",
116
+ "--auto-file-renaming=false",
117
+ ]
118
+ completed = subprocess.run(
119
+ cmd,
120
+ check=False,
121
+ env=_build_env(inherit_proxy=inherit_proxy),
122
+ )
123
+ finally:
124
+ input_file_path.unlink(missing_ok=True)
125
+
126
+ return completed.returncode
127
+
128
+
129
+ def _is_download_complete(request: DownloadRequest) -> bool:
130
+ """Check whether a single download request completed successfully.
131
+
132
+ A file is considered complete when it exists on disk and has no
133
+ leftover aria2 control file (which indicates a partial download).
134
+ """
135
+ if not request.local_path.exists():
136
+ return False
137
+ control_file = request.local_path.with_name(request.local_path.name + ".aria2")
138
+ return not control_file.exists()
139
+
140
+
141
+ def _find_missing_requests(
142
+ requests: Sequence[DownloadRequest],
143
+ ) -> list[DownloadRequest]:
144
+ """Return only the requests whose files were not fully downloaded."""
145
+ return [r for r in requests if not _is_download_complete(r)]
146
+
147
+
148
+ def download_archive_files(
149
+ requests: Sequence[DownloadRequest],
150
+ *,
151
+ inherit_proxy: bool,
152
+ batch_size: int = 4096,
153
+ max_tries: int = 3,
154
+ progress_bar: bool = False,
155
+ ) -> Aria2DownloadResult:
156
+ """Download requests with aria2 using per-file retry semantics.
157
+
158
+ Args:
159
+ requests: Direct download requests to execute.
160
+ inherit_proxy: Whether aria2 should inherit proxy-related env vars.
161
+ batch_size: Number of files per aria2 batch.
162
+ max_tries: Maximum retry rounds for failed batches.
163
+ progress_bar: Whether to render an interactive tqdm progress bar.
164
+
165
+ Returns:
166
+ Aggregated aria2 result.
167
+ """
168
+ if not requests:
169
+ return Aria2DownloadResult(requested=0, failed_requests=[])
170
+
171
+ executable = _find_aria2c()
172
+ pending: list[DownloadRequest] = list(requests)
173
+
174
+ for attempt in range(1, max_tries + 1):
175
+ description = "Download" if attempt == 1 else f"Download retry {attempt - 1}"
176
+ still_missing: list[DownloadRequest] = []
177
+ batches = _chunk_requests(pending, batch_size=batch_size)
178
+
179
+ with make_reporter(
180
+ progress_bar,
181
+ total=len(pending),
182
+ description=description,
183
+ ) as reporter:
184
+ for batch_index, batch in enumerate(batches, start=1):
185
+ batch_label = f"batch {batch_index}/{len(batches)}"
186
+ returncode = _run_batch(executable, batch, inherit_proxy=inherit_proxy)
187
+
188
+ if returncode == 0:
189
+ reporter.tick(ProgressEvent(name=batch_label, ok=True, count=len(batch)))
190
+ continue
191
+
192
+ # aria2 returned non-zero — check which files are actually missing.
193
+ batch_missing = _find_missing_requests(batch)
194
+ batch_ok_count = len(batch) - len(batch_missing)
195
+
196
+ if batch_ok_count > 0:
197
+ reporter.tick(ProgressEvent(name=batch_label, ok=True, count=batch_ok_count))
198
+ if batch_missing:
199
+ reporter.tick(
200
+ ProgressEvent(name=batch_label, ok=False, count=len(batch_missing))
201
+ )
202
+ still_missing.extend(batch_missing)
203
+
204
+ if not still_missing:
205
+ return Aria2DownloadResult(requested=len(requests), failed_requests=[])
206
+
207
+ pending = still_missing
208
+
209
+ return Aria2DownloadResult(
210
+ requested=len(requests),
211
+ failed_requests=pending,
212
+ )
@@ -0,0 +1,138 @@
1
+ """Symbol filters for archive listings."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from typing import TYPE_CHECKING
7
+
8
+ from binance_datatool.common import TradeType
9
+
10
+ if TYPE_CHECKING:
11
+ from collections.abc import Iterable
12
+
13
+ from binance_datatool.common import CmSymbolInfo, ContractType, SpotSymbolInfo, UmSymbolInfo
14
+
15
+
16
+ @dataclass(slots=True)
17
+ class SpotSymbolFilter:
18
+ """Filter for spot symbols."""
19
+
20
+ quote_assets: frozenset[str] | None = None
21
+ exclude_leverage: bool = False
22
+ exclude_stable_pairs: bool = False
23
+
24
+ def matches(self, info: SpotSymbolInfo) -> bool:
25
+ """Return whether the symbol matches the configured spot filters.
26
+
27
+ Args:
28
+ info: Parsed spot symbol metadata.
29
+
30
+ Returns:
31
+ ``True`` when the symbol passes every enabled filter.
32
+ """
33
+ if self.quote_assets is not None and info.quote_asset not in self.quote_assets:
34
+ return False
35
+ if self.exclude_leverage and info.is_leverage:
36
+ return False
37
+ return not (self.exclude_stable_pairs and info.is_stable_pair)
38
+
39
+ def __call__(self, infos: Iterable[SpotSymbolInfo]) -> list[SpotSymbolInfo]:
40
+ """Return the matching spot symbols in input order."""
41
+ return [info for info in infos if self.matches(info)]
42
+
43
+
44
+ @dataclass(slots=True)
45
+ class UmSymbolFilter:
46
+ """Filter for USD-M futures symbols."""
47
+
48
+ quote_assets: frozenset[str] | None = None
49
+ contract_type: ContractType | None = None
50
+ exclude_stable_pairs: bool = False
51
+
52
+ def matches(self, info: UmSymbolInfo) -> bool:
53
+ """Return whether the symbol matches the configured USD-M filters.
54
+
55
+ Args:
56
+ info: Parsed USD-M futures symbol metadata.
57
+
58
+ Returns:
59
+ ``True`` when the symbol passes every enabled filter.
60
+ """
61
+ if self.quote_assets is not None and info.quote_asset not in self.quote_assets:
62
+ return False
63
+ if self.contract_type is not None and info.contract_type is not self.contract_type:
64
+ return False
65
+ return not (self.exclude_stable_pairs and info.is_stable_pair)
66
+
67
+ def __call__(self, infos: Iterable[UmSymbolInfo]) -> list[UmSymbolInfo]:
68
+ """Return the matching USD-M futures symbols in input order."""
69
+ return [info for info in infos if self.matches(info)]
70
+
71
+
72
+ @dataclass(slots=True)
73
+ class CmSymbolFilter:
74
+ """Filter for COIN-M futures symbols."""
75
+
76
+ contract_type: ContractType | None = None
77
+
78
+ def matches(self, info: CmSymbolInfo) -> bool:
79
+ """Return whether the symbol matches the configured COIN-M filters.
80
+
81
+ Args:
82
+ info: Parsed COIN-M futures symbol metadata.
83
+
84
+ Returns:
85
+ ``True`` when the symbol passes every enabled filter.
86
+ """
87
+ return not (self.contract_type is not None and info.contract_type is not self.contract_type)
88
+
89
+ def __call__(self, infos: Iterable[CmSymbolInfo]) -> list[CmSymbolInfo]:
90
+ """Return the matching COIN-M futures symbols in input order."""
91
+ return [info for info in infos if self.matches(info)]
92
+
93
+
94
+ SymbolFilter = SpotSymbolFilter | UmSymbolFilter | CmSymbolFilter
95
+
96
+
97
+ def build_symbol_filter(
98
+ trade_type: TradeType,
99
+ *,
100
+ quote_assets: frozenset[str] | None,
101
+ exclude_leverage: bool,
102
+ exclude_stable_pairs: bool,
103
+ contract_type: ContractType | None,
104
+ ) -> SymbolFilter | None:
105
+ """Build the market-specific symbol filter.
106
+
107
+ Args:
108
+ trade_type: Market segment for the symbol listing.
109
+ quote_assets: Optional normalized quote-asset allowlist.
110
+ exclude_leverage: Whether leveraged spot tokens should be excluded.
111
+ exclude_stable_pairs: Whether stablecoin pairs should be excluded.
112
+ contract_type: Optional futures contract-type constraint.
113
+
114
+ Returns:
115
+ A market-specific symbol filter, or ``None`` when no applicable
116
+ filters are enabled for the given market.
117
+ """
118
+ match trade_type:
119
+ case TradeType.spot:
120
+ if quote_assets is None and not exclude_leverage and not exclude_stable_pairs:
121
+ return None
122
+ return SpotSymbolFilter(
123
+ quote_assets=quote_assets,
124
+ exclude_leverage=exclude_leverage,
125
+ exclude_stable_pairs=exclude_stable_pairs,
126
+ )
127
+ case TradeType.um:
128
+ if quote_assets is None and contract_type is None and not exclude_stable_pairs:
129
+ return None
130
+ return UmSymbolFilter(
131
+ quote_assets=quote_assets,
132
+ contract_type=contract_type,
133
+ exclude_stable_pairs=exclude_stable_pairs,
134
+ )
135
+ case TradeType.cm:
136
+ if contract_type is None:
137
+ return None
138
+ return CmSymbolFilter(contract_type=contract_type)
@@ -0,0 +1,203 @@
1
+ """Local symbol archive directory helpers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ import time
7
+ from collections import defaultdict
8
+ from dataclasses import dataclass
9
+ from typing import TYPE_CHECKING
10
+
11
+ if TYPE_CHECKING:
12
+ from collections.abc import Collection, Sequence
13
+ from pathlib import Path
14
+
15
+ from binance_datatool.common import DataFrequency, DataType, TradeType
16
+
17
+ __all__ = ["SymbolArchiveDir", "create_symbol_archive_dir"]
18
+
19
+
20
+ @dataclass(slots=True)
21
+ class SymbolScanResult:
22
+ """Per-symbol scan outcome used internally by archive workflows."""
23
+
24
+ to_verify: list[Path]
25
+ orphan_zips: list[Path]
26
+ orphan_checksums: list[Path]
27
+ skipped: int
28
+
29
+
30
+ def _match_marker_to_zip(marker_name: str, zip_names: set[str]) -> str | None:
31
+ """Match a marker filename to the zip filename it belongs to."""
32
+ without_suffix = marker_name.removesuffix(".verified")
33
+ if without_suffix in zip_names:
34
+ return without_suffix
35
+
36
+ last_dot = without_suffix.rfind(".")
37
+ if last_dot != -1 and without_suffix[:last_dot] in zip_names:
38
+ return without_suffix[:last_dot]
39
+
40
+ return None
41
+
42
+
43
+ def _parse_marker_timestamps(
44
+ verified_names: set[str],
45
+ zip_names: set[str],
46
+ ) -> dict[str, list[int]]:
47
+ """Parse timestamped verified markers and group them by zip name."""
48
+ markers: dict[str, list[int]] = defaultdict(list)
49
+ for name in verified_names:
50
+ without_suffix = name.removesuffix(".verified")
51
+ last_dot = without_suffix.rfind(".")
52
+ if last_dot == -1:
53
+ continue
54
+
55
+ ts_str = without_suffix[last_dot + 1 :]
56
+ zip_name = without_suffix[:last_dot]
57
+ try:
58
+ ts = int(ts_str)
59
+ except ValueError:
60
+ continue
61
+
62
+ if zip_name in zip_names:
63
+ markers[zip_name].append(ts)
64
+
65
+ return dict(markers)
66
+
67
+
68
+ def collect_markers_by_zip(zip_paths: Sequence[Path]) -> dict[Path, list[Path]]:
69
+ """Collect all verification markers for a list of zip files."""
70
+ dirs_to_zip_names: dict[Path, set[str]] = defaultdict(set)
71
+ for zip_path in zip_paths:
72
+ dirs_to_zip_names[zip_path.parent].add(zip_path.name)
73
+
74
+ markers_by_zip: dict[Path, list[Path]] = defaultdict(list)
75
+ for dir_path, zip_names in dirs_to_zip_names.items():
76
+ for marker_path in dir_path.glob("*.verified"):
77
+ zip_name = _match_marker_to_zip(marker_path.name, zip_names)
78
+ if zip_name is not None:
79
+ markers_by_zip[dir_path / zip_name].append(marker_path)
80
+
81
+ return dict(markers_by_zip)
82
+
83
+
84
+ class SymbolArchiveDir:
85
+ """One local symbol archive directory."""
86
+
87
+ def __init__(self, path: Path) -> None:
88
+ """Initialize the symbol archive directory helper."""
89
+ self.path = path
90
+
91
+ def zip_path(self, zip_name: str) -> Path:
92
+ """Return the local path for a zip file in this directory."""
93
+ return self.path / zip_name
94
+
95
+ def checksum_path(self, zip_name: str) -> Path:
96
+ """Return the local checksum path for a zip file in this directory."""
97
+ return self.path / f"{zip_name}.CHECKSUM"
98
+
99
+ def clear_markers(self, zip_name: str) -> None:
100
+ """Delete all verification markers for one zip file."""
101
+ self.clear_markers_many([zip_name])
102
+
103
+ def clear_markers_many(self, zip_names: Collection[str]) -> None:
104
+ """Delete verification markers for one or more zip files."""
105
+ if not zip_names:
106
+ return
107
+
108
+ if len(zip_names) == 1:
109
+ zip_name = next(iter(zip_names))
110
+ (self.path / f"{zip_name}.verified").unlink(missing_ok=True)
111
+ for marker_path in self.path.glob(f"{zip_name}.*.verified"):
112
+ marker_path.unlink(missing_ok=True)
113
+ return
114
+
115
+ zip_name_set = set(zip_names)
116
+ for marker_path in self.path.glob("*.verified"):
117
+ if _match_marker_to_zip(marker_path.name, zip_name_set) is not None:
118
+ marker_path.unlink(missing_ok=True)
119
+
120
+ def max_source_mtime(self, zip_name: str) -> int:
121
+ """Return the normalized freshness timestamp for a zip/checksum pair."""
122
+ zip_path = self.zip_path(zip_name)
123
+ checksum_path = self.checksum_path(zip_name)
124
+ return math.ceil(max(zip_path.stat().st_mtime, checksum_path.stat().st_mtime))
125
+
126
+ def is_marker_fresh(self, zip_name: str, timestamps: list[int]) -> bool:
127
+ """Return True when any marker timestamp is fresh for this zip file."""
128
+ return bool(timestamps) and max(timestamps) >= self.max_source_mtime(zip_name)
129
+
130
+ def write_marker(self, zip_name: str) -> None:
131
+ """Create a fresh timestamped marker for a verified zip file."""
132
+ timestamp = max(int(time.time()), self.max_source_mtime(zip_name))
133
+ (self.path / f"{zip_name}.{timestamp}.verified").touch()
134
+
135
+ def discard_failed(self, zip_name: str) -> None:
136
+ """Delete a failed zip/checksum pair."""
137
+ self.zip_path(zip_name).unlink(missing_ok=True)
138
+ self.checksum_path(zip_name).unlink(missing_ok=True)
139
+
140
+ def remove_orphan_checksum(self, checksum_name: str) -> None:
141
+ """Delete an orphan checksum file."""
142
+ (self.path / checksum_name).unlink(missing_ok=True)
143
+
144
+ def scan(self) -> SymbolScanResult:
145
+ """Scan the symbol directory and classify local verify work."""
146
+ if not self.path.exists():
147
+ return SymbolScanResult(
148
+ to_verify=[],
149
+ orphan_zips=[],
150
+ orphan_checksums=[],
151
+ skipped=0,
152
+ )
153
+
154
+ zip_paths = sorted(self.path.glob("*.zip"))
155
+ zip_names = {path.name for path in zip_paths}
156
+ checksum_names = {path.name for path in self.path.glob("*.zip.CHECKSUM")}
157
+ verified_names = {path.name for path in self.path.glob("*.verified")}
158
+
159
+ marker_timestamps = _parse_marker_timestamps(verified_names, zip_names)
160
+
161
+ to_verify: list[Path] = []
162
+ orphan_zips: list[Path] = []
163
+ skipped = 0
164
+
165
+ for zip_path in zip_paths:
166
+ zip_name = zip_path.name
167
+ if f"{zip_name}.CHECKSUM" not in checksum_names:
168
+ orphan_zips.append(zip_path)
169
+ continue
170
+
171
+ if self.is_marker_fresh(zip_name, marker_timestamps.get(zip_name, [])):
172
+ skipped += 1
173
+ else:
174
+ to_verify.append(zip_path)
175
+
176
+ orphan_checksums = sorted(
177
+ self.path / checksum_name
178
+ for checksum_name in checksum_names
179
+ if checksum_name.removesuffix(".CHECKSUM") not in zip_names
180
+ )
181
+
182
+ return SymbolScanResult(
183
+ to_verify=to_verify,
184
+ orphan_zips=orphan_zips,
185
+ orphan_checksums=orphan_checksums,
186
+ skipped=skipped,
187
+ )
188
+
189
+
190
+ def create_symbol_archive_dir(
191
+ archive_home: Path,
192
+ trade_type: TradeType,
193
+ data_freq: DataFrequency,
194
+ data_type: DataType,
195
+ symbol: str,
196
+ interval: str | None = None,
197
+ ) -> SymbolArchiveDir:
198
+ """Build the local directory helper that stores files for one symbol."""
199
+ path = archive_home / "data" / trade_type.s3_path / data_freq.value / data_type.value / symbol
200
+ if interval is not None:
201
+ path /= interval
202
+
203
+ return SymbolArchiveDir(path)
@@ -0,0 +1,41 @@
1
+ """Typer application for binance-datatool."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Annotated
6
+
7
+ import typer
8
+
9
+ from binance_datatool.common import configure_cli_logging
10
+
11
+ app = typer.Typer(name="binance-datatool", help="Binance DataTool CLI.", add_completion=False)
12
+
13
+
14
+ @app.callback()
15
+ def root_callback(
16
+ ctx: typer.Context,
17
+ verbose: Annotated[
18
+ int,
19
+ typer.Option(
20
+ "-v",
21
+ "--verbose",
22
+ count=True,
23
+ help="Increase log verbosity (-v=INFO, -vv=DEBUG).",
24
+ ),
25
+ ] = 0,
26
+ archive_home: Annotated[
27
+ str | None,
28
+ typer.Option(
29
+ "--archive-home",
30
+ help="Override the local archive data directory.",
31
+ ),
32
+ ] = None,
33
+ ) -> None:
34
+ """Configure shared CLI state before running a command."""
35
+ configure_cli_logging(verbose)
36
+ ctx.obj = {} if ctx.obj is None else dict(ctx.obj)
37
+ ctx.obj["archive_home_override"] = archive_home
38
+
39
+
40
+ # Register root command modules (side-effect import).
41
+ from binance_datatool.cli import archive as _archive # noqa: F401,E402