binance-datatool 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3 @@
1
+ """binance_datatool package."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,53 @@
1
+ """Archive access helpers."""
2
+
3
+ from binance_datatool.archive.checksum import (
4
+ VerifyFileResult,
5
+ calc_sha256,
6
+ read_expected_checksum,
7
+ verify_single_file,
8
+ )
9
+ from binance_datatool.archive.client import (
10
+ ArchiveClient,
11
+ ArchiveFile,
12
+ SymbolListingResult,
13
+ list_symbols,
14
+ )
15
+ from binance_datatool.archive.downloader import (
16
+ Aria2DownloadResult,
17
+ Aria2NotFoundError,
18
+ DownloadRequest,
19
+ download_archive_files,
20
+ )
21
+ from binance_datatool.archive.filter import (
22
+ CmSymbolFilter,
23
+ SpotSymbolFilter,
24
+ SymbolFilter,
25
+ UmSymbolFilter,
26
+ build_symbol_filter,
27
+ )
28
+ from binance_datatool.archive.symbol_dir import (
29
+ SymbolArchiveDir,
30
+ create_symbol_archive_dir,
31
+ )
32
+
33
+ __all__ = [
34
+ "ArchiveClient",
35
+ "ArchiveFile",
36
+ "Aria2DownloadResult",
37
+ "Aria2NotFoundError",
38
+ "CmSymbolFilter",
39
+ "DownloadRequest",
40
+ "SpotSymbolFilter",
41
+ "SymbolArchiveDir",
42
+ "SymbolListingResult",
43
+ "SymbolFilter",
44
+ "UmSymbolFilter",
45
+ "VerifyFileResult",
46
+ "build_symbol_filter",
47
+ "calc_sha256",
48
+ "create_symbol_archive_dir",
49
+ "download_archive_files",
50
+ "list_symbols",
51
+ "read_expected_checksum",
52
+ "verify_single_file",
53
+ ]
@@ -0,0 +1,55 @@
1
+ """Archive checksum helpers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ from dataclasses import dataclass
7
+ from typing import TYPE_CHECKING
8
+
9
+ if TYPE_CHECKING:
10
+ from pathlib import Path
11
+
12
+
13
+ @dataclass(slots=True)
14
+ class VerifyFileResult:
15
+ """Single-file verification outcome."""
16
+
17
+ zip_path: Path
18
+ passed: bool
19
+ detail: str
20
+
21
+
22
+ def calc_sha256(file_path: Path) -> str:
23
+ """Calculate the SHA256 digest for a file."""
24
+ with file_path.open("rb") as file_obj:
25
+ return hashlib.file_digest(file_obj, "sha256").hexdigest()
26
+
27
+
28
+ def read_expected_checksum(zip_path: Path) -> str:
29
+ """Read the expected checksum from a sibling ``.CHECKSUM`` file.
30
+
31
+ Raises:
32
+ FileNotFoundError: When the ``.CHECKSUM`` sibling does not exist.
33
+ ValueError: When the checksum file is empty or has no parseable token.
34
+ """
35
+ checksum_path = zip_path.parent / f"{zip_path.name}.CHECKSUM"
36
+ line = checksum_path.read_text(encoding="utf-8").strip()
37
+ expected, *_rest = line.split()
38
+ if not expected:
39
+ msg = f"invalid checksum file: {checksum_path}"
40
+ raise ValueError(msg)
41
+ return expected
42
+
43
+
44
+ def verify_single_file(zip_path: Path) -> VerifyFileResult:
45
+ """Verify a single zip file against its sibling checksum file."""
46
+ try:
47
+ expected = read_expected_checksum(zip_path)
48
+ actual = calc_sha256(zip_path)
49
+ except Exception as exc: # noqa: BLE001
50
+ return VerifyFileResult(zip_path=zip_path, passed=False, detail=str(exc))
51
+
52
+ if actual == expected:
53
+ return VerifyFileResult(zip_path=zip_path, passed=True, detail="")
54
+
55
+ return VerifyFileResult(zip_path=zip_path, passed=False, detail="checksum mismatch")
@@ -0,0 +1,391 @@
1
+ """S3 directory listing client for data.binance.vision."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ from dataclasses import dataclass
7
+ from datetime import datetime
8
+ from typing import TYPE_CHECKING, Any, TypeAlias
9
+ from urllib.parse import urlencode
10
+
11
+ import aiohttp
12
+ import xmltodict
13
+ from loguru import logger
14
+ from tenacity import AsyncRetrying, retry_if_exception_type, stop_after_attempt, wait_exponential
15
+
16
+ from binance_datatool.common import S3_HTTP_TIMEOUT_SECONDS, S3_LISTING_PREFIX
17
+ from binance_datatool.common.progress import ProgressEvent, make_reporter
18
+
19
+ if TYPE_CHECKING:
20
+ from collections.abc import Iterable, Sequence
21
+
22
+ from binance_datatool.common.enums import DataFrequency, DataType, TradeType
23
+
24
+
25
+ def _build_prefix(
26
+ trade_type: TradeType,
27
+ data_freq: DataFrequency,
28
+ data_type: DataType,
29
+ ) -> str:
30
+ """Build an S3 prefix for a market archive directory."""
31
+ return f"data/{trade_type.s3_path}/{data_freq.value}/{data_type.value}/"
32
+
33
+
34
+ def _build_listing_url(prefix: str, marker: str | None = None) -> str:
35
+ """Build an S3 bucket listing URL for a prefix."""
36
+ query: dict[str, str] = {"delimiter": "/", "prefix": prefix}
37
+ if marker is not None:
38
+ query["marker"] = marker
39
+ return f"{S3_LISTING_PREFIX}?{urlencode(query)}"
40
+
41
+
42
+ def _normalize_entries(value: Any) -> list[dict[str, Any]]:
43
+ """Normalize xmltodict values that may be absent, a dict, or a list."""
44
+ if value is None:
45
+ return []
46
+ if isinstance(value, list):
47
+ return value
48
+ if isinstance(value, dict):
49
+ return [value]
50
+ msg = f"Unexpected S3 XML entry payload type: {type(value)!r}"
51
+ raise TypeError(msg)
52
+
53
+
54
+ def _extract_prefixes_from_payload(payload: dict[str, Any]) -> list[str]:
55
+ """Extract child prefixes from an S3 listing response payload."""
56
+ result = payload["ListBucketResult"]
57
+ prefixes = _normalize_entries(result.get("CommonPrefixes"))
58
+ return [entry["Prefix"] for entry in prefixes]
59
+
60
+
61
+ def _extract_files_from_payload(payload: dict[str, Any]) -> list[ArchiveFile]:
62
+ """Extract file metadata entries from an S3 listing response payload."""
63
+ result = payload["ListBucketResult"]
64
+ contents = _normalize_entries(result.get("Contents"))
65
+ return [
66
+ ArchiveFile(
67
+ key=entry["Key"],
68
+ size=int(entry["Size"]),
69
+ last_modified=datetime.fromisoformat(entry["LastModified"].replace("Z", "+00:00")),
70
+ )
71
+ for entry in contents
72
+ ]
73
+
74
+
75
+ def _is_truncated(payload: dict[str, Any]) -> bool:
76
+ """Return whether the S3 listing response is truncated."""
77
+ result = payload["ListBucketResult"]
78
+ return result.get("IsTruncated", "false").lower() == "true"
79
+
80
+
81
+ def _next_marker(payload: dict[str, Any], prefixes: Iterable[str]) -> str | None:
82
+ """Return the marker to use for the next S3 listing page."""
83
+ result = payload["ListBucketResult"]
84
+ marker = result.get("NextMarker")
85
+ if marker:
86
+ return marker
87
+
88
+ prefix_list = list(prefixes)
89
+ if prefix_list:
90
+ return prefix_list[-1]
91
+
92
+ contents = _normalize_entries(result.get("Contents"))
93
+ if contents:
94
+ return contents[-1]["Key"]
95
+ return None
96
+
97
+
98
+ def _extract_symbol(prefix: str) -> str:
99
+ """Extract the last non-empty path segment from a prefix."""
100
+ return prefix.rstrip("/").rsplit("/", maxsplit=1)[-1]
101
+
102
+
103
+ @dataclass(slots=True, frozen=True)
104
+ class ArchiveFile:
105
+ """Metadata for a single file on the Binance public data archive."""
106
+
107
+ key: str
108
+ size: int
109
+ last_modified: datetime
110
+
111
+
112
+ SymbolListingResult: TypeAlias = tuple[list[ArchiveFile], str | None]
113
+
114
+
115
+ class ArchiveClient:
116
+ """Client for listing directory contents on the Binance public data archive.
117
+
118
+ Communicates with data.binance.vision over its S3-compatible XML listing
119
+ API. Set ``trust_env=True`` (the default) to honour ``http_proxy`` /
120
+ ``https_proxy`` environment variables.
121
+ """
122
+
123
+ def __init__(
124
+ self,
125
+ *,
126
+ timeout_seconds: int | float = S3_HTTP_TIMEOUT_SECONDS,
127
+ trust_env: bool = True,
128
+ ) -> None:
129
+ """Initialize the archive client.
130
+
131
+ Args:
132
+ timeout_seconds: Total timeout in seconds for each HTTP request.
133
+ trust_env: When ``True``, the underlying ``aiohttp`` session reads
134
+ proxy configuration from standard environment variables
135
+ (``http_proxy``, ``https_proxy``, ``no_proxy``).
136
+ """
137
+ self.timeout_seconds = timeout_seconds
138
+ self.trust_env = trust_env
139
+
140
+ def _create_session(self) -> aiohttp.ClientSession:
141
+ """Create an HTTP session for archive requests."""
142
+ timeout = aiohttp.ClientTimeout(total=self.timeout_seconds)
143
+ return aiohttp.ClientSession(timeout=timeout, trust_env=self.trust_env)
144
+
145
+ async def _fetch_xml(self, session: aiohttp.ClientSession, url: str) -> dict[str, Any]:
146
+ """Fetch and parse a single S3 XML listing page."""
147
+ async for attempt in AsyncRetrying(
148
+ stop=stop_after_attempt(5),
149
+ wait=wait_exponential(multiplier=1, min=1, max=8),
150
+ retry=retry_if_exception_type((aiohttp.ClientError, asyncio.TimeoutError)),
151
+ reraise=True,
152
+ ):
153
+ with attempt:
154
+ async with session.get(url) as response:
155
+ response.raise_for_status()
156
+ text = await response.text()
157
+ return xmltodict.parse(text)
158
+
159
+ msg = "unreachable"
160
+ raise RuntimeError(msg)
161
+
162
+ async def list_dir(self, session: aiohttp.ClientSession, prefix: str) -> list[str]:
163
+ """List all child prefixes under an S3 directory prefix.
164
+
165
+ Automatically follows S3 pagination when the result set exceeds a
166
+ single response page.
167
+
168
+ Args:
169
+ session: An active ``aiohttp`` client session.
170
+ prefix: S3 object key prefix to list
171
+ (e.g. ``"data/spot/daily/klines/"``).
172
+
173
+ Returns:
174
+ Full S3 prefix strings for each child directory.
175
+ """
176
+ prefixes: list[str] = []
177
+ marker: str | None = None
178
+
179
+ while True:
180
+ logger.debug("fetching directory listing page for prefix={} marker={}", prefix, marker)
181
+ payload = await self._fetch_xml(session, _build_listing_url(prefix, marker))
182
+ page_prefixes = _extract_prefixes_from_payload(payload)
183
+ prefixes.extend(page_prefixes)
184
+
185
+ if not _is_truncated(payload):
186
+ break
187
+
188
+ marker = _next_marker(payload, page_prefixes)
189
+ if marker is None:
190
+ break
191
+
192
+ return prefixes
193
+
194
+ async def list_files_in_dir(
195
+ self,
196
+ session: aiohttp.ClientSession,
197
+ prefix: str,
198
+ ) -> list[ArchiveFile]:
199
+ """List all files directly under an S3 directory prefix.
200
+
201
+ Files are listed without recursion. Automatically follows S3
202
+ pagination when the result set exceeds a single response page.
203
+
204
+ Args:
205
+ session: An active ``aiohttp`` client session.
206
+ prefix: S3 object key prefix to list
207
+ (e.g. ``"data/futures/um/monthly/fundingRate/BTCUSDT/"``).
208
+
209
+ Returns:
210
+ Archive file metadata entries for each file directly under
211
+ the prefix.
212
+ """
213
+ files: list[ArchiveFile] = []
214
+ marker: str | None = None
215
+
216
+ while True:
217
+ logger.debug("fetching file listing page for prefix={} marker={}", prefix, marker)
218
+ payload = await self._fetch_xml(session, _build_listing_url(prefix, marker))
219
+ page_files = _extract_files_from_payload(payload)
220
+ files.extend(page_files)
221
+
222
+ if not _is_truncated(payload):
223
+ break
224
+
225
+ marker = _next_marker(payload, [file.key for file in page_files])
226
+ if marker is None:
227
+ break
228
+
229
+ return files
230
+
231
+ async def list_symbols(
232
+ self,
233
+ trade_type: TradeType,
234
+ data_freq: DataFrequency,
235
+ data_type: DataType,
236
+ ) -> list[str]:
237
+ """List available symbols from the Binance archive.
238
+
239
+ Queries data.binance.vision's S3 XML API and returns a sorted list
240
+ of symbol directory names under the given path parameters.
241
+
242
+ Args:
243
+ trade_type: Market segment (spot, um, cm).
244
+ data_freq: Partition frequency (daily, monthly).
245
+ data_type: Dataset type (klines, fundingRate, etc.).
246
+
247
+ Returns:
248
+ Sorted list of symbol names (e.g. ``["BTCUSDT", "ETHUSDT"]``).
249
+ """
250
+ prefix = _build_prefix(trade_type, data_freq, data_type)
251
+
252
+ async with self._create_session() as session:
253
+ child_prefixes = await self.list_dir(session, prefix)
254
+
255
+ return sorted(_extract_symbol(prefix) for prefix in child_prefixes)
256
+
257
+ async def list_symbol_files(
258
+ self,
259
+ trade_type: TradeType,
260
+ data_freq: DataFrequency,
261
+ data_type: DataType,
262
+ symbol: str,
263
+ interval: str | None = None,
264
+ *,
265
+ session: aiohttp.ClientSession | None = None,
266
+ ) -> list[ArchiveFile]:
267
+ """List files for a single symbol directory on the Binance archive.
268
+
269
+ Builds the S3 prefix from the requested path parameters and
270
+ delegates to :meth:`list_files_in_dir`. Callers that run many
271
+ concurrent symbol listings should pass a shared ``session`` so
272
+ every request reuses the same connection pool.
273
+
274
+ Args:
275
+ trade_type: Market segment (spot, um, cm).
276
+ data_freq: Partition frequency (daily, monthly).
277
+ data_type: Dataset type (klines, fundingRate, etc.).
278
+ symbol: Symbol directory to list (e.g. ``"BTCUSDT"``).
279
+ interval: Kline interval directory such as ``"1m"``. Required
280
+ when ``data_type.has_interval_layer`` is ``True`` and must
281
+ be ``None`` otherwise.
282
+ session: Optional pre-existing ``aiohttp`` client session to
283
+ reuse. When ``None`` a short-lived session is created
284
+ and closed inside this call. When provided the caller
285
+ owns its lifecycle and this method does not close it.
286
+
287
+ Returns:
288
+ Archive file metadata entries for each file under the symbol
289
+ directory.
290
+
291
+ Raises:
292
+ ValueError: If ``interval`` does not match
293
+ ``data_type.has_interval_layer``.
294
+ """
295
+ if data_type.has_interval_layer and interval is None:
296
+ msg = "interval is required for kline-class data_type"
297
+ raise ValueError(msg)
298
+ if not data_type.has_interval_layer and interval is not None:
299
+ msg = "interval is not applicable to non-kline data_type"
300
+ raise ValueError(msg)
301
+
302
+ prefix = f"{_build_prefix(trade_type, data_freq, data_type)}{symbol}/"
303
+ if interval is not None:
304
+ prefix = f"{prefix}{interval}/"
305
+
306
+ if session is None:
307
+ async with self._create_session() as local_session:
308
+ return await self.list_files_in_dir(local_session, prefix)
309
+
310
+ return await self.list_files_in_dir(session, prefix)
311
+
312
+ async def list_symbol_files_batch(
313
+ self,
314
+ trade_type: TradeType,
315
+ data_freq: DataFrequency,
316
+ data_type: DataType,
317
+ symbols: Sequence[str],
318
+ interval: str | None = None,
319
+ *,
320
+ progress_bar: bool = False,
321
+ ) -> dict[str, SymbolListingResult]:
322
+ """List files for multiple symbols concurrently via a shared session.
323
+
324
+ Args:
325
+ trade_type: Market segment (spot, um, cm).
326
+ data_freq: Partition frequency (daily, monthly).
327
+ data_type: Dataset type (klines, fundingRate, etc.).
328
+ symbols: Symbols to list in caller-provided order.
329
+ interval: Kline interval directory such as ``"1m"``. Required
330
+ when ``data_type.has_interval_layer`` is ``True`` and must
331
+ be ``None`` otherwise.
332
+ progress_bar: Whether to render an interactive tqdm progress bar.
333
+
334
+ Returns:
335
+ Ordered mapping from each input symbol to ``(files, error)``.
336
+ Successful results store ``([], None)`` for empty directories.
337
+ Failed results store ``([], "<message>")``.
338
+ """
339
+ if not symbols:
340
+ return {}
341
+
342
+ async with self._create_session() as session:
343
+
344
+ async def _one(symbol: str) -> tuple[str, list[ArchiveFile], str | None]:
345
+ try:
346
+ files = await self.list_symbol_files(
347
+ trade_type,
348
+ data_freq,
349
+ data_type,
350
+ symbol,
351
+ interval,
352
+ session=session,
353
+ )
354
+ return symbol, files, None
355
+ except Exception as exc:
356
+ return symbol, [], str(exc)
357
+
358
+ coroutines = [_one(symbol) for symbol in symbols]
359
+ outcomes: dict[str, SymbolListingResult] = {}
360
+ with make_reporter(
361
+ progress_bar,
362
+ total=len(symbols),
363
+ description="List files",
364
+ ) as reporter:
365
+ for coroutine in asyncio.as_completed(coroutines):
366
+ symbol, files, error = await coroutine
367
+ outcomes[symbol] = (files, error)
368
+ reporter.tick(ProgressEvent(name=symbol, ok=error is None))
369
+
370
+ return {symbol: outcomes[symbol] for symbol in symbols}
371
+
372
+
373
+ async def list_symbols(
374
+ trade_type: TradeType,
375
+ data_freq: DataFrequency,
376
+ data_type: DataType,
377
+ ) -> list[str]:
378
+ """List available symbols using the default archive client.
379
+
380
+ Convenience wrapper that creates a temporary :class:`ArchiveClient` and
381
+ delegates to :meth:`ArchiveClient.list_symbols`.
382
+
383
+ Args:
384
+ trade_type: Market segment (spot, um, cm).
385
+ data_freq: Partition frequency (daily, monthly).
386
+ data_type: Dataset type (klines, fundingRate, etc.).
387
+
388
+ Returns:
389
+ Sorted list of symbol names.
390
+ """
391
+ return await ArchiveClient().list_symbols(trade_type, data_freq, data_type)