binance-datatool 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- binance_datatool/__init__.py +3 -0
- binance_datatool/archive/__init__.py +53 -0
- binance_datatool/archive/checksum.py +55 -0
- binance_datatool/archive/client.py +391 -0
- binance_datatool/archive/downloader.py +212 -0
- binance_datatool/archive/filter.py +138 -0
- binance_datatool/archive/symbol_dir.py +203 -0
- binance_datatool/cli/__init__.py +41 -0
- binance_datatool/cli/archive.py +484 -0
- binance_datatool/common/__init__.py +47 -0
- binance_datatool/common/constants.py +61 -0
- binance_datatool/common/enums.py +80 -0
- binance_datatool/common/logging.py +32 -0
- binance_datatool/common/path.py +40 -0
- binance_datatool/common/progress.py +335 -0
- binance_datatool/common/symbols.py +195 -0
- binance_datatool/common/types.py +44 -0
- binance_datatool/py.typed +1 -0
- binance_datatool/workflow/__init__.py +33 -0
- binance_datatool/workflow/_shared.py +39 -0
- binance_datatool/workflow/download.py +193 -0
- binance_datatool/workflow/list_files.py +96 -0
- binance_datatool/workflow/list_symbols.py +125 -0
- binance_datatool/workflow/results.py +184 -0
- binance_datatool/workflow/verify.py +256 -0
- binance_datatool-0.1.0.dist-info/METADATA +172 -0
- binance_datatool-0.1.0.dist-info/RECORD +30 -0
- binance_datatool-0.1.0.dist-info/WHEEL +4 -0
- binance_datatool-0.1.0.dist-info/entry_points.txt +2 -0
- binance_datatool-0.1.0.dist-info/licenses/LICENSE +29 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Archive access helpers."""
|
|
2
|
+
|
|
3
|
+
from binance_datatool.archive.checksum import (
|
|
4
|
+
VerifyFileResult,
|
|
5
|
+
calc_sha256,
|
|
6
|
+
read_expected_checksum,
|
|
7
|
+
verify_single_file,
|
|
8
|
+
)
|
|
9
|
+
from binance_datatool.archive.client import (
|
|
10
|
+
ArchiveClient,
|
|
11
|
+
ArchiveFile,
|
|
12
|
+
SymbolListingResult,
|
|
13
|
+
list_symbols,
|
|
14
|
+
)
|
|
15
|
+
from binance_datatool.archive.downloader import (
|
|
16
|
+
Aria2DownloadResult,
|
|
17
|
+
Aria2NotFoundError,
|
|
18
|
+
DownloadRequest,
|
|
19
|
+
download_archive_files,
|
|
20
|
+
)
|
|
21
|
+
from binance_datatool.archive.filter import (
|
|
22
|
+
CmSymbolFilter,
|
|
23
|
+
SpotSymbolFilter,
|
|
24
|
+
SymbolFilter,
|
|
25
|
+
UmSymbolFilter,
|
|
26
|
+
build_symbol_filter,
|
|
27
|
+
)
|
|
28
|
+
from binance_datatool.archive.symbol_dir import (
|
|
29
|
+
SymbolArchiveDir,
|
|
30
|
+
create_symbol_archive_dir,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
"ArchiveClient",
|
|
35
|
+
"ArchiveFile",
|
|
36
|
+
"Aria2DownloadResult",
|
|
37
|
+
"Aria2NotFoundError",
|
|
38
|
+
"CmSymbolFilter",
|
|
39
|
+
"DownloadRequest",
|
|
40
|
+
"SpotSymbolFilter",
|
|
41
|
+
"SymbolArchiveDir",
|
|
42
|
+
"SymbolListingResult",
|
|
43
|
+
"SymbolFilter",
|
|
44
|
+
"UmSymbolFilter",
|
|
45
|
+
"VerifyFileResult",
|
|
46
|
+
"build_symbol_filter",
|
|
47
|
+
"calc_sha256",
|
|
48
|
+
"create_symbol_archive_dir",
|
|
49
|
+
"download_archive_files",
|
|
50
|
+
"list_symbols",
|
|
51
|
+
"read_expected_checksum",
|
|
52
|
+
"verify_single_file",
|
|
53
|
+
]
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Archive checksum helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import TYPE_CHECKING
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(slots=True)
|
|
14
|
+
class VerifyFileResult:
|
|
15
|
+
"""Single-file verification outcome."""
|
|
16
|
+
|
|
17
|
+
zip_path: Path
|
|
18
|
+
passed: bool
|
|
19
|
+
detail: str
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def calc_sha256(file_path: Path) -> str:
|
|
23
|
+
"""Calculate the SHA256 digest for a file."""
|
|
24
|
+
with file_path.open("rb") as file_obj:
|
|
25
|
+
return hashlib.file_digest(file_obj, "sha256").hexdigest()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def read_expected_checksum(zip_path: Path) -> str:
|
|
29
|
+
"""Read the expected checksum from a sibling ``.CHECKSUM`` file.
|
|
30
|
+
|
|
31
|
+
Raises:
|
|
32
|
+
FileNotFoundError: When the ``.CHECKSUM`` sibling does not exist.
|
|
33
|
+
ValueError: When the checksum file is empty or has no parseable token.
|
|
34
|
+
"""
|
|
35
|
+
checksum_path = zip_path.parent / f"{zip_path.name}.CHECKSUM"
|
|
36
|
+
line = checksum_path.read_text(encoding="utf-8").strip()
|
|
37
|
+
expected, *_rest = line.split()
|
|
38
|
+
if not expected:
|
|
39
|
+
msg = f"invalid checksum file: {checksum_path}"
|
|
40
|
+
raise ValueError(msg)
|
|
41
|
+
return expected
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def verify_single_file(zip_path: Path) -> VerifyFileResult:
|
|
45
|
+
"""Verify a single zip file against its sibling checksum file."""
|
|
46
|
+
try:
|
|
47
|
+
expected = read_expected_checksum(zip_path)
|
|
48
|
+
actual = calc_sha256(zip_path)
|
|
49
|
+
except Exception as exc: # noqa: BLE001
|
|
50
|
+
return VerifyFileResult(zip_path=zip_path, passed=False, detail=str(exc))
|
|
51
|
+
|
|
52
|
+
if actual == expected:
|
|
53
|
+
return VerifyFileResult(zip_path=zip_path, passed=True, detail="")
|
|
54
|
+
|
|
55
|
+
return VerifyFileResult(zip_path=zip_path, passed=False, detail="checksum mismatch")
|
|
@@ -0,0 +1,391 @@
|
|
|
1
|
+
"""S3 directory listing client for data.binance.vision."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from datetime import datetime
|
|
8
|
+
from typing import TYPE_CHECKING, Any, TypeAlias
|
|
9
|
+
from urllib.parse import urlencode
|
|
10
|
+
|
|
11
|
+
import aiohttp
|
|
12
|
+
import xmltodict
|
|
13
|
+
from loguru import logger
|
|
14
|
+
from tenacity import AsyncRetrying, retry_if_exception_type, stop_after_attempt, wait_exponential
|
|
15
|
+
|
|
16
|
+
from binance_datatool.common import S3_HTTP_TIMEOUT_SECONDS, S3_LISTING_PREFIX
|
|
17
|
+
from binance_datatool.common.progress import ProgressEvent, make_reporter
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from collections.abc import Iterable, Sequence
|
|
21
|
+
|
|
22
|
+
from binance_datatool.common.enums import DataFrequency, DataType, TradeType
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _build_prefix(
|
|
26
|
+
trade_type: TradeType,
|
|
27
|
+
data_freq: DataFrequency,
|
|
28
|
+
data_type: DataType,
|
|
29
|
+
) -> str:
|
|
30
|
+
"""Build an S3 prefix for a market archive directory."""
|
|
31
|
+
return f"data/{trade_type.s3_path}/{data_freq.value}/{data_type.value}/"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _build_listing_url(prefix: str, marker: str | None = None) -> str:
|
|
35
|
+
"""Build an S3 bucket listing URL for a prefix."""
|
|
36
|
+
query: dict[str, str] = {"delimiter": "/", "prefix": prefix}
|
|
37
|
+
if marker is not None:
|
|
38
|
+
query["marker"] = marker
|
|
39
|
+
return f"{S3_LISTING_PREFIX}?{urlencode(query)}"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _normalize_entries(value: Any) -> list[dict[str, Any]]:
|
|
43
|
+
"""Normalize xmltodict values that may be absent, a dict, or a list."""
|
|
44
|
+
if value is None:
|
|
45
|
+
return []
|
|
46
|
+
if isinstance(value, list):
|
|
47
|
+
return value
|
|
48
|
+
if isinstance(value, dict):
|
|
49
|
+
return [value]
|
|
50
|
+
msg = f"Unexpected S3 XML entry payload type: {type(value)!r}"
|
|
51
|
+
raise TypeError(msg)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _extract_prefixes_from_payload(payload: dict[str, Any]) -> list[str]:
|
|
55
|
+
"""Extract child prefixes from an S3 listing response payload."""
|
|
56
|
+
result = payload["ListBucketResult"]
|
|
57
|
+
prefixes = _normalize_entries(result.get("CommonPrefixes"))
|
|
58
|
+
return [entry["Prefix"] for entry in prefixes]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _extract_files_from_payload(payload: dict[str, Any]) -> list[ArchiveFile]:
|
|
62
|
+
"""Extract file metadata entries from an S3 listing response payload."""
|
|
63
|
+
result = payload["ListBucketResult"]
|
|
64
|
+
contents = _normalize_entries(result.get("Contents"))
|
|
65
|
+
return [
|
|
66
|
+
ArchiveFile(
|
|
67
|
+
key=entry["Key"],
|
|
68
|
+
size=int(entry["Size"]),
|
|
69
|
+
last_modified=datetime.fromisoformat(entry["LastModified"].replace("Z", "+00:00")),
|
|
70
|
+
)
|
|
71
|
+
for entry in contents
|
|
72
|
+
]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _is_truncated(payload: dict[str, Any]) -> bool:
|
|
76
|
+
"""Return whether the S3 listing response is truncated."""
|
|
77
|
+
result = payload["ListBucketResult"]
|
|
78
|
+
return result.get("IsTruncated", "false").lower() == "true"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _next_marker(payload: dict[str, Any], prefixes: Iterable[str]) -> str | None:
|
|
82
|
+
"""Return the marker to use for the next S3 listing page."""
|
|
83
|
+
result = payload["ListBucketResult"]
|
|
84
|
+
marker = result.get("NextMarker")
|
|
85
|
+
if marker:
|
|
86
|
+
return marker
|
|
87
|
+
|
|
88
|
+
prefix_list = list(prefixes)
|
|
89
|
+
if prefix_list:
|
|
90
|
+
return prefix_list[-1]
|
|
91
|
+
|
|
92
|
+
contents = _normalize_entries(result.get("Contents"))
|
|
93
|
+
if contents:
|
|
94
|
+
return contents[-1]["Key"]
|
|
95
|
+
return None
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _extract_symbol(prefix: str) -> str:
|
|
99
|
+
"""Extract the last non-empty path segment from a prefix."""
|
|
100
|
+
return prefix.rstrip("/").rsplit("/", maxsplit=1)[-1]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@dataclass(slots=True, frozen=True)
|
|
104
|
+
class ArchiveFile:
|
|
105
|
+
"""Metadata for a single file on the Binance public data archive."""
|
|
106
|
+
|
|
107
|
+
key: str
|
|
108
|
+
size: int
|
|
109
|
+
last_modified: datetime
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
SymbolListingResult: TypeAlias = tuple[list[ArchiveFile], str | None]
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class ArchiveClient:
|
|
116
|
+
"""Client for listing directory contents on the Binance public data archive.
|
|
117
|
+
|
|
118
|
+
Communicates with data.binance.vision over its S3-compatible XML listing
|
|
119
|
+
API. Set ``trust_env=True`` (the default) to honour ``http_proxy`` /
|
|
120
|
+
``https_proxy`` environment variables.
|
|
121
|
+
"""
|
|
122
|
+
|
|
123
|
+
def __init__(
|
|
124
|
+
self,
|
|
125
|
+
*,
|
|
126
|
+
timeout_seconds: int | float = S3_HTTP_TIMEOUT_SECONDS,
|
|
127
|
+
trust_env: bool = True,
|
|
128
|
+
) -> None:
|
|
129
|
+
"""Initialize the archive client.
|
|
130
|
+
|
|
131
|
+
Args:
|
|
132
|
+
timeout_seconds: Total timeout in seconds for each HTTP request.
|
|
133
|
+
trust_env: When ``True``, the underlying ``aiohttp`` session reads
|
|
134
|
+
proxy configuration from standard environment variables
|
|
135
|
+
(``http_proxy``, ``https_proxy``, ``no_proxy``).
|
|
136
|
+
"""
|
|
137
|
+
self.timeout_seconds = timeout_seconds
|
|
138
|
+
self.trust_env = trust_env
|
|
139
|
+
|
|
140
|
+
def _create_session(self) -> aiohttp.ClientSession:
|
|
141
|
+
"""Create an HTTP session for archive requests."""
|
|
142
|
+
timeout = aiohttp.ClientTimeout(total=self.timeout_seconds)
|
|
143
|
+
return aiohttp.ClientSession(timeout=timeout, trust_env=self.trust_env)
|
|
144
|
+
|
|
145
|
+
async def _fetch_xml(self, session: aiohttp.ClientSession, url: str) -> dict[str, Any]:
|
|
146
|
+
"""Fetch and parse a single S3 XML listing page."""
|
|
147
|
+
async for attempt in AsyncRetrying(
|
|
148
|
+
stop=stop_after_attempt(5),
|
|
149
|
+
wait=wait_exponential(multiplier=1, min=1, max=8),
|
|
150
|
+
retry=retry_if_exception_type((aiohttp.ClientError, asyncio.TimeoutError)),
|
|
151
|
+
reraise=True,
|
|
152
|
+
):
|
|
153
|
+
with attempt:
|
|
154
|
+
async with session.get(url) as response:
|
|
155
|
+
response.raise_for_status()
|
|
156
|
+
text = await response.text()
|
|
157
|
+
return xmltodict.parse(text)
|
|
158
|
+
|
|
159
|
+
msg = "unreachable"
|
|
160
|
+
raise RuntimeError(msg)
|
|
161
|
+
|
|
162
|
+
async def list_dir(self, session: aiohttp.ClientSession, prefix: str) -> list[str]:
|
|
163
|
+
"""List all child prefixes under an S3 directory prefix.
|
|
164
|
+
|
|
165
|
+
Automatically follows S3 pagination when the result set exceeds a
|
|
166
|
+
single response page.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
session: An active ``aiohttp`` client session.
|
|
170
|
+
prefix: S3 object key prefix to list
|
|
171
|
+
(e.g. ``"data/spot/daily/klines/"``).
|
|
172
|
+
|
|
173
|
+
Returns:
|
|
174
|
+
Full S3 prefix strings for each child directory.
|
|
175
|
+
"""
|
|
176
|
+
prefixes: list[str] = []
|
|
177
|
+
marker: str | None = None
|
|
178
|
+
|
|
179
|
+
while True:
|
|
180
|
+
logger.debug("fetching directory listing page for prefix={} marker={}", prefix, marker)
|
|
181
|
+
payload = await self._fetch_xml(session, _build_listing_url(prefix, marker))
|
|
182
|
+
page_prefixes = _extract_prefixes_from_payload(payload)
|
|
183
|
+
prefixes.extend(page_prefixes)
|
|
184
|
+
|
|
185
|
+
if not _is_truncated(payload):
|
|
186
|
+
break
|
|
187
|
+
|
|
188
|
+
marker = _next_marker(payload, page_prefixes)
|
|
189
|
+
if marker is None:
|
|
190
|
+
break
|
|
191
|
+
|
|
192
|
+
return prefixes
|
|
193
|
+
|
|
194
|
+
async def list_files_in_dir(
|
|
195
|
+
self,
|
|
196
|
+
session: aiohttp.ClientSession,
|
|
197
|
+
prefix: str,
|
|
198
|
+
) -> list[ArchiveFile]:
|
|
199
|
+
"""List all files directly under an S3 directory prefix.
|
|
200
|
+
|
|
201
|
+
Files are listed without recursion. Automatically follows S3
|
|
202
|
+
pagination when the result set exceeds a single response page.
|
|
203
|
+
|
|
204
|
+
Args:
|
|
205
|
+
session: An active ``aiohttp`` client session.
|
|
206
|
+
prefix: S3 object key prefix to list
|
|
207
|
+
(e.g. ``"data/futures/um/monthly/fundingRate/BTCUSDT/"``).
|
|
208
|
+
|
|
209
|
+
Returns:
|
|
210
|
+
Archive file metadata entries for each file directly under
|
|
211
|
+
the prefix.
|
|
212
|
+
"""
|
|
213
|
+
files: list[ArchiveFile] = []
|
|
214
|
+
marker: str | None = None
|
|
215
|
+
|
|
216
|
+
while True:
|
|
217
|
+
logger.debug("fetching file listing page for prefix={} marker={}", prefix, marker)
|
|
218
|
+
payload = await self._fetch_xml(session, _build_listing_url(prefix, marker))
|
|
219
|
+
page_files = _extract_files_from_payload(payload)
|
|
220
|
+
files.extend(page_files)
|
|
221
|
+
|
|
222
|
+
if not _is_truncated(payload):
|
|
223
|
+
break
|
|
224
|
+
|
|
225
|
+
marker = _next_marker(payload, [file.key for file in page_files])
|
|
226
|
+
if marker is None:
|
|
227
|
+
break
|
|
228
|
+
|
|
229
|
+
return files
|
|
230
|
+
|
|
231
|
+
async def list_symbols(
|
|
232
|
+
self,
|
|
233
|
+
trade_type: TradeType,
|
|
234
|
+
data_freq: DataFrequency,
|
|
235
|
+
data_type: DataType,
|
|
236
|
+
) -> list[str]:
|
|
237
|
+
"""List available symbols from the Binance archive.
|
|
238
|
+
|
|
239
|
+
Queries data.binance.vision's S3 XML API and returns a sorted list
|
|
240
|
+
of symbol directory names under the given path parameters.
|
|
241
|
+
|
|
242
|
+
Args:
|
|
243
|
+
trade_type: Market segment (spot, um, cm).
|
|
244
|
+
data_freq: Partition frequency (daily, monthly).
|
|
245
|
+
data_type: Dataset type (klines, fundingRate, etc.).
|
|
246
|
+
|
|
247
|
+
Returns:
|
|
248
|
+
Sorted list of symbol names (e.g. ``["BTCUSDT", "ETHUSDT"]``).
|
|
249
|
+
"""
|
|
250
|
+
prefix = _build_prefix(trade_type, data_freq, data_type)
|
|
251
|
+
|
|
252
|
+
async with self._create_session() as session:
|
|
253
|
+
child_prefixes = await self.list_dir(session, prefix)
|
|
254
|
+
|
|
255
|
+
return sorted(_extract_symbol(prefix) for prefix in child_prefixes)
|
|
256
|
+
|
|
257
|
+
async def list_symbol_files(
|
|
258
|
+
self,
|
|
259
|
+
trade_type: TradeType,
|
|
260
|
+
data_freq: DataFrequency,
|
|
261
|
+
data_type: DataType,
|
|
262
|
+
symbol: str,
|
|
263
|
+
interval: str | None = None,
|
|
264
|
+
*,
|
|
265
|
+
session: aiohttp.ClientSession | None = None,
|
|
266
|
+
) -> list[ArchiveFile]:
|
|
267
|
+
"""List files for a single symbol directory on the Binance archive.
|
|
268
|
+
|
|
269
|
+
Builds the S3 prefix from the requested path parameters and
|
|
270
|
+
delegates to :meth:`list_files_in_dir`. Callers that run many
|
|
271
|
+
concurrent symbol listings should pass a shared ``session`` so
|
|
272
|
+
every request reuses the same connection pool.
|
|
273
|
+
|
|
274
|
+
Args:
|
|
275
|
+
trade_type: Market segment (spot, um, cm).
|
|
276
|
+
data_freq: Partition frequency (daily, monthly).
|
|
277
|
+
data_type: Dataset type (klines, fundingRate, etc.).
|
|
278
|
+
symbol: Symbol directory to list (e.g. ``"BTCUSDT"``).
|
|
279
|
+
interval: Kline interval directory such as ``"1m"``. Required
|
|
280
|
+
when ``data_type.has_interval_layer`` is ``True`` and must
|
|
281
|
+
be ``None`` otherwise.
|
|
282
|
+
session: Optional pre-existing ``aiohttp`` client session to
|
|
283
|
+
reuse. When ``None`` a short-lived session is created
|
|
284
|
+
and closed inside this call. When provided the caller
|
|
285
|
+
owns its lifecycle and this method does not close it.
|
|
286
|
+
|
|
287
|
+
Returns:
|
|
288
|
+
Archive file metadata entries for each file under the symbol
|
|
289
|
+
directory.
|
|
290
|
+
|
|
291
|
+
Raises:
|
|
292
|
+
ValueError: If ``interval`` does not match
|
|
293
|
+
``data_type.has_interval_layer``.
|
|
294
|
+
"""
|
|
295
|
+
if data_type.has_interval_layer and interval is None:
|
|
296
|
+
msg = "interval is required for kline-class data_type"
|
|
297
|
+
raise ValueError(msg)
|
|
298
|
+
if not data_type.has_interval_layer and interval is not None:
|
|
299
|
+
msg = "interval is not applicable to non-kline data_type"
|
|
300
|
+
raise ValueError(msg)
|
|
301
|
+
|
|
302
|
+
prefix = f"{_build_prefix(trade_type, data_freq, data_type)}{symbol}/"
|
|
303
|
+
if interval is not None:
|
|
304
|
+
prefix = f"{prefix}{interval}/"
|
|
305
|
+
|
|
306
|
+
if session is None:
|
|
307
|
+
async with self._create_session() as local_session:
|
|
308
|
+
return await self.list_files_in_dir(local_session, prefix)
|
|
309
|
+
|
|
310
|
+
return await self.list_files_in_dir(session, prefix)
|
|
311
|
+
|
|
312
|
+
async def list_symbol_files_batch(
|
|
313
|
+
self,
|
|
314
|
+
trade_type: TradeType,
|
|
315
|
+
data_freq: DataFrequency,
|
|
316
|
+
data_type: DataType,
|
|
317
|
+
symbols: Sequence[str],
|
|
318
|
+
interval: str | None = None,
|
|
319
|
+
*,
|
|
320
|
+
progress_bar: bool = False,
|
|
321
|
+
) -> dict[str, SymbolListingResult]:
|
|
322
|
+
"""List files for multiple symbols concurrently via a shared session.
|
|
323
|
+
|
|
324
|
+
Args:
|
|
325
|
+
trade_type: Market segment (spot, um, cm).
|
|
326
|
+
data_freq: Partition frequency (daily, monthly).
|
|
327
|
+
data_type: Dataset type (klines, fundingRate, etc.).
|
|
328
|
+
symbols: Symbols to list in caller-provided order.
|
|
329
|
+
interval: Kline interval directory such as ``"1m"``. Required
|
|
330
|
+
when ``data_type.has_interval_layer`` is ``True`` and must
|
|
331
|
+
be ``None`` otherwise.
|
|
332
|
+
progress_bar: Whether to render an interactive tqdm progress bar.
|
|
333
|
+
|
|
334
|
+
Returns:
|
|
335
|
+
Ordered mapping from each input symbol to ``(files, error)``.
|
|
336
|
+
Successful results store ``([], None)`` for empty directories.
|
|
337
|
+
Failed results store ``([], "<message>")``.
|
|
338
|
+
"""
|
|
339
|
+
if not symbols:
|
|
340
|
+
return {}
|
|
341
|
+
|
|
342
|
+
async with self._create_session() as session:
|
|
343
|
+
|
|
344
|
+
async def _one(symbol: str) -> tuple[str, list[ArchiveFile], str | None]:
|
|
345
|
+
try:
|
|
346
|
+
files = await self.list_symbol_files(
|
|
347
|
+
trade_type,
|
|
348
|
+
data_freq,
|
|
349
|
+
data_type,
|
|
350
|
+
symbol,
|
|
351
|
+
interval,
|
|
352
|
+
session=session,
|
|
353
|
+
)
|
|
354
|
+
return symbol, files, None
|
|
355
|
+
except Exception as exc:
|
|
356
|
+
return symbol, [], str(exc)
|
|
357
|
+
|
|
358
|
+
coroutines = [_one(symbol) for symbol in symbols]
|
|
359
|
+
outcomes: dict[str, SymbolListingResult] = {}
|
|
360
|
+
with make_reporter(
|
|
361
|
+
progress_bar,
|
|
362
|
+
total=len(symbols),
|
|
363
|
+
description="List files",
|
|
364
|
+
) as reporter:
|
|
365
|
+
for coroutine in asyncio.as_completed(coroutines):
|
|
366
|
+
symbol, files, error = await coroutine
|
|
367
|
+
outcomes[symbol] = (files, error)
|
|
368
|
+
reporter.tick(ProgressEvent(name=symbol, ok=error is None))
|
|
369
|
+
|
|
370
|
+
return {symbol: outcomes[symbol] for symbol in symbols}
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
async def list_symbols(
|
|
374
|
+
trade_type: TradeType,
|
|
375
|
+
data_freq: DataFrequency,
|
|
376
|
+
data_type: DataType,
|
|
377
|
+
) -> list[str]:
|
|
378
|
+
"""List available symbols using the default archive client.
|
|
379
|
+
|
|
380
|
+
Convenience wrapper that creates a temporary :class:`ArchiveClient` and
|
|
381
|
+
delegates to :meth:`ArchiveClient.list_symbols`.
|
|
382
|
+
|
|
383
|
+
Args:
|
|
384
|
+
trade_type: Market segment (spot, um, cm).
|
|
385
|
+
data_freq: Partition frequency (daily, monthly).
|
|
386
|
+
data_type: Dataset type (klines, fundingRate, etc.).
|
|
387
|
+
|
|
388
|
+
Returns:
|
|
389
|
+
Sorted list of symbol names.
|
|
390
|
+
"""
|
|
391
|
+
return await ArchiveClient().list_symbols(trade_type, data_freq, data_type)
|