binance-datatool 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. binance_datatool-0.1.0/.github/workflows/publish.yml +52 -0
  2. binance_datatool-0.1.0/.gitignore +227 -0
  3. binance_datatool-0.1.0/.pre-commit-config.yaml +18 -0
  4. binance_datatool-0.1.0/AGENTS.md +85 -0
  5. binance_datatool-0.1.0/CLAUDE.md +1 -0
  6. binance_datatool-0.1.0/LICENSE +29 -0
  7. binance_datatool-0.1.0/PKG-INFO +172 -0
  8. binance_datatool-0.1.0/README.md +141 -0
  9. binance_datatool-0.1.0/docs/README.md +20 -0
  10. binance_datatool-0.1.0/docs/architecture.md +83 -0
  11. binance_datatool-0.1.0/docs/extending.md +117 -0
  12. binance_datatool-0.1.0/docs/reference/README.md +53 -0
  13. binance_datatool-0.1.0/docs/reference/archive/README.md +46 -0
  14. binance_datatool-0.1.0/docs/reference/archive/checksum.md +62 -0
  15. binance_datatool-0.1.0/docs/reference/archive/client.md +85 -0
  16. binance_datatool-0.1.0/docs/reference/archive/downloader.md +74 -0
  17. binance_datatool-0.1.0/docs/reference/archive/filter.md +85 -0
  18. binance_datatool-0.1.0/docs/reference/archive/s3-protocol.md +70 -0
  19. binance_datatool-0.1.0/docs/reference/archive/symbol_dir.md +115 -0
  20. binance_datatool-0.1.0/docs/reference/cli/README.md +40 -0
  21. binance_datatool-0.1.0/docs/reference/cli/archive.md +508 -0
  22. binance_datatool-0.1.0/docs/reference/common/constants.md +24 -0
  23. binance_datatool-0.1.0/docs/reference/common/enums.md +53 -0
  24. binance_datatool-0.1.0/docs/reference/common/logging.md +43 -0
  25. binance_datatool-0.1.0/docs/reference/common/path.md +48 -0
  26. binance_datatool-0.1.0/docs/reference/common/progress.md +94 -0
  27. binance_datatool-0.1.0/docs/reference/common/symbols.md +51 -0
  28. binance_datatool-0.1.0/docs/reference/common/types.md +37 -0
  29. binance_datatool-0.1.0/docs/reference/testing.md +42 -0
  30. binance_datatool-0.1.0/docs/reference/workflow/README.md +342 -0
  31. binance_datatool-0.1.0/examples/download_spot_klines_usdt_1m.sh +68 -0
  32. binance_datatool-0.1.0/examples/download_um_funding_usdt_perpetual.sh +68 -0
  33. binance_datatool-0.1.0/examples/download_um_klines_usdt_1m.sh +70 -0
  34. binance_datatool-0.1.0/pyproject.toml +80 -0
  35. binance_datatool-0.1.0/src/binance_datatool/__init__.py +3 -0
  36. binance_datatool-0.1.0/src/binance_datatool/archive/__init__.py +53 -0
  37. binance_datatool-0.1.0/src/binance_datatool/archive/checksum.py +55 -0
  38. binance_datatool-0.1.0/src/binance_datatool/archive/client.py +391 -0
  39. binance_datatool-0.1.0/src/binance_datatool/archive/downloader.py +212 -0
  40. binance_datatool-0.1.0/src/binance_datatool/archive/filter.py +138 -0
  41. binance_datatool-0.1.0/src/binance_datatool/archive/symbol_dir.py +203 -0
  42. binance_datatool-0.1.0/src/binance_datatool/cli/__init__.py +41 -0
  43. binance_datatool-0.1.0/src/binance_datatool/cli/archive.py +484 -0
  44. binance_datatool-0.1.0/src/binance_datatool/common/__init__.py +47 -0
  45. binance_datatool-0.1.0/src/binance_datatool/common/constants.py +61 -0
  46. binance_datatool-0.1.0/src/binance_datatool/common/enums.py +80 -0
  47. binance_datatool-0.1.0/src/binance_datatool/common/logging.py +32 -0
  48. binance_datatool-0.1.0/src/binance_datatool/common/path.py +40 -0
  49. binance_datatool-0.1.0/src/binance_datatool/common/progress.py +335 -0
  50. binance_datatool-0.1.0/src/binance_datatool/common/symbols.py +195 -0
  51. binance_datatool-0.1.0/src/binance_datatool/common/types.py +44 -0
  52. binance_datatool-0.1.0/src/binance_datatool/py.typed +1 -0
  53. binance_datatool-0.1.0/src/binance_datatool/workflow/__init__.py +33 -0
  54. binance_datatool-0.1.0/src/binance_datatool/workflow/_shared.py +39 -0
  55. binance_datatool-0.1.0/src/binance_datatool/workflow/download.py +193 -0
  56. binance_datatool-0.1.0/src/binance_datatool/workflow/list_files.py +96 -0
  57. binance_datatool-0.1.0/src/binance_datatool/workflow/list_symbols.py +125 -0
  58. binance_datatool-0.1.0/src/binance_datatool/workflow/results.py +184 -0
  59. binance_datatool-0.1.0/src/binance_datatool/workflow/verify.py +256 -0
  60. binance_datatool-0.1.0/tests/conftest.py +111 -0
  61. binance_datatool-0.1.0/tests/data/BTCUSDT-1m-2020-01-01.zip +0 -0
  62. binance_datatool-0.1.0/tests/data/BTCUSDT-1m-2020-01-01.zip.CHECKSUM +1 -0
  63. binance_datatool-0.1.0/tests/data/BTCUSDT-1m-2020-01-02.zip +0 -0
  64. binance_datatool-0.1.0/tests/data/BTCUSDT-1m-2020-01-02.zip.CHECKSUM +1 -0
  65. binance_datatool-0.1.0/tests/data/BTCUSDT-1m-2020-01-03.zip +0 -0
  66. binance_datatool-0.1.0/tests/data/BTCUSDT-1m-2020-01-03.zip.CHECKSUM +1 -0
  67. binance_datatool-0.1.0/tests/test_archive_client.py +404 -0
  68. binance_datatool-0.1.0/tests/test_archive_workflow.py +796 -0
  69. binance_datatool-0.1.0/tests/test_checksum.py +54 -0
  70. binance_datatool-0.1.0/tests/test_cli.py +1024 -0
  71. binance_datatool-0.1.0/tests/test_downloader.py +214 -0
  72. binance_datatool-0.1.0/tests/test_enums.py +16 -0
  73. binance_datatool-0.1.0/tests/test_filter.py +150 -0
  74. binance_datatool-0.1.0/tests/test_path.py +37 -0
  75. binance_datatool-0.1.0/tests/test_progress.py +28 -0
  76. binance_datatool-0.1.0/tests/test_symbol_dir.py +318 -0
  77. binance_datatool-0.1.0/tests/test_symbols.py +451 -0
  78. binance_datatool-0.1.0/uv.lock +1765 -0
@@ -0,0 +1,52 @@
1
+ name: Upload Python Package
2
+
3
+ on:
4
+ release:
5
+ types: [published]
6
+
7
+ permissions:
8
+ contents: read
9
+
10
+ jobs:
11
+ release-build:
12
+ runs-on: ubuntu-latest
13
+
14
+ steps:
15
+ - uses: actions/checkout@v5
16
+
17
+ - uses: actions/setup-python@v5
18
+ with:
19
+ python-version: "3.x"
20
+
21
+ - name: Build release distributions
22
+ run: |
23
+ python -m pip install --upgrade pip build
24
+ python -m build
25
+
26
+ - name: Upload distributions
27
+ uses: actions/upload-artifact@v4
28
+ with:
29
+ name: release-dists
30
+ path: dist/
31
+
32
+ pypi-publish:
33
+ runs-on: ubuntu-latest
34
+ needs:
35
+ - release-build
36
+
37
+ permissions:
38
+ id-token: write
39
+
40
+ environment:
41
+ name: pypi
42
+ url: https://pypi.org/p/binance-datatool
43
+
44
+ steps:
45
+ - name: Retrieve release distributions
46
+ uses: actions/download-artifact@v5
47
+ with:
48
+ name: release-dists
49
+ path: dist/
50
+
51
+ - name: Publish release distributions to PyPI
52
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,227 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[codz]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py.cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ # Pipfile.lock
96
+
97
+ # UV
98
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ # uv.lock
102
+
103
+ # poetry
104
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
106
+ # commonly ignored for libraries.
107
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108
+ # poetry.lock
109
+ # poetry.toml
110
+
111
+ # pdm
112
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
113
+ # pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
114
+ # https://pdm-project.org/en/latest/usage/project/#working-with-version-control
115
+ # pdm.lock
116
+ # pdm.toml
117
+ .pdm-python
118
+ .pdm-build/
119
+
120
+ # pixi
121
+ # Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
122
+ # pixi.lock
123
+ # Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
124
+ # in the .venv directory. It is recommended not to include this directory in version control.
125
+ .pixi
126
+
127
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
128
+ __pypackages__/
129
+
130
+ # Celery stuff
131
+ celerybeat-schedule
132
+ celerybeat.pid
133
+
134
+ # Redis
135
+ *.rdb
136
+ *.aof
137
+ *.pid
138
+
139
+ # RabbitMQ
140
+ mnesia/
141
+ rabbitmq/
142
+ rabbitmq-data/
143
+
144
+ # ActiveMQ
145
+ activemq-data/
146
+
147
+ # SageMath parsed files
148
+ *.sage.py
149
+
150
+ # Environments
151
+ .env
152
+ .envrc
153
+ .venv
154
+ env/
155
+ venv/
156
+ ENV/
157
+ env.bak/
158
+ venv.bak/
159
+
160
+ # Spyder project settings
161
+ .spyderproject
162
+ .spyproject
163
+
164
+ # Rope project settings
165
+ .ropeproject
166
+
167
+ # mkdocs documentation
168
+ /site
169
+
170
+ # mypy
171
+ .mypy_cache/
172
+ .dmypy.json
173
+ dmypy.json
174
+
175
+ # Pyre type checker
176
+ .pyre/
177
+
178
+ # pytype static type analyzer
179
+ .pytype/
180
+
181
+ # Cython debug symbols
182
+ cython_debug/
183
+
184
+ # PyCharm
185
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
186
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
187
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
188
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
189
+ # .idea/
190
+
191
+ # Abstra
192
+ # Abstra is an AI-powered process automation framework.
193
+ # Ignore directories containing user credentials, local state, and settings.
194
+ # Learn more at https://abstra.io/docs
195
+ .abstra/
196
+
197
+ # Visual Studio Code
198
+ # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
199
+ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
200
+ # and can be added to the global gitignore or merged into this file. However, if you prefer,
201
+ # you could uncomment the following to ignore the entire vscode folder
202
+ .vscode/
203
+
204
+ # Ruff stuff:
205
+ .ruff_cache/
206
+
207
+ # PyPI configuration file
208
+ .pypirc
209
+
210
+ # Marimo
211
+ marimo/_static/
212
+ marimo/_lsp/
213
+ __marimo__/
214
+
215
+ # Streamlit
216
+ .streamlit/secrets.toml
217
+
218
+ # Local files
219
+ .DS_Store
220
+ temp/
221
+
222
+ # AI
223
+ .codex
224
+ .AGENTS.local.md
225
+ .agents/
226
+ .claude/
227
+ skills-lock.json
@@ -0,0 +1,18 @@
1
+ repos:
2
+ - repo: https://github.com/astral-sh/ruff-pre-commit
3
+ rev: v0.11.13
4
+ hooks:
5
+ - id: ruff
6
+ args: [--fix]
7
+ - id: ruff-format
8
+ - repo: https://github.com/astral-sh/uv-pre-commit
9
+ rev: 0.7.22
10
+ hooks:
11
+ - id: uv-lock
12
+ - repo: https://github.com/pre-commit/pre-commit-hooks
13
+ rev: v5.0.0
14
+ hooks:
15
+ - id: trailing-whitespace
16
+ - id: end-of-file-fixer
17
+ - id: check-yaml
18
+ - id: check-added-large-files
@@ -0,0 +1,85 @@
1
+ # AGENTS.md
2
+
3
+ ## Purpose
4
+ - This repository hosts the `binance_datatool` Python package.
5
+ - Treat checked-in source code, tests, configuration, and public documentation as the source of
6
+ truth for current behavior.
7
+ - Keep the repository easy for both human developers and AI agents to navigate.
8
+
9
+ ## Local Instructions
10
+ - Before making changes, check whether `.AGENTS.local.md` exists at the repository root.
11
+ - If `.AGENTS.local.md` exists, read it and follow it as additional local guidance.
12
+ - Public instructions in this file apply to everyone; local instructions add developer-specific context.
13
+
14
+ ## Documentation Navigation
15
+ - Read `docs/architecture.md` before making structural or cross-layer changes.
16
+ - Read `docs/extending.md` before adding a new command, workflow, enum member, or sub-command group.
17
+ - Read `docs/reference/testing.md` before changing tests, fixtures, or test layout conventions.
18
+ - Use `docs/reference/README.md` as the entry point for module and CLI reference details.
19
+ - Treat checked-in code and tests as higher priority than documentation if they conflict.
20
+ - When a change alters behavior, public interfaces, or contributor workflows, update the relevant
21
+ `docs/` promptly after the code change is committed.
22
+ - Unless explicitly requested otherwise, do not mix code changes and documentation updates in the
23
+ same commit.
24
+
25
+ ## Working Model
26
+ - Build the project as a modern Python package named `binance_datatool`.
27
+ - Use the intended package layout: shared code under `binance_datatool.common`, archive access
28
+ under `binance_datatool.archive`, CLI entrypoints under `binance_datatool.cli`, and business
29
+ workflows under `binance_datatool.workflow.archive`.
30
+ - Prefer clear, composable workflows and thin CLI entrypoints.
31
+ - Keep the root package minimal. Export only version metadata from `binance_datatool.__init__`.
32
+
33
+ ## Toolchain
34
+ - Package and dependency management: `uv`
35
+ - Build backend: `hatchling`
36
+ - Linting and formatting: `ruff`
37
+ - Testing: `pytest`
38
+ - Git hooks: `pre-commit`
39
+ - External downloader: `aria2`
40
+
41
+ ## Expected Commands
42
+ - Environment setup: `uv sync`
43
+ - Lint: `uv run ruff check .`
44
+ - Format: `uv run ruff format .`
45
+ - Tests: `uv run pytest`
46
+ - Targeted tests: `uv run pytest tests/path_to_test.py`
47
+ - CLI entrypoint target: `uv run binance-datatool --help`
48
+
49
+ ## Code Standards
50
+ - Use written English for code comments, docstrings, logs, CLI text, and developer-facing messages.
51
+ - Write clear comments only where they add real value; do not narrate obvious code.
52
+ - Use Google-style docstrings for public modules, classes, and functions.
53
+ - Use modern Python syntax compatible with Python 3.11, but **MUST NOT** use syntax introduced after 3.11
54
+ - Keep Python source line length at 100 characters or less. This limit does not apply to
55
+ Markdown files under `docs/`, where long table rows and wide ASCII diagrams are allowed.
56
+ - Keep imports at module top level unless a local import is technically necessary.
57
+ - Prefer explicit types and clear names over implicit behavior.
58
+ - Prefer small, focused modules with straightforward responsibilities.
59
+ - Follow a let-it-crash mindset. Do not add casual fallback logic, silent recovery, or defensive
60
+ branches unless there is a concrete and justified failure mode.
61
+ - Replace hand-rolled infrastructure with mature third-party libraries when the design already
62
+ names a standard dependency.
63
+
64
+ ## Architecture Expectations
65
+ - Keep CLI modules thin: parse arguments, construct workflows, and present output.
66
+ - Put business logic in importable workflow or domain modules, not inside CLI command functions.
67
+ - Prefer Polars LazyFrame-based data processing when working with tabular pipelines.
68
+ - Preserve a clear separation between shared utilities, archive access, parsing, completion,
69
+ metadata tracking, and holographic kline generation.
70
+ - Design commands to be atomic and composable so an agent can inspect state and then choose the
71
+ next step.
72
+
73
+ ## Testing Expectations
74
+ - Add or update tests for every meaningful behavior change.
75
+ - Prefer the smallest test scope that proves the change.
76
+ - Use `pytest` as the test runner and keep tests readable enough for agent-assisted maintenance.
77
+ - When adding data-processing behavior, include representative fixtures or focused regression
78
+ coverage.
79
+ - Focus tests on functional correctness and observable behavior, not on logging implementation
80
+ details such as how loguru renders or routes messages to stderr.
81
+
82
+ ## Repository Boundaries
83
+ - `temp/` is git-ignored and may contain temporary or non-public materials.
84
+ - Do not treat `temp/` as part of the public package surface or public project documentation.
85
+ - Keep checked-in guidance focused on the package and stable workflows, not on transient working files.
@@ -0,0 +1 @@
1
+ AGENTS.md
@@ -0,0 +1,29 @@
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2026, lostleaf
4
+ All rights reserved.
5
+
6
+ Redistribution and use in source and binary forms, with or without
7
+ modification, are permitted provided that the following conditions are met:
8
+
9
+ 1. Redistributions of source code must retain the above copyright notice, this
10
+ list of conditions and the following disclaimer.
11
+
12
+ 2. Redistributions in binary form must reproduce the above copyright notice,
13
+ this list of conditions and the following disclaimer in the documentation
14
+ and/or other materials provided with the distribution.
15
+
16
+ 3. Neither the name of the copyright holder nor the names of its
17
+ contributors may be used to endorse or promote products derived from
18
+ this software without specific prior written permission.
19
+
20
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
24
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
26
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
27
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,172 @@
1
+ Metadata-Version: 2.4
2
+ Name: binance-datatool
3
+ Version: 0.1.0
4
+ Summary: A non-official Python toolkit for managing historical market data from data.binance.vision
5
+ Project-URL: Repository, https://github.com/lostleaf/binance-datatool
6
+ Author: lostleaf
7
+ License-Expression: BSD-3-Clause
8
+ License-File: LICENSE
9
+ Keywords: binance,crypto,historical-data,market-data,trading
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Intended Audience :: Financial and Insurance Industry
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3 :: Only
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Programming Language :: Python :: 3.14
18
+ Classifier: Programming Language :: Python :: Implementation :: CPython
19
+ Classifier: Topic :: Office/Business :: Financial
20
+ Classifier: Topic :: Office/Business :: Financial :: Investment
21
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
22
+ Requires-Python: >=3.11
23
+ Requires-Dist: aiohttp>=3.13.5
24
+ Requires-Dist: loguru>=0.7.3
25
+ Requires-Dist: python-binance>=1.0.32
26
+ Requires-Dist: tenacity>=9.1.4
27
+ Requires-Dist: tqdm>=4.67.1
28
+ Requires-Dist: typer>=0.24.1
29
+ Requires-Dist: xmltodict>=1.0.4
30
+ Description-Content-Type: text/markdown
31
+
32
+ # Binance DataTool - Built for Agents, Data Delivered.
33
+
34
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue?logo=python&logoColor=white)](https://www.python.org/)
35
+ [![License: BSD-3-Clause](https://img.shields.io/badge/license-BSD--3--Clause-green)](LICENSE)
36
+ [![Ruff](https://img.shields.io/badge/code%20style-ruff-d4aa00?logo=ruff&logoColor=white)](https://docs.astral.sh/ruff/)
37
+ [![uv](https://img.shields.io/badge/package%20manager-uv-de5fe9?logo=uv&logoColor=white)](https://docs.astral.sh/uv/)
38
+
39
+ A non-official Python toolkit for managing historical market data from
40
+ [data.binance.vision](https://data.binance.vision/) — designed
41
+ from the ground up for AI agents and quantitative workflows.
42
+
43
+ ## Why Agent-First?
44
+
45
+ Most market-data tools are built for humans clicking through GUIs.
46
+ **binance-datatool** is different — every command is designed to be
47
+ called by AI agents just as easily as by a human:
48
+
49
+ - **Composable CLI** — stdin/stdout pipelines let agents chain
50
+ commands without glue code
51
+ - **Atomic operations** — each command does one thing well; agents
52
+ can inspect, decide, and act step by step
53
+ - **Deterministic output** — structured, parseable results that
54
+ agents can reason about
55
+ - **Zero interaction** — no prompts, no confirmations; dry-run mode
56
+ for safe previews
57
+
58
+ ## Features
59
+
60
+ - **Multi-market coverage** — Spot, USD-M Futures, and COIN-M Futures
61
+ - **Smart symbol filtering** — filter by quote asset, exclude stablecoins
62
+ and leveraged tokens, select contract types
63
+ - **Resumable batch downloads** — diff-based sync via
64
+ [aria2](https://aria2.github.io/) that only fetches new or updated files
65
+ - **Data integrity verification** — SHA256 checksum validation with
66
+ timestamped marker caching
67
+ - **Composable pipelines** — Unix-friendly design; commands read from
68
+ stdin and write to stdout
69
+ - **Async I/O** — concurrent S3 listing and parallel checksum verification
70
+
71
+ ## Quick Start
72
+
73
+ ### Prerequisites
74
+
75
+ - [Python](https://www.python.org/) 3.11+
76
+ - [uv](https://docs.astral.sh/uv/) — fast Python package manager
77
+ - [aria2](https://aria2.github.io/) — required for the `download` command
78
+
79
+ ### Install
80
+
81
+ ```bash
82
+ pip install binance-datatool
83
+ export BINANCE_DATATOOL_ARCHIVE_HOME=$HOME/crypto_data/binance_archive
84
+ ```
85
+
86
+ For development:
87
+
88
+ ```bash
89
+ git clone https://github.com/lostleaf/binance-datatool.git
90
+ cd binance-datatool
91
+ uv sync
92
+ export BINANCE_DATATOOL_ARCHIVE_HOME=$HOME/crypto_data/binance_archive
93
+ ```
94
+
95
+ ### Try it
96
+
97
+ ```bash
98
+ # List USDT-quoted spot symbols, excluding stablecoins
99
+ uv run binance-datatool list-symbols spot --quote USDT --exclude-stables
100
+
101
+ # Download daily 1m klines
102
+ uv run binance-datatool list-symbols spot --quote USDT --exclude-stables \
103
+ | uv run binance-datatool download spot --type klines --interval 1m
104
+
105
+ # Verify integrity
106
+ uv run binance-datatool verify spot --type klines --interval 1m BTCUSDT
107
+ ```
108
+
109
+ ## Supported Data Types
110
+
111
+ | Category | Types |
112
+ |----------|-------|
113
+ | Trade data | `klines`, `aggTrades`, `trades` |
114
+ | Derivatives | `fundingRate`, `liquidationSnapshot`, `metrics` |
115
+ | Order book | `bookDepth`, `bookTicker` |
116
+ | Index data | `indexPriceKlines`, `markPriceKlines`, `premiumIndexKlines` |
117
+
118
+ Each type is available in **daily** or **monthly** partitions. Kline types
119
+ support all standard intervals (`1m` `3m` `5m` `15m` `30m` `1h` `2h` `4h`
120
+ `6h` `8h` `12h` `1d` `3d` `1w` `1mo`).
121
+
122
+ ## CLI Overview
123
+
124
+ The entry point is `binance-datatool`. All data commands are available
125
+ directly on the root app:
126
+
127
+ | Command | Description |
128
+ |---------|-------------|
129
+ | `list-symbols` | List available symbols on the remote archive |
130
+ | `list-files` | List archive files for given symbols |
131
+ | `download` | Download new or updated archive files |
132
+ | `verify` | Verify files against SHA256 checksums |
133
+
134
+ > [!TIP]
135
+ > Every command accepts symbols from **stdin** or as **positional args**,
136
+ > making it trivial to compose pipelines:
137
+ >
138
+ > ```bash
139
+ > binance-datatool list-symbols um --quote USDT \
140
+ > | binance-datatool download um --type klines --interval 1m
141
+ > ```
142
+
143
+ Run `uv run binance-datatool --help` for the full option reference.
144
+
145
+ ## Architecture
146
+
147
+ ```
148
+ CLI Layer Thin Typer commands (argument parsing, output formatting)
149
+ │
150
+ Workflow Layer Business logic (diff computation, verification protocol)
151
+ │
152
+ Archive Layer S3 communication (HTTP listing, aria2 downloading, checksums)
153
+ │
154
+ Common Layer Shared types, enums, constants, and utilities
155
+ ```
156
+
157
+ See [docs/architecture.md](docs/architecture.md) for a detailed design overview.
158
+
159
+ ## Vision
160
+
161
+ binance-datatool is the data foundation for agent-driven quantitative
162
+ research. Next up: **Agent skills** — packaged capabilities that let AI
163
+ agents autonomously discover, fetch, and validate market data end-to-end.
164
+
165
+ ## Documentation
166
+
167
+ | Document | Description |
168
+ |----------|-------------|
169
+ | [Architecture](docs/architecture.md) | Layered design, data flow, and key decisions |
170
+ | [Module Reference](docs/reference/README.md) | Per-subpackage API reference |
171
+ | [Extending](docs/extending.md) | How to add commands, enums, workflows, and tests |
172
+ | [Testing Guide](docs/reference/testing.md) | Test organization and conventions |