binance-datatool 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- binance_datatool-0.1.0/.github/workflows/publish.yml +52 -0
- binance_datatool-0.1.0/.gitignore +227 -0
- binance_datatool-0.1.0/.pre-commit-config.yaml +18 -0
- binance_datatool-0.1.0/AGENTS.md +85 -0
- binance_datatool-0.1.0/CLAUDE.md +1 -0
- binance_datatool-0.1.0/LICENSE +29 -0
- binance_datatool-0.1.0/PKG-INFO +172 -0
- binance_datatool-0.1.0/README.md +141 -0
- binance_datatool-0.1.0/docs/README.md +20 -0
- binance_datatool-0.1.0/docs/architecture.md +83 -0
- binance_datatool-0.1.0/docs/extending.md +117 -0
- binance_datatool-0.1.0/docs/reference/README.md +53 -0
- binance_datatool-0.1.0/docs/reference/archive/README.md +46 -0
- binance_datatool-0.1.0/docs/reference/archive/checksum.md +62 -0
- binance_datatool-0.1.0/docs/reference/archive/client.md +85 -0
- binance_datatool-0.1.0/docs/reference/archive/downloader.md +74 -0
- binance_datatool-0.1.0/docs/reference/archive/filter.md +85 -0
- binance_datatool-0.1.0/docs/reference/archive/s3-protocol.md +70 -0
- binance_datatool-0.1.0/docs/reference/archive/symbol_dir.md +115 -0
- binance_datatool-0.1.0/docs/reference/cli/README.md +40 -0
- binance_datatool-0.1.0/docs/reference/cli/archive.md +508 -0
- binance_datatool-0.1.0/docs/reference/common/constants.md +24 -0
- binance_datatool-0.1.0/docs/reference/common/enums.md +53 -0
- binance_datatool-0.1.0/docs/reference/common/logging.md +43 -0
- binance_datatool-0.1.0/docs/reference/common/path.md +48 -0
- binance_datatool-0.1.0/docs/reference/common/progress.md +94 -0
- binance_datatool-0.1.0/docs/reference/common/symbols.md +51 -0
- binance_datatool-0.1.0/docs/reference/common/types.md +37 -0
- binance_datatool-0.1.0/docs/reference/testing.md +42 -0
- binance_datatool-0.1.0/docs/reference/workflow/README.md +342 -0
- binance_datatool-0.1.0/examples/download_spot_klines_usdt_1m.sh +68 -0
- binance_datatool-0.1.0/examples/download_um_funding_usdt_perpetual.sh +68 -0
- binance_datatool-0.1.0/examples/download_um_klines_usdt_1m.sh +70 -0
- binance_datatool-0.1.0/pyproject.toml +80 -0
- binance_datatool-0.1.0/src/binance_datatool/__init__.py +3 -0
- binance_datatool-0.1.0/src/binance_datatool/archive/__init__.py +53 -0
- binance_datatool-0.1.0/src/binance_datatool/archive/checksum.py +55 -0
- binance_datatool-0.1.0/src/binance_datatool/archive/client.py +391 -0
- binance_datatool-0.1.0/src/binance_datatool/archive/downloader.py +212 -0
- binance_datatool-0.1.0/src/binance_datatool/archive/filter.py +138 -0
- binance_datatool-0.1.0/src/binance_datatool/archive/symbol_dir.py +203 -0
- binance_datatool-0.1.0/src/binance_datatool/cli/__init__.py +41 -0
- binance_datatool-0.1.0/src/binance_datatool/cli/archive.py +484 -0
- binance_datatool-0.1.0/src/binance_datatool/common/__init__.py +47 -0
- binance_datatool-0.1.0/src/binance_datatool/common/constants.py +61 -0
- binance_datatool-0.1.0/src/binance_datatool/common/enums.py +80 -0
- binance_datatool-0.1.0/src/binance_datatool/common/logging.py +32 -0
- binance_datatool-0.1.0/src/binance_datatool/common/path.py +40 -0
- binance_datatool-0.1.0/src/binance_datatool/common/progress.py +335 -0
- binance_datatool-0.1.0/src/binance_datatool/common/symbols.py +195 -0
- binance_datatool-0.1.0/src/binance_datatool/common/types.py +44 -0
- binance_datatool-0.1.0/src/binance_datatool/py.typed +1 -0
- binance_datatool-0.1.0/src/binance_datatool/workflow/__init__.py +33 -0
- binance_datatool-0.1.0/src/binance_datatool/workflow/_shared.py +39 -0
- binance_datatool-0.1.0/src/binance_datatool/workflow/download.py +193 -0
- binance_datatool-0.1.0/src/binance_datatool/workflow/list_files.py +96 -0
- binance_datatool-0.1.0/src/binance_datatool/workflow/list_symbols.py +125 -0
- binance_datatool-0.1.0/src/binance_datatool/workflow/results.py +184 -0
- binance_datatool-0.1.0/src/binance_datatool/workflow/verify.py +256 -0
- binance_datatool-0.1.0/tests/conftest.py +111 -0
- binance_datatool-0.1.0/tests/data/BTCUSDT-1m-2020-01-01.zip +0 -0
- binance_datatool-0.1.0/tests/data/BTCUSDT-1m-2020-01-01.zip.CHECKSUM +1 -0
- binance_datatool-0.1.0/tests/data/BTCUSDT-1m-2020-01-02.zip +0 -0
- binance_datatool-0.1.0/tests/data/BTCUSDT-1m-2020-01-02.zip.CHECKSUM +1 -0
- binance_datatool-0.1.0/tests/data/BTCUSDT-1m-2020-01-03.zip +0 -0
- binance_datatool-0.1.0/tests/data/BTCUSDT-1m-2020-01-03.zip.CHECKSUM +1 -0
- binance_datatool-0.1.0/tests/test_archive_client.py +404 -0
- binance_datatool-0.1.0/tests/test_archive_workflow.py +796 -0
- binance_datatool-0.1.0/tests/test_checksum.py +54 -0
- binance_datatool-0.1.0/tests/test_cli.py +1024 -0
- binance_datatool-0.1.0/tests/test_downloader.py +214 -0
- binance_datatool-0.1.0/tests/test_enums.py +16 -0
- binance_datatool-0.1.0/tests/test_filter.py +150 -0
- binance_datatool-0.1.0/tests/test_path.py +37 -0
- binance_datatool-0.1.0/tests/test_progress.py +28 -0
- binance_datatool-0.1.0/tests/test_symbol_dir.py +318 -0
- binance_datatool-0.1.0/tests/test_symbols.py +451 -0
- binance_datatool-0.1.0/uv.lock +1765 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
name: Upload Python Package
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
release-build:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v5
|
|
16
|
+
|
|
17
|
+
- uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: "3.x"
|
|
20
|
+
|
|
21
|
+
- name: Build release distributions
|
|
22
|
+
run: |
|
|
23
|
+
python -m pip install --upgrade pip build
|
|
24
|
+
python -m build
|
|
25
|
+
|
|
26
|
+
- name: Upload distributions
|
|
27
|
+
uses: actions/upload-artifact@v4
|
|
28
|
+
with:
|
|
29
|
+
name: release-dists
|
|
30
|
+
path: dist/
|
|
31
|
+
|
|
32
|
+
pypi-publish:
|
|
33
|
+
runs-on: ubuntu-latest
|
|
34
|
+
needs:
|
|
35
|
+
- release-build
|
|
36
|
+
|
|
37
|
+
permissions:
|
|
38
|
+
id-token: write
|
|
39
|
+
|
|
40
|
+
environment:
|
|
41
|
+
name: pypi
|
|
42
|
+
url: https://pypi.org/p/binance-datatool
|
|
43
|
+
|
|
44
|
+
steps:
|
|
45
|
+
- name: Retrieve release distributions
|
|
46
|
+
uses: actions/download-artifact@v5
|
|
47
|
+
with:
|
|
48
|
+
name: release-dists
|
|
49
|
+
path: dist/
|
|
50
|
+
|
|
51
|
+
- name: Publish release distributions to PyPI
|
|
52
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py.cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
# Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# UV
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
# uv.lock
|
|
102
|
+
|
|
103
|
+
# poetry
|
|
104
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
105
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
106
|
+
# commonly ignored for libraries.
|
|
107
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
108
|
+
# poetry.lock
|
|
109
|
+
# poetry.toml
|
|
110
|
+
|
|
111
|
+
# pdm
|
|
112
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
113
|
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
114
|
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
115
|
+
# pdm.lock
|
|
116
|
+
# pdm.toml
|
|
117
|
+
.pdm-python
|
|
118
|
+
.pdm-build/
|
|
119
|
+
|
|
120
|
+
# pixi
|
|
121
|
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
122
|
+
# pixi.lock
|
|
123
|
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
124
|
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
125
|
+
.pixi
|
|
126
|
+
|
|
127
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
128
|
+
__pypackages__/
|
|
129
|
+
|
|
130
|
+
# Celery stuff
|
|
131
|
+
celerybeat-schedule
|
|
132
|
+
celerybeat.pid
|
|
133
|
+
|
|
134
|
+
# Redis
|
|
135
|
+
*.rdb
|
|
136
|
+
*.aof
|
|
137
|
+
*.pid
|
|
138
|
+
|
|
139
|
+
# RabbitMQ
|
|
140
|
+
mnesia/
|
|
141
|
+
rabbitmq/
|
|
142
|
+
rabbitmq-data/
|
|
143
|
+
|
|
144
|
+
# ActiveMQ
|
|
145
|
+
activemq-data/
|
|
146
|
+
|
|
147
|
+
# SageMath parsed files
|
|
148
|
+
*.sage.py
|
|
149
|
+
|
|
150
|
+
# Environments
|
|
151
|
+
.env
|
|
152
|
+
.envrc
|
|
153
|
+
.venv
|
|
154
|
+
env/
|
|
155
|
+
venv/
|
|
156
|
+
ENV/
|
|
157
|
+
env.bak/
|
|
158
|
+
venv.bak/
|
|
159
|
+
|
|
160
|
+
# Spyder project settings
|
|
161
|
+
.spyderproject
|
|
162
|
+
.spyproject
|
|
163
|
+
|
|
164
|
+
# Rope project settings
|
|
165
|
+
.ropeproject
|
|
166
|
+
|
|
167
|
+
# mkdocs documentation
|
|
168
|
+
/site
|
|
169
|
+
|
|
170
|
+
# mypy
|
|
171
|
+
.mypy_cache/
|
|
172
|
+
.dmypy.json
|
|
173
|
+
dmypy.json
|
|
174
|
+
|
|
175
|
+
# Pyre type checker
|
|
176
|
+
.pyre/
|
|
177
|
+
|
|
178
|
+
# pytype static type analyzer
|
|
179
|
+
.pytype/
|
|
180
|
+
|
|
181
|
+
# Cython debug symbols
|
|
182
|
+
cython_debug/
|
|
183
|
+
|
|
184
|
+
# PyCharm
|
|
185
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
186
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
187
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
188
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
189
|
+
# .idea/
|
|
190
|
+
|
|
191
|
+
# Abstra
|
|
192
|
+
# Abstra is an AI-powered process automation framework.
|
|
193
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
194
|
+
# Learn more at https://abstra.io/docs
|
|
195
|
+
.abstra/
|
|
196
|
+
|
|
197
|
+
# Visual Studio Code
|
|
198
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
|
199
|
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
200
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
|
201
|
+
# you could uncomment the following to ignore the entire vscode folder
|
|
202
|
+
.vscode/
|
|
203
|
+
|
|
204
|
+
# Ruff stuff:
|
|
205
|
+
.ruff_cache/
|
|
206
|
+
|
|
207
|
+
# PyPI configuration file
|
|
208
|
+
.pypirc
|
|
209
|
+
|
|
210
|
+
# Marimo
|
|
211
|
+
marimo/_static/
|
|
212
|
+
marimo/_lsp/
|
|
213
|
+
__marimo__/
|
|
214
|
+
|
|
215
|
+
# Streamlit
|
|
216
|
+
.streamlit/secrets.toml
|
|
217
|
+
|
|
218
|
+
# Local files
|
|
219
|
+
.DS_Store
|
|
220
|
+
temp/
|
|
221
|
+
|
|
222
|
+
# AI
|
|
223
|
+
.codex
|
|
224
|
+
.AGENTS.local.md
|
|
225
|
+
.agents/
|
|
226
|
+
.claude/
|
|
227
|
+
skills-lock.json
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
3
|
+
rev: v0.11.13
|
|
4
|
+
hooks:
|
|
5
|
+
- id: ruff
|
|
6
|
+
args: [--fix]
|
|
7
|
+
- id: ruff-format
|
|
8
|
+
- repo: https://github.com/astral-sh/uv-pre-commit
|
|
9
|
+
rev: 0.7.22
|
|
10
|
+
hooks:
|
|
11
|
+
- id: uv-lock
|
|
12
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
13
|
+
rev: v5.0.0
|
|
14
|
+
hooks:
|
|
15
|
+
- id: trailing-whitespace
|
|
16
|
+
- id: end-of-file-fixer
|
|
17
|
+
- id: check-yaml
|
|
18
|
+
- id: check-added-large-files
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# AGENTS.md
|
|
2
|
+
|
|
3
|
+
## Purpose
|
|
4
|
+
- This repository hosts the `binance_datatool` Python package.
|
|
5
|
+
- Treat checked-in source code, tests, configuration, and public documentation as the source of
|
|
6
|
+
truth for current behavior.
|
|
7
|
+
- Keep the repository easy for both human developers and AI agents to navigate.
|
|
8
|
+
|
|
9
|
+
## Local Instructions
|
|
10
|
+
- Before making changes, check whether `.AGENTS.local.md` exists at the repository root.
|
|
11
|
+
- If `.AGENTS.local.md` exists, read it and follow it as additional local guidance.
|
|
12
|
+
- Public instructions in this file apply to everyone; local instructions add developer-specific context.
|
|
13
|
+
|
|
14
|
+
## Documentation Navigation
|
|
15
|
+
- Read `docs/architecture.md` before making structural or cross-layer changes.
|
|
16
|
+
- Read `docs/extending.md` before adding a new command, workflow, enum member, or sub-command group.
|
|
17
|
+
- Read `docs/reference/testing.md` before changing tests, fixtures, or test layout conventions.
|
|
18
|
+
- Use `docs/reference/README.md` as the entry point for module and CLI reference details.
|
|
19
|
+
- Treat checked-in code and tests as higher priority than documentation if they conflict.
|
|
20
|
+
- When a change alters behavior, public interfaces, or contributor workflows, update the relevant
|
|
21
|
+
`docs/` promptly after the code change is committed.
|
|
22
|
+
- Unless explicitly requested otherwise, do not mix code changes and documentation updates in the
|
|
23
|
+
same commit.
|
|
24
|
+
|
|
25
|
+
## Working Model
|
|
26
|
+
- Build the project as a modern Python package named `binance_datatool`.
|
|
27
|
+
- Use the intended package layout: shared code under `binance_datatool.common`, archive access
|
|
28
|
+
under `binance_datatool.archive`, CLI entrypoints under `binance_datatool.cli`, and business
|
|
29
|
+
workflows under `binance_datatool.workflow.archive`.
|
|
30
|
+
- Prefer clear, composable workflows and thin CLI entrypoints.
|
|
31
|
+
- Keep the root package minimal. Export only version metadata from `binance_datatool.__init__`.
|
|
32
|
+
|
|
33
|
+
## Toolchain
|
|
34
|
+
- Package and dependency management: `uv`
|
|
35
|
+
- Build backend: `hatchling`
|
|
36
|
+
- Linting and formatting: `ruff`
|
|
37
|
+
- Testing: `pytest`
|
|
38
|
+
- Git hooks: `pre-commit`
|
|
39
|
+
- External downloader: `aria2`
|
|
40
|
+
|
|
41
|
+
## Expected Commands
|
|
42
|
+
- Environment setup: `uv sync`
|
|
43
|
+
- Lint: `uv run ruff check .`
|
|
44
|
+
- Format: `uv run ruff format .`
|
|
45
|
+
- Tests: `uv run pytest`
|
|
46
|
+
- Targeted tests: `uv run pytest tests/path_to_test.py`
|
|
47
|
+
- CLI entrypoint target: `uv run binance-datatool --help`
|
|
48
|
+
|
|
49
|
+
## Code Standards
|
|
50
|
+
- Use written English for code comments, docstrings, logs, CLI text, and developer-facing messages.
|
|
51
|
+
- Write clear comments only where they add real value; do not narrate obvious code.
|
|
52
|
+
- Use Google-style docstrings for public modules, classes, and functions.
|
|
53
|
+
- Use modern Python syntax compatible with Python 3.11, but **MUST NOT** use syntax introduced after 3.11
|
|
54
|
+
- Keep Python source line length at 100 characters or less. This limit does not apply to
|
|
55
|
+
Markdown files under `docs/`, where long table rows and wide ASCII diagrams are allowed.
|
|
56
|
+
- Keep imports at module top level unless a local import is technically necessary.
|
|
57
|
+
- Prefer explicit types and clear names over implicit behavior.
|
|
58
|
+
- Prefer small, focused modules with straightforward responsibilities.
|
|
59
|
+
- Follow a let-it-crash mindset. Do not add casual fallback logic, silent recovery, or defensive
|
|
60
|
+
branches unless there is a concrete and justified failure mode.
|
|
61
|
+
- Replace hand-rolled infrastructure with mature third-party libraries when the design already
|
|
62
|
+
names a standard dependency.
|
|
63
|
+
|
|
64
|
+
## Architecture Expectations
|
|
65
|
+
- Keep CLI modules thin: parse arguments, construct workflows, and present output.
|
|
66
|
+
- Put business logic in importable workflow or domain modules, not inside CLI command functions.
|
|
67
|
+
- Prefer Polars LazyFrame-based data processing when working with tabular pipelines.
|
|
68
|
+
- Preserve a clear separation between shared utilities, archive access, parsing, completion,
|
|
69
|
+
metadata tracking, and holographic kline generation.
|
|
70
|
+
- Design commands to be atomic and composable so an agent can inspect state and then choose the
|
|
71
|
+
next step.
|
|
72
|
+
|
|
73
|
+
## Testing Expectations
|
|
74
|
+
- Add or update tests for every meaningful behavior change.
|
|
75
|
+
- Prefer the smallest test scope that proves the change.
|
|
76
|
+
- Use `pytest` as the test runner and keep tests readable enough for agent-assisted maintenance.
|
|
77
|
+
- When adding data-processing behavior, include representative fixtures or focused regression
|
|
78
|
+
coverage.
|
|
79
|
+
- Focus tests on functional correctness and observable behavior, not on logging implementation
|
|
80
|
+
details such as how loguru renders or routes messages to stderr.
|
|
81
|
+
|
|
82
|
+
## Repository Boundaries
|
|
83
|
+
- `temp/` is git-ignored and may contain temporary or non-public materials.
|
|
84
|
+
- Do not treat `temp/` as part of the public package surface or public project documentation.
|
|
85
|
+
- Keep checked-in guidance focused on the package and stable workflows, not on transient working files.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
AGENTS.md
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, lostleaf
|
|
4
|
+
All rights reserved.
|
|
5
|
+
|
|
6
|
+
Redistribution and use in source and binary forms, with or without
|
|
7
|
+
modification, are permitted provided that the following conditions are met:
|
|
8
|
+
|
|
9
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
10
|
+
list of conditions and the following disclaimer.
|
|
11
|
+
|
|
12
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
13
|
+
this list of conditions and the following disclaimer in the documentation
|
|
14
|
+
and/or other materials provided with the distribution.
|
|
15
|
+
|
|
16
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
17
|
+
contributors may be used to endorse or promote products derived from
|
|
18
|
+
this software without specific prior written permission.
|
|
19
|
+
|
|
20
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
21
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
22
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
23
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
24
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
25
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
26
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
27
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
28
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: binance-datatool
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A non-official Python toolkit for managing historical market data from data.binance.vision
|
|
5
|
+
Project-URL: Repository, https://github.com/lostleaf/binance-datatool
|
|
6
|
+
Author: lostleaf
|
|
7
|
+
License-Expression: BSD-3-Clause
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: binance,crypto,historical-data,market-data,trading
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
19
|
+
Classifier: Topic :: Office/Business :: Financial
|
|
20
|
+
Classifier: Topic :: Office/Business :: Financial :: Investment
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Requires-Python: >=3.11
|
|
23
|
+
Requires-Dist: aiohttp>=3.13.5
|
|
24
|
+
Requires-Dist: loguru>=0.7.3
|
|
25
|
+
Requires-Dist: python-binance>=1.0.32
|
|
26
|
+
Requires-Dist: tenacity>=9.1.4
|
|
27
|
+
Requires-Dist: tqdm>=4.67.1
|
|
28
|
+
Requires-Dist: typer>=0.24.1
|
|
29
|
+
Requires-Dist: xmltodict>=1.0.4
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# Binance DataTool - Built for Agents, Data Delivered.
|
|
33
|
+
|
|
34
|
+
[](https://www.python.org/)
|
|
35
|
+
[](LICENSE)
|
|
36
|
+
[](https://docs.astral.sh/ruff/)
|
|
37
|
+
[](https://docs.astral.sh/uv/)
|
|
38
|
+
|
|
39
|
+
A non-official Python toolkit for managing historical market data from
|
|
40
|
+
[data.binance.vision](https://data.binance.vision/) — designed
|
|
41
|
+
from the ground up for AI agents and quantitative workflows.
|
|
42
|
+
|
|
43
|
+
## Why Agent-First?
|
|
44
|
+
|
|
45
|
+
Most market-data tools are built for humans clicking through GUIs.
|
|
46
|
+
**binance-datatool** is different — every command is designed to be
|
|
47
|
+
called by AI agents just as easily as by a human:
|
|
48
|
+
|
|
49
|
+
- **Composable CLI** — stdin/stdout pipelines let agents chain
|
|
50
|
+
commands without glue code
|
|
51
|
+
- **Atomic operations** — each command does one thing well; agents
|
|
52
|
+
can inspect, decide, and act step by step
|
|
53
|
+
- **Deterministic output** — structured, parseable results that
|
|
54
|
+
agents can reason about
|
|
55
|
+
- **Zero interaction** — no prompts, no confirmations; dry-run mode
|
|
56
|
+
for safe previews
|
|
57
|
+
|
|
58
|
+
## Features
|
|
59
|
+
|
|
60
|
+
- **Multi-market coverage** — Spot, USD-M Futures, and COIN-M Futures
|
|
61
|
+
- **Smart symbol filtering** — filter by quote asset, exclude stablecoins
|
|
62
|
+
and leveraged tokens, select contract types
|
|
63
|
+
- **Resumable batch downloads** — diff-based sync via
|
|
64
|
+
[aria2](https://aria2.github.io/) that only fetches new or updated files
|
|
65
|
+
- **Data integrity verification** — SHA256 checksum validation with
|
|
66
|
+
timestamped marker caching
|
|
67
|
+
- **Composable pipelines** — Unix-friendly design; commands read from
|
|
68
|
+
stdin and write to stdout
|
|
69
|
+
- **Async I/O** — concurrent S3 listing and parallel checksum verification
|
|
70
|
+
|
|
71
|
+
## Quick Start
|
|
72
|
+
|
|
73
|
+
### Prerequisites
|
|
74
|
+
|
|
75
|
+
- [Python](https://www.python.org/) 3.11+
|
|
76
|
+
- [uv](https://docs.astral.sh/uv/) — fast Python package manager
|
|
77
|
+
- [aria2](https://aria2.github.io/) — required for the `download` command
|
|
78
|
+
|
|
79
|
+
### Install
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
pip install binance-datatool
|
|
83
|
+
export BINANCE_DATATOOL_ARCHIVE_HOME=$HOME/crypto_data/binance_archive
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
For development:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
git clone https://github.com/lostleaf/binance-datatool.git
|
|
90
|
+
cd binance-datatool
|
|
91
|
+
uv sync
|
|
92
|
+
export BINANCE_DATATOOL_ARCHIVE_HOME=$HOME/crypto_data/binance_archive
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Try it
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
# List USDT-quoted spot symbols, excluding stablecoins
|
|
99
|
+
uv run binance-datatool list-symbols spot --quote USDT --exclude-stables
|
|
100
|
+
|
|
101
|
+
# Download daily 1m klines
|
|
102
|
+
uv run binance-datatool list-symbols spot --quote USDT --exclude-stables \
|
|
103
|
+
| uv run binance-datatool download spot --type klines --interval 1m
|
|
104
|
+
|
|
105
|
+
# Verify integrity
|
|
106
|
+
uv run binance-datatool verify spot --type klines --interval 1m BTCUSDT
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
## Supported Data Types
|
|
110
|
+
|
|
111
|
+
| Category | Types |
|
|
112
|
+
|----------|-------|
|
|
113
|
+
| Trade data | `klines`, `aggTrades`, `trades` |
|
|
114
|
+
| Derivatives | `fundingRate`, `liquidationSnapshot`, `metrics` |
|
|
115
|
+
| Order book | `bookDepth`, `bookTicker` |
|
|
116
|
+
| Index data | `indexPriceKlines`, `markPriceKlines`, `premiumIndexKlines` |
|
|
117
|
+
|
|
118
|
+
Each type is available in **daily** or **monthly** partitions. Kline types
|
|
119
|
+
support all standard intervals (`1m` `3m` `5m` `15m` `30m` `1h` `2h` `4h`
|
|
120
|
+
`6h` `8h` `12h` `1d` `3d` `1w` `1mo`).
|
|
121
|
+
|
|
122
|
+
## CLI Overview
|
|
123
|
+
|
|
124
|
+
The entry point is `binance-datatool`. All data commands are available
|
|
125
|
+
directly on the root app:
|
|
126
|
+
|
|
127
|
+
| Command | Description |
|
|
128
|
+
|---------|-------------|
|
|
129
|
+
| `list-symbols` | List available symbols on the remote archive |
|
|
130
|
+
| `list-files` | List archive files for given symbols |
|
|
131
|
+
| `download` | Download new or updated archive files |
|
|
132
|
+
| `verify` | Verify files against SHA256 checksums |
|
|
133
|
+
|
|
134
|
+
> [!TIP]
|
|
135
|
+
> Every command accepts symbols from **stdin** or as **positional args**,
|
|
136
|
+
> making it trivial to compose pipelines:
|
|
137
|
+
>
|
|
138
|
+
> ```bash
|
|
139
|
+
> binance-datatool list-symbols um --quote USDT \
|
|
140
|
+
> | binance-datatool download um --type klines --interval 1m
|
|
141
|
+
> ```
|
|
142
|
+
|
|
143
|
+
Run `uv run binance-datatool --help` for the full option reference.
|
|
144
|
+
|
|
145
|
+
## Architecture
|
|
146
|
+
|
|
147
|
+
```
|
|
148
|
+
CLI Layer Thin Typer commands (argument parsing, output formatting)
|
|
149
|
+
│
|
|
150
|
+
Workflow Layer Business logic (diff computation, verification protocol)
|
|
151
|
+
│
|
|
152
|
+
Archive Layer S3 communication (HTTP listing, aria2 downloading, checksums)
|
|
153
|
+
│
|
|
154
|
+
Common Layer Shared types, enums, constants, and utilities
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
See [docs/architecture.md](docs/architecture.md) for a detailed design overview.
|
|
158
|
+
|
|
159
|
+
## Vision
|
|
160
|
+
|
|
161
|
+
binance-datatool is the data foundation for agent-driven quantitative
|
|
162
|
+
research. Next up: **Agent skills** — packaged capabilities that let AI
|
|
163
|
+
agents autonomously discover, fetch, and validate market data end-to-end.
|
|
164
|
+
|
|
165
|
+
## Documentation
|
|
166
|
+
|
|
167
|
+
| Document | Description |
|
|
168
|
+
|----------|-------------|
|
|
169
|
+
| [Architecture](docs/architecture.md) | Layered design, data flow, and key decisions |
|
|
170
|
+
| [Module Reference](docs/reference/README.md) | Per-subpackage API reference |
|
|
171
|
+
| [Extending](docs/extending.md) | How to add commands, enums, workflows, and tests |
|
|
172
|
+
| [Testing Guide](docs/reference/testing.md) | Test organization and conventions |
|