dccd 2.3.2__tar.gz → 2.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dccd-2.3.2 → dccd-2.4.0}/CHANGELOG.md +39 -0
- dccd-2.4.0/PKG-INFO +300 -0
- dccd-2.4.0/README.md +227 -0
- dccd-2.4.0/dccd/daemon/api.py +702 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/backfill.py +141 -20
- {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/cli.py +132 -6
- {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/config.py +11 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/stream_manager.py +161 -21
- dccd-2.4.0/dccd/daemon/ui/__init__.py +4 -0
- dccd-2.4.0/dccd/daemon/ui/static/favicon.svg +27 -0
- dccd-2.4.0/dccd/daemon/ui/static/htmx.min.js +1 -0
- dccd-2.4.0/dccd/daemon/ui/static/logo.svg +59 -0
- dccd-2.4.0/dccd/daemon/ui/templates/base.html +160 -0
- dccd-2.4.0/dccd/daemon/ui/templates/config.html +225 -0
- dccd-2.4.0/dccd/daemon/ui/templates/dashboard.html +159 -0
- dccd-2.4.0/dccd/daemon/ui/templates/inventory.html +92 -0
- dccd-2.4.0/dccd/daemon/ui/templates/jobs.html +141 -0
- dccd-2.4.0/dccd/daemon/ui/templates/logs.html +19 -0
- dccd-2.4.0/dccd/daemon/ui/templates/partials/backfill_modal.html +123 -0
- dccd-2.4.0/dccd/daemon/ui/templates/storage.html +51 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/histo_dl/binance.py +16 -12
- {dccd-2.3.2 → dccd-2.4.0}/dccd/histo_dl/bybit.py +13 -9
- {dccd-2.3.2 → dccd-2.4.0}/dccd/histo_dl/coinbase.py +45 -35
- {dccd-2.3.2 → dccd-2.4.0}/dccd/histo_dl/kraken.py +15 -11
- {dccd-2.3.2 → dccd-2.4.0}/dccd/histo_dl/okx.py +14 -10
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_backfill.py +42 -4
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_coinbase.py +23 -0
- dccd-2.4.0/dccd/tests/test_daemon_api.py +438 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_daemon_stream_manager.py +172 -4
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_okx.py +24 -0
- dccd-2.4.0/dccd.egg-info/PKG-INFO +300 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd.egg-info/SOURCES.txt +15 -1
- {dccd-2.3.2 → dccd-2.4.0}/dccd.egg-info/requires.txt +12 -0
- {dccd-2.3.2 → dccd-2.4.0}/pyproject.toml +8 -4
- dccd-2.3.2/PKG-INFO +0 -316
- dccd-2.3.2/README.rst +0 -254
- dccd-2.3.2/dccd.egg-info/PKG-INFO +0 -316
- {dccd-2.3.2 → dccd-2.4.0}/CONTRIBUTING.md +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/LICENSE.txt +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/MANIFEST.in +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/__init__.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/__init__.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/binance.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/bitfinex.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/bitmex.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/bybit.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/exchange.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/kraken.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/okx.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/__init__.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/health.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/scheduler.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/storage.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/histo_dl/__init__.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/histo_dl/exchange.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/models.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/process_data.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/storage.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/__init__.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/conftest.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_binance.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_binance_ws.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_bitfinex.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_bitmex.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_bybit.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_bybit_ws.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_daemon_cli.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_daemon_config.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_daemon_health.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_daemon_scheduler.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_daemon_storage.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_date_time.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_histo_dl.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_io.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_kraken.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_kraken_ws.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_models.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_okx_ws.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_process_data.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_storage.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_websocket.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tools/__init__.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tools/date_time.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tools/io.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd/tools/websocket.py +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd.egg-info/dependency_links.txt +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd.egg-info/entry_points.txt +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/dccd.egg-info/top_level.txt +0 -0
- {dccd-2.3.2 → dccd-2.4.0}/setup.cfg +0 -0
|
@@ -6,6 +6,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [2.4.0] - 2026-06-04
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- `dccd/daemon/api.py` — web UI and JSON API (FastAPI + Jinja2 + htmx): a thin
|
|
14
|
+
HTTP layer over the existing daemon modules exposing dashboard (live health
|
|
15
|
+
metrics), inventory (stored data coverage), jobs (histo/stream list + add/remove
|
|
16
|
+
+ live backfill progress), logs (tail), config (view/validate/save the YAML),
|
|
17
|
+
and storage (rclone status + manual sync). JSON-only API (`/api/*`) with
|
|
18
|
+
dumb-shell templates, so the front-end can be swapped without touching the API.
|
|
19
|
+
Optional Bearer-token auth via `settings.ui_auth_token`
|
|
20
|
+
- `dccd/daemon/cli.py` — `dccd ui`: serve the web UI standalone; the UI is also
|
|
21
|
+
started automatically (background thread) by `dccd start` when the `[ui]` extra
|
|
22
|
+
is installed
|
|
23
|
+
- `dccd/daemon/config.py` — `SettingsConfig.ui_host`, `ui_port`, `ui_auth_token`:
|
|
24
|
+
web UI bind address, port, and optional auth token
|
|
25
|
+
- `dccd/daemon/backfill.py` — `progress_callback` and `stop_event` on
|
|
26
|
+
`_BackfillBase.run()` / `run_backfill()`: let the UI report live progress and
|
|
27
|
+
cancel a running backfill (defaults keep CLI behaviour unchanged)
|
|
28
|
+
- `dccd/daemon/stream_manager.py` — `SyncService` writes
|
|
29
|
+
`{local_path}/.dccd/last_sync.json` after each successful remote push, so the UI
|
|
30
|
+
can display the last sync time
|
|
31
|
+
- `pyproject.toml` — new optional extra `[ui]` (`fastapi`, `uvicorn[standard]`,
|
|
32
|
+
`jinja2`); install with `pip install dccd[daemon,ui]`
|
|
33
|
+
|
|
34
|
+
## [2.3.3] - 2026-05-31
|
|
35
|
+
|
|
36
|
+
### Added
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
- `doc/source/` — complete Sphinx documentation overhaul: redesigned homepage
|
|
40
|
+
with sphinx-design cards, captioned toctrees (Getting Started / Data Collection /
|
|
41
|
+
Reference), new pages (`installation`, `quickstart`, `changelog`, `cli`,
|
|
42
|
+
`configuration`, `models`, `storage`, `tools`, per-exchange histo/continuous pages),
|
|
43
|
+
adaptive light/dark logo and favicon, sticky top navbar with PyPI/GitHub/Fynance
|
|
44
|
+
links, hero header with inline logo+title, responsive layout (#59, #61)
|
|
45
|
+
- `README.md` — converted from RST to Markdown; inline logo+title header with
|
|
46
|
+
`<picture>` for light/dark mode switching; badges on two rows (#60)
|
|
47
|
+
|
|
9
48
|
## [2.3.2] - 2026-05-25
|
|
10
49
|
|
|
11
50
|
### Added
|
dccd-2.4.0/PKG-INFO
ADDED
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dccd
|
|
3
|
+
Version: 2.4.0
|
|
4
|
+
Summary: Download Crypto Currency Data from different exchanges.
|
|
5
|
+
Author-email: Arthur Bernard <arthur.bernard.92@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
|
|
8
|
+
Project-URL: Documentation, https://download-crypto-currencies-data.readthedocs.io/
|
|
9
|
+
Project-URL: Changelog, https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/CHANGELOG.md
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Database
|
|
20
|
+
Classifier: Topic :: Office/Business :: Financial
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE.txt
|
|
24
|
+
Requires-Dist: numpy>=1.26
|
|
25
|
+
Requires-Dist: polars>=0.20
|
|
26
|
+
Requires-Dist: pyarrow>=13
|
|
27
|
+
Requires-Dist: requests>=2.28
|
|
28
|
+
Requires-Dist: openpyxl>=3.1
|
|
29
|
+
Requires-Dist: websockets>=12.0
|
|
30
|
+
Requires-Dist: scipy>=1.10
|
|
31
|
+
Requires-Dist: SQLAlchemy>=2.0
|
|
32
|
+
Requires-Dist: tenacity>=8.0
|
|
33
|
+
Requires-Dist: pydantic>=2.0
|
|
34
|
+
Provides-Extra: io
|
|
35
|
+
Requires-Dist: polars>=0.20; extra == "io"
|
|
36
|
+
Requires-Dist: pyarrow>=13; extra == "io"
|
|
37
|
+
Provides-Extra: daemon
|
|
38
|
+
Requires-Dist: pyyaml>=6.0; extra == "daemon"
|
|
39
|
+
Requires-Dist: apscheduler<4,>=3.10; extra == "daemon"
|
|
40
|
+
Requires-Dist: typer>=0.12; extra == "daemon"
|
|
41
|
+
Requires-Dist: tqdm>=4.64; extra == "daemon"
|
|
42
|
+
Provides-Extra: ui
|
|
43
|
+
Requires-Dist: fastapi>=0.110; extra == "ui"
|
|
44
|
+
Requires-Dist: uvicorn[standard]>=0.29; extra == "ui"
|
|
45
|
+
Requires-Dist: jinja2>=3.1; extra == "ui"
|
|
46
|
+
Provides-Extra: dev
|
|
47
|
+
Requires-Dist: pytest>=7.4; extra == "dev"
|
|
48
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
|
|
49
|
+
Requires-Dist: pytest-cov>=4.1; extra == "dev"
|
|
50
|
+
Requires-Dist: ruff>=0.4; extra == "dev"
|
|
51
|
+
Requires-Dist: interrogate>=1.5; extra == "dev"
|
|
52
|
+
Requires-Dist: mypy>=1.0; extra == "dev"
|
|
53
|
+
Requires-Dist: pyyaml>=6.0; extra == "dev"
|
|
54
|
+
Requires-Dist: apscheduler<4,>=3.10; extra == "dev"
|
|
55
|
+
Requires-Dist: typer>=0.12; extra == "dev"
|
|
56
|
+
Requires-Dist: tqdm>=4.64; extra == "dev"
|
|
57
|
+
Requires-Dist: fastapi>=0.110; extra == "dev"
|
|
58
|
+
Requires-Dist: uvicorn[standard]>=0.29; extra == "dev"
|
|
59
|
+
Requires-Dist: jinja2>=3.1; extra == "dev"
|
|
60
|
+
Requires-Dist: httpx>=0.27; extra == "dev"
|
|
61
|
+
Provides-Extra: doc
|
|
62
|
+
Requires-Dist: sphinx>=7.0; extra == "doc"
|
|
63
|
+
Requires-Dist: furo; extra == "doc"
|
|
64
|
+
Requires-Dist: numpydoc; extra == "doc"
|
|
65
|
+
Requires-Dist: sphinx-design; extra == "doc"
|
|
66
|
+
Requires-Dist: sphinx-copybutton; extra == "doc"
|
|
67
|
+
Requires-Dist: pyyaml>=6.0; extra == "doc"
|
|
68
|
+
Requires-Dist: apscheduler<4,>=3.10; extra == "doc"
|
|
69
|
+
Requires-Dist: fastapi>=0.110; extra == "doc"
|
|
70
|
+
Requires-Dist: uvicorn[standard]>=0.29; extra == "doc"
|
|
71
|
+
Requires-Dist: jinja2>=3.1; extra == "doc"
|
|
72
|
+
Dynamic: license-file
|
|
73
|
+
|
|
74
|
+
<picture>
|
|
75
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-dark-transparent.svg">
|
|
76
|
+
<img alt="dccd logo" src="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-light-transparent.svg" height="180px" align="left">
|
|
77
|
+
</picture>
|
|
78
|
+
|
|
79
|
+
# **Download Crypto-Currency Data**
|
|
80
|
+
|
|
81
|
+
[](https://pypi.org/project/dccd/)
|
|
82
|
+
[](https://pypi.org/project/dccd/)
|
|
83
|
+
[](https://pypi.org/project/dccd/)
|
|
84
|
+
[](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/actions/workflows/ci.yml)
|
|
85
|
+
[](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/LICENSE.txt)<br>
|
|
86
|
+
[](https://download-crypto-currencies-data.readthedocs.io/en/latest/)
|
|
87
|
+
[](https://codecov.io/gh/ArthurBernard/Download_Crypto_Currencies_Data)
|
|
88
|
+
[](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data)
|
|
89
|
+
[](https://pepy.tech/project/dccd)
|
|
90
|
+
|
|
91
|
+
___
|
|
92
|
+
|
|
93
|
+
Python package to download crypto-currency data (OHLCV, trades, order book) from multiple
|
|
94
|
+
exchanges via REST and WebSocket APIs. Data can be saved to CSV, Excel, SQLite, PostgreSQL,
|
|
95
|
+
or Parquet.
|
|
96
|
+
|
|
97
|
+
## Installation
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
pip install dccd
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
With autonomous daemon support (APScheduler + PyYAML):
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
pip install "dccd[daemon]"
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
With the web UI (FastAPI + htmx — `dccd ui`):
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
pip install "dccd[daemon,ui]"
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
From source:
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
git clone https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
|
|
119
|
+
cd Download_Crypto_Currencies_Data
|
|
120
|
+
pip install -e .
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Supported exchanges
|
|
124
|
+
|
|
125
|
+
| Exchange | REST OHLCV | REST Trades | REST Order Book | WS OHLCV | WS Trades | WS Order Book |
|
|
126
|
+
|----------|:----------:|:-----------:|:---------------:|:--------:|:---------:|:-------------:|
|
|
127
|
+
| Binance | ✓ | ✓ | ✓ | | ✓ | ✓ |
|
|
128
|
+
| Coinbase | ✓ | ✓ † | ✓ | | | |
|
|
129
|
+
| Kraken | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
|
|
130
|
+
| Bybit | ✓ | ✓ † | ✓ | | ✓ | ✓ |
|
|
131
|
+
| OKX | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
|
|
132
|
+
| Bitfinex | | | | ✓ \* | ✓ | ✓ |
|
|
133
|
+
| Bitmex | | | | | ✓ | ✓ |
|
|
134
|
+
|
|
135
|
+
\* Bitfinex WS OHLCV is aggregated from the trades stream via `get_ohlc_bitfinex`.
|
|
136
|
+
† Recent trades only (Bybit ≤ 1 000, Coinbase ≤ 100) — no deep historical pagination via the public REST API.
|
|
137
|
+
|
|
138
|
+
## Presentation
|
|
139
|
+
|
|
140
|
+
**Historical Downloader** `dccd.histo_dl`
|
|
141
|
+
Download OHLCV data via REST APIs and save to disk. Supports chunked requests, automatic retry on rate-limit (HTTP 429), and incremental updates from the last saved timestamp.
|
|
142
|
+
|
|
143
|
+
**Continuous Downloader** `dccd.continuous_dl`
|
|
144
|
+
Stream real-time data (order book, trades) via WebSocket with automatic reconnection and configurable processing/saving callbacks.
|
|
145
|
+
|
|
146
|
+
**Daemon** `dccd.daemon`
|
|
147
|
+
Autonomous, server-side collector driven by a YAML config. Runs REST jobs on a schedule (APScheduler), opens WebSocket streams for real-time collection, and periodically syncs all local data to one or more remote destinations (NAS, S3, SFTP, …) via rclone. Multiple remotes and a configurable sync interval are supported; collection is never blocked by remote availability.
|
|
148
|
+
|
|
149
|
+
**Web UI** `dccd.daemon.api`
|
|
150
|
+
Optional browser interface (FastAPI + htmx) mirroring the CLI: dashboard of live health metrics, data inventory, job management (add/remove pairs, launch and cancel backfills), log tail, config editor, and remote-storage status. Runs standalone (`dccd ui`) or embedded in `dccd start`.
|
|
151
|
+
|
|
152
|
+
### Output formats
|
|
153
|
+
|
|
154
|
+
Historical data can be saved as **CSV**, **Excel** (`.xlsx`), **SQLite**, **PostgreSQL** (via SQLAlchemy), or **Parquet**.
|
|
155
|
+
All DataFrames are native `polars.DataFrame`. A `pandas.DataFrame` can be obtained via `get_data(format='pandas')`.
|
|
156
|
+
|
|
157
|
+
## Quick start
|
|
158
|
+
|
|
159
|
+
Historical data:
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
from dccd.histo_dl import FromBinance
|
|
163
|
+
|
|
164
|
+
obj = FromBinance('/path/to/data/', 'BTC', 3600, fiat='USDT')
|
|
165
|
+
obj.import_data(start='2024-01-01 00:00:00', end='2024-12-31 00:00:00')
|
|
166
|
+
obj.save(form='parquet')
|
|
167
|
+
df = obj.get_data() # polars DataFrame (default)
|
|
168
|
+
df_pd = obj.get_data(format='pandas') # pandas DataFrame (optional)
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
Incremental update (resume from last saved point):
|
|
172
|
+
|
|
173
|
+
```python
|
|
174
|
+
obj.import_data(start='last', end='now').save(form='parquet')
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
Other exchanges:
|
|
178
|
+
|
|
179
|
+
```python
|
|
180
|
+
from dccd.histo_dl import FromKraken, FromBybit, FromOKX
|
|
181
|
+
|
|
182
|
+
FromKraken('/path/', 'ETH', 3600).import_data(start='2024-01-01', end='now').save()
|
|
183
|
+
FromBybit('/path/', 'BTC', 86400).import_data(start='2024-01-01', end='now').save()
|
|
184
|
+
FromOKX('/path/', 'BTC', 3600).import_data(start='2024-01-01', end='now').save()
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Trades (historical or recent):
|
|
188
|
+
|
|
189
|
+
```python
|
|
190
|
+
from dccd.histo_dl import FromBinance, FromKraken
|
|
191
|
+
|
|
192
|
+
obj = FromBinance('/path/', 'BTC', 3600, fiat='USDT')
|
|
193
|
+
obj.import_trades(start='2024-01-01 00:00:00', end='2024-01-02 00:00:00')
|
|
194
|
+
obj.save_trades(form='csv')
|
|
195
|
+
df = obj.trades_df # polars DataFrame — columns: TS, price, amount, type, tid
|
|
196
|
+
|
|
197
|
+
# Kraken also supports full history; Bybit/Coinbase return recent-only snapshots
|
|
198
|
+
FromKraken('/path/', 'BTC', 3600).import_trades(start='2024-01-01', end='2024-01-02').save_trades()
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
Order book snapshot:
|
|
202
|
+
|
|
203
|
+
```python
|
|
204
|
+
from dccd.histo_dl import FromOKX
|
|
205
|
+
|
|
206
|
+
obj = FromOKX('/path/', 'BTC', 3600)
|
|
207
|
+
obj.import_orderbook(depth=50)
|
|
208
|
+
obj.save_orderbook(form='csv')
|
|
209
|
+
df = obj.orderbook_df # columns: side, price, amount, count
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
Daemon (autonomous collector) — `config.yml`:
|
|
213
|
+
|
|
214
|
+
```yaml
|
|
215
|
+
settings:
|
|
216
|
+
data_path: /data/crypto/
|
|
217
|
+
timezone: UTC
|
|
218
|
+
|
|
219
|
+
storage:
|
|
220
|
+
remotes:
|
|
221
|
+
- provider: rclone
|
|
222
|
+
remote: "mynas:crypto/"
|
|
223
|
+
sync_interval: 3600
|
|
224
|
+
|
|
225
|
+
histo_jobs:
|
|
226
|
+
- exchange: binance
|
|
227
|
+
pairs: [BTC/USDT, ETH/USDT]
|
|
228
|
+
span: 3600
|
|
229
|
+
format: parquet
|
|
230
|
+
|
|
231
|
+
stream_jobs:
|
|
232
|
+
- exchange: binance
|
|
233
|
+
pairs: [BTC/USDT]
|
|
234
|
+
channels: [trades, book]
|
|
235
|
+
time_step: 60
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
CLI quick start:
|
|
239
|
+
|
|
240
|
+
```bash
|
|
241
|
+
# Validate the config
|
|
242
|
+
dccd validate --config config.yml
|
|
243
|
+
|
|
244
|
+
# Backfill all OHLC history defined in config (resumable)
|
|
245
|
+
dccd backfill --config config.yml --start "2020-01-01 00:00:00"
|
|
246
|
+
|
|
247
|
+
# Dry run — estimate windows and time without downloading
|
|
248
|
+
dccd backfill --config config.yml --dry-run
|
|
249
|
+
|
|
250
|
+
# Backfill only one exchange
|
|
251
|
+
dccd backfill --config config.yml --exchange kraken
|
|
252
|
+
|
|
253
|
+
# One incremental batch per job, then exit (for cron)
|
|
254
|
+
dccd collect --config config.yml
|
|
255
|
+
|
|
256
|
+
# Continuous daemon (Ctrl-C to stop)
|
|
257
|
+
dccd start --config config.yml
|
|
258
|
+
|
|
259
|
+
# Add / remove a histo job in-place
|
|
260
|
+
dccd add --exchange kraken --pair ETH/USD --span 86400 --config config.yml
|
|
261
|
+
dccd remove --exchange kraken --pair ETH/USD --span 86400 --config config.yml
|
|
262
|
+
|
|
263
|
+
# Inspect all data on disk (OHLC, trades, orderbook)
|
|
264
|
+
dccd inventory --config config.yml
|
|
265
|
+
|
|
266
|
+
# Web UI — dashboard, inventory, jobs, logs, config (needs: pip install "dccd[daemon,ui]")
|
|
267
|
+
dccd ui --config config.yml # http://127.0.0.1:8080
|
|
268
|
+
|
|
269
|
+
# Enable shell tab-completion (run once after install)
|
|
270
|
+
dccd --install-completion
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
> `--config` is optional — dccd searches `./config.yml` then `~/.config/dccd/config.yml` when omitted.
|
|
274
|
+
|
|
275
|
+
Python API:
|
|
276
|
+
|
|
277
|
+
```python
|
|
278
|
+
from dccd.daemon.config import load_config
|
|
279
|
+
from dccd.daemon.scheduler import run_once, build_histo_scheduler
|
|
280
|
+
from dccd.daemon.stream_manager import StreamManager
|
|
281
|
+
|
|
282
|
+
config = load_config('config.yml')
|
|
283
|
+
|
|
284
|
+
# One-shot: download all histo jobs once, then exit
|
|
285
|
+
run_once(config)
|
|
286
|
+
|
|
287
|
+
# Daemon mode: periodic REST + live WebSocket streams
|
|
288
|
+
scheduler = build_histo_scheduler(config)
|
|
289
|
+
scheduler.start()
|
|
290
|
+
|
|
291
|
+
mgr = StreamManager(config)
|
|
292
|
+
mgr.start() # runs until mgr.stop() is called
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
## Links
|
|
296
|
+
|
|
297
|
+
- PyPI: https://pypi.org/project/dccd/
|
|
298
|
+
- Documentation: https://download-crypto-currencies-data.readthedocs.io/
|
|
299
|
+
- Source: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
|
|
300
|
+
- Changelog: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/CHANGELOG.md
|
dccd-2.4.0/README.md
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
<picture>
|
|
2
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-dark-transparent.svg">
|
|
3
|
+
<img alt="dccd logo" src="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-light-transparent.svg" height="180px" align="left">
|
|
4
|
+
</picture>
|
|
5
|
+
|
|
6
|
+
# **Download Crypto-Currency Data**
|
|
7
|
+
|
|
8
|
+
[](https://pypi.org/project/dccd/)
|
|
9
|
+
[](https://pypi.org/project/dccd/)
|
|
10
|
+
[](https://pypi.org/project/dccd/)
|
|
11
|
+
[](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/actions/workflows/ci.yml)
|
|
12
|
+
[](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/LICENSE.txt)<br>
|
|
13
|
+
[](https://download-crypto-currencies-data.readthedocs.io/en/latest/)
|
|
14
|
+
[](https://codecov.io/gh/ArthurBernard/Download_Crypto_Currencies_Data)
|
|
15
|
+
[](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data)
|
|
16
|
+
[](https://pepy.tech/project/dccd)
|
|
17
|
+
|
|
18
|
+
___
|
|
19
|
+
|
|
20
|
+
Python package to download crypto-currency data (OHLCV, trades, order book) from multiple
|
|
21
|
+
exchanges via REST and WebSocket APIs. Data can be saved to CSV, Excel, SQLite, PostgreSQL,
|
|
22
|
+
or Parquet.
|
|
23
|
+
|
|
24
|
+
## Installation
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
pip install dccd
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
With autonomous daemon support (APScheduler + PyYAML):
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install "dccd[daemon]"
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
With the web UI (FastAPI + htmx — `dccd ui`):
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install "dccd[daemon,ui]"
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
From source:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
git clone https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
|
|
46
|
+
cd Download_Crypto_Currencies_Data
|
|
47
|
+
pip install -e .
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Supported exchanges
|
|
51
|
+
|
|
52
|
+
| Exchange | REST OHLCV | REST Trades | REST Order Book | WS OHLCV | WS Trades | WS Order Book |
|
|
53
|
+
|----------|:----------:|:-----------:|:---------------:|:--------:|:---------:|:-------------:|
|
|
54
|
+
| Binance | ✓ | ✓ | ✓ | | ✓ | ✓ |
|
|
55
|
+
| Coinbase | ✓ | ✓ † | ✓ | | | |
|
|
56
|
+
| Kraken | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
|
|
57
|
+
| Bybit | ✓ | ✓ † | ✓ | | ✓ | ✓ |
|
|
58
|
+
| OKX | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
|
|
59
|
+
| Bitfinex | | | | ✓ \* | ✓ | ✓ |
|
|
60
|
+
| Bitmex | | | | | ✓ | ✓ |
|
|
61
|
+
|
|
62
|
+
\* Bitfinex WS OHLCV is aggregated from the trades stream via `get_ohlc_bitfinex`.
|
|
63
|
+
† Recent trades only (Bybit ≤ 1 000, Coinbase ≤ 100) — no deep historical pagination via the public REST API.
|
|
64
|
+
|
|
65
|
+
## Presentation
|
|
66
|
+
|
|
67
|
+
**Historical Downloader** `dccd.histo_dl`
|
|
68
|
+
Download OHLCV data via REST APIs and save to disk. Supports chunked requests, automatic retry on rate-limit (HTTP 429), and incremental updates from the last saved timestamp.
|
|
69
|
+
|
|
70
|
+
**Continuous Downloader** `dccd.continuous_dl`
|
|
71
|
+
Stream real-time data (order book, trades) via WebSocket with automatic reconnection and configurable processing/saving callbacks.
|
|
72
|
+
|
|
73
|
+
**Daemon** `dccd.daemon`
|
|
74
|
+
Autonomous, server-side collector driven by a YAML config. Runs REST jobs on a schedule (APScheduler), opens WebSocket streams for real-time collection, and periodically syncs all local data to one or more remote destinations (NAS, S3, SFTP, …) via rclone. Multiple remotes and a configurable sync interval are supported; collection is never blocked by remote availability.
|
|
75
|
+
|
|
76
|
+
**Web UI** `dccd.daemon.api`
|
|
77
|
+
Optional browser interface (FastAPI + htmx) mirroring the CLI: dashboard of live health metrics, data inventory, job management (add/remove pairs, launch and cancel backfills), log tail, config editor, and remote-storage status. Runs standalone (`dccd ui`) or embedded in `dccd start`.
|
|
78
|
+
|
|
79
|
+
### Output formats
|
|
80
|
+
|
|
81
|
+
Historical data can be saved as **CSV**, **Excel** (`.xlsx`), **SQLite**, **PostgreSQL** (via SQLAlchemy), or **Parquet**.
|
|
82
|
+
All DataFrames are native `polars.DataFrame`. A `pandas.DataFrame` can be obtained via `get_data(format='pandas')`.
|
|
83
|
+
|
|
84
|
+
## Quick start
|
|
85
|
+
|
|
86
|
+
Historical data:
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from dccd.histo_dl import FromBinance
|
|
90
|
+
|
|
91
|
+
obj = FromBinance('/path/to/data/', 'BTC', 3600, fiat='USDT')
|
|
92
|
+
obj.import_data(start='2024-01-01 00:00:00', end='2024-12-31 00:00:00')
|
|
93
|
+
obj.save(form='parquet')
|
|
94
|
+
df = obj.get_data() # polars DataFrame (default)
|
|
95
|
+
df_pd = obj.get_data(format='pandas') # pandas DataFrame (optional)
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Incremental update (resume from last saved point):
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
obj.import_data(start='last', end='now').save(form='parquet')
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Other exchanges:
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
from dccd.histo_dl import FromKraken, FromBybit, FromOKX
|
|
108
|
+
|
|
109
|
+
FromKraken('/path/', 'ETH', 3600).import_data(start='2024-01-01', end='now').save()
|
|
110
|
+
FromBybit('/path/', 'BTC', 86400).import_data(start='2024-01-01', end='now').save()
|
|
111
|
+
FromOKX('/path/', 'BTC', 3600).import_data(start='2024-01-01', end='now').save()
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Trades (historical or recent):
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from dccd.histo_dl import FromBinance, FromKraken
|
|
118
|
+
|
|
119
|
+
obj = FromBinance('/path/', 'BTC', 3600, fiat='USDT')
|
|
120
|
+
obj.import_trades(start='2024-01-01 00:00:00', end='2024-01-02 00:00:00')
|
|
121
|
+
obj.save_trades(form='csv')
|
|
122
|
+
df = obj.trades_df # polars DataFrame — columns: TS, price, amount, type, tid
|
|
123
|
+
|
|
124
|
+
# Kraken also supports full history; Bybit/Coinbase return recent-only snapshots
|
|
125
|
+
FromKraken('/path/', 'BTC', 3600).import_trades(start='2024-01-01', end='2024-01-02').save_trades()
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Order book snapshot:
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from dccd.histo_dl import FromOKX
|
|
132
|
+
|
|
133
|
+
obj = FromOKX('/path/', 'BTC', 3600)
|
|
134
|
+
obj.import_orderbook(depth=50)
|
|
135
|
+
obj.save_orderbook(form='csv')
|
|
136
|
+
df = obj.orderbook_df # columns: side, price, amount, count
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Daemon (autonomous collector) — `config.yml`:
|
|
140
|
+
|
|
141
|
+
```yaml
|
|
142
|
+
settings:
|
|
143
|
+
data_path: /data/crypto/
|
|
144
|
+
timezone: UTC
|
|
145
|
+
|
|
146
|
+
storage:
|
|
147
|
+
remotes:
|
|
148
|
+
- provider: rclone
|
|
149
|
+
remote: "mynas:crypto/"
|
|
150
|
+
sync_interval: 3600
|
|
151
|
+
|
|
152
|
+
histo_jobs:
|
|
153
|
+
- exchange: binance
|
|
154
|
+
pairs: [BTC/USDT, ETH/USDT]
|
|
155
|
+
span: 3600
|
|
156
|
+
format: parquet
|
|
157
|
+
|
|
158
|
+
stream_jobs:
|
|
159
|
+
- exchange: binance
|
|
160
|
+
pairs: [BTC/USDT]
|
|
161
|
+
channels: [trades, book]
|
|
162
|
+
time_step: 60
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
CLI quick start:
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
# Validate the config
|
|
169
|
+
dccd validate --config config.yml
|
|
170
|
+
|
|
171
|
+
# Backfill all OHLC history defined in config (resumable)
|
|
172
|
+
dccd backfill --config config.yml --start "2020-01-01 00:00:00"
|
|
173
|
+
|
|
174
|
+
# Dry run — estimate windows and time without downloading
|
|
175
|
+
dccd backfill --config config.yml --dry-run
|
|
176
|
+
|
|
177
|
+
# Backfill only one exchange
|
|
178
|
+
dccd backfill --config config.yml --exchange kraken
|
|
179
|
+
|
|
180
|
+
# One incremental batch per job, then exit (for cron)
|
|
181
|
+
dccd collect --config config.yml
|
|
182
|
+
|
|
183
|
+
# Continuous daemon (Ctrl-C to stop)
|
|
184
|
+
dccd start --config config.yml
|
|
185
|
+
|
|
186
|
+
# Add / remove a histo job in-place
|
|
187
|
+
dccd add --exchange kraken --pair ETH/USD --span 86400 --config config.yml
|
|
188
|
+
dccd remove --exchange kraken --pair ETH/USD --span 86400 --config config.yml
|
|
189
|
+
|
|
190
|
+
# Inspect all data on disk (OHLC, trades, orderbook)
|
|
191
|
+
dccd inventory --config config.yml
|
|
192
|
+
|
|
193
|
+
# Web UI — dashboard, inventory, jobs, logs, config (needs: pip install "dccd[daemon,ui]")
|
|
194
|
+
dccd ui --config config.yml # http://127.0.0.1:8080
|
|
195
|
+
|
|
196
|
+
# Enable shell tab-completion (run once after install)
|
|
197
|
+
dccd --install-completion
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
> `--config` is optional — dccd searches `./config.yml` then `~/.config/dccd/config.yml` when omitted.
|
|
201
|
+
|
|
202
|
+
Python API:
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
from dccd.daemon.config import load_config
|
|
206
|
+
from dccd.daemon.scheduler import run_once, build_histo_scheduler
|
|
207
|
+
from dccd.daemon.stream_manager import StreamManager
|
|
208
|
+
|
|
209
|
+
config = load_config('config.yml')
|
|
210
|
+
|
|
211
|
+
# One-shot: download all histo jobs once, then exit
|
|
212
|
+
run_once(config)
|
|
213
|
+
|
|
214
|
+
# Daemon mode: periodic REST + live WebSocket streams
|
|
215
|
+
scheduler = build_histo_scheduler(config)
|
|
216
|
+
scheduler.start()
|
|
217
|
+
|
|
218
|
+
mgr = StreamManager(config)
|
|
219
|
+
mgr.start() # runs until mgr.stop() is called
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
## Links
|
|
223
|
+
|
|
224
|
+
- PyPI: https://pypi.org/project/dccd/
|
|
225
|
+
- Documentation: https://download-crypto-currencies-data.readthedocs.io/
|
|
226
|
+
- Source: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
|
|
227
|
+
- Changelog: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/CHANGELOG.md
|