dccd 2.3.1__tar.gz → 2.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dccd-2.3.1 → dccd-2.3.3}/CHANGELOG.md +32 -1
- dccd-2.3.3/PKG-INFO +275 -0
- dccd-2.3.3/README.md +213 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/bitfinex.py +1 -1
- {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/backfill.py +68 -33
- {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/cli.py +235 -27
- {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/config.py +47 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/stream_manager.py +3 -4
- {dccd-2.3.1 → dccd-2.3.3}/dccd/histo_dl/binance.py +17 -13
- {dccd-2.3.1 → dccd-2.3.3}/dccd/histo_dl/bybit.py +14 -10
- {dccd-2.3.1 → dccd-2.3.3}/dccd/histo_dl/coinbase.py +15 -12
- {dccd-2.3.1 → dccd-2.3.3}/dccd/histo_dl/exchange.py +43 -49
- {dccd-2.3.1 → dccd-2.3.3}/dccd/histo_dl/kraken.py +15 -11
- {dccd-2.3.1 → dccd-2.3.3}/dccd/histo_dl/okx.py +15 -11
- dccd-2.3.3/dccd/process_data.py +139 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/storage.py +33 -36
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_backfill.py +51 -3
- dccd-2.3.3/dccd/tests/test_daemon_cli.py +358 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_daemon_config.py +48 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_histo_dl.py +13 -5
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_io.py +26 -26
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_okx.py +24 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_process_data.py +19 -17
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_storage.py +12 -11
- dccd-2.3.3/dccd/tools/io.py +240 -0
- dccd-2.3.3/dccd.egg-info/PKG-INFO +275 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd.egg-info/SOURCES.txt +1 -1
- {dccd-2.3.1 → dccd-2.3.3}/dccd.egg-info/requires.txt +3 -3
- {dccd-2.3.1 → dccd-2.3.3}/pyproject.toml +6 -5
- dccd-2.3.1/PKG-INFO +0 -315
- dccd-2.3.1/README.rst +0 -253
- dccd-2.3.1/dccd/process_data.py +0 -166
- dccd-2.3.1/dccd/tests/test_daemon_cli.py +0 -108
- dccd-2.3.1/dccd/tools/io.py +0 -495
- dccd-2.3.1/dccd.egg-info/PKG-INFO +0 -315
- {dccd-2.3.1 → dccd-2.3.3}/CONTRIBUTING.md +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/LICENSE.txt +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/MANIFEST.in +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/__init__.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/__init__.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/binance.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/bitmex.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/bybit.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/exchange.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/kraken.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/okx.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/__init__.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/health.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/scheduler.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/storage.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/histo_dl/__init__.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/models.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/__init__.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/conftest.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_binance.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_binance_ws.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_bitfinex.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_bitmex.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_bybit.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_bybit_ws.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_coinbase.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_daemon_health.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_daemon_scheduler.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_daemon_storage.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_daemon_stream_manager.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_date_time.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_kraken.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_kraken_ws.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_models.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_okx_ws.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_websocket.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tools/__init__.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tools/date_time.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd/tools/websocket.py +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd.egg-info/dependency_links.txt +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd.egg-info/entry_points.txt +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/dccd.egg-info/top_level.txt +0 -0
- {dccd-2.3.1 → dccd-2.3.3}/setup.cfg +0 -0
|
@@ -6,10 +6,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
-
## [2.3.
|
|
9
|
+
## [2.3.3] - 2026-05-31
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- `doc/source/` — complete Sphinx documentation overhaul: redesigned homepage
|
|
14
|
+
with sphinx-design cards, captioned toctrees (Getting Started / Data Collection /
|
|
15
|
+
Reference), new pages (`installation`, `quickstart`, `changelog`, `cli`,
|
|
16
|
+
`configuration`, `models`, `storage`, `tools`, per-exchange histo/continuous pages),
|
|
17
|
+
adaptive light/dark logo and favicon, sticky top navbar with PyPI/GitHub/Fynance
|
|
18
|
+
links, hero header with inline logo+title, responsive layout (#59, #61)
|
|
19
|
+
- `README.md` — converted from RST to Markdown; inline logo+title header with
|
|
20
|
+
`<picture>` for light/dark mode switching; badges on two rows (#60)
|
|
21
|
+
|
|
22
|
+
## [2.3.2] - 2026-05-25
|
|
23
|
+
|
|
24
|
+
### Added
|
|
25
|
+
|
|
26
|
+
- `dccd/daemon/cli.py` — `dccd status --json`: emit raw metrics as a JSON object on stdout, suitable for piping into Grafana / jq (#53)
|
|
27
|
+
- `dccd/daemon/config.py` — `HistoJob.max_retries` (int, 1–10, default 3) and `HistoJob.retry_delay` (float ≥ 0, default 2.0): per-job retry configuration for transient network errors; delay is exponential (`retry_delay * 2^(attempt-1)`) (#53)
|
|
28
|
+
- `dccd/daemon/config.py` — `resolve_config_path()` and `DEFAULT_CONFIG_PATH`: CLI commands now fall back to `$XDG_CONFIG_HOME/dccd/config.yml` (default `~/.config/dccd/config.yml`) when no `--config` option is provided and `./config.yml` does not exist (#49)
|
|
29
|
+
- `dccd/daemon/cli.py` — `dccd inventory`: scans `data_path` and prints a table of all stored OHLC, trades, and orderbook data with date range, row count, and gap count per series; uses Polars for fast columnar reads (#50)
|
|
30
|
+
- `dccd/daemon/cli.py` — `dccd remove --exchange X --pair Y --span N`: removes a pair from a histo_job (or the whole job if it was the last pair) and re-validates the config before writing (#50)
|
|
31
|
+
|
|
32
|
+
### Changed
|
|
33
|
+
|
|
34
|
+
- `dccd/storage.py`, `dccd/histo_dl/exchange.py`, `dccd/daemon/backfill.py`, `dccd/process_data.py`, `dccd/daemon/stream_manager.py` — replace pandas with polars throughout; `DataStore.save/load` accept/return `pl.DataFrame`; `get_data()` defaults to `format='polars'`; `set_marketdepth` returns a flat long-format `pl.DataFrame`; stream savers write parquet via `DataStore`; `pandas` removed from core dependencies (#52)
|
|
35
|
+
- `dccd/daemon/backfill.py` — backfill progress bar now shows the current window date (`YYYY-MM-DD → YYYY-MM-DD`) instead of a raw window count (`n win`); makes it easy to see which period is being downloaded at a glance (#48)
|
|
10
36
|
|
|
11
37
|
### Fixed
|
|
12
38
|
|
|
39
|
+
- `dccd/histo_dl/exchange.py` — `_sort_data` no longer raises `ColumnNotFoundError: "date"` when the exchange API returns an empty candle list; the polars migration (PR #52) had re-introduced a variant of the empty-data crash from v2.3.1; now returns early with an empty `self.df` (#54)
|
|
40
|
+
|
|
41
|
+
## [2.3.1] - 2026-05-24
|
|
42
|
+
|
|
43
|
+
### Fixed
|
|
13
44
|
- `dccd/storage.py` — `DataStore.missing_intervals` now detects the gap **before** the first saved row when the requested `start` predates `file_min`; previously only the trailing gap (after `file_max`) was returned, causing `dccd backfill --start <early-date>` to silently skip all historical data before the first existing candle (#46)
|
|
14
45
|
- `dccd/histo_dl/coinbase.py` — raise `RuntimeError` when Coinbase returns HTTP 200 with a JSON dict (e.g. `{"message": "..."}` for near-future windows) instead of silently iterating dict keys and crashing with `ValueError` (#45)
|
|
15
46
|
- `dccd/histo_dl/coinbase.py` — additional guard: raise `RuntimeError` when Coinbase returns a JSON list whose first element is not itself a list/tuple (e.g. `["message"]`); previously caused `float("m")` `ValueError` (#45)
|
dccd-2.3.3/PKG-INFO
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dccd
|
|
3
|
+
Version: 2.3.3
|
|
4
|
+
Summary: Download Crypto Currency Data from different exchanges.
|
|
5
|
+
Author-email: Arthur Bernard <arthur.bernard.92@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
|
|
8
|
+
Project-URL: Documentation, https://download-crypto-currencies-data.readthedocs.io/
|
|
9
|
+
Project-URL: Changelog, https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/CHANGELOG.md
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Database
|
|
20
|
+
Classifier: Topic :: Office/Business :: Financial
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE.txt
|
|
24
|
+
Requires-Dist: numpy>=1.26
|
|
25
|
+
Requires-Dist: polars>=0.20
|
|
26
|
+
Requires-Dist: pyarrow>=13
|
|
27
|
+
Requires-Dist: requests>=2.28
|
|
28
|
+
Requires-Dist: openpyxl>=3.1
|
|
29
|
+
Requires-Dist: websockets>=12.0
|
|
30
|
+
Requires-Dist: scipy>=1.10
|
|
31
|
+
Requires-Dist: SQLAlchemy>=2.0
|
|
32
|
+
Requires-Dist: tenacity>=8.0
|
|
33
|
+
Requires-Dist: pydantic>=2.0
|
|
34
|
+
Provides-Extra: io
|
|
35
|
+
Requires-Dist: polars>=0.20; extra == "io"
|
|
36
|
+
Requires-Dist: pyarrow>=13; extra == "io"
|
|
37
|
+
Provides-Extra: daemon
|
|
38
|
+
Requires-Dist: pyyaml>=6.0; extra == "daemon"
|
|
39
|
+
Requires-Dist: apscheduler<4,>=3.10; extra == "daemon"
|
|
40
|
+
Requires-Dist: typer>=0.12; extra == "daemon"
|
|
41
|
+
Requires-Dist: tqdm>=4.64; extra == "daemon"
|
|
42
|
+
Provides-Extra: dev
|
|
43
|
+
Requires-Dist: pytest>=7.4; extra == "dev"
|
|
44
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
|
|
45
|
+
Requires-Dist: pytest-cov>=4.1; extra == "dev"
|
|
46
|
+
Requires-Dist: ruff>=0.4; extra == "dev"
|
|
47
|
+
Requires-Dist: interrogate>=1.5; extra == "dev"
|
|
48
|
+
Requires-Dist: mypy>=1.0; extra == "dev"
|
|
49
|
+
Requires-Dist: pyyaml>=6.0; extra == "dev"
|
|
50
|
+
Requires-Dist: apscheduler<4,>=3.10; extra == "dev"
|
|
51
|
+
Requires-Dist: typer>=0.12; extra == "dev"
|
|
52
|
+
Requires-Dist: tqdm>=4.64; extra == "dev"
|
|
53
|
+
Provides-Extra: doc
|
|
54
|
+
Requires-Dist: sphinx>=7.0; extra == "doc"
|
|
55
|
+
Requires-Dist: furo; extra == "doc"
|
|
56
|
+
Requires-Dist: numpydoc; extra == "doc"
|
|
57
|
+
Requires-Dist: sphinx-design; extra == "doc"
|
|
58
|
+
Requires-Dist: sphinx-copybutton; extra == "doc"
|
|
59
|
+
Requires-Dist: pyyaml>=6.0; extra == "doc"
|
|
60
|
+
Requires-Dist: apscheduler<4,>=3.10; extra == "doc"
|
|
61
|
+
Dynamic: license-file
|
|
62
|
+
|
|
63
|
+
<picture>
|
|
64
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-dark-transparent.svg">
|
|
65
|
+
<img alt="dccd logo" src="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-light-transparent.svg" height="130px" align="left">
|
|
66
|
+
</picture>
|
|
67
|
+
|
|
68
|
+
# Download Crypto-Currency Data
|
|
69
|
+
|
|
70
|
+
[](https://pypi.org/project/dccd/)
|
|
71
|
+
[](https://pypi.org/project/dccd/)
|
|
72
|
+
[](https://pypi.org/project/dccd/)
|
|
73
|
+
[](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/actions/workflows/ci.yml)
|
|
74
|
+
[](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/LICENSE.txt)<br>
|
|
75
|
+
[](https://download-crypto-currencies-data.readthedocs.io/en/latest/)
|
|
76
|
+
[](https://codecov.io/gh/ArthurBernard/Download_Crypto_Currencies_Data)
|
|
77
|
+
[](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data)
|
|
78
|
+
[](https://pepy.tech/project/dccd)
|
|
79
|
+
|
|
80
|
+
Python package to download crypto-currency data (OHLCV, trades, order book) from multiple
|
|
81
|
+
exchanges via REST and WebSocket APIs. Data can be saved to CSV, Excel, SQLite, PostgreSQL,
|
|
82
|
+
or Parquet.
|
|
83
|
+
|
|
84
|
+
## Installation
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
pip install dccd
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
With autonomous daemon support (APScheduler + PyYAML):
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
pip install "dccd[daemon]"
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
From source:
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
git clone https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
|
|
100
|
+
cd Download_Crypto_Currencies_Data
|
|
101
|
+
pip install -e .
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## Supported exchanges
|
|
105
|
+
|
|
106
|
+
| Exchange | REST OHLCV | REST Trades | REST Order Book | WS OHLCV | WS Trades | WS Order Book |
|
|
107
|
+
|----------|:----------:|:-----------:|:---------------:|:--------:|:---------:|:-------------:|
|
|
108
|
+
| Binance | ✓ | ✓ | ✓ | | ✓ | ✓ |
|
|
109
|
+
| Coinbase | ✓ | ✓ † | ✓ | | | |
|
|
110
|
+
| Kraken | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
|
|
111
|
+
| Bybit | ✓ | ✓ † | ✓ | | ✓ | ✓ |
|
|
112
|
+
| OKX | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
|
|
113
|
+
| Bitfinex | | | | ✓ \* | ✓ | ✓ |
|
|
114
|
+
| Bitmex | | | | | ✓ | ✓ |
|
|
115
|
+
|
|
116
|
+
\* Bitfinex WS OHLCV is aggregated from the trades stream via `get_ohlc_bitfinex`.
|
|
117
|
+
† Recent trades only (Bybit ≤ 1 000, Coinbase ≤ 100) — no deep historical pagination via the public REST API.
|
|
118
|
+
|
|
119
|
+
## Presentation
|
|
120
|
+
|
|
121
|
+
**Historical Downloader** `dccd.histo_dl`
|
|
122
|
+
Download OHLCV data via REST APIs and save to disk. Supports chunked requests, automatic retry on rate-limit (HTTP 429), and incremental updates from the last saved timestamp.
|
|
123
|
+
|
|
124
|
+
**Continuous Downloader** `dccd.continuous_dl`
|
|
125
|
+
Stream real-time data (order book, trades) via WebSocket with automatic reconnection and configurable processing/saving callbacks.
|
|
126
|
+
|
|
127
|
+
**Daemon** `dccd.daemon`
|
|
128
|
+
Autonomous, server-side collector driven by a YAML config. Runs REST jobs on a schedule (APScheduler), opens WebSocket streams for real-time collection, and periodically syncs all local data to one or more remote destinations (NAS, S3, SFTP, …) via rclone. Multiple remotes and a configurable sync interval are supported; collection is never blocked by remote availability.
|
|
129
|
+
|
|
130
|
+
### Output formats
|
|
131
|
+
|
|
132
|
+
Historical data can be saved as **CSV**, **Excel** (`.xlsx`), **SQLite**, **PostgreSQL** (via SQLAlchemy), or **Parquet**.
|
|
133
|
+
All DataFrames are native `polars.DataFrame`. A `pandas.DataFrame` can be obtained via `get_data(format='pandas')`.
|
|
134
|
+
|
|
135
|
+
## Quick start
|
|
136
|
+
|
|
137
|
+
Historical data:
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
from dccd.histo_dl import FromBinance
|
|
141
|
+
|
|
142
|
+
obj = FromBinance('/path/to/data/', 'BTC', 3600, fiat='USDT')
|
|
143
|
+
obj.import_data(start='2024-01-01 00:00:00', end='2024-12-31 00:00:00')
|
|
144
|
+
obj.save(form='parquet')
|
|
145
|
+
df = obj.get_data() # polars DataFrame (default)
|
|
146
|
+
df_pd = obj.get_data(format='pandas') # pandas DataFrame (optional)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Incremental update (resume from last saved point):
|
|
150
|
+
|
|
151
|
+
```python
|
|
152
|
+
obj.import_data(start='last', end='now').save(form='parquet')
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Other exchanges:
|
|
156
|
+
|
|
157
|
+
```python
|
|
158
|
+
from dccd.histo_dl import FromKraken, FromBybit, FromOKX
|
|
159
|
+
|
|
160
|
+
FromKraken('/path/', 'ETH', 3600).import_data(start='2024-01-01', end='now').save()
|
|
161
|
+
FromBybit('/path/', 'BTC', 86400).import_data(start='2024-01-01', end='now').save()
|
|
162
|
+
FromOKX('/path/', 'BTC', 3600).import_data(start='2024-01-01', end='now').save()
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
Trades (historical or recent):
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
from dccd.histo_dl import FromBinance, FromKraken
|
|
169
|
+
|
|
170
|
+
obj = FromBinance('/path/', 'BTC', 3600, fiat='USDT')
|
|
171
|
+
obj.import_trades(start='2024-01-01 00:00:00', end='2024-01-02 00:00:00')
|
|
172
|
+
obj.save_trades(form='csv')
|
|
173
|
+
df = obj.trades_df # polars DataFrame — columns: TS, price, amount, type, tid
|
|
174
|
+
|
|
175
|
+
# Kraken also supports full history; Bybit/Coinbase return recent-only snapshots
|
|
176
|
+
FromKraken('/path/', 'BTC', 3600).import_trades(start='2024-01-01', end='2024-01-02').save_trades()
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
Order book snapshot:
|
|
180
|
+
|
|
181
|
+
```python
|
|
182
|
+
from dccd.histo_dl import FromOKX
|
|
183
|
+
|
|
184
|
+
obj = FromOKX('/path/', 'BTC', 3600)
|
|
185
|
+
obj.import_orderbook(depth=50)
|
|
186
|
+
obj.save_orderbook(form='csv')
|
|
187
|
+
df = obj.orderbook_df # columns: side, price, amount, count
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
Daemon (autonomous collector) — `config.yml`:
|
|
191
|
+
|
|
192
|
+
```yaml
|
|
193
|
+
settings:
|
|
194
|
+
data_path: /data/crypto/
|
|
195
|
+
timezone: UTC
|
|
196
|
+
|
|
197
|
+
storage:
|
|
198
|
+
remotes:
|
|
199
|
+
- provider: rclone
|
|
200
|
+
remote: "mynas:crypto/"
|
|
201
|
+
sync_interval: 3600
|
|
202
|
+
|
|
203
|
+
histo_jobs:
|
|
204
|
+
- exchange: binance
|
|
205
|
+
pairs: [BTC/USDT, ETH/USDT]
|
|
206
|
+
span: 3600
|
|
207
|
+
format: parquet
|
|
208
|
+
|
|
209
|
+
stream_jobs:
|
|
210
|
+
- exchange: binance
|
|
211
|
+
pairs: [BTC/USDT]
|
|
212
|
+
channels: [trades, book]
|
|
213
|
+
time_step: 60
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
CLI quick start:
|
|
217
|
+
|
|
218
|
+
```bash
|
|
219
|
+
# Validate the config
|
|
220
|
+
dccd validate --config config.yml
|
|
221
|
+
|
|
222
|
+
# Backfill all OHLC history defined in config (resumable)
|
|
223
|
+
dccd backfill --config config.yml --start "2020-01-01 00:00:00"
|
|
224
|
+
|
|
225
|
+
# Dry run — estimate windows and time without downloading
|
|
226
|
+
dccd backfill --config config.yml --dry-run
|
|
227
|
+
|
|
228
|
+
# Backfill only one exchange
|
|
229
|
+
dccd backfill --config config.yml --exchange kraken
|
|
230
|
+
|
|
231
|
+
# One incremental batch per job, then exit (for cron)
|
|
232
|
+
dccd collect --config config.yml
|
|
233
|
+
|
|
234
|
+
# Continuous daemon (Ctrl-C to stop)
|
|
235
|
+
dccd start --config config.yml
|
|
236
|
+
|
|
237
|
+
# Add / remove a histo job in-place
|
|
238
|
+
dccd add --exchange kraken --pair ETH/USD --span 86400 --config config.yml
|
|
239
|
+
dccd remove --exchange kraken --pair ETH/USD --span 86400 --config config.yml
|
|
240
|
+
|
|
241
|
+
# Inspect all data on disk (OHLC, trades, orderbook)
|
|
242
|
+
dccd inventory --config config.yml
|
|
243
|
+
|
|
244
|
+
# Enable shell tab-completion (run once after install)
|
|
245
|
+
dccd --install-completion
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
> `--config` is optional — dccd searches `./config.yml` then `~/.config/dccd/config.yml` when omitted.
|
|
249
|
+
|
|
250
|
+
Python API:
|
|
251
|
+
|
|
252
|
+
```python
|
|
253
|
+
from dccd.daemon.config import load_config
|
|
254
|
+
from dccd.daemon.scheduler import run_once, build_histo_scheduler
|
|
255
|
+
from dccd.daemon.stream_manager import StreamManager
|
|
256
|
+
|
|
257
|
+
config = load_config('config.yml')
|
|
258
|
+
|
|
259
|
+
# One-shot: download all histo jobs once, then exit
|
|
260
|
+
run_once(config)
|
|
261
|
+
|
|
262
|
+
# Daemon mode: periodic REST + live WebSocket streams
|
|
263
|
+
scheduler = build_histo_scheduler(config)
|
|
264
|
+
scheduler.start()
|
|
265
|
+
|
|
266
|
+
mgr = StreamManager(config)
|
|
267
|
+
mgr.start() # runs until mgr.stop() is called
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
## Links
|
|
271
|
+
|
|
272
|
+
- PyPI: https://pypi.org/project/dccd/
|
|
273
|
+
- Documentation: https://download-crypto-currencies-data.readthedocs.io/
|
|
274
|
+
- Source: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
|
|
275
|
+
- Changelog: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/CHANGELOG.md
|
dccd-2.3.3/README.md
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
<picture>
|
|
2
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-dark-transparent.svg">
|
|
3
|
+
<img alt="dccd logo" src="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-light-transparent.svg" height="130px" align="left">
|
|
4
|
+
</picture>
|
|
5
|
+
|
|
6
|
+
# Download Crypto-Currency Data
|
|
7
|
+
|
|
8
|
+
[](https://pypi.org/project/dccd/)
|
|
9
|
+
[](https://pypi.org/project/dccd/)
|
|
10
|
+
[](https://pypi.org/project/dccd/)
|
|
11
|
+
[](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/actions/workflows/ci.yml)
|
|
12
|
+
[](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/LICENSE.txt)<br>
|
|
13
|
+
[](https://download-crypto-currencies-data.readthedocs.io/en/latest/)
|
|
14
|
+
[](https://codecov.io/gh/ArthurBernard/Download_Crypto_Currencies_Data)
|
|
15
|
+
[](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data)
|
|
16
|
+
[](https://pepy.tech/project/dccd)
|
|
17
|
+
|
|
18
|
+
Python package to download crypto-currency data (OHLCV, trades, order book) from multiple
|
|
19
|
+
exchanges via REST and WebSocket APIs. Data can be saved to CSV, Excel, SQLite, PostgreSQL,
|
|
20
|
+
or Parquet.
|
|
21
|
+
|
|
22
|
+
## Installation
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install dccd
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
With autonomous daemon support (APScheduler + PyYAML):
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install "dccd[daemon]"
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
From source:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
git clone https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
|
|
38
|
+
cd Download_Crypto_Currencies_Data
|
|
39
|
+
pip install -e .
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Supported exchanges
|
|
43
|
+
|
|
44
|
+
| Exchange | REST OHLCV | REST Trades | REST Order Book | WS OHLCV | WS Trades | WS Order Book |
|
|
45
|
+
|----------|:----------:|:-----------:|:---------------:|:--------:|:---------:|:-------------:|
|
|
46
|
+
| Binance | ✓ | ✓ | ✓ | | ✓ | ✓ |
|
|
47
|
+
| Coinbase | ✓ | ✓ † | ✓ | | | |
|
|
48
|
+
| Kraken | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
|
|
49
|
+
| Bybit | ✓ | ✓ † | ✓ | | ✓ | ✓ |
|
|
50
|
+
| OKX | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
|
|
51
|
+
| Bitfinex | | | | ✓ \* | ✓ | ✓ |
|
|
52
|
+
| Bitmex | | | | | ✓ | ✓ |
|
|
53
|
+
|
|
54
|
+
\* Bitfinex WS OHLCV is aggregated from the trades stream via `get_ohlc_bitfinex`.
|
|
55
|
+
† Recent trades only (Bybit ≤ 1 000, Coinbase ≤ 100) — no deep historical pagination via the public REST API.
|
|
56
|
+
|
|
57
|
+
## Presentation
|
|
58
|
+
|
|
59
|
+
**Historical Downloader** `dccd.histo_dl`
|
|
60
|
+
Download OHLCV data via REST APIs and save to disk. Supports chunked requests, automatic retry on rate-limit (HTTP 429), and incremental updates from the last saved timestamp.
|
|
61
|
+
|
|
62
|
+
**Continuous Downloader** `dccd.continuous_dl`
|
|
63
|
+
Stream real-time data (order book, trades) via WebSocket with automatic reconnection and configurable processing/saving callbacks.
|
|
64
|
+
|
|
65
|
+
**Daemon** `dccd.daemon`
|
|
66
|
+
Autonomous, server-side collector driven by a YAML config. Runs REST jobs on a schedule (APScheduler), opens WebSocket streams for real-time collection, and periodically syncs all local data to one or more remote destinations (NAS, S3, SFTP, …) via rclone. Multiple remotes and a configurable sync interval are supported; collection is never blocked by remote availability.
|
|
67
|
+
|
|
68
|
+
### Output formats
|
|
69
|
+
|
|
70
|
+
Historical data can be saved as **CSV**, **Excel** (`.xlsx`), **SQLite**, **PostgreSQL** (via SQLAlchemy), or **Parquet**.
|
|
71
|
+
All DataFrames are native `polars.DataFrame`. A `pandas.DataFrame` can be obtained via `get_data(format='pandas')`.
|
|
72
|
+
|
|
73
|
+
## Quick start
|
|
74
|
+
|
|
75
|
+
Historical data:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from dccd.histo_dl import FromBinance
|
|
79
|
+
|
|
80
|
+
obj = FromBinance('/path/to/data/', 'BTC', 3600, fiat='USDT')
|
|
81
|
+
obj.import_data(start='2024-01-01 00:00:00', end='2024-12-31 00:00:00')
|
|
82
|
+
obj.save(form='parquet')
|
|
83
|
+
df = obj.get_data() # polars DataFrame (default)
|
|
84
|
+
df_pd = obj.get_data(format='pandas') # pandas DataFrame (optional)
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Incremental update (resume from last saved point):
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
obj.import_data(start='last', end='now').save(form='parquet')
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Other exchanges:
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
from dccd.histo_dl import FromKraken, FromBybit, FromOKX
|
|
97
|
+
|
|
98
|
+
FromKraken('/path/', 'ETH', 3600).import_data(start='2024-01-01', end='now').save()
|
|
99
|
+
FromBybit('/path/', 'BTC', 86400).import_data(start='2024-01-01', end='now').save()
|
|
100
|
+
FromOKX('/path/', 'BTC', 3600).import_data(start='2024-01-01', end='now').save()
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Trades (historical or recent):
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from dccd.histo_dl import FromBinance, FromKraken
|
|
107
|
+
|
|
108
|
+
obj = FromBinance('/path/', 'BTC', 3600, fiat='USDT')
|
|
109
|
+
obj.import_trades(start='2024-01-01 00:00:00', end='2024-01-02 00:00:00')
|
|
110
|
+
obj.save_trades(form='csv')
|
|
111
|
+
df = obj.trades_df # polars DataFrame — columns: TS, price, amount, type, tid
|
|
112
|
+
|
|
113
|
+
# Kraken also supports full history; Bybit/Coinbase return recent-only snapshots
|
|
114
|
+
FromKraken('/path/', 'BTC', 3600).import_trades(start='2024-01-01', end='2024-01-02').save_trades()
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Order book snapshot:
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from dccd.histo_dl import FromOKX
|
|
121
|
+
|
|
122
|
+
obj = FromOKX('/path/', 'BTC', 3600)
|
|
123
|
+
obj.import_orderbook(depth=50)
|
|
124
|
+
obj.save_orderbook(form='csv')
|
|
125
|
+
df = obj.orderbook_df # columns: side, price, amount, count
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Daemon (autonomous collector) — `config.yml`:
|
|
129
|
+
|
|
130
|
+
```yaml
|
|
131
|
+
settings:
|
|
132
|
+
data_path: /data/crypto/
|
|
133
|
+
timezone: UTC
|
|
134
|
+
|
|
135
|
+
storage:
|
|
136
|
+
remotes:
|
|
137
|
+
- provider: rclone
|
|
138
|
+
remote: "mynas:crypto/"
|
|
139
|
+
sync_interval: 3600
|
|
140
|
+
|
|
141
|
+
histo_jobs:
|
|
142
|
+
- exchange: binance
|
|
143
|
+
pairs: [BTC/USDT, ETH/USDT]
|
|
144
|
+
span: 3600
|
|
145
|
+
format: parquet
|
|
146
|
+
|
|
147
|
+
stream_jobs:
|
|
148
|
+
- exchange: binance
|
|
149
|
+
pairs: [BTC/USDT]
|
|
150
|
+
channels: [trades, book]
|
|
151
|
+
time_step: 60
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
CLI quick start:
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
# Validate the config
|
|
158
|
+
dccd validate --config config.yml
|
|
159
|
+
|
|
160
|
+
# Backfill all OHLC history defined in config (resumable)
|
|
161
|
+
dccd backfill --config config.yml --start "2020-01-01 00:00:00"
|
|
162
|
+
|
|
163
|
+
# Dry run — estimate windows and time without downloading
|
|
164
|
+
dccd backfill --config config.yml --dry-run
|
|
165
|
+
|
|
166
|
+
# Backfill only one exchange
|
|
167
|
+
dccd backfill --config config.yml --exchange kraken
|
|
168
|
+
|
|
169
|
+
# One incremental batch per job, then exit (for cron)
|
|
170
|
+
dccd collect --config config.yml
|
|
171
|
+
|
|
172
|
+
# Continuous daemon (Ctrl-C to stop)
|
|
173
|
+
dccd start --config config.yml
|
|
174
|
+
|
|
175
|
+
# Add / remove a histo job in-place
|
|
176
|
+
dccd add --exchange kraken --pair ETH/USD --span 86400 --config config.yml
|
|
177
|
+
dccd remove --exchange kraken --pair ETH/USD --span 86400 --config config.yml
|
|
178
|
+
|
|
179
|
+
# Inspect all data on disk (OHLC, trades, orderbook)
|
|
180
|
+
dccd inventory --config config.yml
|
|
181
|
+
|
|
182
|
+
# Enable shell tab-completion (run once after install)
|
|
183
|
+
dccd --install-completion
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
> `--config` is optional — dccd searches `./config.yml` then `~/.config/dccd/config.yml` when omitted.
|
|
187
|
+
|
|
188
|
+
Python API:
|
|
189
|
+
|
|
190
|
+
```python
|
|
191
|
+
from dccd.daemon.config import load_config
|
|
192
|
+
from dccd.daemon.scheduler import run_once, build_histo_scheduler
|
|
193
|
+
from dccd.daemon.stream_manager import StreamManager
|
|
194
|
+
|
|
195
|
+
config = load_config('config.yml')
|
|
196
|
+
|
|
197
|
+
# One-shot: download all histo jobs once, then exit
|
|
198
|
+
run_once(config)
|
|
199
|
+
|
|
200
|
+
# Daemon mode: periodic REST + live WebSocket streams
|
|
201
|
+
scheduler = build_histo_scheduler(config)
|
|
202
|
+
scheduler.start()
|
|
203
|
+
|
|
204
|
+
mgr = StreamManager(config)
|
|
205
|
+
mgr.start() # runs until mgr.stop() is called
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
## Links
|
|
209
|
+
|
|
210
|
+
- PyPI: https://pypi.org/project/dccd/
|
|
211
|
+
- Documentation: https://download-crypto-currencies-data.readthedocs.io/
|
|
212
|
+
- Source: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
|
|
213
|
+
- Changelog: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/CHANGELOG.md
|
|
@@ -286,7 +286,7 @@ def get_data_bitfinex(channel: str, process_func: Any, process_params: dict[str,
|
|
|
286
286
|
'Oracle', 'MSSQL', 'MySQL'},
|
|
287
287
|
It will create an IODataBase object to save/update the database in the
|
|
288
288
|
specified format `save_method`, default is 'DataFrame' it save as
|
|
289
|
-
binary
|
|
289
|
+
binary pl.DataFrame object. More informations are available into
|
|
290
290
|
:mod:`dccd.tools.io`.
|
|
291
291
|
io_params : dict, optional
|
|
292
292
|
Dictionary of the keyword arguments available to the
|