dccd 2.3.1__tar.gz → 2.3.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. {dccd-2.3.1 → dccd-2.3.3}/CHANGELOG.md +32 -1
  2. dccd-2.3.3/PKG-INFO +275 -0
  3. dccd-2.3.3/README.md +213 -0
  4. {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/bitfinex.py +1 -1
  5. {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/backfill.py +68 -33
  6. {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/cli.py +235 -27
  7. {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/config.py +47 -0
  8. {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/stream_manager.py +3 -4
  9. {dccd-2.3.1 → dccd-2.3.3}/dccd/histo_dl/binance.py +17 -13
  10. {dccd-2.3.1 → dccd-2.3.3}/dccd/histo_dl/bybit.py +14 -10
  11. {dccd-2.3.1 → dccd-2.3.3}/dccd/histo_dl/coinbase.py +15 -12
  12. {dccd-2.3.1 → dccd-2.3.3}/dccd/histo_dl/exchange.py +43 -49
  13. {dccd-2.3.1 → dccd-2.3.3}/dccd/histo_dl/kraken.py +15 -11
  14. {dccd-2.3.1 → dccd-2.3.3}/dccd/histo_dl/okx.py +15 -11
  15. dccd-2.3.3/dccd/process_data.py +139 -0
  16. {dccd-2.3.1 → dccd-2.3.3}/dccd/storage.py +33 -36
  17. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_backfill.py +51 -3
  18. dccd-2.3.3/dccd/tests/test_daemon_cli.py +358 -0
  19. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_daemon_config.py +48 -0
  20. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_histo_dl.py +13 -5
  21. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_io.py +26 -26
  22. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_okx.py +24 -0
  23. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_process_data.py +19 -17
  24. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_storage.py +12 -11
  25. dccd-2.3.3/dccd/tools/io.py +240 -0
  26. dccd-2.3.3/dccd.egg-info/PKG-INFO +275 -0
  27. {dccd-2.3.1 → dccd-2.3.3}/dccd.egg-info/SOURCES.txt +1 -1
  28. {dccd-2.3.1 → dccd-2.3.3}/dccd.egg-info/requires.txt +3 -3
  29. {dccd-2.3.1 → dccd-2.3.3}/pyproject.toml +6 -5
  30. dccd-2.3.1/PKG-INFO +0 -315
  31. dccd-2.3.1/README.rst +0 -253
  32. dccd-2.3.1/dccd/process_data.py +0 -166
  33. dccd-2.3.1/dccd/tests/test_daemon_cli.py +0 -108
  34. dccd-2.3.1/dccd/tools/io.py +0 -495
  35. dccd-2.3.1/dccd.egg-info/PKG-INFO +0 -315
  36. {dccd-2.3.1 → dccd-2.3.3}/CONTRIBUTING.md +0 -0
  37. {dccd-2.3.1 → dccd-2.3.3}/LICENSE.txt +0 -0
  38. {dccd-2.3.1 → dccd-2.3.3}/MANIFEST.in +0 -0
  39. {dccd-2.3.1 → dccd-2.3.3}/dccd/__init__.py +0 -0
  40. {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/__init__.py +0 -0
  41. {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/binance.py +0 -0
  42. {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/bitmex.py +0 -0
  43. {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/bybit.py +0 -0
  44. {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/exchange.py +0 -0
  45. {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/kraken.py +0 -0
  46. {dccd-2.3.1 → dccd-2.3.3}/dccd/continuous_dl/okx.py +0 -0
  47. {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/__init__.py +0 -0
  48. {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/health.py +0 -0
  49. {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/scheduler.py +0 -0
  50. {dccd-2.3.1 → dccd-2.3.3}/dccd/daemon/storage.py +0 -0
  51. {dccd-2.3.1 → dccd-2.3.3}/dccd/histo_dl/__init__.py +0 -0
  52. {dccd-2.3.1 → dccd-2.3.3}/dccd/models.py +0 -0
  53. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/__init__.py +0 -0
  54. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/conftest.py +0 -0
  55. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_binance.py +0 -0
  56. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_binance_ws.py +0 -0
  57. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_bitfinex.py +0 -0
  58. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_bitmex.py +0 -0
  59. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_bybit.py +0 -0
  60. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_bybit_ws.py +0 -0
  61. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_coinbase.py +0 -0
  62. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_daemon_health.py +0 -0
  63. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_daemon_scheduler.py +0 -0
  64. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_daemon_storage.py +0 -0
  65. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_daemon_stream_manager.py +0 -0
  66. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_date_time.py +0 -0
  67. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_kraken.py +0 -0
  68. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_kraken_ws.py +0 -0
  69. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_models.py +0 -0
  70. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_okx_ws.py +0 -0
  71. {dccd-2.3.1 → dccd-2.3.3}/dccd/tests/test_websocket.py +0 -0
  72. {dccd-2.3.1 → dccd-2.3.3}/dccd/tools/__init__.py +0 -0
  73. {dccd-2.3.1 → dccd-2.3.3}/dccd/tools/date_time.py +0 -0
  74. {dccd-2.3.1 → dccd-2.3.3}/dccd/tools/websocket.py +0 -0
  75. {dccd-2.3.1 → dccd-2.3.3}/dccd.egg-info/dependency_links.txt +0 -0
  76. {dccd-2.3.1 → dccd-2.3.3}/dccd.egg-info/entry_points.txt +0 -0
  77. {dccd-2.3.1 → dccd-2.3.3}/dccd.egg-info/top_level.txt +0 -0
  78. {dccd-2.3.1 → dccd-2.3.3}/setup.cfg +0 -0
@@ -6,10 +6,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
- ## [2.3.1] - 2026-05-24
9
+ ## [2.3.3] - 2026-05-31
10
+
11
+ ### Added
12
+
13
+ - `doc/source/` — complete Sphinx documentation overhaul: redesigned homepage
14
+ with sphinx-design cards, captioned toctrees (Getting Started / Data Collection /
15
+ Reference), new pages (`installation`, `quickstart`, `changelog`, `cli`,
16
+ `configuration`, `models`, `storage`, `tools`, per-exchange histo/continuous pages),
17
+ adaptive light/dark logo and favicon, sticky top navbar with PyPI/GitHub/Fynance
18
+ links, hero header with inline logo+title, responsive layout (#59, #61)
19
+ - `README.md` — converted from RST to Markdown; inline logo+title header with
20
+ `<picture>` for light/dark mode switching; badges on two rows (#60)
21
+
22
+ ## [2.3.2] - 2026-05-25
23
+
24
+ ### Added
25
+
26
+ - `dccd/daemon/cli.py` — `dccd status --json`: emit raw metrics as a JSON object on stdout, suitable for piping into Grafana / jq (#53)
27
+ - `dccd/daemon/config.py` — `HistoJob.max_retries` (int, 1–10, default 3) and `HistoJob.retry_delay` (float ≥ 0, default 2.0): per-job retry configuration for transient network errors; delay is exponential (`retry_delay * 2^(attempt-1)`) (#53)
28
+ - `dccd/daemon/config.py` — `resolve_config_path()` and `DEFAULT_CONFIG_PATH`: CLI commands now fall back to `$XDG_CONFIG_HOME/dccd/config.yml` (default `~/.config/dccd/config.yml`) when no `--config` option is provided and `./config.yml` does not exist (#49)
29
+ - `dccd/daemon/cli.py` — `dccd inventory`: scans `data_path` and prints a table of all stored OHLC, trades, and orderbook data with date range, row count, and gap count per series; uses Polars for fast columnar reads (#50)
30
+ - `dccd/daemon/cli.py` — `dccd remove --exchange X --pair Y --span N`: removes a pair from a histo_job (or the whole job if it was the last pair) and re-validates the config before writing (#50)
31
+
32
+ ### Changed
33
+
34
+ - `dccd/storage.py`, `dccd/histo_dl/exchange.py`, `dccd/daemon/backfill.py`, `dccd/process_data.py`, `dccd/daemon/stream_manager.py` — replace pandas with polars throughout; `DataStore.save/load` accept/return `pl.DataFrame`; `get_data()` defaults to `format='polars'`; `set_marketdepth` returns a flat long-format `pl.DataFrame`; stream savers write parquet via `DataStore`; `pandas` removed from core dependencies (#52)
35
+ - `dccd/daemon/backfill.py` — backfill progress bar now shows the current window date (`YYYY-MM-DD → YYYY-MM-DD`) instead of a raw window count (`n win`); makes it easy to see which period is being downloaded at a glance (#48)
10
36
 
11
37
  ### Fixed
12
38
 
39
+ - `dccd/histo_dl/exchange.py` — `_sort_data` no longer raises `ColumnNotFoundError: "date"` when the exchange API returns an empty candle list; the polars migration (PR #52) had re-introduced a variant of the empty-data crash from v2.3.1; now returns early with an empty `self.df` (#54)
40
+
41
+ ## [2.3.1] - 2026-05-24
42
+
43
+ ### Fixed
13
44
  - `dccd/storage.py` — `DataStore.missing_intervals` now detects the gap **before** the first saved row when the requested `start` predates `file_min`; previously only the trailing gap (after `file_max`) was returned, causing `dccd backfill --start <early-date>` to silently skip all historical data before the first existing candle (#46)
14
45
  - `dccd/histo_dl/coinbase.py` — raise `RuntimeError` when Coinbase returns HTTP 200 with a JSON dict (e.g. `{"message": "..."}` for near-future windows) instead of silently iterating dict keys and crashing with `ValueError` (#45)
15
46
  - `dccd/histo_dl/coinbase.py` — additional guard: raise `RuntimeError` when Coinbase returns a JSON list whose first element is not itself a list/tuple (e.g. `["message"]`); previously caused `float("m")` `ValueError` (#45)
dccd-2.3.3/PKG-INFO ADDED
@@ -0,0 +1,275 @@
1
+ Metadata-Version: 2.4
2
+ Name: dccd
3
+ Version: 2.3.3
4
+ Summary: Download Crypto Currency Data from different exchanges.
5
+ Author-email: Arthur Bernard <arthur.bernard.92@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
8
+ Project-URL: Documentation, https://download-crypto-currencies-data.readthedocs.io/
9
+ Project-URL: Changelog, https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/CHANGELOG.md
10
+ Classifier: Development Status :: 5 - Production/Stable
11
+ Classifier: Intended Audience :: Financial and Insurance Industry
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Database
20
+ Classifier: Topic :: Office/Business :: Financial
21
+ Requires-Python: >=3.10
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE.txt
24
+ Requires-Dist: numpy>=1.26
25
+ Requires-Dist: polars>=0.20
26
+ Requires-Dist: pyarrow>=13
27
+ Requires-Dist: requests>=2.28
28
+ Requires-Dist: openpyxl>=3.1
29
+ Requires-Dist: websockets>=12.0
30
+ Requires-Dist: scipy>=1.10
31
+ Requires-Dist: SQLAlchemy>=2.0
32
+ Requires-Dist: tenacity>=8.0
33
+ Requires-Dist: pydantic>=2.0
34
+ Provides-Extra: io
35
+ Requires-Dist: polars>=0.20; extra == "io"
36
+ Requires-Dist: pyarrow>=13; extra == "io"
37
+ Provides-Extra: daemon
38
+ Requires-Dist: pyyaml>=6.0; extra == "daemon"
39
+ Requires-Dist: apscheduler<4,>=3.10; extra == "daemon"
40
+ Requires-Dist: typer>=0.12; extra == "daemon"
41
+ Requires-Dist: tqdm>=4.64; extra == "daemon"
42
+ Provides-Extra: dev
43
+ Requires-Dist: pytest>=7.4; extra == "dev"
44
+ Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
45
+ Requires-Dist: pytest-cov>=4.1; extra == "dev"
46
+ Requires-Dist: ruff>=0.4; extra == "dev"
47
+ Requires-Dist: interrogate>=1.5; extra == "dev"
48
+ Requires-Dist: mypy>=1.0; extra == "dev"
49
+ Requires-Dist: pyyaml>=6.0; extra == "dev"
50
+ Requires-Dist: apscheduler<4,>=3.10; extra == "dev"
51
+ Requires-Dist: typer>=0.12; extra == "dev"
52
+ Requires-Dist: tqdm>=4.64; extra == "dev"
53
+ Provides-Extra: doc
54
+ Requires-Dist: sphinx>=7.0; extra == "doc"
55
+ Requires-Dist: furo; extra == "doc"
56
+ Requires-Dist: numpydoc; extra == "doc"
57
+ Requires-Dist: sphinx-design; extra == "doc"
58
+ Requires-Dist: sphinx-copybutton; extra == "doc"
59
+ Requires-Dist: pyyaml>=6.0; extra == "doc"
60
+ Requires-Dist: apscheduler<4,>=3.10; extra == "doc"
61
+ Dynamic: license-file
62
+
63
+ <picture>
64
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-dark-transparent.svg">
65
+ <img alt="dccd logo" src="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-light-transparent.svg" height="130px" align="left">
66
+ </picture>
67
+
68
+ # Download Crypto-Currency Data
69
+
70
+ [![Python versions](https://img.shields.io/pypi/pyversions/dccd)](https://pypi.org/project/dccd/)
71
+ [![PyPI](https://img.shields.io/pypi/v/dccd.svg)](https://pypi.org/project/dccd/)
72
+ [![PyPI status](https://img.shields.io/pypi/status/dccd.svg?colorB=blue)](https://pypi.org/project/dccd/)
73
+ [![CI](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/actions/workflows/ci.yml/badge.svg)](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/actions/workflows/ci.yml)
74
+ [![License](https://img.shields.io/github/license/ArthurBernard/Download_Crypto_Currencies_Data.svg)](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/LICENSE.txt)<br>
75
+ [![Documentation](https://readthedocs.org/projects/download-crypto-currencies-data/badge/?version=latest)](https://download-crypto-currencies-data.readthedocs.io/en/latest/)
76
+ [![Coverage](https://codecov.io/gh/ArthurBernard/Download_Crypto_Currencies_Data/branch/master/graph/badge.svg)](https://codecov.io/gh/ArthurBernard/Download_Crypto_Currencies_Data)
77
+ [![Docstring coverage](https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/badges/interrogate_badge.svg)](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data)
78
+ [![Downloads](https://pepy.tech/badge/dccd)](https://pepy.tech/project/dccd)
79
+
80
+ Python package to download crypto-currency data (OHLCV, trades, order book) from multiple
81
+ exchanges via REST and WebSocket APIs. Data can be saved to CSV, Excel, SQLite, PostgreSQL,
82
+ or Parquet.
83
+
84
+ ## Installation
85
+
86
+ ```bash
87
+ pip install dccd
88
+ ```
89
+
90
+ With autonomous daemon support (APScheduler + PyYAML):
91
+
92
+ ```bash
93
+ pip install "dccd[daemon]"
94
+ ```
95
+
96
+ From source:
97
+
98
+ ```bash
99
+ git clone https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
100
+ cd Download_Crypto_Currencies_Data
101
+ pip install -e .
102
+ ```
103
+
104
+ ## Supported exchanges
105
+
106
+ | Exchange | REST OHLCV | REST Trades | REST Order Book | WS OHLCV | WS Trades | WS Order Book |
107
+ |----------|:----------:|:-----------:|:---------------:|:--------:|:---------:|:-------------:|
108
+ | Binance | ✓ | ✓ | ✓ | | ✓ | ✓ |
109
+ | Coinbase | ✓ | ✓ † | ✓ | | | |
110
+ | Kraken | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
111
+ | Bybit | ✓ | ✓ † | ✓ | | ✓ | ✓ |
112
+ | OKX | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
113
+ | Bitfinex | | | | ✓ \* | ✓ | ✓ |
114
+ | Bitmex | | | | | ✓ | ✓ |
115
+
116
+ \* Bitfinex WS OHLCV is aggregated from the trades stream via `get_ohlc_bitfinex`.
117
+ † Recent trades only (Bybit ≤ 1 000, Coinbase ≤ 100) — no deep historical pagination via the public REST API.
118
+
119
+ ## Presentation
120
+
121
+ **Historical Downloader** `dccd.histo_dl`
122
+ Download OHLCV data via REST APIs and save to disk. Supports chunked requests, automatic retry on rate-limit (HTTP 429), and incremental updates from the last saved timestamp.
123
+
124
+ **Continuous Downloader** `dccd.continuous_dl`
125
+ Stream real-time data (order book, trades) via WebSocket with automatic reconnection and configurable processing/saving callbacks.
126
+
127
+ **Daemon** `dccd.daemon`
128
+ Autonomous, server-side collector driven by a YAML config. Runs REST jobs on a schedule (APScheduler), opens WebSocket streams for real-time collection, and periodically syncs all local data to one or more remote destinations (NAS, S3, SFTP, …) via rclone. Multiple remotes and a configurable sync interval are supported; collection is never blocked by remote availability.
129
+
130
+ ### Output formats
131
+
132
+ Historical data can be saved as **CSV**, **Excel** (`.xlsx`), **SQLite**, **PostgreSQL** (via SQLAlchemy), or **Parquet**.
133
+ All DataFrames are native `polars.DataFrame`. A `pandas.DataFrame` can be obtained via `get_data(format='pandas')`.
134
+
135
+ ## Quick start
136
+
137
+ Historical data:
138
+
139
+ ```python
140
+ from dccd.histo_dl import FromBinance
141
+
142
+ obj = FromBinance('/path/to/data/', 'BTC', 3600, fiat='USDT')
143
+ obj.import_data(start='2024-01-01 00:00:00', end='2024-12-31 00:00:00')
144
+ obj.save(form='parquet')
145
+ df = obj.get_data() # polars DataFrame (default)
146
+ df_pd = obj.get_data(format='pandas') # pandas DataFrame (optional)
147
+ ```
148
+
149
+ Incremental update (resume from last saved point):
150
+
151
+ ```python
152
+ obj.import_data(start='last', end='now').save(form='parquet')
153
+ ```
154
+
155
+ Other exchanges:
156
+
157
+ ```python
158
+ from dccd.histo_dl import FromKraken, FromBybit, FromOKX
159
+
160
+ FromKraken('/path/', 'ETH', 3600).import_data(start='2024-01-01', end='now').save()
161
+ FromBybit('/path/', 'BTC', 86400).import_data(start='2024-01-01', end='now').save()
162
+ FromOKX('/path/', 'BTC', 3600).import_data(start='2024-01-01', end='now').save()
163
+ ```
164
+
165
+ Trades (historical or recent):
166
+
167
+ ```python
168
+ from dccd.histo_dl import FromBinance, FromKraken
169
+
170
+ obj = FromBinance('/path/', 'BTC', 3600, fiat='USDT')
171
+ obj.import_trades(start='2024-01-01 00:00:00', end='2024-01-02 00:00:00')
172
+ obj.save_trades(form='csv')
173
+ df = obj.trades_df # polars DataFrame — columns: TS, price, amount, type, tid
174
+
175
+ # Kraken also supports full history; Bybit/Coinbase return recent-only snapshots
176
+ FromKraken('/path/', 'BTC', 3600).import_trades(start='2024-01-01', end='2024-01-02').save_trades()
177
+ ```
178
+
179
+ Order book snapshot:
180
+
181
+ ```python
182
+ from dccd.histo_dl import FromOKX
183
+
184
+ obj = FromOKX('/path/', 'BTC', 3600)
185
+ obj.import_orderbook(depth=50)
186
+ obj.save_orderbook(form='csv')
187
+ df = obj.orderbook_df # columns: side, price, amount, count
188
+ ```
189
+
190
+ Daemon (autonomous collector) — `config.yml`:
191
+
192
+ ```yaml
193
+ settings:
194
+ data_path: /data/crypto/
195
+ timezone: UTC
196
+
197
+ storage:
198
+ remotes:
199
+ - provider: rclone
200
+ remote: "mynas:crypto/"
201
+ sync_interval: 3600
202
+
203
+ histo_jobs:
204
+ - exchange: binance
205
+ pairs: [BTC/USDT, ETH/USDT]
206
+ span: 3600
207
+ format: parquet
208
+
209
+ stream_jobs:
210
+ - exchange: binance
211
+ pairs: [BTC/USDT]
212
+ channels: [trades, book]
213
+ time_step: 60
214
+ ```
215
+
216
+ CLI quick start:
217
+
218
+ ```bash
219
+ # Validate the config
220
+ dccd validate --config config.yml
221
+
222
+ # Backfill all OHLC history defined in config (resumable)
223
+ dccd backfill --config config.yml --start "2020-01-01 00:00:00"
224
+
225
+ # Dry run — estimate windows and time without downloading
226
+ dccd backfill --config config.yml --dry-run
227
+
228
+ # Backfill only one exchange
229
+ dccd backfill --config config.yml --exchange kraken
230
+
231
+ # One incremental batch per job, then exit (for cron)
232
+ dccd collect --config config.yml
233
+
234
+ # Continuous daemon (Ctrl-C to stop)
235
+ dccd start --config config.yml
236
+
237
+ # Add / remove a histo job in-place
238
+ dccd add --exchange kraken --pair ETH/USD --span 86400 --config config.yml
239
+ dccd remove --exchange kraken --pair ETH/USD --span 86400 --config config.yml
240
+
241
+ # Inspect all data on disk (OHLC, trades, orderbook)
242
+ dccd inventory --config config.yml
243
+
244
+ # Enable shell tab-completion (run once after install)
245
+ dccd --install-completion
246
+ ```
247
+
248
+ > `--config` is optional — dccd searches `./config.yml` then `~/.config/dccd/config.yml` when omitted.
249
+
250
+ Python API:
251
+
252
+ ```python
253
+ from dccd.daemon.config import load_config
254
+ from dccd.daemon.scheduler import run_once, build_histo_scheduler
255
+ from dccd.daemon.stream_manager import StreamManager
256
+
257
+ config = load_config('config.yml')
258
+
259
+ # One-shot: download all histo jobs once, then exit
260
+ run_once(config)
261
+
262
+ # Daemon mode: periodic REST + live WebSocket streams
263
+ scheduler = build_histo_scheduler(config)
264
+ scheduler.start()
265
+
266
+ mgr = StreamManager(config)
267
+ mgr.start() # runs until mgr.stop() is called
268
+ ```
269
+
270
+ ## Links
271
+
272
+ - PyPI: https://pypi.org/project/dccd/
273
+ - Documentation: https://download-crypto-currencies-data.readthedocs.io/
274
+ - Source: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
275
+ - Changelog: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/CHANGELOG.md
dccd-2.3.3/README.md ADDED
@@ -0,0 +1,213 @@
1
+ <picture>
2
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-dark-transparent.svg">
3
+ <img alt="dccd logo" src="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-light-transparent.svg" height="130px" align="left">
4
+ </picture>
5
+
6
+ # Download Crypto-Currency Data
7
+
8
+ [![Python versions](https://img.shields.io/pypi/pyversions/dccd)](https://pypi.org/project/dccd/)
9
+ [![PyPI](https://img.shields.io/pypi/v/dccd.svg)](https://pypi.org/project/dccd/)
10
+ [![PyPI status](https://img.shields.io/pypi/status/dccd.svg?colorB=blue)](https://pypi.org/project/dccd/)
11
+ [![CI](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/actions/workflows/ci.yml/badge.svg)](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/actions/workflows/ci.yml)
12
+ [![License](https://img.shields.io/github/license/ArthurBernard/Download_Crypto_Currencies_Data.svg)](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/LICENSE.txt)<br>
13
+ [![Documentation](https://readthedocs.org/projects/download-crypto-currencies-data/badge/?version=latest)](https://download-crypto-currencies-data.readthedocs.io/en/latest/)
14
+ [![Coverage](https://codecov.io/gh/ArthurBernard/Download_Crypto_Currencies_Data/branch/master/graph/badge.svg)](https://codecov.io/gh/ArthurBernard/Download_Crypto_Currencies_Data)
15
+ [![Docstring coverage](https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/badges/interrogate_badge.svg)](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data)
16
+ [![Downloads](https://pepy.tech/badge/dccd)](https://pepy.tech/project/dccd)
17
+
18
+ Python package to download crypto-currency data (OHLCV, trades, order book) from multiple
19
+ exchanges via REST and WebSocket APIs. Data can be saved to CSV, Excel, SQLite, PostgreSQL,
20
+ or Parquet.
21
+
22
+ ## Installation
23
+
24
+ ```bash
25
+ pip install dccd
26
+ ```
27
+
28
+ With autonomous daemon support (APScheduler + PyYAML):
29
+
30
+ ```bash
31
+ pip install "dccd[daemon]"
32
+ ```
33
+
34
+ From source:
35
+
36
+ ```bash
37
+ git clone https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
38
+ cd Download_Crypto_Currencies_Data
39
+ pip install -e .
40
+ ```
41
+
42
+ ## Supported exchanges
43
+
44
+ | Exchange | REST OHLCV | REST Trades | REST Order Book | WS OHLCV | WS Trades | WS Order Book |
45
+ |----------|:----------:|:-----------:|:---------------:|:--------:|:---------:|:-------------:|
46
+ | Binance | ✓ | ✓ | ✓ | | ✓ | ✓ |
47
+ | Coinbase | ✓ | ✓ † | ✓ | | | |
48
+ | Kraken | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
49
+ | Bybit | ✓ | ✓ † | ✓ | | ✓ | ✓ |
50
+ | OKX | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
51
+ | Bitfinex | | | | ✓ \* | ✓ | ✓ |
52
+ | Bitmex | | | | | ✓ | ✓ |
53
+
54
+ \* Bitfinex WS OHLCV is aggregated from the trades stream via `get_ohlc_bitfinex`.
55
+ † Recent trades only (Bybit ≤ 1 000, Coinbase ≤ 100) — no deep historical pagination via the public REST API.
56
+
57
+ ## Presentation
58
+
59
+ **Historical Downloader** `dccd.histo_dl`
60
+ Download OHLCV data via REST APIs and save to disk. Supports chunked requests, automatic retry on rate-limit (HTTP 429), and incremental updates from the last saved timestamp.
61
+
62
+ **Continuous Downloader** `dccd.continuous_dl`
63
+ Stream real-time data (order book, trades) via WebSocket with automatic reconnection and configurable processing/saving callbacks.
64
+
65
+ **Daemon** `dccd.daemon`
66
+ Autonomous, server-side collector driven by a YAML config. Runs REST jobs on a schedule (APScheduler), opens WebSocket streams for real-time collection, and periodically syncs all local data to one or more remote destinations (NAS, S3, SFTP, …) via rclone. Multiple remotes and a configurable sync interval are supported; collection is never blocked by remote availability.
67
+
68
+ ### Output formats
69
+
70
+ Historical data can be saved as **CSV**, **Excel** (`.xlsx`), **SQLite**, **PostgreSQL** (via SQLAlchemy), or **Parquet**.
71
+ All DataFrames are native `polars.DataFrame`. A `pandas.DataFrame` can be obtained via `get_data(format='pandas')`.
72
+
73
+ ## Quick start
74
+
75
+ Historical data:
76
+
77
+ ```python
78
+ from dccd.histo_dl import FromBinance
79
+
80
+ obj = FromBinance('/path/to/data/', 'BTC', 3600, fiat='USDT')
81
+ obj.import_data(start='2024-01-01 00:00:00', end='2024-12-31 00:00:00')
82
+ obj.save(form='parquet')
83
+ df = obj.get_data() # polars DataFrame (default)
84
+ df_pd = obj.get_data(format='pandas') # pandas DataFrame (optional)
85
+ ```
86
+
87
+ Incremental update (resume from last saved point):
88
+
89
+ ```python
90
+ obj.import_data(start='last', end='now').save(form='parquet')
91
+ ```
92
+
93
+ Other exchanges:
94
+
95
+ ```python
96
+ from dccd.histo_dl import FromKraken, FromBybit, FromOKX
97
+
98
+ FromKraken('/path/', 'ETH', 3600).import_data(start='2024-01-01', end='now').save()
99
+ FromBybit('/path/', 'BTC', 86400).import_data(start='2024-01-01', end='now').save()
100
+ FromOKX('/path/', 'BTC', 3600).import_data(start='2024-01-01', end='now').save()
101
+ ```
102
+
103
+ Trades (historical or recent):
104
+
105
+ ```python
106
+ from dccd.histo_dl import FromBinance, FromKraken
107
+
108
+ obj = FromBinance('/path/', 'BTC', 3600, fiat='USDT')
109
+ obj.import_trades(start='2024-01-01 00:00:00', end='2024-01-02 00:00:00')
110
+ obj.save_trades(form='csv')
111
+ df = obj.trades_df # polars DataFrame — columns: TS, price, amount, type, tid
112
+
113
+ # Kraken also supports full history; Bybit/Coinbase return recent-only snapshots
114
+ FromKraken('/path/', 'BTC', 3600).import_trades(start='2024-01-01', end='2024-01-02').save_trades()
115
+ ```
116
+
117
+ Order book snapshot:
118
+
119
+ ```python
120
+ from dccd.histo_dl import FromOKX
121
+
122
+ obj = FromOKX('/path/', 'BTC', 3600)
123
+ obj.import_orderbook(depth=50)
124
+ obj.save_orderbook(form='csv')
125
+ df = obj.orderbook_df # columns: side, price, amount, count
126
+ ```
127
+
128
+ Daemon (autonomous collector) — `config.yml`:
129
+
130
+ ```yaml
131
+ settings:
132
+ data_path: /data/crypto/
133
+ timezone: UTC
134
+
135
+ storage:
136
+ remotes:
137
+ - provider: rclone
138
+ remote: "mynas:crypto/"
139
+ sync_interval: 3600
140
+
141
+ histo_jobs:
142
+ - exchange: binance
143
+ pairs: [BTC/USDT, ETH/USDT]
144
+ span: 3600
145
+ format: parquet
146
+
147
+ stream_jobs:
148
+ - exchange: binance
149
+ pairs: [BTC/USDT]
150
+ channels: [trades, book]
151
+ time_step: 60
152
+ ```
153
+
154
+ CLI quick start:
155
+
156
+ ```bash
157
+ # Validate the config
158
+ dccd validate --config config.yml
159
+
160
+ # Backfill all OHLC history defined in config (resumable)
161
+ dccd backfill --config config.yml --start "2020-01-01 00:00:00"
162
+
163
+ # Dry run — estimate windows and time without downloading
164
+ dccd backfill --config config.yml --dry-run
165
+
166
+ # Backfill only one exchange
167
+ dccd backfill --config config.yml --exchange kraken
168
+
169
+ # One incremental batch per job, then exit (for cron)
170
+ dccd collect --config config.yml
171
+
172
+ # Continuous daemon (Ctrl-C to stop)
173
+ dccd start --config config.yml
174
+
175
+ # Add / remove a histo job in-place
176
+ dccd add --exchange kraken --pair ETH/USD --span 86400 --config config.yml
177
+ dccd remove --exchange kraken --pair ETH/USD --span 86400 --config config.yml
178
+
179
+ # Inspect all data on disk (OHLC, trades, orderbook)
180
+ dccd inventory --config config.yml
181
+
182
+ # Enable shell tab-completion (run once after install)
183
+ dccd --install-completion
184
+ ```
185
+
186
+ > `--config` is optional — dccd searches `./config.yml` then `~/.config/dccd/config.yml` when omitted.
187
+
188
+ Python API:
189
+
190
+ ```python
191
+ from dccd.daemon.config import load_config
192
+ from dccd.daemon.scheduler import run_once, build_histo_scheduler
193
+ from dccd.daemon.stream_manager import StreamManager
194
+
195
+ config = load_config('config.yml')
196
+
197
+ # One-shot: download all histo jobs once, then exit
198
+ run_once(config)
199
+
200
+ # Daemon mode: periodic REST + live WebSocket streams
201
+ scheduler = build_histo_scheduler(config)
202
+ scheduler.start()
203
+
204
+ mgr = StreamManager(config)
205
+ mgr.start() # runs until mgr.stop() is called
206
+ ```
207
+
208
+ ## Links
209
+
210
+ - PyPI: https://pypi.org/project/dccd/
211
+ - Documentation: https://download-crypto-currencies-data.readthedocs.io/
212
+ - Source: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
213
+ - Changelog: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/CHANGELOG.md
@@ -286,7 +286,7 @@ def get_data_bitfinex(channel: str, process_func: Any, process_params: dict[str,
286
286
  'Oracle', 'MSSQL', 'MySQL'},
287
287
  It will create an IODataBase object to save/update the database in the
288
288
  specified format `save_method`, default is 'DataFrame' it save as
289
- binary pd.DataFrame object. More informations are available into
289
+ binary pl.DataFrame object. More informations are available into
290
290
  :mod:`dccd.tools.io`.
291
291
  io_params : dict, optional
292
292
  Dictionary of the keyword arguments available to the