dccd 2.3.2__tar.gz → 2.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. {dccd-2.3.2 → dccd-2.4.0}/CHANGELOG.md +39 -0
  2. dccd-2.4.0/PKG-INFO +300 -0
  3. dccd-2.4.0/README.md +227 -0
  4. dccd-2.4.0/dccd/daemon/api.py +702 -0
  5. {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/backfill.py +141 -20
  6. {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/cli.py +132 -6
  7. {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/config.py +11 -0
  8. {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/stream_manager.py +161 -21
  9. dccd-2.4.0/dccd/daemon/ui/__init__.py +4 -0
  10. dccd-2.4.0/dccd/daemon/ui/static/favicon.svg +27 -0
  11. dccd-2.4.0/dccd/daemon/ui/static/htmx.min.js +1 -0
  12. dccd-2.4.0/dccd/daemon/ui/static/logo.svg +59 -0
  13. dccd-2.4.0/dccd/daemon/ui/templates/base.html +160 -0
  14. dccd-2.4.0/dccd/daemon/ui/templates/config.html +225 -0
  15. dccd-2.4.0/dccd/daemon/ui/templates/dashboard.html +159 -0
  16. dccd-2.4.0/dccd/daemon/ui/templates/inventory.html +92 -0
  17. dccd-2.4.0/dccd/daemon/ui/templates/jobs.html +141 -0
  18. dccd-2.4.0/dccd/daemon/ui/templates/logs.html +19 -0
  19. dccd-2.4.0/dccd/daemon/ui/templates/partials/backfill_modal.html +123 -0
  20. dccd-2.4.0/dccd/daemon/ui/templates/storage.html +51 -0
  21. {dccd-2.3.2 → dccd-2.4.0}/dccd/histo_dl/binance.py +16 -12
  22. {dccd-2.3.2 → dccd-2.4.0}/dccd/histo_dl/bybit.py +13 -9
  23. {dccd-2.3.2 → dccd-2.4.0}/dccd/histo_dl/coinbase.py +45 -35
  24. {dccd-2.3.2 → dccd-2.4.0}/dccd/histo_dl/kraken.py +15 -11
  25. {dccd-2.3.2 → dccd-2.4.0}/dccd/histo_dl/okx.py +14 -10
  26. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_backfill.py +42 -4
  27. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_coinbase.py +23 -0
  28. dccd-2.4.0/dccd/tests/test_daemon_api.py +438 -0
  29. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_daemon_stream_manager.py +172 -4
  30. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_okx.py +24 -0
  31. dccd-2.4.0/dccd.egg-info/PKG-INFO +300 -0
  32. {dccd-2.3.2 → dccd-2.4.0}/dccd.egg-info/SOURCES.txt +15 -1
  33. {dccd-2.3.2 → dccd-2.4.0}/dccd.egg-info/requires.txt +12 -0
  34. {dccd-2.3.2 → dccd-2.4.0}/pyproject.toml +8 -4
  35. dccd-2.3.2/PKG-INFO +0 -316
  36. dccd-2.3.2/README.rst +0 -254
  37. dccd-2.3.2/dccd.egg-info/PKG-INFO +0 -316
  38. {dccd-2.3.2 → dccd-2.4.0}/CONTRIBUTING.md +0 -0
  39. {dccd-2.3.2 → dccd-2.4.0}/LICENSE.txt +0 -0
  40. {dccd-2.3.2 → dccd-2.4.0}/MANIFEST.in +0 -0
  41. {dccd-2.3.2 → dccd-2.4.0}/dccd/__init__.py +0 -0
  42. {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/__init__.py +0 -0
  43. {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/binance.py +0 -0
  44. {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/bitfinex.py +0 -0
  45. {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/bitmex.py +0 -0
  46. {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/bybit.py +0 -0
  47. {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/exchange.py +0 -0
  48. {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/kraken.py +0 -0
  49. {dccd-2.3.2 → dccd-2.4.0}/dccd/continuous_dl/okx.py +0 -0
  50. {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/__init__.py +0 -0
  51. {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/health.py +0 -0
  52. {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/scheduler.py +0 -0
  53. {dccd-2.3.2 → dccd-2.4.0}/dccd/daemon/storage.py +0 -0
  54. {dccd-2.3.2 → dccd-2.4.0}/dccd/histo_dl/__init__.py +0 -0
  55. {dccd-2.3.2 → dccd-2.4.0}/dccd/histo_dl/exchange.py +0 -0
  56. {dccd-2.3.2 → dccd-2.4.0}/dccd/models.py +0 -0
  57. {dccd-2.3.2 → dccd-2.4.0}/dccd/process_data.py +0 -0
  58. {dccd-2.3.2 → dccd-2.4.0}/dccd/storage.py +0 -0
  59. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/__init__.py +0 -0
  60. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/conftest.py +0 -0
  61. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_binance.py +0 -0
  62. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_binance_ws.py +0 -0
  63. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_bitfinex.py +0 -0
  64. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_bitmex.py +0 -0
  65. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_bybit.py +0 -0
  66. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_bybit_ws.py +0 -0
  67. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_daemon_cli.py +0 -0
  68. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_daemon_config.py +0 -0
  69. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_daemon_health.py +0 -0
  70. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_daemon_scheduler.py +0 -0
  71. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_daemon_storage.py +0 -0
  72. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_date_time.py +0 -0
  73. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_histo_dl.py +0 -0
  74. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_io.py +0 -0
  75. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_kraken.py +0 -0
  76. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_kraken_ws.py +0 -0
  77. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_models.py +0 -0
  78. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_okx_ws.py +0 -0
  79. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_process_data.py +0 -0
  80. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_storage.py +0 -0
  81. {dccd-2.3.2 → dccd-2.4.0}/dccd/tests/test_websocket.py +0 -0
  82. {dccd-2.3.2 → dccd-2.4.0}/dccd/tools/__init__.py +0 -0
  83. {dccd-2.3.2 → dccd-2.4.0}/dccd/tools/date_time.py +0 -0
  84. {dccd-2.3.2 → dccd-2.4.0}/dccd/tools/io.py +0 -0
  85. {dccd-2.3.2 → dccd-2.4.0}/dccd/tools/websocket.py +0 -0
  86. {dccd-2.3.2 → dccd-2.4.0}/dccd.egg-info/dependency_links.txt +0 -0
  87. {dccd-2.3.2 → dccd-2.4.0}/dccd.egg-info/entry_points.txt +0 -0
  88. {dccd-2.3.2 → dccd-2.4.0}/dccd.egg-info/top_level.txt +0 -0
  89. {dccd-2.3.2 → dccd-2.4.0}/setup.cfg +0 -0
@@ -6,6 +6,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [2.4.0] - 2026-06-04
10
+
11
+ ### Added
12
+
13
+ - `dccd/daemon/api.py` — web UI and JSON API (FastAPI + Jinja2 + htmx): a thin
14
+ HTTP layer over the existing daemon modules exposing dashboard (live health
15
+ metrics), inventory (stored data coverage), jobs (histo/stream list + add/remove
16
+ + live backfill progress), logs (tail), config (view/validate/save the YAML),
17
+ and storage (rclone status + manual sync). JSON-only API (`/api/*`) with
18
+ dumb-shell templates, so the front-end can be swapped without touching the API.
19
+ Optional Bearer-token auth via `settings.ui_auth_token`
20
+ - `dccd/daemon/cli.py` — `dccd ui`: serve the web UI standalone; the UI is also
21
+ started automatically (background thread) by `dccd start` when the `[ui]` extra
22
+ is installed
23
+ - `dccd/daemon/config.py` — `SettingsConfig.ui_host`, `ui_port`, `ui_auth_token`:
24
+ web UI bind address, port, and optional auth token
25
+ - `dccd/daemon/backfill.py` — `progress_callback` and `stop_event` on
26
+ `_BackfillBase.run()` / `run_backfill()`: let the UI report live progress and
27
+ cancel a running backfill (defaults keep CLI behaviour unchanged)
28
+ - `dccd/daemon/stream_manager.py` — `SyncService` writes
29
+ `{local_path}/.dccd/last_sync.json` after each successful remote push, so the UI
30
+ can display the last sync time
31
+ - `pyproject.toml` — new optional extra `[ui]` (`fastapi`, `uvicorn[standard]`,
32
+ `jinja2`); install with `pip install dccd[daemon,ui]`
33
+
34
+ ## [2.3.3] - 2026-05-31
35
+
36
+ ### Added
37
+
38
+
39
+ - `doc/source/` — complete Sphinx documentation overhaul: redesigned homepage
40
+ with sphinx-design cards, captioned toctrees (Getting Started / Data Collection /
41
+ Reference), new pages (`installation`, `quickstart`, `changelog`, `cli`,
42
+ `configuration`, `models`, `storage`, `tools`, per-exchange histo/continuous pages),
43
+ adaptive light/dark logo and favicon, sticky top navbar with PyPI/GitHub/Fynance
44
+ links, hero header with inline logo+title, responsive layout (#59, #61)
45
+ - `README.md` — converted from RST to Markdown; inline logo+title header with
46
+ `<picture>` for light/dark mode switching; badges on two rows (#60)
47
+
9
48
  ## [2.3.2] - 2026-05-25
10
49
 
11
50
  ### Added
dccd-2.4.0/PKG-INFO ADDED
@@ -0,0 +1,300 @@
1
+ Metadata-Version: 2.4
2
+ Name: dccd
3
+ Version: 2.4.0
4
+ Summary: Download Crypto Currency Data from different exchanges.
5
+ Author-email: Arthur Bernard <arthur.bernard.92@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
8
+ Project-URL: Documentation, https://download-crypto-currencies-data.readthedocs.io/
9
+ Project-URL: Changelog, https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/CHANGELOG.md
10
+ Classifier: Development Status :: 5 - Production/Stable
11
+ Classifier: Intended Audience :: Financial and Insurance Industry
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Database
20
+ Classifier: Topic :: Office/Business :: Financial
21
+ Requires-Python: >=3.10
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE.txt
24
+ Requires-Dist: numpy>=1.26
25
+ Requires-Dist: polars>=0.20
26
+ Requires-Dist: pyarrow>=13
27
+ Requires-Dist: requests>=2.28
28
+ Requires-Dist: openpyxl>=3.1
29
+ Requires-Dist: websockets>=12.0
30
+ Requires-Dist: scipy>=1.10
31
+ Requires-Dist: SQLAlchemy>=2.0
32
+ Requires-Dist: tenacity>=8.0
33
+ Requires-Dist: pydantic>=2.0
34
+ Provides-Extra: io
35
+ Requires-Dist: polars>=0.20; extra == "io"
36
+ Requires-Dist: pyarrow>=13; extra == "io"
37
+ Provides-Extra: daemon
38
+ Requires-Dist: pyyaml>=6.0; extra == "daemon"
39
+ Requires-Dist: apscheduler<4,>=3.10; extra == "daemon"
40
+ Requires-Dist: typer>=0.12; extra == "daemon"
41
+ Requires-Dist: tqdm>=4.64; extra == "daemon"
42
+ Provides-Extra: ui
43
+ Requires-Dist: fastapi>=0.110; extra == "ui"
44
+ Requires-Dist: uvicorn[standard]>=0.29; extra == "ui"
45
+ Requires-Dist: jinja2>=3.1; extra == "ui"
46
+ Provides-Extra: dev
47
+ Requires-Dist: pytest>=7.4; extra == "dev"
48
+ Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
49
+ Requires-Dist: pytest-cov>=4.1; extra == "dev"
50
+ Requires-Dist: ruff>=0.4; extra == "dev"
51
+ Requires-Dist: interrogate>=1.5; extra == "dev"
52
+ Requires-Dist: mypy>=1.0; extra == "dev"
53
+ Requires-Dist: pyyaml>=6.0; extra == "dev"
54
+ Requires-Dist: apscheduler<4,>=3.10; extra == "dev"
55
+ Requires-Dist: typer>=0.12; extra == "dev"
56
+ Requires-Dist: tqdm>=4.64; extra == "dev"
57
+ Requires-Dist: fastapi>=0.110; extra == "dev"
58
+ Requires-Dist: uvicorn[standard]>=0.29; extra == "dev"
59
+ Requires-Dist: jinja2>=3.1; extra == "dev"
60
+ Requires-Dist: httpx>=0.27; extra == "dev"
61
+ Provides-Extra: doc
62
+ Requires-Dist: sphinx>=7.0; extra == "doc"
63
+ Requires-Dist: furo; extra == "doc"
64
+ Requires-Dist: numpydoc; extra == "doc"
65
+ Requires-Dist: sphinx-design; extra == "doc"
66
+ Requires-Dist: sphinx-copybutton; extra == "doc"
67
+ Requires-Dist: pyyaml>=6.0; extra == "doc"
68
+ Requires-Dist: apscheduler<4,>=3.10; extra == "doc"
69
+ Requires-Dist: fastapi>=0.110; extra == "doc"
70
+ Requires-Dist: uvicorn[standard]>=0.29; extra == "doc"
71
+ Requires-Dist: jinja2>=3.1; extra == "doc"
72
+ Dynamic: license-file
73
+
74
+ <picture>
75
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-dark-transparent.svg">
76
+ <img alt="dccd logo" src="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-light-transparent.svg" height="180px" align="left">
77
+ </picture>
78
+
79
+ # **Download Crypto-Currency Data**
80
+
81
+ [![Python versions](https://img.shields.io/pypi/pyversions/dccd)](https://pypi.org/project/dccd/)
82
+ [![PyPI](https://img.shields.io/pypi/v/dccd.svg)](https://pypi.org/project/dccd/)
83
+ [![PyPI status](https://img.shields.io/pypi/status/dccd.svg?colorB=blue)](https://pypi.org/project/dccd/)
84
+ [![CI](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/actions/workflows/ci.yml/badge.svg)](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/actions/workflows/ci.yml)
85
+ [![License](https://img.shields.io/github/license/ArthurBernard/Download_Crypto_Currencies_Data.svg)](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/LICENSE.txt)<br>
86
+ [![Documentation](https://readthedocs.org/projects/download-crypto-currencies-data/badge/?version=latest)](https://download-crypto-currencies-data.readthedocs.io/en/latest/)
87
+ [![Coverage](https://codecov.io/gh/ArthurBernard/Download_Crypto_Currencies_Data/branch/master/graph/badge.svg)](https://codecov.io/gh/ArthurBernard/Download_Crypto_Currencies_Data)
88
+ [![Docstring coverage](https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/badges/interrogate_badge.svg)](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data)
89
+ [![Downloads](https://pepy.tech/badge/dccd)](https://pepy.tech/project/dccd)
90
+
91
+ ___
92
+
93
+ Python package to download crypto-currency data (OHLCV, trades, order book) from multiple
94
+ exchanges via REST and WebSocket APIs. Data can be saved to CSV, Excel, SQLite, PostgreSQL,
95
+ or Parquet.
96
+
97
+ ## Installation
98
+
99
+ ```bash
100
+ pip install dccd
101
+ ```
102
+
103
+ With autonomous daemon support (APScheduler + PyYAML):
104
+
105
+ ```bash
106
+ pip install "dccd[daemon]"
107
+ ```
108
+
109
+ With the web UI (FastAPI + htmx — `dccd ui`):
110
+
111
+ ```bash
112
+ pip install "dccd[daemon,ui]"
113
+ ```
114
+
115
+ From source:
116
+
117
+ ```bash
118
+ git clone https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
119
+ cd Download_Crypto_Currencies_Data
120
+ pip install -e .
121
+ ```
122
+
123
+ ## Supported exchanges
124
+
125
+ | Exchange | REST OHLCV | REST Trades | REST Order Book | WS OHLCV | WS Trades | WS Order Book |
126
+ |----------|:----------:|:-----------:|:---------------:|:--------:|:---------:|:-------------:|
127
+ | Binance | ✓ | ✓ | ✓ | | ✓ | ✓ |
128
+ | Coinbase | ✓ | ✓ † | ✓ | | | |
129
+ | Kraken | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
130
+ | Bybit | ✓ | ✓ † | ✓ | | ✓ | ✓ |
131
+ | OKX | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
132
+ | Bitfinex | | | | ✓ \* | ✓ | ✓ |
133
+ | Bitmex | | | | | ✓ | ✓ |
134
+
135
+ \* Bitfinex WS OHLCV is aggregated from the trades stream via `get_ohlc_bitfinex`.
136
+ † Recent trades only (Bybit ≤ 1 000, Coinbase ≤ 100) — no deep historical pagination via the public REST API.
137
+
138
+ ## Presentation
139
+
140
+ **Historical Downloader** `dccd.histo_dl`
141
+ Download OHLCV data via REST APIs and save to disk. Supports chunked requests, automatic retry on rate-limit (HTTP 429), and incremental updates from the last saved timestamp.
142
+
143
+ **Continuous Downloader** `dccd.continuous_dl`
144
+ Stream real-time data (order book, trades) via WebSocket with automatic reconnection and configurable processing/saving callbacks.
145
+
146
+ **Daemon** `dccd.daemon`
147
+ Autonomous, server-side collector driven by a YAML config. Runs REST jobs on a schedule (APScheduler), opens WebSocket streams for real-time collection, and periodically syncs all local data to one or more remote destinations (NAS, S3, SFTP, …) via rclone. Multiple remotes and a configurable sync interval are supported; collection is never blocked by remote availability.
148
+
149
+ **Web UI** `dccd.daemon.api`
150
+ Optional browser interface (FastAPI + htmx) mirroring the CLI: dashboard of live health metrics, data inventory, job management (add/remove pairs, launch and cancel backfills), log tail, config editor, and remote-storage status. Runs standalone (`dccd ui`) or embedded in `dccd start`.
151
+
152
+ ### Output formats
153
+
154
+ Historical data can be saved as **CSV**, **Excel** (`.xlsx`), **SQLite**, **PostgreSQL** (via SQLAlchemy), or **Parquet**.
155
+ All DataFrames are native `polars.DataFrame`. A `pandas.DataFrame` can be obtained via `get_data(format='pandas')`.
156
+
157
+ ## Quick start
158
+
159
+ Historical data:
160
+
161
+ ```python
162
+ from dccd.histo_dl import FromBinance
163
+
164
+ obj = FromBinance('/path/to/data/', 'BTC', 3600, fiat='USDT')
165
+ obj.import_data(start='2024-01-01 00:00:00', end='2024-12-31 00:00:00')
166
+ obj.save(form='parquet')
167
+ df = obj.get_data() # polars DataFrame (default)
168
+ df_pd = obj.get_data(format='pandas') # pandas DataFrame (optional)
169
+ ```
170
+
171
+ Incremental update (resume from last saved point):
172
+
173
+ ```python
174
+ obj.import_data(start='last', end='now').save(form='parquet')
175
+ ```
176
+
177
+ Other exchanges:
178
+
179
+ ```python
180
+ from dccd.histo_dl import FromKraken, FromBybit, FromOKX
181
+
182
+ FromKraken('/path/', 'ETH', 3600).import_data(start='2024-01-01', end='now').save()
183
+ FromBybit('/path/', 'BTC', 86400).import_data(start='2024-01-01', end='now').save()
184
+ FromOKX('/path/', 'BTC', 3600).import_data(start='2024-01-01', end='now').save()
185
+ ```
186
+
187
+ Trades (historical or recent):
188
+
189
+ ```python
190
+ from dccd.histo_dl import FromBinance, FromKraken
191
+
192
+ obj = FromBinance('/path/', 'BTC', 3600, fiat='USDT')
193
+ obj.import_trades(start='2024-01-01 00:00:00', end='2024-01-02 00:00:00')
194
+ obj.save_trades(form='csv')
195
+ df = obj.trades_df # polars DataFrame — columns: TS, price, amount, type, tid
196
+
197
+ # Kraken also supports full history; Bybit/Coinbase return recent-only snapshots
198
+ FromKraken('/path/', 'BTC', 3600).import_trades(start='2024-01-01', end='2024-01-02').save_trades()
199
+ ```
200
+
201
+ Order book snapshot:
202
+
203
+ ```python
204
+ from dccd.histo_dl import FromOKX
205
+
206
+ obj = FromOKX('/path/', 'BTC', 3600)
207
+ obj.import_orderbook(depth=50)
208
+ obj.save_orderbook(form='csv')
209
+ df = obj.orderbook_df # columns: side, price, amount, count
210
+ ```
211
+
212
+ Daemon (autonomous collector) — `config.yml`:
213
+
214
+ ```yaml
215
+ settings:
216
+ data_path: /data/crypto/
217
+ timezone: UTC
218
+
219
+ storage:
220
+ remotes:
221
+ - provider: rclone
222
+ remote: "mynas:crypto/"
223
+ sync_interval: 3600
224
+
225
+ histo_jobs:
226
+ - exchange: binance
227
+ pairs: [BTC/USDT, ETH/USDT]
228
+ span: 3600
229
+ format: parquet
230
+
231
+ stream_jobs:
232
+ - exchange: binance
233
+ pairs: [BTC/USDT]
234
+ channels: [trades, book]
235
+ time_step: 60
236
+ ```
237
+
238
+ CLI quick start:
239
+
240
+ ```bash
241
+ # Validate the config
242
+ dccd validate --config config.yml
243
+
244
+ # Backfill all OHLC history defined in config (resumable)
245
+ dccd backfill --config config.yml --start "2020-01-01 00:00:00"
246
+
247
+ # Dry run — estimate windows and time without downloading
248
+ dccd backfill --config config.yml --dry-run
249
+
250
+ # Backfill only one exchange
251
+ dccd backfill --config config.yml --exchange kraken
252
+
253
+ # One incremental batch per job, then exit (for cron)
254
+ dccd collect --config config.yml
255
+
256
+ # Continuous daemon (Ctrl-C to stop)
257
+ dccd start --config config.yml
258
+
259
+ # Add / remove a histo job in-place
260
+ dccd add --exchange kraken --pair ETH/USD --span 86400 --config config.yml
261
+ dccd remove --exchange kraken --pair ETH/USD --span 86400 --config config.yml
262
+
263
+ # Inspect all data on disk (OHLC, trades, orderbook)
264
+ dccd inventory --config config.yml
265
+
266
+ # Web UI — dashboard, inventory, jobs, logs, config (needs: pip install "dccd[daemon,ui]")
267
+ dccd ui --config config.yml # http://127.0.0.1:8080
268
+
269
+ # Enable shell tab-completion (run once after install)
270
+ dccd --install-completion
271
+ ```
272
+
273
+ > `--config` is optional — dccd searches `./config.yml` then `~/.config/dccd/config.yml` when omitted.
274
+
275
+ Python API:
276
+
277
+ ```python
278
+ from dccd.daemon.config import load_config
279
+ from dccd.daemon.scheduler import run_once, build_histo_scheduler
280
+ from dccd.daemon.stream_manager import StreamManager
281
+
282
+ config = load_config('config.yml')
283
+
284
+ # One-shot: download all histo jobs once, then exit
285
+ run_once(config)
286
+
287
+ # Daemon mode: periodic REST + live WebSocket streams
288
+ scheduler = build_histo_scheduler(config)
289
+ scheduler.start()
290
+
291
+ mgr = StreamManager(config)
292
+ mgr.start() # runs until mgr.stop() is called
293
+ ```
294
+
295
+ ## Links
296
+
297
+ - PyPI: https://pypi.org/project/dccd/
298
+ - Documentation: https://download-crypto-currencies-data.readthedocs.io/
299
+ - Source: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
300
+ - Changelog: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/CHANGELOG.md
dccd-2.4.0/README.md ADDED
@@ -0,0 +1,227 @@
1
+ <picture>
2
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-dark-transparent.svg">
3
+ <img alt="dccd logo" src="https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/develop/doc/source/_static/logo-light-transparent.svg" height="180px" align="left">
4
+ </picture>
5
+
6
+ # **Download Crypto-Currency Data**
7
+
8
+ [![Python versions](https://img.shields.io/pypi/pyversions/dccd)](https://pypi.org/project/dccd/)
9
+ [![PyPI](https://img.shields.io/pypi/v/dccd.svg)](https://pypi.org/project/dccd/)
10
+ [![PyPI status](https://img.shields.io/pypi/status/dccd.svg?colorB=blue)](https://pypi.org/project/dccd/)
11
+ [![CI](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/actions/workflows/ci.yml/badge.svg)](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/actions/workflows/ci.yml)
12
+ [![License](https://img.shields.io/github/license/ArthurBernard/Download_Crypto_Currencies_Data.svg)](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/LICENSE.txt)<br>
13
+ [![Documentation](https://readthedocs.org/projects/download-crypto-currencies-data/badge/?version=latest)](https://download-crypto-currencies-data.readthedocs.io/en/latest/)
14
+ [![Coverage](https://codecov.io/gh/ArthurBernard/Download_Crypto_Currencies_Data/branch/master/graph/badge.svg)](https://codecov.io/gh/ArthurBernard/Download_Crypto_Currencies_Data)
15
+ [![Docstring coverage](https://raw.githubusercontent.com/ArthurBernard/Download_Crypto_Currencies_Data/badges/interrogate_badge.svg)](https://github.com/ArthurBernard/Download_Crypto_Currencies_Data)
16
+ [![Downloads](https://pepy.tech/badge/dccd)](https://pepy.tech/project/dccd)
17
+
18
+ ___
19
+
20
+ Python package to download crypto-currency data (OHLCV, trades, order book) from multiple
21
+ exchanges via REST and WebSocket APIs. Data can be saved to CSV, Excel, SQLite, PostgreSQL,
22
+ or Parquet.
23
+
24
+ ## Installation
25
+
26
+ ```bash
27
+ pip install dccd
28
+ ```
29
+
30
+ With autonomous daemon support (APScheduler + PyYAML):
31
+
32
+ ```bash
33
+ pip install "dccd[daemon]"
34
+ ```
35
+
36
+ With the web UI (FastAPI + htmx — `dccd ui`):
37
+
38
+ ```bash
39
+ pip install "dccd[daemon,ui]"
40
+ ```
41
+
42
+ From source:
43
+
44
+ ```bash
45
+ git clone https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
46
+ cd Download_Crypto_Currencies_Data
47
+ pip install -e .
48
+ ```
49
+
50
+ ## Supported exchanges
51
+
52
+ | Exchange | REST OHLCV | REST Trades | REST Order Book | WS OHLCV | WS Trades | WS Order Book |
53
+ |----------|:----------:|:-----------:|:---------------:|:--------:|:---------:|:-------------:|
54
+ | Binance | ✓ | ✓ | ✓ | | ✓ | ✓ |
55
+ | Coinbase | ✓ | ✓ † | ✓ | | | |
56
+ | Kraken | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
57
+ | Bybit | ✓ | ✓ † | ✓ | | ✓ | ✓ |
58
+ | OKX | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
59
+ | Bitfinex | | | | ✓ \* | ✓ | ✓ |
60
+ | Bitmex | | | | | ✓ | ✓ |
61
+
62
+ \* Bitfinex WS OHLCV is aggregated from the trades stream via `get_ohlc_bitfinex`.
63
+ † Recent trades only (Bybit ≤ 1 000, Coinbase ≤ 100) — no deep historical pagination via the public REST API.
64
+
65
+ ## Presentation
66
+
67
+ **Historical Downloader** `dccd.histo_dl`
68
+ Download OHLCV data via REST APIs and save to disk. Supports chunked requests, automatic retry on rate-limit (HTTP 429), and incremental updates from the last saved timestamp.
69
+
70
+ **Continuous Downloader** `dccd.continuous_dl`
71
+ Stream real-time data (order book, trades) via WebSocket with automatic reconnection and configurable processing/saving callbacks.
72
+
73
+ **Daemon** `dccd.daemon`
74
+ Autonomous, server-side collector driven by a YAML config. Runs REST jobs on a schedule (APScheduler), opens WebSocket streams for real-time collection, and periodically syncs all local data to one or more remote destinations (NAS, S3, SFTP, …) via rclone. Multiple remotes and a configurable sync interval are supported; collection is never blocked by remote availability.
75
+
76
+ **Web UI** `dccd.daemon.api`
77
+ Optional browser interface (FastAPI + htmx) mirroring the CLI: dashboard of live health metrics, data inventory, job management (add/remove pairs, launch and cancel backfills), log tail, config editor, and remote-storage status. Runs standalone (`dccd ui`) or embedded in `dccd start`.
78
+
79
+ ### Output formats
80
+
81
+ Historical data can be saved as **CSV**, **Excel** (`.xlsx`), **SQLite**, **PostgreSQL** (via SQLAlchemy), or **Parquet**.
82
+ All DataFrames are native `polars.DataFrame`. A `pandas.DataFrame` can be obtained via `get_data(format='pandas')`.
83
+
84
+ ## Quick start
85
+
86
+ Historical data:
87
+
88
+ ```python
89
+ from dccd.histo_dl import FromBinance
90
+
91
+ obj = FromBinance('/path/to/data/', 'BTC', 3600, fiat='USDT')
92
+ obj.import_data(start='2024-01-01 00:00:00', end='2024-12-31 00:00:00')
93
+ obj.save(form='parquet')
94
+ df = obj.get_data() # polars DataFrame (default)
95
+ df_pd = obj.get_data(format='pandas') # pandas DataFrame (optional)
96
+ ```
97
+
98
+ Incremental update (resume from last saved point):
99
+
100
+ ```python
101
+ obj.import_data(start='last', end='now').save(form='parquet')
102
+ ```
103
+
104
+ Other exchanges:
105
+
106
+ ```python
107
+ from dccd.histo_dl import FromKraken, FromBybit, FromOKX
108
+
109
+ FromKraken('/path/', 'ETH', 3600).import_data(start='2024-01-01', end='now').save()
110
+ FromBybit('/path/', 'BTC', 86400).import_data(start='2024-01-01', end='now').save()
111
+ FromOKX('/path/', 'BTC', 3600).import_data(start='2024-01-01', end='now').save()
112
+ ```
113
+
114
+ Trades (historical or recent):
115
+
116
+ ```python
117
+ from dccd.histo_dl import FromBinance, FromKraken
118
+
119
+ obj = FromBinance('/path/', 'BTC', 3600, fiat='USDT')
120
+ obj.import_trades(start='2024-01-01 00:00:00', end='2024-01-02 00:00:00')
121
+ obj.save_trades(form='csv')
122
+ df = obj.trades_df # polars DataFrame — columns: TS, price, amount, type, tid
123
+
124
+ # Kraken also supports full history; Bybit/Coinbase return recent-only snapshots
125
+ FromKraken('/path/', 'BTC', 3600).import_trades(start='2024-01-01', end='2024-01-02').save_trades()
126
+ ```
127
+
128
+ Order book snapshot:
129
+
130
+ ```python
131
+ from dccd.histo_dl import FromOKX
132
+
133
+ obj = FromOKX('/path/', 'BTC', 3600)
134
+ obj.import_orderbook(depth=50)
135
+ obj.save_orderbook(form='csv')
136
+ df = obj.orderbook_df # columns: side, price, amount, count
137
+ ```
138
+
139
+ Daemon (autonomous collector) — `config.yml`:
140
+
141
+ ```yaml
142
+ settings:
143
+ data_path: /data/crypto/
144
+ timezone: UTC
145
+
146
+ storage:
147
+ remotes:
148
+ - provider: rclone
149
+ remote: "mynas:crypto/"
150
+ sync_interval: 3600
151
+
152
+ histo_jobs:
153
+ - exchange: binance
154
+ pairs: [BTC/USDT, ETH/USDT]
155
+ span: 3600
156
+ format: parquet
157
+
158
+ stream_jobs:
159
+ - exchange: binance
160
+ pairs: [BTC/USDT]
161
+ channels: [trades, book]
162
+ time_step: 60
163
+ ```
164
+
165
+ CLI quick start:
166
+
167
+ ```bash
168
+ # Validate the config
169
+ dccd validate --config config.yml
170
+
171
+ # Backfill all OHLC history defined in config (resumable)
172
+ dccd backfill --config config.yml --start "2020-01-01 00:00:00"
173
+
174
+ # Dry run — estimate windows and time without downloading
175
+ dccd backfill --config config.yml --dry-run
176
+
177
+ # Backfill only one exchange
178
+ dccd backfill --config config.yml --exchange kraken
179
+
180
+ # One incremental batch per job, then exit (for cron)
181
+ dccd collect --config config.yml
182
+
183
+ # Continuous daemon (Ctrl-C to stop)
184
+ dccd start --config config.yml
185
+
186
+ # Add / remove a histo job in-place
187
+ dccd add --exchange kraken --pair ETH/USD --span 86400 --config config.yml
188
+ dccd remove --exchange kraken --pair ETH/USD --span 86400 --config config.yml
189
+
190
+ # Inspect all data on disk (OHLC, trades, orderbook)
191
+ dccd inventory --config config.yml
192
+
193
+ # Web UI — dashboard, inventory, jobs, logs, config (needs: pip install "dccd[daemon,ui]")
194
+ dccd ui --config config.yml # http://127.0.0.1:8080
195
+
196
+ # Enable shell tab-completion (run once after install)
197
+ dccd --install-completion
198
+ ```
199
+
200
+ > `--config` is optional — dccd searches `./config.yml` then `~/.config/dccd/config.yml` when omitted.
201
+
202
+ Python API:
203
+
204
+ ```python
205
+ from dccd.daemon.config import load_config
206
+ from dccd.daemon.scheduler import run_once, build_histo_scheduler
207
+ from dccd.daemon.stream_manager import StreamManager
208
+
209
+ config = load_config('config.yml')
210
+
211
+ # One-shot: download all histo jobs once, then exit
212
+ run_once(config)
213
+
214
+ # Daemon mode: periodic REST + live WebSocket streams
215
+ scheduler = build_histo_scheduler(config)
216
+ scheduler.start()
217
+
218
+ mgr = StreamManager(config)
219
+ mgr.start() # runs until mgr.stop() is called
220
+ ```
221
+
222
+ ## Links
223
+
224
+ - PyPI: https://pypi.org/project/dccd/
225
+ - Documentation: https://download-crypto-currencies-data.readthedocs.io/
226
+ - Source: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data
227
+ - Changelog: https://github.com/ArthurBernard/Download_Crypto_Currencies_Data/blob/master/CHANGELOG.md