cryptofeed 3.0.0__tar.gz → 3.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/CHANGES.md +48 -2
- {cryptofeed-3.0.0/cryptofeed.egg-info → cryptofeed-3.0.2}/PKG-INFO +26 -4
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/README.md +23 -2
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/backend.py +71 -24
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/mongo.py +12 -7
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/postgres.py +22 -2
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/redis.py +57 -43
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/capture/playback.py +164 -37
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/capture/recorder.py +2 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/config.py +15 -12
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/connection.py +84 -76
- cryptofeed-3.0.2/cryptofeed/connection_handler.py +192 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/defines.py +2 -2
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/__init__.py +4 -4
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/binance.py +92 -19
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/binance_delivery.py +8 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/binance_futures.py +8 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/bitfinex.py +56 -47
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/bitget.py +38 -9
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/bithumb.py +19 -6
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/bybit.py +20 -45
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/coinbase.py +102 -35
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/cryptodotcom.py +58 -50
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/deribit.py +16 -19
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/dydx.py +61 -14
- cryptofeed-3.0.0/cryptofeed/exchanges/gateio.py → cryptofeed-3.0.2/cryptofeed/exchanges/gate.py +66 -16
- cryptofeed-3.0.0/cryptofeed/exchanges/gateio_futures.py → cryptofeed-3.0.2/cryptofeed/exchanges/gate_futures.py +68 -42
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/gemini.py +42 -8
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/htx.py +12 -4
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/htx_swap.py +10 -4
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/hyperliquid.py +20 -6
- cryptofeed-3.0.2/cryptofeed/exchanges/independent_reserve.py +359 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/kraken.py +89 -23
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/kraken_futures.py +55 -19
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/kucoin.py +37 -12
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/kucoin_futures.py +11 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/mexc.py +59 -42
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/okx.py +16 -9
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/phemex.py +51 -28
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/poloniex.py +14 -18
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/feed.py +79 -32
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/feedhandler.py +24 -2
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/log.py +1 -1
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/symbols.py +3 -2
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/types.pyi +22 -4
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/types.pyx +7 -1
- cryptofeed-3.0.2/cryptofeed/util/history.py +156 -0
- cryptofeed-3.0.2/cryptofeed/util/snapshots.py +151 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2/cryptofeed.egg-info}/PKG-INFO +26 -4
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed.egg-info/SOURCES.txt +4 -2
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/pyproject.toml +4 -3
- cryptofeed-3.0.0/cryptofeed/connection_handler.py +0 -157
- cryptofeed-3.0.0/cryptofeed/exchanges/independent_reserve.py +0 -267
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/LICENSE +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/MANIFEST.in +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/__init__.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/_json.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/__init__.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/_line_protocol.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/http.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/influxdb.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/kafka.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/quest.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/socket.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/zmq.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/callback.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/capture/__init__.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/capture/_zstd.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/capture/pcap.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exceptions.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchange.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/binance_us.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/bitflyer.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/bitstamp.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/upbit.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/nbbo.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/py.typed +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/util/__init__.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/util/time.py +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed.egg-info/dependency_links.txt +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed.egg-info/requires.txt +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed.egg-info/top_level.txt +0 -0
- {cryptofeed-3.0.0 → cryptofeed-3.0.2}/setup.cfg +0 -0
|
@@ -1,5 +1,51 @@
|
|
|
1
1
|
## Changelog
|
|
2
2
|
|
|
3
|
+
### 3.0.2 (2026-10-29)
|
|
4
|
+
* Bugfix: Bybit liquidation side is now the side of the liquidation order (SELL when a long is liquidated), as for Binance and OKX
|
|
5
|
+
* Bugfix: Deribit liquidations flagged `M` (maker liquidated) reported the taker's side
|
|
6
|
+
* Bugfix: Bybit open interest now uses `singleOpenInterest`; `openInterest` counts both sides (2x other exchanges)
|
|
7
|
+
* Bugfix: reconnection of unstable feeds adjusted with backoffs
|
|
8
|
+
* Bugfix: the silence watchdog starts only after subscribe
|
|
9
|
+
* Bugfix: HTTP requests time out after 60s (10 s to connect)
|
|
10
|
+
* Bugfix: log_message_on_error raised UnboundLocalError when the read failed before the first message
|
|
11
|
+
* Bugfix: improvements to `log` section in config
|
|
12
|
+
* Update: the default Feed `retries` is now 15
|
|
13
|
+
* Update: Python 3.15 and free-threaded 3.15t are built, tested and shipped as wheels
|
|
14
|
+
* Bugfix: Gemini no longer emits trades for symbols that were only subscribed to L2_BOOK
|
|
15
|
+
* Update: OKX websocket endpoints no longer use port 8443, which OKX retires on 2026-10-31
|
|
16
|
+
* Bugfix: Bithumb no longer emits book or ticker updates for symbols that were not subscribed to that channel
|
|
17
|
+
* Bugfix: Various data improvements across exchanges
|
|
18
|
+
* Bugfix: connection hardening, improved retries, improved snapshot syncing
|
|
19
|
+
* Bugfix: Poloniex trades corrected
|
|
20
|
+
* Bugfix: Phemex spot values were scaled incorrectly
|
|
21
|
+
* Bugfix: set Gate Futures decimal request header correctly
|
|
22
|
+
* Bugfix: Gemini perpetual instruments resolved correctly
|
|
23
|
+
* Bugfix: Hyperliquid spot instruments resolved correctly
|
|
24
|
+
* Bugfix: Poloniex instruments resolved correctly
|
|
25
|
+
* Bugfix: Bitfinex max_depth limited correctly when checksum validation is enabled
|
|
26
|
+
* Bugfix: Bybit ticker subscription fixed
|
|
27
|
+
* Bugfix: Binance Funding data correction
|
|
28
|
+
* Bugfix: backends handle many feeds, single failure scenario correctly
|
|
29
|
+
* Bugfix: Postgres backend retries through a PostgreSQL restart instead of dying
|
|
30
|
+
* Bugfix: book snapshots serialized with numeric_type=Decimal were not valid JSON. price keys now strings
|
|
31
|
+
* Bugfix: Redis incorrectly ignored `socket` path
|
|
32
|
+
* Update: expiry dates in normalized symbols now zero padded for every exchange (eg. BTC-USD-26H05)
|
|
33
|
+
* Bugfix: the shutdown enforced on backend writers
|
|
34
|
+
* Bugfix: cancelling a backend writer no longer loses messages from the stats counters
|
|
35
|
+
* Bugfix: a backend instance can be reused across runs
|
|
36
|
+
* Bugfix: BookMongo stores deltas as book documents and accept numeric_type=float
|
|
37
|
+
* Bugfix: Redis Stream and ZSet backends retry only rejected commands
|
|
38
|
+
* Bugfix: Postgres backends render dict and list values as JSON
|
|
39
|
+
* Feature: Postgres backends define table layouts and provide create_table_sql()
|
|
40
|
+
* Update: Postgres backends default numeric_type to Decimal, so NUMERIC columns receive exact values
|
|
41
|
+
|
|
42
|
+
### 3.0.1 (2026-09-27)
|
|
43
|
+
* Update: Gate.io -> Gate rebranding
|
|
44
|
+
* Bugfix: Binance Futures and Binance Delivery L2_BOOK with `depth_interval='250ms'` subscribed to `<symbol>@depth@250ms`, which Binance accepts but never sends data on. The default interval now uses the bare `<symbol>@depth` stream.
|
|
45
|
+
* Bugfix: `depth_interval=None` on Binance Futures and Binance Delivery produced an invalid subscription. `None` now resolves to the venue default
|
|
46
|
+
* Bugfix: Coinbase, OKX, Gate, Kraken data quality issues
|
|
47
|
+
* Bugfix: dYdX subscribe/reconnect connection issues
|
|
48
|
+
|
|
3
49
|
### 3.0.0 (2026-09-21)
|
|
4
50
|
* Feature: New asyncio core
|
|
5
51
|
* Update: REST endpoints removed (trading functionality moves to the cryptotrade project)
|
|
@@ -120,7 +166,7 @@
|
|
|
120
166
|
* Update: Exchange name change OKEx -> OKX
|
|
121
167
|
* Bugfix: OKX candle REST code was setting values incorrectly
|
|
122
168
|
* Update: OKX now uses v5 for all connections (REST and WS). Update endpoints to new exchange name: okex.com -> okx.com
|
|
123
|
-
|
|
169
|
+
|
|
124
170
|
### 2.2.0 (2021-02-16)
|
|
125
171
|
* Feature: New exchange: Bit.com
|
|
126
172
|
* Feature: Rework how exchanges that have multiple websocket endpoints are managed and configured.
|
|
@@ -147,7 +193,7 @@
|
|
|
147
193
|
* Feature: Add ByBit sandbox endpoints.
|
|
148
194
|
* Bugfix: Fix calculation in OrderInfo on Binance.
|
|
149
195
|
* Feature: Support list of bootstrap servers for Kafka backend.
|
|
150
|
-
* Feature: Add OrderInfo and Fills zmq callbacks
|
|
196
|
+
* Feature: Add OrderInfo and Fills zmq callbacks
|
|
151
197
|
|
|
152
198
|
### 2.1.1 (2021-11-29)
|
|
153
199
|
* Bugfix: Position data type missing side field.
|
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cryptofeed
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.2
|
|
4
4
|
Summary: Cryptocurrency Exchange Websocket Data Feed Handler
|
|
5
5
|
Author-email: Bryant Moscon <bmoscon@gmail.com>
|
|
6
6
|
License-Expression: AGPL-3.0-or-later
|
|
7
7
|
Project-URL: Homepage, https://github.com/bmoscon/cryptofeed
|
|
8
|
-
Keywords: cryptocurrency,bitcoin,btc,feed handler,market feed,market data,crypto assets,Trades,Tickers,BBO,Funding,Open Interest,Liquidation,Order book,Bid,Ask,Binance,Binance Delivery,Binance Futures,Binance US,Bitfinex,bitFlyer,Bitget,Bithumb,Bitstamp,Bybit,Coinbase,Crypto.com,Deribit,Gate
|
|
8
|
+
Keywords: cryptocurrency,bitcoin,btc,feed handler,market feed,market data,crypto assets,Trades,Tickers,BBO,Funding,Open Interest,Liquidation,Order book,Bid,Ask,Binance,Binance Delivery,Binance Futures,Binance US,Bitfinex,bitFlyer,Bitget,Bithumb,Bitstamp,Bybit,Coinbase,Crypto.com,Deribit,Gate,Gate Futures,dYdX,Gemini,Hyperliquid,HTX,HTX Swap,Independent Reserve,Kraken,Kraken Futures,KuCoin,MEXC,OKX,Phemex,Poloniex,Upbit
|
|
9
9
|
Classifier: Intended Audience :: Developers
|
|
10
10
|
Classifier: Development Status :: 4 - Beta
|
|
11
11
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.13
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.14
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
14
15
|
Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta
|
|
15
16
|
Classifier: Framework :: AsyncIO
|
|
16
17
|
Classifier: Operating System :: POSIX :: Linux
|
|
@@ -70,8 +71,8 @@ Handles multiple cryptocurrency exchange data feeds and returns normalized and s
|
|
|
70
71
|
* [Coinbase](https://www.coinbase.com/)
|
|
71
72
|
* [Crypto.com](https://www.crypto.com)
|
|
72
73
|
* [Deribit](https://www.deribit.com/)
|
|
73
|
-
* [Gate
|
|
74
|
-
* [Gate
|
|
74
|
+
* [Gate](https://www.gate.com/)
|
|
75
|
+
* [Gate Futures](https://www.gate.com/futures)
|
|
75
76
|
* [dYdX v4](https://dydx.trade/)
|
|
76
77
|
* [Gemini](https://gemini.com/)
|
|
77
78
|
* [Hyperliquid](https://hyperliquid.xyz/)
|
|
@@ -152,6 +153,27 @@ Cryptofeed supports the following public data channels from exchanges
|
|
|
152
153
|
* INDEX
|
|
153
154
|
* CANDLES - Candlestick / K-Line data.
|
|
154
155
|
|
|
156
|
+
## Normalized data
|
|
157
|
+
|
|
158
|
+
Every callback receives two arguments: a normalized object and `receipt_timestamp`, the time (float epoch seconds)
|
|
159
|
+
cryptofeed received the message that produced the callback. The objects are defined in `cryptofeed/types.pyx` (`Trade`,
|
|
160
|
+
`Ticker`, `OrderBook`, `L1Book`, `Candle`, `Funding`, `OpenInterest`, `Liquidation`, `Index`) and share these rules:
|
|
161
|
+
|
|
162
|
+
* `exchange` is the venue id and `symbol` the normalized name: `BASE-QUOTE` for spot, `BASE-QUOTE-PERP` for
|
|
163
|
+
perpetuals, `BASE-QUOTE-<expiry>` for dated futures with the expiry as a two-digit year, a futures month code and
|
|
164
|
+
a zero-padded day (`BTC-USD-26H05`), and `BASE-QUOTE-<strike>-<expiry>-CALL|PUT` for options.
|
|
165
|
+
* Prices, sizes, rates and volumes are `Decimal`. `numeric_type` on a backend or `to_dict()` converts them.
|
|
166
|
+
* `timestamp` is the venue's event time in float epoch seconds. `timestamp` is `None` when the venue does not provide
|
|
167
|
+
a timestamp and backends then store the receipt time in its place.
|
|
168
|
+
* `Trade.side` is the taker's side. `Trade.id` is the venue's id or `None`.
|
|
169
|
+
* `OrderBook.book` holds the full local book (`.bids`, `.asks`). `delta` is `None` for a snapshot and otherwise lists
|
|
170
|
+
the levels that changed as `(price, size)` with size 0 for a removed level. `sequence_number` and `checksum` are
|
|
171
|
+
set when the venue provides them.
|
|
172
|
+
* `Candle.closed` is `True` once the bar is final, by default only closed candles are delivered.
|
|
173
|
+
(`candle_closed_only=False` also delivers each update of the open bar).
|
|
174
|
+
* `trade_history` and `candle_history` control whether the batch of recent trades or candles some venues send on
|
|
175
|
+
subscribe is delivered, by default it is `False`.
|
|
176
|
+
|
|
155
177
|
## Backends
|
|
156
178
|
|
|
157
179
|
Cryptofeed supports `backend` callbacks that will write directly to storage or other interfaces.
|
|
@@ -21,8 +21,8 @@ Handles multiple cryptocurrency exchange data feeds and returns normalized and s
|
|
|
21
21
|
* [Coinbase](https://www.coinbase.com/)
|
|
22
22
|
* [Crypto.com](https://www.crypto.com)
|
|
23
23
|
* [Deribit](https://www.deribit.com/)
|
|
24
|
-
* [Gate
|
|
25
|
-
* [Gate
|
|
24
|
+
* [Gate](https://www.gate.com/)
|
|
25
|
+
* [Gate Futures](https://www.gate.com/futures)
|
|
26
26
|
* [dYdX v4](https://dydx.trade/)
|
|
27
27
|
* [Gemini](https://gemini.com/)
|
|
28
28
|
* [Hyperliquid](https://hyperliquid.xyz/)
|
|
@@ -103,6 +103,27 @@ Cryptofeed supports the following public data channels from exchanges
|
|
|
103
103
|
* INDEX
|
|
104
104
|
* CANDLES - Candlestick / K-Line data.
|
|
105
105
|
|
|
106
|
+
## Normalized data
|
|
107
|
+
|
|
108
|
+
Every callback receives two arguments: a normalized object and `receipt_timestamp`, the time (float epoch seconds)
|
|
109
|
+
cryptofeed received the message that produced the callback. The objects are defined in `cryptofeed/types.pyx` (`Trade`,
|
|
110
|
+
`Ticker`, `OrderBook`, `L1Book`, `Candle`, `Funding`, `OpenInterest`, `Liquidation`, `Index`) and share these rules:
|
|
111
|
+
|
|
112
|
+
* `exchange` is the venue id and `symbol` the normalized name: `BASE-QUOTE` for spot, `BASE-QUOTE-PERP` for
|
|
113
|
+
perpetuals, `BASE-QUOTE-<expiry>` for dated futures with the expiry as a two-digit year, a futures month code and
|
|
114
|
+
a zero-padded day (`BTC-USD-26H05`), and `BASE-QUOTE-<strike>-<expiry>-CALL|PUT` for options.
|
|
115
|
+
* Prices, sizes, rates and volumes are `Decimal`. `numeric_type` on a backend or `to_dict()` converts them.
|
|
116
|
+
* `timestamp` is the venue's event time in float epoch seconds. `timestamp` is `None` when the venue does not provide
|
|
117
|
+
a timestamp and backends then store the receipt time in its place.
|
|
118
|
+
* `Trade.side` is the taker's side. `Trade.id` is the venue's id or `None`.
|
|
119
|
+
* `OrderBook.book` holds the full local book (`.bids`, `.asks`). `delta` is `None` for a snapshot and otherwise lists
|
|
120
|
+
the levels that changed as `(price, size)` with size 0 for a removed level. `sequence_number` and `checksum` are
|
|
121
|
+
set when the venue provides them.
|
|
122
|
+
* `Candle.closed` is `True` once the bar is final, by default only closed candles are delivered.
|
|
123
|
+
(`candle_closed_only=False` also delivers each update of the open bar).
|
|
124
|
+
* `trade_history` and `candle_history` control whether the batch of recent trades or candles some venues send on
|
|
125
|
+
subscribe is delivered, by default it is `False`.
|
|
126
|
+
|
|
106
127
|
## Backends
|
|
107
128
|
|
|
108
129
|
Cryptofeed supports `backend` callbacks that will write directly to storage or other interfaces.
|
|
@@ -35,11 +35,12 @@ class RetryPolicy:
|
|
|
35
35
|
max_attempts: int = 5
|
|
36
36
|
base: float = 0.5
|
|
37
37
|
jitter: bool = True
|
|
38
|
+
max_delay: float = 60.0
|
|
38
39
|
|
|
39
40
|
def delay(self, attempt: int) -> float:
|
|
40
|
-
d = self.base * (2 ** (attempt - 1))
|
|
41
|
+
d = min(self.base * (2 ** (attempt - 1)), self.max_delay)
|
|
41
42
|
if self.jitter:
|
|
42
|
-
d *= random.uniform(0
|
|
43
|
+
d *= random.uniform(1.0, 1.5)
|
|
43
44
|
return d
|
|
44
45
|
|
|
45
46
|
|
|
@@ -61,6 +62,24 @@ class BackendStats:
|
|
|
61
62
|
return self.written + self.conflicts
|
|
62
63
|
|
|
63
64
|
|
|
65
|
+
async def finish_writers(backends: list, deadline: float):
|
|
66
|
+
if not backends:
|
|
67
|
+
return
|
|
68
|
+
try:
|
|
69
|
+
async with asyncio.timeout(deadline):
|
|
70
|
+
for backend in backends:
|
|
71
|
+
await backend.signal()
|
|
72
|
+
|
|
73
|
+
pending = [backend.worker for backend in backends if not backend.worker.done()]
|
|
74
|
+
if pending:
|
|
75
|
+
await asyncio.wait(pending)
|
|
76
|
+
except TimeoutError:
|
|
77
|
+
pending = [backend for backend in backends if not backend.worker.done()]
|
|
78
|
+
LOG.warning('backend flush exceeded the %.1fs shutdown deadline - cancelling %d writer(s)', deadline, len(pending))
|
|
79
|
+
for backend in pending:
|
|
80
|
+
backend.worker.cancel()
|
|
81
|
+
|
|
82
|
+
|
|
64
83
|
class BackendQueue:
|
|
65
84
|
hand_off = False
|
|
66
85
|
max_depth = 10_000
|
|
@@ -71,6 +90,7 @@ class BackendQueue:
|
|
|
71
90
|
retry = RetryPolicy()
|
|
72
91
|
retryable_exceptions: tuple = ()
|
|
73
92
|
SENTINEL_TIMEOUT = 1.0
|
|
93
|
+
CLOSE_TIMEOUT = 0.25
|
|
74
94
|
BLOCK_POLL = 0.25
|
|
75
95
|
FAILURE_LEVEL = logging.ERROR
|
|
76
96
|
|
|
@@ -105,6 +125,7 @@ class BackendQueue:
|
|
|
105
125
|
self._orphaned = False
|
|
106
126
|
self._owners = 0
|
|
107
127
|
self._owners_seen = []
|
|
128
|
+
self._shared = False
|
|
108
129
|
self._inflight = []
|
|
109
130
|
|
|
110
131
|
@property
|
|
@@ -117,17 +138,22 @@ class BackendQueue:
|
|
|
117
138
|
conflicts=self._conflicts, retries=self._retries, last_error=self._last_error,
|
|
118
139
|
last_write_ts=self._last_write_ts, hand_off=bool(getattr(self, 'hand_off', False)))
|
|
119
140
|
|
|
120
|
-
def start_writer(self, tg: asyncio.TaskGroup, name: str, owner=None):
|
|
141
|
+
def start_writer(self, tg: asyncio.TaskGroup, name: str, owner=None, shared: bool = False):
|
|
121
142
|
self._init_state()
|
|
122
143
|
self._owners += 1
|
|
123
|
-
|
|
144
|
+
worker = getattr(self, 'worker', None)
|
|
145
|
+
|
|
146
|
+
if worker is not None and not worker.done():
|
|
124
147
|
if owner is not None and any(seen is owner for seen in self._owners_seen):
|
|
125
148
|
LOG.info('%s: one feed is using this backend instance for more than one channel', type(self).__name__)
|
|
126
|
-
|
|
127
|
-
LOG.warning('%s: a second feed is sharing this backend instance', type(self).__name__)
|
|
149
|
+
elif not self._shared:
|
|
150
|
+
LOG.warning('%s: a second feed is sharing this backend instance - if the first feed fails, the writer stops with it', type(self).__name__)
|
|
128
151
|
self._owners_seen.append(owner)
|
|
129
152
|
return None
|
|
130
|
-
|
|
153
|
+
|
|
154
|
+
self._owners_seen = [owner]
|
|
155
|
+
self._shared = shared
|
|
156
|
+
self._orphaned = False
|
|
131
157
|
self.queue = asyncio.Queue(maxsize=self.max_depth)
|
|
132
158
|
self._stopping = asyncio.Event()
|
|
133
159
|
self.worker = tg.create_task(self.writer(), name=name)
|
|
@@ -139,32 +165,39 @@ class BackendQueue:
|
|
|
139
165
|
if stopping is not None:
|
|
140
166
|
stopping.set()
|
|
141
167
|
|
|
142
|
-
|
|
168
|
+
def release_owner(self) -> bool:
|
|
143
169
|
if not getattr(self, 'started', False):
|
|
144
170
|
return False
|
|
145
|
-
|
|
146
171
|
self._owners -= 1
|
|
147
172
|
if self._owners_seen:
|
|
148
173
|
self._owners_seen.pop()
|
|
149
|
-
|
|
150
|
-
return False
|
|
174
|
+
return self._owners <= 0
|
|
151
175
|
|
|
176
|
+
async def signal(self):
|
|
152
177
|
self._stopping.set()
|
|
178
|
+
if self.worker.done():
|
|
179
|
+
return
|
|
153
180
|
try:
|
|
154
181
|
self.queue.put_nowait(SHUTDOWN_SENTINEL)
|
|
155
|
-
return True
|
|
156
182
|
except asyncio.QueueFull:
|
|
157
|
-
|
|
183
|
+
if self.overflow == 'block':
|
|
184
|
+
await self.queue.put(SHUTDOWN_SENTINEL)
|
|
185
|
+
else:
|
|
186
|
+
self.queue.get_nowait()
|
|
187
|
+
self._count_overflow_drop()
|
|
188
|
+
self.queue.put_nowait(SHUTDOWN_SENTINEL)
|
|
158
189
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
190
|
+
async def stop(self) -> bool:
|
|
191
|
+
if not self.release_owner():
|
|
192
|
+
return False
|
|
193
|
+
try:
|
|
194
|
+
async with asyncio.timeout(self.SENTINEL_TIMEOUT):
|
|
195
|
+
await self.signal()
|
|
196
|
+
except TimeoutError:
|
|
163
197
|
LOG.warning('%s: did not drain within %.1fs of shutdown', type(self).__name__, self.SENTINEL_TIMEOUT)
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
self.queue.put_nowait(SHUTDOWN_SENTINEL)
|
|
198
|
+
self.queue.get_nowait()
|
|
199
|
+
self._count_overflow_drop()
|
|
200
|
+
self.queue.put_nowait(SHUTDOWN_SENTINEL)
|
|
168
201
|
return True
|
|
169
202
|
|
|
170
203
|
async def write(self, data):
|
|
@@ -219,12 +252,17 @@ class BackendQueue:
|
|
|
219
252
|
|
|
220
253
|
async def writer(self):
|
|
221
254
|
self.running = True
|
|
255
|
+
cancelled = False
|
|
256
|
+
|
|
222
257
|
try:
|
|
223
258
|
while self.running:
|
|
224
259
|
batch = await self._next_batch()
|
|
225
260
|
if batch:
|
|
226
261
|
await self._write_with_retry(batch)
|
|
227
262
|
await self.flush(self.flush_deadline)
|
|
263
|
+
except asyncio.CancelledError:
|
|
264
|
+
cancelled = True
|
|
265
|
+
raise
|
|
228
266
|
finally:
|
|
229
267
|
self.running = False
|
|
230
268
|
abandoned = len(self._inflight)
|
|
@@ -239,7 +277,14 @@ class BackendQueue:
|
|
|
239
277
|
LOG.warning('%s: writer stopped with %d message(s) undelivered - counted as dropped',
|
|
240
278
|
type(self).__name__, abandoned)
|
|
241
279
|
try:
|
|
242
|
-
|
|
280
|
+
if cancelled:
|
|
281
|
+
async with asyncio.timeout(self.CLOSE_TIMEOUT):
|
|
282
|
+
await self.close()
|
|
283
|
+
else:
|
|
284
|
+
await self.close()
|
|
285
|
+
except TimeoutError:
|
|
286
|
+
self._last_error = 'close() timed out'
|
|
287
|
+
LOG.warning('%s: close() did not return within %.2fs of cancellation', type(self).__name__, self.CLOSE_TIMEOUT)
|
|
243
288
|
except Exception as e:
|
|
244
289
|
self._last_error = f'{type(e).__name__}: {e}'
|
|
245
290
|
LOG.exception('%s: close() failed - data not yet flushed may be lost', type(self).__name__)
|
|
@@ -251,6 +296,7 @@ class BackendQueue:
|
|
|
251
296
|
return []
|
|
252
297
|
|
|
253
298
|
batch = [first]
|
|
299
|
+
self._inflight = batch
|
|
254
300
|
loop = asyncio.get_running_loop()
|
|
255
301
|
deadline = loop.time() + self.batch_interval
|
|
256
302
|
|
|
@@ -351,10 +397,11 @@ class BackendQueue:
|
|
|
351
397
|
await self._write_with_retry(batch)
|
|
352
398
|
del pending[:len(batch)]
|
|
353
399
|
except TimeoutError:
|
|
354
|
-
self._inflight = []
|
|
355
|
-
self._dropped_failed += len(pending)
|
|
356
400
|
LOG.warning('%s: flush deadline (%.1fs) hit - %d messages dropped',
|
|
357
401
|
type(self).__name__, deadline, len(pending))
|
|
402
|
+
finally:
|
|
403
|
+
self._inflight = []
|
|
404
|
+
self._dropped_failed += len(pending)
|
|
358
405
|
|
|
359
406
|
async def connect(self):
|
|
360
407
|
pass
|
|
@@ -38,7 +38,7 @@ class MongoCallback(BackendQueue):
|
|
|
38
38
|
none_to: any
|
|
39
39
|
if not None, convert None values to this value before inserting into MongoDB
|
|
40
40
|
numeric_type: type
|
|
41
|
-
type to convert numeric values to before inserting into MongoDB. Defaults to str
|
|
41
|
+
type to convert numeric values to before inserting into MongoDB. Defaults to str, but can be float
|
|
42
42
|
client_options: dict
|
|
43
43
|
passed straight to pymongo's AsyncMongoClient - username/password, tls,
|
|
44
44
|
connectTimeoutMS, serverSelectionTimeoutMS and anything else the driver accepts.
|
|
@@ -80,21 +80,26 @@ class MongoCallback(BackendQueue):
|
|
|
80
80
|
return value
|
|
81
81
|
return dt.fromtimestamp(value, tz=timezone.utc)
|
|
82
82
|
|
|
83
|
+
@staticmethod
|
|
84
|
+
def _side(levels) -> bytes:
|
|
85
|
+
pairs = levels.items() if isinstance(levels, dict) else levels
|
|
86
|
+
return bson.encode({str(price): size for price, size in pairs})
|
|
87
|
+
|
|
83
88
|
def _document(self, update: dict) -> dict:
|
|
84
89
|
update['timestamp'] = self._datetime(update['timestamp'])
|
|
85
90
|
update['receipt_timestamp'] = self._datetime(update['receipt_timestamp'])
|
|
86
91
|
|
|
87
|
-
if 'book' in update:
|
|
88
|
-
|
|
89
|
-
source = update['
|
|
92
|
+
if 'book' in update or 'delta' in update:
|
|
93
|
+
snapshot = 'book' in update
|
|
94
|
+
source = update['book'] if snapshot else update['delta']
|
|
90
95
|
return {
|
|
91
96
|
'exchange': update['exchange'],
|
|
92
97
|
'symbol': update['symbol'],
|
|
93
98
|
'timestamp': update['timestamp'],
|
|
94
99
|
'receipt_timestamp': update['receipt_timestamp'],
|
|
95
|
-
'delta':
|
|
96
|
-
'bid':
|
|
97
|
-
'ask':
|
|
100
|
+
'delta': not snapshot,
|
|
101
|
+
'bid': self._side(source['bid']),
|
|
102
|
+
'ask': self._side(source['ask']),
|
|
98
103
|
}
|
|
99
104
|
return update
|
|
100
105
|
|
|
@@ -5,6 +5,7 @@ Please see the LICENSE file for the terms and conditions
|
|
|
5
5
|
associated with this software.
|
|
6
6
|
'''
|
|
7
7
|
import re
|
|
8
|
+
from decimal import Decimal
|
|
8
9
|
from collections import defaultdict
|
|
9
10
|
from datetime import datetime as dt, timezone
|
|
10
11
|
|
|
@@ -16,6 +17,7 @@ from cryptofeed.defines import CANDLES, FUNDING, OPEN_INTEREST, TICKER, TRADES,
|
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
_COMMON_FIELDS = frozenset({'exchange', 'symbol', 'timestamp', 'receipt_timestamp', 'receipt'})
|
|
20
|
+
_COMMON_COLUMNS = (('timestamp', 'TIMESTAMPTZ'), ('receipt_timestamp', 'TIMESTAMPTZ'), ('exchange', 'TEXT'), ('symbol', 'TEXT'))
|
|
19
21
|
_REJECTED = (asyncpg.exceptions.DataError, asyncpg.exceptions.IntegrityConstraintViolationError)
|
|
20
22
|
|
|
21
23
|
|
|
@@ -30,6 +32,8 @@ def _render(value):
|
|
|
30
32
|
return None
|
|
31
33
|
if isinstance(value, bool):
|
|
32
34
|
return 'true' if value else 'false'
|
|
35
|
+
if isinstance(value, (dict, list)):
|
|
36
|
+
return json.dumps(value)
|
|
33
37
|
return str(value)
|
|
34
38
|
|
|
35
39
|
|
|
@@ -47,10 +51,16 @@ def _ident(data: dict) -> str:
|
|
|
47
51
|
|
|
48
52
|
|
|
49
53
|
class PostgresCallback(BackendQueue):
|
|
50
|
-
retryable_exceptions = (OSError, asyncpg.PostgresConnectionError)
|
|
54
|
+
retryable_exceptions = (OSError, asyncpg.PostgresConnectionError, asyncpg.OperatorInterventionError)
|
|
51
55
|
fields = frozenset()
|
|
56
|
+
COLUMNS = (('data', 'JSONB'),)
|
|
52
57
|
|
|
53
|
-
|
|
58
|
+
@classmethod
|
|
59
|
+
def create_table_sql(cls, table: str = None) -> str:
|
|
60
|
+
columns = ', '.join(f'"{name}" {sql_type}' for name, sql_type in _COMMON_COLUMNS + cls.COLUMNS)
|
|
61
|
+
return f'CREATE TABLE IF NOT EXISTS {table or cls.default_table} (id serial PRIMARY KEY, {columns})'
|
|
62
|
+
|
|
63
|
+
def __init__(self, host='127.0.0.1', user=None, pw=None, db=None, port=None, table=None, custom_columns: dict = None, none_to=None, numeric_type=Decimal, **kwargs):
|
|
54
64
|
"""
|
|
55
65
|
host: str
|
|
56
66
|
Database host address
|
|
@@ -68,6 +78,8 @@ class PostgresCallback(BackendQueue):
|
|
|
68
78
|
Note: to store BOOK data in a JSONB column, include a 'data' field, e.g. {'symbol': 'symbol', 'data': 'json_data'}
|
|
69
79
|
Both halves are checked when the backend first connects: an unknown column name
|
|
70
80
|
or a field this callback never produces is an error, not a column of NULLs.
|
|
81
|
+
numeric_type: type
|
|
82
|
+
cast numerics to this value. Decimal (default) for NUMERIC columns
|
|
71
83
|
"""
|
|
72
84
|
super().__init__(**kwargs)
|
|
73
85
|
self.pool = None
|
|
@@ -189,6 +201,7 @@ class PostgresCallback(BackendQueue):
|
|
|
189
201
|
|
|
190
202
|
class TradePostgres(PostgresCallback, BackendCallback):
|
|
191
203
|
default_table = TRADES
|
|
204
|
+
COLUMNS = (('side', 'TEXT'), ('amount', 'NUMERIC'), ('price', 'NUMERIC'), ('trade_id', 'TEXT'), ('order_type', 'TEXT'))
|
|
192
205
|
fields = _COMMON_FIELDS | {'side', 'amount', 'price', 'id', 'type'}
|
|
193
206
|
|
|
194
207
|
def _default_row(self, data, ts, rts):
|
|
@@ -197,6 +210,7 @@ class TradePostgres(PostgresCallback, BackendCallback):
|
|
|
197
210
|
|
|
198
211
|
class FundingPostgres(PostgresCallback, BackendCallback):
|
|
199
212
|
default_table = FUNDING
|
|
213
|
+
COLUMNS = (('mark_price', 'NUMERIC'), ('rate', 'NUMERIC'), ('next_funding_time', 'TIMESTAMPTZ'), ('predicted_rate', 'NUMERIC'))
|
|
200
214
|
fields = _COMMON_FIELDS | {'mark_price', 'rate', 'next_funding_time', 'predicted_rate'}
|
|
201
215
|
|
|
202
216
|
def _prepare(self, data):
|
|
@@ -208,6 +222,7 @@ class FundingPostgres(PostgresCallback, BackendCallback):
|
|
|
208
222
|
|
|
209
223
|
class TickerPostgres(PostgresCallback, BackendCallback):
|
|
210
224
|
default_table = TICKER
|
|
225
|
+
COLUMNS = (('bid', 'NUMERIC'), ('ask', 'NUMERIC'))
|
|
211
226
|
fields = _COMMON_FIELDS | {'bid', 'ask'}
|
|
212
227
|
|
|
213
228
|
def _default_row(self, data, ts, rts):
|
|
@@ -216,6 +231,7 @@ class TickerPostgres(PostgresCallback, BackendCallback):
|
|
|
216
231
|
|
|
217
232
|
class OpenInterestPostgres(PostgresCallback, BackendCallback):
|
|
218
233
|
default_table = OPEN_INTEREST
|
|
234
|
+
COLUMNS = (('open_interest', 'NUMERIC'),)
|
|
219
235
|
fields = _COMMON_FIELDS | {'open_interest'}
|
|
220
236
|
|
|
221
237
|
def _default_row(self, data, ts, rts):
|
|
@@ -224,6 +240,7 @@ class OpenInterestPostgres(PostgresCallback, BackendCallback):
|
|
|
224
240
|
|
|
225
241
|
class IndexPostgres(PostgresCallback, BackendCallback):
|
|
226
242
|
default_table = INDEX
|
|
243
|
+
COLUMNS = (('price', 'NUMERIC'),)
|
|
227
244
|
fields = _COMMON_FIELDS | {'price'}
|
|
228
245
|
|
|
229
246
|
def _default_row(self, data, ts, rts):
|
|
@@ -232,6 +249,7 @@ class IndexPostgres(PostgresCallback, BackendCallback):
|
|
|
232
249
|
|
|
233
250
|
class LiquidationsPostgres(PostgresCallback, BackendCallback):
|
|
234
251
|
default_table = LIQUIDATIONS
|
|
252
|
+
COLUMNS = (('side', 'TEXT'), ('quantity', 'NUMERIC'), ('price', 'NUMERIC'), ('trade_id', 'TEXT'), ('status', 'TEXT'))
|
|
235
253
|
fields = _COMMON_FIELDS | {'side', 'quantity', 'price', 'id', 'status'}
|
|
236
254
|
|
|
237
255
|
def _default_row(self, data, ts, rts):
|
|
@@ -240,6 +258,7 @@ class LiquidationsPostgres(PostgresCallback, BackendCallback):
|
|
|
240
258
|
|
|
241
259
|
class BookPostgres(PostgresCallback, BackendBookCallback):
|
|
242
260
|
default_table = 'book'
|
|
261
|
+
COLUMNS = (('data', 'JSONB'),)
|
|
243
262
|
fields = _COMMON_FIELDS | {'data', 'book', 'delta'}
|
|
244
263
|
|
|
245
264
|
def __init__(self, *args, snapshots_only=False, snapshot_interval=1000, **kwargs):
|
|
@@ -261,6 +280,7 @@ class BookPostgres(PostgresCallback, BackendBookCallback):
|
|
|
261
280
|
|
|
262
281
|
class CandlesPostgres(PostgresCallback, BackendCallback):
|
|
263
282
|
default_table = CANDLES
|
|
283
|
+
COLUMNS = (('candle_start', 'TIMESTAMPTZ'), ('candle_stop', 'TIMESTAMPTZ'), ('interval', 'TEXT'), ('trades', 'INTEGER'), ('open', 'NUMERIC'), ('close', 'NUMERIC'), ('high', 'NUMERIC'), ('low', 'NUMERIC'), ('volume', 'NUMERIC'), ('closed', 'BOOLEAN'))
|
|
264
284
|
fields = _COMMON_FIELDS | {'start', 'stop', 'interval', 'trades', 'open', 'close', 'high', 'low', 'volume', 'closed'}
|
|
265
285
|
|
|
266
286
|
def _prepare(self, data):
|