cryptofeed 3.0.0__tar.gz → 3.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/CHANGES.md +48 -2
  2. {cryptofeed-3.0.0/cryptofeed.egg-info → cryptofeed-3.0.2}/PKG-INFO +26 -4
  3. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/README.md +23 -2
  4. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/backend.py +71 -24
  5. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/mongo.py +12 -7
  6. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/postgres.py +22 -2
  7. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/redis.py +57 -43
  8. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/capture/playback.py +164 -37
  9. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/capture/recorder.py +2 -0
  10. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/config.py +15 -12
  11. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/connection.py +84 -76
  12. cryptofeed-3.0.2/cryptofeed/connection_handler.py +192 -0
  13. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/defines.py +2 -2
  14. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/__init__.py +4 -4
  15. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/binance.py +92 -19
  16. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/binance_delivery.py +8 -0
  17. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/binance_futures.py +8 -0
  18. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/bitfinex.py +56 -47
  19. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/bitget.py +38 -9
  20. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/bithumb.py +19 -6
  21. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/bybit.py +20 -45
  22. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/coinbase.py +102 -35
  23. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/cryptodotcom.py +58 -50
  24. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/deribit.py +16 -19
  25. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/dydx.py +61 -14
  26. cryptofeed-3.0.0/cryptofeed/exchanges/gateio.py → cryptofeed-3.0.2/cryptofeed/exchanges/gate.py +66 -16
  27. cryptofeed-3.0.0/cryptofeed/exchanges/gateio_futures.py → cryptofeed-3.0.2/cryptofeed/exchanges/gate_futures.py +68 -42
  28. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/gemini.py +42 -8
  29. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/htx.py +12 -4
  30. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/htx_swap.py +10 -4
  31. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/hyperliquid.py +20 -6
  32. cryptofeed-3.0.2/cryptofeed/exchanges/independent_reserve.py +359 -0
  33. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/kraken.py +89 -23
  34. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/kraken_futures.py +55 -19
  35. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/kucoin.py +37 -12
  36. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/kucoin_futures.py +11 -0
  37. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/mexc.py +59 -42
  38. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/okx.py +16 -9
  39. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/phemex.py +51 -28
  40. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/poloniex.py +14 -18
  41. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/feed.py +79 -32
  42. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/feedhandler.py +24 -2
  43. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/log.py +1 -1
  44. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/symbols.py +3 -2
  45. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/types.pyi +22 -4
  46. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/types.pyx +7 -1
  47. cryptofeed-3.0.2/cryptofeed/util/history.py +156 -0
  48. cryptofeed-3.0.2/cryptofeed/util/snapshots.py +151 -0
  49. {cryptofeed-3.0.0 → cryptofeed-3.0.2/cryptofeed.egg-info}/PKG-INFO +26 -4
  50. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed.egg-info/SOURCES.txt +4 -2
  51. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/pyproject.toml +4 -3
  52. cryptofeed-3.0.0/cryptofeed/connection_handler.py +0 -157
  53. cryptofeed-3.0.0/cryptofeed/exchanges/independent_reserve.py +0 -267
  54. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/LICENSE +0 -0
  55. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/MANIFEST.in +0 -0
  56. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/__init__.py +0 -0
  57. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/_json.py +0 -0
  58. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/__init__.py +0 -0
  59. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/_line_protocol.py +0 -0
  60. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/http.py +0 -0
  61. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/influxdb.py +0 -0
  62. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/kafka.py +0 -0
  63. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/quest.py +0 -0
  64. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/socket.py +0 -0
  65. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/backends/zmq.py +0 -0
  66. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/callback.py +0 -0
  67. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/capture/__init__.py +0 -0
  68. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/capture/_zstd.py +0 -0
  69. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/capture/pcap.py +0 -0
  70. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exceptions.py +0 -0
  71. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchange.py +0 -0
  72. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/binance_us.py +0 -0
  73. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/bitflyer.py +0 -0
  74. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/bitstamp.py +0 -0
  75. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/exchanges/upbit.py +0 -0
  76. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/nbbo.py +0 -0
  77. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/py.typed +0 -0
  78. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/util/__init__.py +0 -0
  79. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed/util/time.py +0 -0
  80. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed.egg-info/dependency_links.txt +0 -0
  81. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed.egg-info/requires.txt +0 -0
  82. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/cryptofeed.egg-info/top_level.txt +0 -0
  83. {cryptofeed-3.0.0 → cryptofeed-3.0.2}/setup.cfg +0 -0
@@ -1,5 +1,51 @@
1
1
  ## Changelog
2
2
 
3
+ ### 3.0.2 (2026-10-29)
4
+ * Bugfix: Bybit liquidation side is now the side of the liquidation order (SELL when a long is liquidated), as for Binance and OKX
5
+ * Bugfix: Deribit liquidations flagged `M` (maker liquidated) reported the taker's side
6
+ * Bugfix: Bybit open interest now uses `singleOpenInterest`; `openInterest` counts both sides (2x other exchanges)
7
+ * Bugfix: reconnection of unstable feeds adjusted with backoffs
8
+ * Bugfix: the silence watchdog starts only after subscribe
9
+ * Bugfix: HTTP requests time out after 60s (10 s to connect)
10
+ * Bugfix: log_message_on_error raised UnboundLocalError when the read failed before the first message
11
+ * Bugfix: improvements to `log` section in config
12
+ * Update: the default Feed `retries` is now 15
13
+ * Update: Python 3.15 and free-threaded 3.15t are built, tested and shipped as wheels
14
+ * Bugfix: Gemini no longer emits trades for symbols that were only subscribed to L2_BOOK
15
+ * Update: OKX websocket endpoints no longer use port 8443, which OKX retires on 2026-10-31
16
+ * Bugfix: Bithumb no longer emits book or ticker updates for symbols that were not subscribed to that channel
17
+ * Bugfix: Various data improvements across exchanges
18
+ * Bugfix: connection hardening, improved retries, improved snapshot syncing
19
+ * Bugfix: Poloniex trades corrected
20
+ * Bugfix: Phemex spot values were scaled incorrectly
21
+ * Bugfix: set Gate Futures decimal request header correctly
22
+ * Bugfix: Gemini perpetual instruments resolved correctly
23
+ * Bugfix: Hyperliquid spot instruments resolved correctly
24
+ * Bugfix: Poloniex instruments resolved correctly
25
+ * Bugfix: Bitfinex max_depth limited correctly when checksum validation is enabled
26
+ * Bugfix: Bybit ticker subscription fixed
27
+ * Bugfix: Binance Funding data correction
28
+ * Bugfix: backends handle many feeds, single failure scenario correctly
29
+ * Bugfix: Postgres backend retries through a PostgreSQL restart instead of dying
30
+ * Bugfix: book snapshots serialized with numeric_type=Decimal were not valid JSON. price keys now strings
31
+ * Bugfix: Redis incorrectly ignored `socket` path
32
+ * Update: expiry dates in normalized symbols now zero padded for every exchange (eg. BTC-USD-26H05)
33
+ * Bugfix: the shutdown enforced on backend writers
34
+ * Bugfix: cancelling a backend writer no longer loses messages from the stats counters
35
+ * Bugfix: a backend instance can be reused across runs
36
+ * Bugfix: BookMongo stores deltas as book documents and accept numeric_type=float
37
+ * Bugfix: Redis Stream and ZSet backends retry only rejected commands
38
+ * Bugfix: Postgres backends render dict and list values as JSON
39
+ * Feature: Postgres backends define table layouts and provide create_table_sql()
40
+ * Update: Postgres backends default numeric_type to Decimal, so NUMERIC columns receive exact values
41
+
42
+ ### 3.0.1 (2026-09-27)
43
+ * Update: Gate.io -> Gate rebranding
44
+ * Bugfix: Binance Futures and Binance Delivery L2_BOOK with `depth_interval='250ms'` subscribed to `<symbol>@depth@250ms`, which Binance accepts but never sends data on. The default interval now uses the bare `<symbol>@depth` stream.
45
+ * Bugfix: `depth_interval=None` on Binance Futures and Binance Delivery produced an invalid subscription. `None` now resolves to the venue default
46
+ * Bugfix: Coinbase, OKX, Gate, Kraken data quality issues
47
+ * Bugfix: dYdX subscribe/reconnect connection issues
48
+
3
49
  ### 3.0.0 (2026-09-21)
4
50
  * Feature: New asyncio core
5
51
  * Update: REST endpoints removed (trading functionality moves to the cryptotrade project)
@@ -120,7 +166,7 @@
120
166
  * Update: Exchange name change OKEx -> OKX
121
167
  * Bugfix: OKX candle REST code was setting values incorrectly
122
168
  * Update: OKX now uses v5 for all connections (REST and WS). Update endpoints to new exchange name: okex.com -> okx.com
123
-
169
+
124
170
  ### 2.2.0 (2021-02-16)
125
171
  * Feature: New exchange: Bit.com
126
172
  * Feature: Rework how exchanges that have multiple websocket endpoints are managed and configured.
@@ -147,7 +193,7 @@
147
193
  * Feature: Add ByBit sandbox endpoints.
148
194
  * Bugfix: Fix calculation in OrderInfo on Binance.
149
195
  * Feature: Support list of bootstrap servers for Kafka backend.
150
- * Feature: Add OrderInfo and Fills zmq callbacks
196
+ * Feature: Add OrderInfo and Fills zmq callbacks
151
197
 
152
198
  ### 2.1.1 (2021-11-29)
153
199
  * Bugfix: Position data type missing side field.
@@ -1,16 +1,17 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cryptofeed
3
- Version: 3.0.0
3
+ Version: 3.0.2
4
4
  Summary: Cryptocurrency Exchange Websocket Data Feed Handler
5
5
  Author-email: Bryant Moscon <bmoscon@gmail.com>
6
6
  License-Expression: AGPL-3.0-or-later
7
7
  Project-URL: Homepage, https://github.com/bmoscon/cryptofeed
8
- Keywords: cryptocurrency,bitcoin,btc,feed handler,market feed,market data,crypto assets,Trades,Tickers,BBO,Funding,Open Interest,Liquidation,Order book,Bid,Ask,Binance,Binance Delivery,Binance Futures,Binance US,Bitfinex,bitFlyer,Bitget,Bithumb,Bitstamp,Bybit,Coinbase,Crypto.com,Deribit,Gate.io,Gate.io Futures,dYdX,Gemini,Hyperliquid,HTX,HTX Swap,Independent Reserve,Kraken,Kraken Futures,KuCoin,MEXC,OKX,Phemex,Poloniex,Upbit
8
+ Keywords: cryptocurrency,bitcoin,btc,feed handler,market feed,market data,crypto assets,Trades,Tickers,BBO,Funding,Open Interest,Liquidation,Order book,Bid,Ask,Binance,Binance Delivery,Binance Futures,Binance US,Bitfinex,bitFlyer,Bitget,Bithumb,Bitstamp,Bybit,Coinbase,Crypto.com,Deribit,Gate,Gate Futures,dYdX,Gemini,Hyperliquid,HTX,HTX Swap,Independent Reserve,Kraken,Kraken Futures,KuCoin,MEXC,OKX,Phemex,Poloniex,Upbit
9
9
  Classifier: Intended Audience :: Developers
10
10
  Classifier: Development Status :: 4 - Beta
11
11
  Classifier: Programming Language :: Python :: 3 :: Only
12
12
  Classifier: Programming Language :: Python :: 3.13
13
13
  Classifier: Programming Language :: Python :: 3.14
14
+ Classifier: Programming Language :: Python :: 3.15
14
15
  Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta
15
16
  Classifier: Framework :: AsyncIO
16
17
  Classifier: Operating System :: POSIX :: Linux
@@ -70,8 +71,8 @@ Handles multiple cryptocurrency exchange data feeds and returns normalized and s
70
71
  * [Coinbase](https://www.coinbase.com/)
71
72
  * [Crypto.com](https://www.crypto.com)
72
73
  * [Deribit](https://www.deribit.com/)
73
- * [Gate.io](https://www.gate.io/)
74
- * [Gate.io Futures](https://www.gate.io/futures_center)
74
+ * [Gate](https://www.gate.com/)
75
+ * [Gate Futures](https://www.gate.com/futures)
75
76
  * [dYdX v4](https://dydx.trade/)
76
77
  * [Gemini](https://gemini.com/)
77
78
  * [Hyperliquid](https://hyperliquid.xyz/)
@@ -152,6 +153,27 @@ Cryptofeed supports the following public data channels from exchanges
152
153
  * INDEX
153
154
  * CANDLES - Candlestick / K-Line data.
154
155
 
156
+ ## Normalized data
157
+
158
+ Every callback receives two arguments: a normalized object and `receipt_timestamp`, the time (float epoch seconds)
159
+ cryptofeed received the message that produced the callback. The objects are defined in `cryptofeed/types.pyx` (`Trade`,
160
+ `Ticker`, `OrderBook`, `L1Book`, `Candle`, `Funding`, `OpenInterest`, `Liquidation`, `Index`) and share these rules:
161
+
162
+ * `exchange` is the venue id and `symbol` the normalized name: `BASE-QUOTE` for spot, `BASE-QUOTE-PERP` for
163
+ perpetuals, `BASE-QUOTE-<expiry>` for dated futures with the expiry as a two-digit year, a futures month code and
164
+ a zero-padded day (`BTC-USD-26H05`), and `BASE-QUOTE-<strike>-<expiry>-CALL|PUT` for options.
165
+ * Prices, sizes, rates and volumes are `Decimal`. `numeric_type` on a backend or `to_dict()` converts them.
166
+ * `timestamp` is the venue's event time in float epoch seconds. `timestamp` is `None` when the venue does not provide
167
+ a timestamp and backends then store the receipt time in its place.
168
+ * `Trade.side` is the taker's side. `Trade.id` is the venue's id or `None`.
169
+ * `OrderBook.book` holds the full local book (`.bids`, `.asks`). `delta` is `None` for a snapshot and otherwise lists
170
+ the levels that changed as `(price, size)` with size 0 for a removed level. `sequence_number` and `checksum` are
171
+ set when the venue provides them.
172
+ * `Candle.closed` is `True` once the bar is final, by default only closed candles are delivered.
173
+ (`candle_closed_only=False` also delivers each update of the open bar).
174
+ * `trade_history` and `candle_history` control whether the batch of recent trades or candles some venues send on
175
+ subscribe is delivered, by default it is `False`.
176
+
155
177
  ## Backends
156
178
 
157
179
  Cryptofeed supports `backend` callbacks that will write directly to storage or other interfaces.
@@ -21,8 +21,8 @@ Handles multiple cryptocurrency exchange data feeds and returns normalized and s
21
21
  * [Coinbase](https://www.coinbase.com/)
22
22
  * [Crypto.com](https://www.crypto.com)
23
23
  * [Deribit](https://www.deribit.com/)
24
- * [Gate.io](https://www.gate.io/)
25
- * [Gate.io Futures](https://www.gate.io/futures_center)
24
+ * [Gate](https://www.gate.com/)
25
+ * [Gate Futures](https://www.gate.com/futures)
26
26
  * [dYdX v4](https://dydx.trade/)
27
27
  * [Gemini](https://gemini.com/)
28
28
  * [Hyperliquid](https://hyperliquid.xyz/)
@@ -103,6 +103,27 @@ Cryptofeed supports the following public data channels from exchanges
103
103
  * INDEX
104
104
  * CANDLES - Candlestick / K-Line data.
105
105
 
106
+ ## Normalized data
107
+
108
+ Every callback receives two arguments: a normalized object and `receipt_timestamp`, the time (float epoch seconds)
109
+ cryptofeed received the message that produced the callback. The objects are defined in `cryptofeed/types.pyx` (`Trade`,
110
+ `Ticker`, `OrderBook`, `L1Book`, `Candle`, `Funding`, `OpenInterest`, `Liquidation`, `Index`) and share these rules:
111
+
112
+ * `exchange` is the venue id and `symbol` the normalized name: `BASE-QUOTE` for spot, `BASE-QUOTE-PERP` for
113
+ perpetuals, `BASE-QUOTE-<expiry>` for dated futures with the expiry as a two-digit year, a futures month code and
114
+ a zero-padded day (`BTC-USD-26H05`), and `BASE-QUOTE-<strike>-<expiry>-CALL|PUT` for options.
115
+ * Prices, sizes, rates and volumes are `Decimal`. `numeric_type` on a backend or `to_dict()` converts them.
116
+ * `timestamp` is the venue's event time in float epoch seconds. `timestamp` is `None` when the venue does not provide
117
+ a timestamp and backends then store the receipt time in its place.
118
+ * `Trade.side` is the taker's side. `Trade.id` is the venue's id or `None`.
119
+ * `OrderBook.book` holds the full local book (`.bids`, `.asks`). `delta` is `None` for a snapshot and otherwise lists
120
+ the levels that changed as `(price, size)` with size 0 for a removed level. `sequence_number` and `checksum` are
121
+ set when the venue provides them.
122
+ * `Candle.closed` is `True` once the bar is final, by default only closed candles are delivered.
123
+ (`candle_closed_only=False` also delivers each update of the open bar).
124
+ * `trade_history` and `candle_history` control whether the batch of recent trades or candles some venues send on
125
+ subscribe is delivered, by default it is `False`.
126
+
106
127
  ## Backends
107
128
 
108
129
  Cryptofeed supports `backend` callbacks that will write directly to storage or other interfaces.
@@ -35,11 +35,12 @@ class RetryPolicy:
35
35
  max_attempts: int = 5
36
36
  base: float = 0.5
37
37
  jitter: bool = True
38
+ max_delay: float = 60.0
38
39
 
39
40
  def delay(self, attempt: int) -> float:
40
- d = self.base * (2 ** (attempt - 1))
41
+ d = min(self.base * (2 ** (attempt - 1)), self.max_delay)
41
42
  if self.jitter:
42
- d *= random.uniform(0.5, 1.5)
43
+ d *= random.uniform(1.0, 1.5)
43
44
  return d
44
45
 
45
46
 
@@ -61,6 +62,24 @@ class BackendStats:
61
62
  return self.written + self.conflicts
62
63
 
63
64
 
65
+ async def finish_writers(backends: list, deadline: float):
66
+ if not backends:
67
+ return
68
+ try:
69
+ async with asyncio.timeout(deadline):
70
+ for backend in backends:
71
+ await backend.signal()
72
+
73
+ pending = [backend.worker for backend in backends if not backend.worker.done()]
74
+ if pending:
75
+ await asyncio.wait(pending)
76
+ except TimeoutError:
77
+ pending = [backend for backend in backends if not backend.worker.done()]
78
+ LOG.warning('backend flush exceeded the %.1fs shutdown deadline - cancelling %d writer(s)', deadline, len(pending))
79
+ for backend in pending:
80
+ backend.worker.cancel()
81
+
82
+
64
83
  class BackendQueue:
65
84
  hand_off = False
66
85
  max_depth = 10_000
@@ -71,6 +90,7 @@ class BackendQueue:
71
90
  retry = RetryPolicy()
72
91
  retryable_exceptions: tuple = ()
73
92
  SENTINEL_TIMEOUT = 1.0
93
+ CLOSE_TIMEOUT = 0.25
74
94
  BLOCK_POLL = 0.25
75
95
  FAILURE_LEVEL = logging.ERROR
76
96
 
@@ -105,6 +125,7 @@ class BackendQueue:
105
125
  self._orphaned = False
106
126
  self._owners = 0
107
127
  self._owners_seen = []
128
+ self._shared = False
108
129
  self._inflight = []
109
130
 
110
131
  @property
@@ -117,17 +138,22 @@ class BackendQueue:
117
138
  conflicts=self._conflicts, retries=self._retries, last_error=self._last_error,
118
139
  last_write_ts=self._last_write_ts, hand_off=bool(getattr(self, 'hand_off', False)))
119
140
 
120
- def start_writer(self, tg: asyncio.TaskGroup, name: str, owner=None):
141
+ def start_writer(self, tg: asyncio.TaskGroup, name: str, owner=None, shared: bool = False):
121
142
  self._init_state()
122
143
  self._owners += 1
123
- if getattr(self, 'started', False):
144
+ worker = getattr(self, 'worker', None)
145
+
146
+ if worker is not None and not worker.done():
124
147
  if owner is not None and any(seen is owner for seen in self._owners_seen):
125
148
  LOG.info('%s: one feed is using this backend instance for more than one channel', type(self).__name__)
126
- else:
127
- LOG.warning('%s: a second feed is sharing this backend instance', type(self).__name__)
149
+ elif not self._shared:
150
+ LOG.warning('%s: a second feed is sharing this backend instance - if the first feed fails, the writer stops with it', type(self).__name__)
128
151
  self._owners_seen.append(owner)
129
152
  return None
130
- self._owners_seen.append(owner)
153
+
154
+ self._owners_seen = [owner]
155
+ self._shared = shared
156
+ self._orphaned = False
131
157
  self.queue = asyncio.Queue(maxsize=self.max_depth)
132
158
  self._stopping = asyncio.Event()
133
159
  self.worker = tg.create_task(self.writer(), name=name)
@@ -139,32 +165,39 @@ class BackendQueue:
139
165
  if stopping is not None:
140
166
  stopping.set()
141
167
 
142
- async def stop(self):
168
+ def release_owner(self) -> bool:
143
169
  if not getattr(self, 'started', False):
144
170
  return False
145
-
146
171
  self._owners -= 1
147
172
  if self._owners_seen:
148
173
  self._owners_seen.pop()
149
- if self._owners > 0:
150
- return False
174
+ return self._owners <= 0
151
175
 
176
+ async def signal(self):
152
177
  self._stopping.set()
178
+ if self.worker.done():
179
+ return
153
180
  try:
154
181
  self.queue.put_nowait(SHUTDOWN_SENTINEL)
155
- return True
156
182
  except asyncio.QueueFull:
157
- pass
183
+ if self.overflow == 'block':
184
+ await self.queue.put(SHUTDOWN_SENTINEL)
185
+ else:
186
+ self.queue.get_nowait()
187
+ self._count_overflow_drop()
188
+ self.queue.put_nowait(SHUTDOWN_SENTINEL)
158
189
 
159
- if self.overflow == 'block':
160
- with suppress(TimeoutError):
161
- await asyncio.wait_for(self.queue.put(SHUTDOWN_SENTINEL), self.SENTINEL_TIMEOUT)
162
- return True
190
+ async def stop(self) -> bool:
191
+ if not self.release_owner():
192
+ return False
193
+ try:
194
+ async with asyncio.timeout(self.SENTINEL_TIMEOUT):
195
+ await self.signal()
196
+ except TimeoutError:
163
197
  LOG.warning('%s: did not drain within %.1fs of shutdown', type(self).__name__, self.SENTINEL_TIMEOUT)
164
-
165
- self.queue.get_nowait()
166
- self._count_overflow_drop()
167
- self.queue.put_nowait(SHUTDOWN_SENTINEL)
198
+ self.queue.get_nowait()
199
+ self._count_overflow_drop()
200
+ self.queue.put_nowait(SHUTDOWN_SENTINEL)
168
201
  return True
169
202
 
170
203
  async def write(self, data):
@@ -219,12 +252,17 @@ class BackendQueue:
219
252
 
220
253
  async def writer(self):
221
254
  self.running = True
255
+ cancelled = False
256
+
222
257
  try:
223
258
  while self.running:
224
259
  batch = await self._next_batch()
225
260
  if batch:
226
261
  await self._write_with_retry(batch)
227
262
  await self.flush(self.flush_deadline)
263
+ except asyncio.CancelledError:
264
+ cancelled = True
265
+ raise
228
266
  finally:
229
267
  self.running = False
230
268
  abandoned = len(self._inflight)
@@ -239,7 +277,14 @@ class BackendQueue:
239
277
  LOG.warning('%s: writer stopped with %d message(s) undelivered - counted as dropped',
240
278
  type(self).__name__, abandoned)
241
279
  try:
242
- await self.close()
280
+ if cancelled:
281
+ async with asyncio.timeout(self.CLOSE_TIMEOUT):
282
+ await self.close()
283
+ else:
284
+ await self.close()
285
+ except TimeoutError:
286
+ self._last_error = 'close() timed out'
287
+ LOG.warning('%s: close() did not return within %.2fs of cancellation', type(self).__name__, self.CLOSE_TIMEOUT)
243
288
  except Exception as e:
244
289
  self._last_error = f'{type(e).__name__}: {e}'
245
290
  LOG.exception('%s: close() failed - data not yet flushed may be lost', type(self).__name__)
@@ -251,6 +296,7 @@ class BackendQueue:
251
296
  return []
252
297
 
253
298
  batch = [first]
299
+ self._inflight = batch
254
300
  loop = asyncio.get_running_loop()
255
301
  deadline = loop.time() + self.batch_interval
256
302
 
@@ -351,10 +397,11 @@ class BackendQueue:
351
397
  await self._write_with_retry(batch)
352
398
  del pending[:len(batch)]
353
399
  except TimeoutError:
354
- self._inflight = []
355
- self._dropped_failed += len(pending)
356
400
  LOG.warning('%s: flush deadline (%.1fs) hit - %d messages dropped',
357
401
  type(self).__name__, deadline, len(pending))
402
+ finally:
403
+ self._inflight = []
404
+ self._dropped_failed += len(pending)
358
405
 
359
406
  async def connect(self):
360
407
  pass
@@ -38,7 +38,7 @@ class MongoCallback(BackendQueue):
38
38
  none_to: any
39
39
  if not None, convert None values to this value before inserting into MongoDB
40
40
  numeric_type: type
41
- type to convert numeric values to before inserting into MongoDB. Defaults to str
41
+ type to convert numeric values to before inserting into MongoDB. Defaults to str, but can be float
42
42
  client_options: dict
43
43
  passed straight to pymongo's AsyncMongoClient - username/password, tls,
44
44
  connectTimeoutMS, serverSelectionTimeoutMS and anything else the driver accepts.
@@ -80,21 +80,26 @@ class MongoCallback(BackendQueue):
80
80
  return value
81
81
  return dt.fromtimestamp(value, tz=timezone.utc)
82
82
 
83
+ @staticmethod
84
+ def _side(levels) -> bytes:
85
+ pairs = levels.items() if isinstance(levels, dict) else levels
86
+ return bson.encode({str(price): size for price, size in pairs})
87
+
83
88
  def _document(self, update: dict) -> dict:
84
89
  update['timestamp'] = self._datetime(update['timestamp'])
85
90
  update['receipt_timestamp'] = self._datetime(update['receipt_timestamp'])
86
91
 
87
- if 'book' in update:
88
- delta = 'delta' in update
89
- source = update['delta'] if delta else update['book']
92
+ if 'book' in update or 'delta' in update:
93
+ snapshot = 'book' in update
94
+ source = update['book'] if snapshot else update['delta']
90
95
  return {
91
96
  'exchange': update['exchange'],
92
97
  'symbol': update['symbol'],
93
98
  'timestamp': update['timestamp'],
94
99
  'receipt_timestamp': update['receipt_timestamp'],
95
- 'delta': delta,
96
- 'bid': bson.encode(source['bid']),
97
- 'ask': bson.encode(source['ask']),
100
+ 'delta': not snapshot,
101
+ 'bid': self._side(source['bid']),
102
+ 'ask': self._side(source['ask']),
98
103
  }
99
104
  return update
100
105
 
@@ -5,6 +5,7 @@ Please see the LICENSE file for the terms and conditions
5
5
  associated with this software.
6
6
  '''
7
7
  import re
8
+ from decimal import Decimal
8
9
  from collections import defaultdict
9
10
  from datetime import datetime as dt, timezone
10
11
 
@@ -16,6 +17,7 @@ from cryptofeed.defines import CANDLES, FUNDING, OPEN_INTEREST, TICKER, TRADES,
16
17
 
17
18
 
18
19
  _COMMON_FIELDS = frozenset({'exchange', 'symbol', 'timestamp', 'receipt_timestamp', 'receipt'})
20
+ _COMMON_COLUMNS = (('timestamp', 'TIMESTAMPTZ'), ('receipt_timestamp', 'TIMESTAMPTZ'), ('exchange', 'TEXT'), ('symbol', 'TEXT'))
19
21
  _REJECTED = (asyncpg.exceptions.DataError, asyncpg.exceptions.IntegrityConstraintViolationError)
20
22
 
21
23
 
@@ -30,6 +32,8 @@ def _render(value):
30
32
  return None
31
33
  if isinstance(value, bool):
32
34
  return 'true' if value else 'false'
35
+ if isinstance(value, (dict, list)):
36
+ return json.dumps(value)
33
37
  return str(value)
34
38
 
35
39
 
@@ -47,10 +51,16 @@ def _ident(data: dict) -> str:
47
51
 
48
52
 
49
53
  class PostgresCallback(BackendQueue):
50
- retryable_exceptions = (OSError, asyncpg.PostgresConnectionError)
54
+ retryable_exceptions = (OSError, asyncpg.PostgresConnectionError, asyncpg.OperatorInterventionError)
51
55
  fields = frozenset()
56
+ COLUMNS = (('data', 'JSONB'),)
52
57
 
53
- def __init__(self, host='127.0.0.1', user=None, pw=None, db=None, port=None, table=None, custom_columns: dict = None, none_to=None, numeric_type=float, **kwargs):
58
+ @classmethod
59
+ def create_table_sql(cls, table: str = None) -> str:
60
+ columns = ', '.join(f'"{name}" {sql_type}' for name, sql_type in _COMMON_COLUMNS + cls.COLUMNS)
61
+ return f'CREATE TABLE IF NOT EXISTS {table or cls.default_table} (id serial PRIMARY KEY, {columns})'
62
+
63
+ def __init__(self, host='127.0.0.1', user=None, pw=None, db=None, port=None, table=None, custom_columns: dict = None, none_to=None, numeric_type=Decimal, **kwargs):
54
64
  """
55
65
  host: str
56
66
  Database host address
@@ -68,6 +78,8 @@ class PostgresCallback(BackendQueue):
68
78
  Note: to store BOOK data in a JSONB column, include a 'data' field, e.g. {'symbol': 'symbol', 'data': 'json_data'}
69
79
  Both halves are checked when the backend first connects: an unknown column name
70
80
  or a field this callback never produces is an error, not a column of NULLs.
81
+ numeric_type: type
82
+ cast numerics to this value. Decimal (default) for NUMERIC columns
71
83
  """
72
84
  super().__init__(**kwargs)
73
85
  self.pool = None
@@ -189,6 +201,7 @@ class PostgresCallback(BackendQueue):
189
201
 
190
202
  class TradePostgres(PostgresCallback, BackendCallback):
191
203
  default_table = TRADES
204
+ COLUMNS = (('side', 'TEXT'), ('amount', 'NUMERIC'), ('price', 'NUMERIC'), ('trade_id', 'TEXT'), ('order_type', 'TEXT'))
192
205
  fields = _COMMON_FIELDS | {'side', 'amount', 'price', 'id', 'type'}
193
206
 
194
207
  def _default_row(self, data, ts, rts):
@@ -197,6 +210,7 @@ class TradePostgres(PostgresCallback, BackendCallback):
197
210
 
198
211
  class FundingPostgres(PostgresCallback, BackendCallback):
199
212
  default_table = FUNDING
213
+ COLUMNS = (('mark_price', 'NUMERIC'), ('rate', 'NUMERIC'), ('next_funding_time', 'TIMESTAMPTZ'), ('predicted_rate', 'NUMERIC'))
200
214
  fields = _COMMON_FIELDS | {'mark_price', 'rate', 'next_funding_time', 'predicted_rate'}
201
215
 
202
216
  def _prepare(self, data):
@@ -208,6 +222,7 @@ class FundingPostgres(PostgresCallback, BackendCallback):
208
222
 
209
223
  class TickerPostgres(PostgresCallback, BackendCallback):
210
224
  default_table = TICKER
225
+ COLUMNS = (('bid', 'NUMERIC'), ('ask', 'NUMERIC'))
211
226
  fields = _COMMON_FIELDS | {'bid', 'ask'}
212
227
 
213
228
  def _default_row(self, data, ts, rts):
@@ -216,6 +231,7 @@ class TickerPostgres(PostgresCallback, BackendCallback):
216
231
 
217
232
  class OpenInterestPostgres(PostgresCallback, BackendCallback):
218
233
  default_table = OPEN_INTEREST
234
+ COLUMNS = (('open_interest', 'NUMERIC'),)
219
235
  fields = _COMMON_FIELDS | {'open_interest'}
220
236
 
221
237
  def _default_row(self, data, ts, rts):
@@ -224,6 +240,7 @@ class OpenInterestPostgres(PostgresCallback, BackendCallback):
224
240
 
225
241
  class IndexPostgres(PostgresCallback, BackendCallback):
226
242
  default_table = INDEX
243
+ COLUMNS = (('price', 'NUMERIC'),)
227
244
  fields = _COMMON_FIELDS | {'price'}
228
245
 
229
246
  def _default_row(self, data, ts, rts):
@@ -232,6 +249,7 @@ class IndexPostgres(PostgresCallback, BackendCallback):
232
249
 
233
250
  class LiquidationsPostgres(PostgresCallback, BackendCallback):
234
251
  default_table = LIQUIDATIONS
252
+ COLUMNS = (('side', 'TEXT'), ('quantity', 'NUMERIC'), ('price', 'NUMERIC'), ('trade_id', 'TEXT'), ('status', 'TEXT'))
235
253
  fields = _COMMON_FIELDS | {'side', 'quantity', 'price', 'id', 'status'}
236
254
 
237
255
  def _default_row(self, data, ts, rts):
@@ -240,6 +258,7 @@ class LiquidationsPostgres(PostgresCallback, BackendCallback):
240
258
 
241
259
  class BookPostgres(PostgresCallback, BackendBookCallback):
242
260
  default_table = 'book'
261
+ COLUMNS = (('data', 'JSONB'),)
243
262
  fields = _COMMON_FIELDS | {'data', 'book', 'delta'}
244
263
 
245
264
  def __init__(self, *args, snapshots_only=False, snapshot_interval=1000, **kwargs):
@@ -261,6 +280,7 @@ class BookPostgres(PostgresCallback, BackendBookCallback):
261
280
 
262
281
  class CandlesPostgres(PostgresCallback, BackendCallback):
263
282
  default_table = CANDLES
283
+ COLUMNS = (('candle_start', 'TIMESTAMPTZ'), ('candle_stop', 'TIMESTAMPTZ'), ('interval', 'TEXT'), ('trades', 'INTEGER'), ('open', 'NUMERIC'), ('close', 'NUMERIC'), ('high', 'NUMERIC'), ('low', 'NUMERIC'), ('volume', 'NUMERIC'), ('closed', 'BOOLEAN'))
264
284
  fields = _COMMON_FIELDS | {'start', 'stop', 'interval', 'trades', 'open', 'close', 'high', 'low', 'volume', 'closed'}
265
285
 
266
286
  def _prepare(self, data):