cryptofeed 2.3.1__tar.gz → 2.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/CHANGES.md +13 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/LICENSE +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/PKG-INFO +15 -4
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/README.md +0 -2
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/__init__.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/_util.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/aggregate.py +5 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/arctic.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/backend.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/gcppubsub.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/http.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/influxdb.py +1 -1
- cryptofeed-2.3.2/cryptofeed/backends/kafka.py +158 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/mongo.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/postgres.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/quest.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/rabbitmq.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/redis.py +27 -2
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/socket.py +3 -3
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/zmq.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/callback.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/config.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/connection.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/connection_handler.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/defines.py +2 -4
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exceptions.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchange.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/__init__.py +4 -8
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/ascendex.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/bequant.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/binance.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/binance_delivery.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/binance_futures.py +1 -1
- cryptofeed-2.3.2/cryptofeed/exchanges/binance_tr.py +21 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/binance_us.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/bitfinex.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/bitflyer.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/bitget.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/bithumb.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/bitmex.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/bitstamp.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/blockchain.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/bybit.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/coinbase.py +5 -2
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/cryptodotcom.py +2 -2
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/delta.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/dydx.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/fmfw.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/gateio.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/gemini.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/hitbtc.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/huobi.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/huobi_dm.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/huobi_swap.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/independent_reserve.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/kraken.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/kraken_futures.py +2 -2
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/kucoin.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/mixins/binance_rest.py +8 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/mixins/bitfinex_rest.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/mixins/bitmex_rest.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/mixins/bitstamp_rest.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/mixins/coinbase_rest.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/mixins/deribit_rest.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/mixins/dydx_rest.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/mixins/gemini_rest.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/mixins/kraken_rest.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/mixins/okx_rest.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/mixins/upbit_rest.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/okcoin.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/okx.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/phemex.py +4 -2
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/poloniex.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/probit.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/upbit.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/feed.py +10 -5
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/feedhandler.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/log.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/nbbo.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/raw_data_collection.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/symbols.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/types.c +4 -4
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/types.pyx +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/util/book.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/util/perf.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/util/time.py +1 -1
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed.egg-info/PKG-INFO +15 -4
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed.egg-info/SOURCES.txt +1 -6
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed.egg-info/requires.txt +4 -5
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/setup.py +8 -10
- cryptofeed-2.3.1/cryptofeed/backends/kafka.py +0 -106
- cryptofeed-2.3.1/cryptofeed/exchanges/ftx.py +0 -437
- cryptofeed-2.3.1/cryptofeed/exchanges/ftx_tr.py +0 -29
- cryptofeed-2.3.1/cryptofeed/exchanges/ftx_us.py +0 -29
- cryptofeed-2.3.1/cryptofeed/exchanges/mixins/ftx_rest.py +0 -135
- cryptofeed-2.3.1/cryptofeed/exchanges/mixins/ftx_rest_tr.py +0 -21
- cryptofeed-2.3.1/cryptofeed/exchanges/mixins/ftx_rest_us.py +0 -21
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/AUTHORS.md +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/INSTALL.md +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/MANIFEST.in +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/backends/__init__.py +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/ascendex_futures.py +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/bitdotcom.py +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/bittrex.py +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/deribit.py +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/exx.py +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/exchanges/mixins/__init__.py +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/util/__init__.py +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed/util/split.py +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed.egg-info/dependency_links.txt +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/cryptofeed.egg-info/top_level.txt +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/pyproject.toml +0 -0
- {cryptofeed-2.3.1 → cryptofeed-2.3.2}/setup.cfg +0 -0
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
## Changelog
|
|
2
2
|
|
|
3
|
+
### 2.3.2 (2023-05-27)
|
|
4
|
+
* Bugfix: Fix Socket backend
|
|
5
|
+
* Bugfix: Fix AUCTION symbol parsing on Coinbase
|
|
6
|
+
* Bugfix: Fix PERPETUAL symbol parsing on Phemex
|
|
7
|
+
* Bugfix: Fix PERPETUAL symbol parsing on Kraken Futures
|
|
8
|
+
* Feature: Access to all AIOKafka configuration options
|
|
9
|
+
* Feature: Use backend Queue for Kafka
|
|
10
|
+
* Feature: Add support for storing book snapshots in Redis as key-value
|
|
11
|
+
* Update: Switch from unmaintained aioredis to redis-py
|
|
12
|
+
* Bugfix: Correct value for Crypto.com Ask price
|
|
13
|
+
* Update: Remove cChardet dependency
|
|
14
|
+
* Feature: Binance TR support
|
|
15
|
+
|
|
3
16
|
### 2.3.1 (2022-10-31)
|
|
4
17
|
* Bugfix: timestamp not reset correctly on reconnect
|
|
5
18
|
* Bugfix: Arctic backend failing to write Trades when trade type was not present in data
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
Copyright (C) 2017-
|
|
1
|
+
Copyright (C) 2017-2023 Bryant Moscon - bmoscon@gmail.com
|
|
2
2
|
|
|
3
3
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
4
4
|
of this software and associated documentation files (the "Software"), to
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: cryptofeed
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.2
|
|
4
4
|
Summary: Cryptocurrency Exchange Websocket Data Feed Handler
|
|
5
5
|
Home-page: https://github.com/bmoscon/cryptofeed
|
|
6
6
|
Author: Bryant Moscon
|
|
7
7
|
Author-email: bmoscon@gmail.com
|
|
8
8
|
License: XFree86
|
|
9
|
-
Keywords: cryptocurrency,bitcoin,btc,feed handler,market feed,market data,crypto assets,Trades,Tickers,BBO,Funding,Open Interest,Liquidation,Order book,Bid,Ask,fmfw.io,Bitfinex,bitFlyer,AscendEX,Bitstamp,Bittrex,Blockchain.com,Bybit,Binance,Binance Delivery,Binance Futures,Binance US,BitMEX,Coinbase,Deribit,EXX,
|
|
9
|
+
Keywords: cryptocurrency,bitcoin,btc,feed handler,market feed,market data,crypto assets,Trades,Tickers,BBO,Funding,Open Interest,Liquidation,Order book,Bid,Ask,fmfw.io,Bitfinex,bitFlyer,AscendEX,Bitstamp,Bittrex,Blockchain.com,Bybit,Binance,Binance Delivery,Binance Futures,Binance US,BitMEX,Coinbase,Deribit,EXX,Gate.io,Gemini,HitBTC,Huobi,Huobi DM,Huobi Swap,Kraken,Kraken Futures,OKCoin,OKX,Poloniex,ProBit,Upbit
|
|
10
10
|
Platform: UNKNOWN
|
|
11
11
|
Classifier: Intended Audience :: Developers
|
|
12
12
|
Classifier: Development Status :: 4 - Beta
|
|
@@ -63,8 +63,6 @@ Handles multiple cryptocurrency exchange data feeds and returns normalized and s
|
|
|
63
63
|
* [dYdX](https://dydx.exchange/)
|
|
64
64
|
* [FMFW.io](https://www.fmfw.io/)
|
|
65
65
|
* [EXX](https://www.exx.com/)
|
|
66
|
-
* [FTX](https://ftx.com/)
|
|
67
|
-
* [FTX US](https://ftx.us/)
|
|
68
66
|
* [Gate.io](https://www.gate.io/)
|
|
69
67
|
* [Gemini](https://gemini.com/)
|
|
70
68
|
* [HitBTC](https://hitbtc.com/)
|
|
@@ -300,6 +298,19 @@ Your Pull Requests are also welcome, even for minor changes.
|
|
|
300
298
|
|
|
301
299
|
## Changelog
|
|
302
300
|
|
|
301
|
+
### 2.3.2 (2023-05-27)
|
|
302
|
+
* Bugfix: Fix Socket backend
|
|
303
|
+
* Bugfix: Fix AUCTION symbol parsing on Coinbase
|
|
304
|
+
* Bugfix: Fix PERPETUAL symbol parsing on Phemex
|
|
305
|
+
* Bugfix: Fix PERPETUAL symbol parsing on Kraken Futures
|
|
306
|
+
* Feature: Access to all AIOKafka configuration options
|
|
307
|
+
* Feature: Use backend Queue for Kafka
|
|
308
|
+
* Feature: Add support for storing book snapshots in Redis as key-value
|
|
309
|
+
* Update: Switch from unmaintained aioredis to redis-py
|
|
310
|
+
* Bugfix: Correct value for Crypto.com Ask price
|
|
311
|
+
* Update: Remove cChardet dependency
|
|
312
|
+
* Feature: Binance TR support
|
|
313
|
+
|
|
303
314
|
### 2.3.1 (2022-10-31)
|
|
304
315
|
* Bugfix: timestamp not reset correctly on reconnect
|
|
305
316
|
* Bugfix: Arctic backend failing to write Trades when trade type was not present in data
|
|
@@ -32,8 +32,6 @@ Handles multiple cryptocurrency exchange data feeds and returns normalized and s
|
|
|
32
32
|
* [dYdX](https://dydx.exchange/)
|
|
33
33
|
* [FMFW.io](https://www.fmfw.io/)
|
|
34
34
|
* [EXX](https://www.exx.com/)
|
|
35
|
-
* [FTX](https://ftx.com/)
|
|
36
|
-
* [FTX US](https://ftx.us/)
|
|
37
35
|
* [Gate.io](https://www.gate.io/)
|
|
38
36
|
* [Gemini](https://gemini.com/)
|
|
39
37
|
* [HitBTC](https://hitbtc.com/)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
'''
|
|
2
|
-
Copyright (C) 2017-
|
|
2
|
+
Copyright (C) 2017-2023 Bryant Moscon - bmoscon@gmail.com
|
|
3
3
|
|
|
4
4
|
Please see the LICENSE file for the terms and conditions
|
|
5
5
|
associated with this software.
|
|
@@ -13,6 +13,10 @@ import numpy as np
|
|
|
13
13
|
class AggregateCallback:
|
|
14
14
|
def __init__(self, handler):
|
|
15
15
|
self.handler = handler
|
|
16
|
+
if hasattr(self.handler, '__class__'):
|
|
17
|
+
setattr(self, 'start', self.handler.start)
|
|
18
|
+
setattr(self, 'stop', self.handler.stop)
|
|
19
|
+
self.__name__ = self.handler.__class__
|
|
16
20
|
|
|
17
21
|
|
|
18
22
|
class Throttle(AggregateCallback):
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Copyright (C) 2017-2023 Bryant Moscon - bmoscon@gmail.com
|
|
3
|
+
|
|
4
|
+
Please see the LICENSE file for the terms and conditions
|
|
5
|
+
associated with this software.
|
|
6
|
+
'''
|
|
7
|
+
from collections import defaultdict
|
|
8
|
+
import asyncio
|
|
9
|
+
import logging
|
|
10
|
+
from typing import Optional, ByteString
|
|
11
|
+
|
|
12
|
+
from aiokafka import AIOKafkaProducer
|
|
13
|
+
from aiokafka.errors import RequestTimedOutError, KafkaConnectionError, NodeNotReadyError
|
|
14
|
+
from yapic import json
|
|
15
|
+
|
|
16
|
+
from cryptofeed.backends.backend import BackendBookCallback, BackendCallback, BackendQueue
|
|
17
|
+
|
|
18
|
+
LOG = logging.getLogger('feedhandler')
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class KafkaCallback(BackendQueue):
|
|
22
|
+
def __init__(self, key=None, numeric_type=float, none_to=None, **kwargs):
|
|
23
|
+
"""
|
|
24
|
+
You can pass configuration options to AIOKafkaProducer as keyword arguments.
|
|
25
|
+
(either individual kwargs, an unpacked dictionary `**config_dict`, or both)
|
|
26
|
+
A full list of configuration parameters can be found at
|
|
27
|
+
https://aiokafka.readthedocs.io/en/stable/api.html#aiokafka.AIOKafkaProducer
|
|
28
|
+
|
|
29
|
+
A 'value_serializer' option allows use of other schemas such as Avro, Protobuf etc.
|
|
30
|
+
The default serialization is JSON Bytes
|
|
31
|
+
|
|
32
|
+
Example:
|
|
33
|
+
|
|
34
|
+
**{'bootstrap_servers': '127.0.0.1:9092',
|
|
35
|
+
'client_id': 'cryptofeed',
|
|
36
|
+
'acks': 1,
|
|
37
|
+
'value_serializer': your_serialization_function}
|
|
38
|
+
|
|
39
|
+
(Passing the event loop is already handled)
|
|
40
|
+
"""
|
|
41
|
+
self.producer_config = kwargs
|
|
42
|
+
self.producer = None
|
|
43
|
+
self.key: str = key or self.default_key
|
|
44
|
+
self.numeric_type = numeric_type
|
|
45
|
+
self.none_to = none_to
|
|
46
|
+
# Do not allow writer to send messages until connection confirmed
|
|
47
|
+
self.running = False
|
|
48
|
+
|
|
49
|
+
def _default_serializer(self, to_bytes: dict | str) -> ByteString:
|
|
50
|
+
if isinstance(to_bytes, dict):
|
|
51
|
+
return json.dumpb(to_bytes)
|
|
52
|
+
elif isinstance(to_bytes, str):
|
|
53
|
+
return to_bytes.encode()
|
|
54
|
+
else:
|
|
55
|
+
raise TypeError(f'{type(to_bytes)} is not a valid Serialization type')
|
|
56
|
+
|
|
57
|
+
async def _connect(self):
|
|
58
|
+
if not self.producer:
|
|
59
|
+
loop = asyncio.get_event_loop()
|
|
60
|
+
try:
|
|
61
|
+
config_keys = ', '.join([k for k in self.producer_config.keys()])
|
|
62
|
+
LOG.info(f'{self.__class__.__name__}: Configuring AIOKafka with the following parameters: {config_keys}')
|
|
63
|
+
self.producer = AIOKafkaProducer(**self.producer_config, loop=loop)
|
|
64
|
+
# Quit if invalid config option passed to AIOKafka
|
|
65
|
+
except (TypeError, ValueError) as e:
|
|
66
|
+
LOG.error(f'{self.__class__.__name__}: Invalid AIOKafka configuration: {e.args}{chr(10)}See https://aiokafka.readthedocs.io/en/stable/api.html#aiokafka.AIOKafkaProducer for list of configuration options')
|
|
67
|
+
raise SystemExit
|
|
68
|
+
else:
|
|
69
|
+
while not self.running:
|
|
70
|
+
try:
|
|
71
|
+
await self.producer.start()
|
|
72
|
+
except KafkaConnectionError:
|
|
73
|
+
LOG.error(f'{self.__class__.__name__}: Unable to bootstrap from host(s)')
|
|
74
|
+
await asyncio.sleep(10)
|
|
75
|
+
else:
|
|
76
|
+
LOG.info(f'{self.__class__.__name__}: "{self.producer.client._client_id}" connected to cluster containing {len(self.producer.client.cluster.brokers())} broker(s)')
|
|
77
|
+
self.running = True
|
|
78
|
+
|
|
79
|
+
def topic(self, data: dict) -> str:
|
|
80
|
+
return f"{self.key}-{data['exchange']}-{data['symbol']}"
|
|
81
|
+
|
|
82
|
+
def partition_key(self, data: dict) -> Optional[bytes]:
|
|
83
|
+
return None
|
|
84
|
+
|
|
85
|
+
def partition(self, data: dict) -> Optional[int]:
|
|
86
|
+
return None
|
|
87
|
+
|
|
88
|
+
async def writer(self):
|
|
89
|
+
await self._connect()
|
|
90
|
+
while self.running:
|
|
91
|
+
async with self.read_queue() as updates:
|
|
92
|
+
for index in range(len(updates)):
|
|
93
|
+
topic = self.topic(updates[index])
|
|
94
|
+
# Check for user-provided serializers, otherwise use default
|
|
95
|
+
value = updates[index] if self.producer_config.get('value_serializer') else self._default_serializer(updates[index])
|
|
96
|
+
key = self.key if self.producer_config.get('key_serializer') else self._default_serializer(self.key)
|
|
97
|
+
partition = self.partition(updates[index])
|
|
98
|
+
try:
|
|
99
|
+
send_future = await self.producer.send(topic, value, key, partition)
|
|
100
|
+
await send_future
|
|
101
|
+
except RequestTimedOutError:
|
|
102
|
+
LOG.error(f'{self.__class__.__name__}: No response received from server within {self.producer._request_timeout_ms} ms. Messages may not have been delivered')
|
|
103
|
+
except NodeNotReadyError:
|
|
104
|
+
LOG.error(f'{self.__class__.__name__}: Node not ready')
|
|
105
|
+
except Exception as e:
|
|
106
|
+
LOG.info(f'{self.__class__.__name__}: Encountered an error:{chr(10)}{e}')
|
|
107
|
+
LOG.info(f"{self.__class__.__name__}: sending last messages and closing connection '{self.producer.client._client_id}'")
|
|
108
|
+
await self.producer.stop()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class TradeKafka(KafkaCallback, BackendCallback):
|
|
112
|
+
default_key = 'trades'
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class FundingKafka(KafkaCallback, BackendCallback):
|
|
116
|
+
default_key = 'funding'
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class BookKafka(KafkaCallback, BackendBookCallback):
|
|
120
|
+
default_key = 'book'
|
|
121
|
+
|
|
122
|
+
def __init__(self, *args, snapshots_only=False, snapshot_interval=1000, **kwargs):
|
|
123
|
+
self.snapshots_only = snapshots_only
|
|
124
|
+
self.snapshot_interval = snapshot_interval
|
|
125
|
+
self.snapshot_count = defaultdict(int)
|
|
126
|
+
super().__init__(*args, **kwargs)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class TickerKafka(KafkaCallback, BackendCallback):
|
|
130
|
+
default_key = 'ticker'
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class OpenInterestKafka(KafkaCallback, BackendCallback):
|
|
134
|
+
default_key = 'open_interest'
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class LiquidationsKafka(KafkaCallback, BackendCallback):
|
|
138
|
+
default_key = 'liquidations'
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class CandlesKafka(KafkaCallback, BackendCallback):
|
|
142
|
+
default_key = 'candles'
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class OrderInfoKafka(KafkaCallback, BackendCallback):
|
|
146
|
+
default_key = 'order_info'
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class TransactionsKafka(KafkaCallback, BackendCallback):
|
|
150
|
+
default_key = 'transactions'
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class BalancesKafka(KafkaCallback, BackendCallback):
|
|
154
|
+
default_key = 'balances'
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class FillsKafka(KafkaCallback, BackendCallback):
|
|
158
|
+
default_key = 'fills'
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
'''
|
|
2
|
-
Copyright (C) 2017-
|
|
2
|
+
Copyright (C) 2017-2023 Bryant Moscon - bmoscon@gmail.com
|
|
3
3
|
|
|
4
4
|
Please see the LICENSE file for the terms and conditions
|
|
5
5
|
associated with this software.
|
|
6
6
|
'''
|
|
7
7
|
from collections import defaultdict
|
|
8
8
|
|
|
9
|
-
import aioredis
|
|
9
|
+
from redis import asyncio as aioredis
|
|
10
10
|
from yapic import json
|
|
11
11
|
|
|
12
12
|
from cryptofeed.backends.backend import BackendBookCallback, BackendCallback, BackendQueue
|
|
@@ -78,6 +78,21 @@ class RedisStreamCallback(RedisCallback):
|
|
|
78
78
|
await conn.connection_pool.disconnect()
|
|
79
79
|
|
|
80
80
|
|
|
81
|
+
class RedisKeyCallback(RedisCallback):
|
|
82
|
+
|
|
83
|
+
async def writer(self):
|
|
84
|
+
conn = aioredis.from_url(self.redis)
|
|
85
|
+
|
|
86
|
+
while self.running:
|
|
87
|
+
async with self.read_queue() as updates:
|
|
88
|
+
update = list(updates)[-1]
|
|
89
|
+
if update:
|
|
90
|
+
await conn.set(f"{self.key}-{update['exchange']}-{update['symbol']}", json.dumps(update))
|
|
91
|
+
|
|
92
|
+
await conn.close()
|
|
93
|
+
await conn.connection_pool.disconnect()
|
|
94
|
+
|
|
95
|
+
|
|
81
96
|
class TradeRedis(RedisZSetCallback, BackendCallback):
|
|
82
97
|
default_key = 'trades'
|
|
83
98
|
|
|
@@ -114,6 +129,16 @@ class BookStream(RedisStreamCallback, BackendBookCallback):
|
|
|
114
129
|
super().__init__(*args, **kwargs)
|
|
115
130
|
|
|
116
131
|
|
|
132
|
+
class BookSnapshotRedisKey(RedisKeyCallback, BackendBookCallback):
|
|
133
|
+
default_key = 'book'
|
|
134
|
+
|
|
135
|
+
def __init__(self, *args, snapshot_interval=1000, score_key='receipt_timestamp', **kwargs):
|
|
136
|
+
kwargs['snapshots_only'] = True
|
|
137
|
+
self.snapshot_interval = snapshot_interval
|
|
138
|
+
self.snapshot_count = defaultdict(int)
|
|
139
|
+
super().__init__(*args, score_key=score_key, **kwargs)
|
|
140
|
+
|
|
141
|
+
|
|
117
142
|
class TickerRedis(RedisZSetCallback, BackendCallback):
|
|
118
143
|
default_key = 'ticker'
|
|
119
144
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
'''
|
|
2
|
-
Copyright (C) 2017-
|
|
2
|
+
Copyright (C) 2017-2023 Bryant Moscon - bmoscon@gmail.com
|
|
3
3
|
|
|
4
4
|
Please see the LICENSE file for the terms and conditions
|
|
5
5
|
associated with this software.
|
|
@@ -85,9 +85,9 @@ class SocketCallback(BackendQueue):
|
|
|
85
85
|
msg = json.dumps({'type': 'chunked', 'chunks': len(chunks), 'data': chunk}).encode()
|
|
86
86
|
self.conn.sendto(msg)
|
|
87
87
|
else:
|
|
88
|
-
self.conn.sendto(
|
|
88
|
+
self.conn.sendto(data.encode())
|
|
89
89
|
else:
|
|
90
|
-
self.conn.write(
|
|
90
|
+
self.conn.write(data.encode())
|
|
91
91
|
|
|
92
92
|
async def connect(self):
|
|
93
93
|
if not self.conn:
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
'''
|
|
2
|
-
Copyright (C) 2017-
|
|
2
|
+
Copyright (C) 2017-2023 Bryant Moscon - bmoscon@gmail.com
|
|
3
3
|
|
|
4
4
|
Please see the LICENSE file for the terms and conditions
|
|
5
5
|
associated with this software.
|
|
@@ -17,6 +17,7 @@ BITHUMB = 'BITHUMB'
|
|
|
17
17
|
BITMEX = 'BITMEX'
|
|
18
18
|
BINANCE = 'BINANCE'
|
|
19
19
|
BINANCE_US = 'BINANCE_US'
|
|
20
|
+
BINANCE_TR = 'BINANCE_TR'
|
|
20
21
|
BINANCE_FUTURES = 'BINANCE_FUTURES'
|
|
21
22
|
BINANCE_DELIVERY = 'BINANCE_DELIVERY'
|
|
22
23
|
BITDOTCOM = 'BIT.COM'
|
|
@@ -32,9 +33,6 @@ DELTA = 'DELTA'
|
|
|
32
33
|
DERIBIT = 'DERIBIT'
|
|
33
34
|
DYDX = 'DYDX'
|
|
34
35
|
EXX = 'EXX'
|
|
35
|
-
FTX = 'FTX'
|
|
36
|
-
FTX_US = 'FTX_US'
|
|
37
|
-
FTX_TR = 'FTX_TR'
|
|
38
36
|
FMFW = 'FMFW'
|
|
39
37
|
GATEIO = 'GATEIO'
|
|
40
38
|
GEMINI = 'GEMINI'
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
'''
|
|
2
|
-
Copyright (C) 2017-
|
|
2
|
+
Copyright (C) 2017-2023 Bryant Moscon - bmoscon@gmail.com
|
|
3
3
|
|
|
4
4
|
Please see the LICENSE file for the terms and conditions
|
|
5
5
|
associated with this software.
|
|
6
6
|
'''
|
|
7
7
|
from cryptofeed.defines import *
|
|
8
|
-
from cryptofeed.defines import
|
|
8
|
+
from cryptofeed.defines import EXX as EXX_str, FMFW as FMFW_str, OKX as OKX_str
|
|
9
9
|
from .bitdotcom import BitDotCom
|
|
10
10
|
from .phemex import Phemex
|
|
11
11
|
from .ascendex import AscendEX
|
|
@@ -15,6 +15,7 @@ from .binance import Binance
|
|
|
15
15
|
from .binance_delivery import BinanceDelivery
|
|
16
16
|
from .binance_futures import BinanceFutures
|
|
17
17
|
from .binance_us import BinanceUS
|
|
18
|
+
from .binance_tr import BinanceTR
|
|
18
19
|
from .fmfw import FMFW
|
|
19
20
|
from .bitfinex import Bitfinex
|
|
20
21
|
from .bitflyer import Bitflyer
|
|
@@ -31,9 +32,6 @@ from .delta import Delta
|
|
|
31
32
|
from .deribit import Deribit
|
|
32
33
|
from .dydx import dYdX
|
|
33
34
|
from .exx import EXX
|
|
34
|
-
from .ftx import FTX
|
|
35
|
-
from .ftx_us import FTXUS
|
|
36
|
-
from .ftx_tr import FTXTR
|
|
37
35
|
from .gateio import Gateio
|
|
38
36
|
from .gemini import Gemini
|
|
39
37
|
from .hitbtc import HitBTC
|
|
@@ -58,6 +56,7 @@ EXCHANGE_MAP = {
|
|
|
58
56
|
BINANCE_DELIVERY: BinanceDelivery,
|
|
59
57
|
BINANCE_FUTURES: BinanceFutures,
|
|
60
58
|
BINANCE_US: BinanceUS,
|
|
59
|
+
BINANCE_TR: BinanceTR,
|
|
61
60
|
BINANCE: Binance,
|
|
62
61
|
FMFW_str: FMFW,
|
|
63
62
|
BITDOTCOM: BitDotCom,
|
|
@@ -76,9 +75,6 @@ EXCHANGE_MAP = {
|
|
|
76
75
|
DELTA: Delta,
|
|
77
76
|
DYDX: dYdX,
|
|
78
77
|
EXX_str: EXX,
|
|
79
|
-
FTX_str: FTX,
|
|
80
|
-
FTX_US: FTXUS,
|
|
81
|
-
FTX_TR: FTXTR,
|
|
82
78
|
GATEIO: Gateio,
|
|
83
79
|
GEMINI: Gemini,
|
|
84
80
|
HITBTC: HitBTC,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Copyright (C) 2017-2023 Bryant Moscon - bmoscon@gmail.com
|
|
3
|
+
|
|
4
|
+
Please see the LICENSE file for the terms and conditions
|
|
5
|
+
associated with this software.
|
|
6
|
+
'''
|
|
7
|
+
import logging
|
|
8
|
+
|
|
9
|
+
from cryptofeed.connection import RestEndpoint, WebsocketEndpoint, Routes
|
|
10
|
+
from cryptofeed.defines import BINANCE_TR
|
|
11
|
+
from cryptofeed.exchanges.binance import Binance
|
|
12
|
+
from cryptofeed.exchanges.mixins.binance_rest import BinanceTRRestMixin
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
LOG = logging.getLogger('feedhandler')
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class BinanceTR(Binance, BinanceTRRestMixin):
|
|
19
|
+
id = BINANCE_TR
|
|
20
|
+
websocket_endpoints = [WebsocketEndpoint('wss://stream-cloud.trbinance.com')]
|
|
21
|
+
rest_endpoints = [RestEndpoint('https://api.binance.me', routes=Routes('/api/v3/exchangeInfo', l2book='/api/v3/depth?symbol={}&limit={}'))]
|