Qubx 0.0.1__cp311-cp311-manylinux_2_35_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of Qubx might be problematic. Click here for more details.
- qubx/__init__.py +164 -0
- qubx/_nb_magic.py +69 -0
- qubx/core/__init__.py +0 -0
- qubx/core/basics.py +224 -0
- qubx/core/lookups.py +152 -0
- qubx/core/series.cpython-311-x86_64-linux-gnu.so +0 -0
- qubx/core/series.pxd +94 -0
- qubx/core/series.pyx +763 -0
- qubx/core/strategy.py +89 -0
- qubx/core/utils.cpython-311-x86_64-linux-gnu.so +0 -0
- qubx/core/utils.pyx +54 -0
- qubx/data/readers.py +387 -0
- qubx/math/__init__.py +1 -0
- qubx/math/stats.py +42 -0
- qubx/ta/__init__.py +0 -0
- qubx/ta/indicators.cpython-311-x86_64-linux-gnu.so +0 -0
- qubx/ta/indicators.pyx +258 -0
- qubx/utils/__init__.py +3 -0
- qubx/utils/_pyxreloader.py +271 -0
- qubx/utils/charting/mpl_helpers.py +182 -0
- qubx/utils/marketdata/binance.py +212 -0
- qubx/utils/misc.py +234 -0
- qubx/utils/pandas.py +206 -0
- qubx/utils/time.py +145 -0
- qubx-0.0.1.dist-info/METADATA +39 -0
- qubx-0.0.1.dist-info/RECORD +27 -0
- qubx-0.0.1.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Misc graphics handy utilitites to be used in interactive analysis
|
|
3
|
+
"""
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
6
|
+
import statsmodels.tsa.stattools as st
|
|
7
|
+
import matplotlib
|
|
8
|
+
import matplotlib.pyplot as plt
|
|
9
|
+
from cycler import cycler
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
DARK_MATLPLOT_THEME = [
|
|
13
|
+
('backend', 'module://matplotlib_inline.backend_inline'),
|
|
14
|
+
('interactive', True),
|
|
15
|
+
('lines.color', '#5050f0'),
|
|
16
|
+
('text.color', '#d0d0d0'),
|
|
17
|
+
('axes.facecolor', '#000000'),
|
|
18
|
+
('axes.edgecolor', '#404040'),
|
|
19
|
+
('axes.grid', True),
|
|
20
|
+
('axes.labelsize', 'large'),
|
|
21
|
+
('axes.labelcolor', 'green'),
|
|
22
|
+
('axes.prop_cycle', cycler('color', ['#08F7FE', '#00ff41', '#FE53BB', '#F5D300', '#449AcD', 'g',
|
|
23
|
+
'#f62841', 'y', '#088487', '#E24A33', '#f01010'])),
|
|
24
|
+
('legend.fontsize', 'small'),
|
|
25
|
+
('legend.fancybox', False),
|
|
26
|
+
('legend.edgecolor', '#305030'),
|
|
27
|
+
('legend.shadow', False),
|
|
28
|
+
('lines.antialiased', True),
|
|
29
|
+
('lines.linewidth', 0.8), # reduced line width
|
|
30
|
+
('patch.linewidth', 0.5),
|
|
31
|
+
('patch.antialiased', True),
|
|
32
|
+
('xtick.color', '#909090'),
|
|
33
|
+
('ytick.color', '#909090'),
|
|
34
|
+
('xtick.labelsize', 'large'),
|
|
35
|
+
('ytick.labelsize', 'large'),
|
|
36
|
+
('grid.color', '#404040'),
|
|
37
|
+
('grid.linestyle', '--'),
|
|
38
|
+
('grid.linewidth', 0.5),
|
|
39
|
+
('grid.alpha', 0.8),
|
|
40
|
+
('figure.figsize', [12.0, 5.0]),
|
|
41
|
+
('figure.dpi', 80.0),
|
|
42
|
+
('figure.facecolor', '#050505'),
|
|
43
|
+
('figure.edgecolor', (1, 1, 1, 0)),
|
|
44
|
+
('figure.subplot.bottom', 0.125),
|
|
45
|
+
('savefig.facecolor', '#000000'),
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
LIGHT_MATPLOT_THEME = [
|
|
49
|
+
('backend', 'module://matplotlib_inline.backend_inline'),
|
|
50
|
+
('interactive', True),
|
|
51
|
+
('lines.color', '#101010'),
|
|
52
|
+
('text.color', '#303030'),
|
|
53
|
+
('lines.antialiased', True),
|
|
54
|
+
('lines.linewidth', 1),
|
|
55
|
+
('patch.linewidth', 0.5),
|
|
56
|
+
('patch.facecolor', '#348ABD'),
|
|
57
|
+
('patch.edgecolor', '#eeeeee'),
|
|
58
|
+
('patch.antialiased', True),
|
|
59
|
+
('axes.facecolor', '#fafafa'),
|
|
60
|
+
('axes.edgecolor', '#d0d0d0'),
|
|
61
|
+
('axes.linewidth', 1),
|
|
62
|
+
('axes.titlesize', 'x-large'),
|
|
63
|
+
('axes.labelsize', 'large'),
|
|
64
|
+
('axes.labelcolor', '#555555'),
|
|
65
|
+
('axes.axisbelow', True),
|
|
66
|
+
('axes.grid', True),
|
|
67
|
+
('axes.prop_cycle', cycler('color', ['#6792E0', '#27ae60', '#c44e52', '#975CC3', '#ff914d', '#77BEDB',
|
|
68
|
+
'#303030', '#4168B7', '#93B851', '#e74c3c', '#bc89e0', '#ff711a',
|
|
69
|
+
'#3498db', '#6C7A89'])),
|
|
70
|
+
('legend.fontsize', 'small'),
|
|
71
|
+
('legend.fancybox', False),
|
|
72
|
+
('xtick.color', '#707070'),
|
|
73
|
+
('ytick.color', '#707070'),
|
|
74
|
+
('grid.color', '#606060'),
|
|
75
|
+
('grid.linestyle', '--'),
|
|
76
|
+
('grid.linewidth', 0.5),
|
|
77
|
+
('grid.alpha', 0.3),
|
|
78
|
+
('figure.figsize', [8.0, 5.0]),
|
|
79
|
+
('figure.dpi', 80.0),
|
|
80
|
+
('figure.facecolor', '#ffffff'),
|
|
81
|
+
('figure.edgecolor', '#ffffff'),
|
|
82
|
+
('figure.subplot.bottom', 0.1)
|
|
83
|
+
]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def fig(w=16, h=5, dpi=96, facecolor=None, edgecolor=None, num=None):
|
|
87
|
+
"""
|
|
88
|
+
Simple helper for creating figure
|
|
89
|
+
"""
|
|
90
|
+
return plt.figure(num=num, figsize=(w, h), dpi=dpi, facecolor=facecolor, edgecolor=edgecolor)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def subplot(shape, loc, rowspan=2, colspan=1):
|
|
94
|
+
"""
|
|
95
|
+
Some handy grid splitting for plots. Example for 2x2:
|
|
96
|
+
|
|
97
|
+
>>> subplot(22, 1); plt.plot([-1,2,-3])
|
|
98
|
+
>>> subplot(22, 2); plt.plot([1,2,3])
|
|
99
|
+
>>> subplot(22, 3); plt.plot([1,2,3])
|
|
100
|
+
>>> subplot(22, 4); plt.plot([3,-2,1])
|
|
101
|
+
|
|
102
|
+
same as following
|
|
103
|
+
|
|
104
|
+
>>> subplot((2,2), (0,0)); plt.plot([-1,2,-3])
|
|
105
|
+
>>> subplot((2,2), (0,1)); plt.plot([1,2,3])
|
|
106
|
+
>>> subplot((2,2), (1,0)); plt.plot([1,2,3])
|
|
107
|
+
>>> subplot((2,2), (1,1)); plt.plot([3,-2,1])
|
|
108
|
+
|
|
109
|
+
:param shape: scalar (like matlab subplot) or tuple
|
|
110
|
+
:param loc: scalar (like matlab subplot) or tuple
|
|
111
|
+
:param rowspan: rows spanned
|
|
112
|
+
:param colspan: columns spanned
|
|
113
|
+
"""
|
|
114
|
+
isscalar = lambda x: not isinstance(x, (list, tuple, dict, np.ndarray))
|
|
115
|
+
|
|
116
|
+
if isscalar(shape):
|
|
117
|
+
if 0 < shape < 100:
|
|
118
|
+
shape = (max(shape // 10, 1), max(shape % 10, 1))
|
|
119
|
+
else:
|
|
120
|
+
raise ValueError("Wrong scalar value for shape. It should be in range (1...99)")
|
|
121
|
+
|
|
122
|
+
if isscalar(loc):
|
|
123
|
+
nm = max(shape[0], 1) * max(shape[1], 1)
|
|
124
|
+
if 0 < loc <= nm:
|
|
125
|
+
x = (loc - 1) // shape[1]
|
|
126
|
+
y = loc - 1 - shape[1] * x
|
|
127
|
+
loc = (x, y)
|
|
128
|
+
else:
|
|
129
|
+
raise ValueError("Wrong scalar value for location. It should be in range (1...%d)" % nm)
|
|
130
|
+
|
|
131
|
+
return plt.subplot2grid(shape, loc=loc, rowspan=rowspan, colspan=colspan)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def sbp(shape, loc, r=1, c=1):
|
|
135
|
+
"""
|
|
136
|
+
Just shortcut for subplot(...) function
|
|
137
|
+
|
|
138
|
+
:param shape: scalar (like matlab subplot) or tuple
|
|
139
|
+
:param loc: scalar (like matlab subplot) or tuple
|
|
140
|
+
:param r: rows spanned
|
|
141
|
+
:param c: columns spanned
|
|
142
|
+
:return:
|
|
143
|
+
"""
|
|
144
|
+
return subplot(shape, loc, rowspan=r, colspan=c)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def vline(ax, x, c, lw=1, ls='--'):
|
|
148
|
+
x = pd.to_datetime(x) if isinstance(x, str) else x
|
|
149
|
+
if not isinstance(ax, (list, tuple)):
|
|
150
|
+
ax = [ax]
|
|
151
|
+
for a in ax:
|
|
152
|
+
a.axvline(x, 0, 1, c=c, lw=1, linestyle=ls)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def hline(*zs, mirror=True):
|
|
156
|
+
[plt.axhline(z, ls='--', c='r', lw=0.5) for z in zs]
|
|
157
|
+
if mirror:
|
|
158
|
+
[plt.axhline(-z, ls='--', c='r', lw=0.5) for z in zs]
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def ellips(ax, x, y, c='r', r=2.5, lw=2, ls='-'):
|
|
162
|
+
"""
|
|
163
|
+
Draw ellips annotation on specified plot at (x,y) point
|
|
164
|
+
"""
|
|
165
|
+
from matplotlib.patches import Ellipse
|
|
166
|
+
x = pd.to_datetime(x) if isinstance(x, str) else x
|
|
167
|
+
w, h = (r, r) if np.isscalar(r) else (r[0], r[1])
|
|
168
|
+
ax.add_artist(Ellipse(xy=[x, y], width=w, height=h, angle=0, fill=False, color=c, lw=lw, ls=ls))
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def set_mpl_theme(theme: str):
|
|
172
|
+
import plotly.io as pio
|
|
173
|
+
|
|
174
|
+
if 'dark' in theme.lower():
|
|
175
|
+
pio.templates.default = "plotly_dark"
|
|
176
|
+
for (k, v) in DARK_MATLPLOT_THEME:
|
|
177
|
+
matplotlib.rcParams[k] = v
|
|
178
|
+
|
|
179
|
+
elif 'light' in theme.lower():
|
|
180
|
+
pio.templates.default = "plotly_white"
|
|
181
|
+
for (k, v) in LIGHT_MATPLOT_THEME:
|
|
182
|
+
matplotlib.rcParams[k] = v
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pandas as pd
|
|
3
|
+
from os.path import exists, join, split, basename
|
|
4
|
+
# from dateutil import parser
|
|
5
|
+
from tqdm.notebook import tqdm
|
|
6
|
+
from typing import Any, Callable, Dict, List
|
|
7
|
+
from binance.client import Client, HistoricalKlinesType, BinanceAPIException
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
from qubx import logger
|
|
11
|
+
from qubx.utils.misc import makedirs, get_local_qubx_folder
|
|
12
|
+
|
|
13
|
+
DEFALT_LOCAL_FILE_STORAGE = makedirs(get_local_qubx_folder(), 'data/export/binance_trades/')
|
|
14
|
+
|
|
15
|
+
_DEFAULT_MARKET_DATA_DB = 'md'
|
|
16
|
+
BINANCE_DATA_STORAGE = "https://s3-ap-northeast-1.amazonaws.com"
|
|
17
|
+
BINANCE_DATA_URL = "https://data.binance.vision/"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def get_binance_symbol_info_for_type(market_types: List[str]) -> Dict[str, Dict[str, Any]]:
|
|
21
|
+
"""
|
|
22
|
+
Get list of all symbols from binance for given list of market types:
|
|
23
|
+
possible types are: SPOT, FUTURES, COINSFUTURES
|
|
24
|
+
|
|
25
|
+
>>> get_binance_symbol_info_for_type('FUTURES')
|
|
26
|
+
|
|
27
|
+
:param market_type: SPOT, FUTURES (UM) or COINSFUTURES (CM)
|
|
28
|
+
"""
|
|
29
|
+
client = Client()
|
|
30
|
+
infos = {}
|
|
31
|
+
for market_type in (market_types if not isinstance(market_types, str) else [market_types]):
|
|
32
|
+
if market_type in ['FUTURES', 'UM']:
|
|
33
|
+
infos['binance.um'] = client.futures_exchange_info()
|
|
34
|
+
|
|
35
|
+
elif market_type in ['COINSFUTURES', 'CM']:
|
|
36
|
+
infos['binance.cm'] = client.futures_coin_exchange_info()
|
|
37
|
+
|
|
38
|
+
elif market_type == 'SPOT':
|
|
39
|
+
infos['binance'] = client.get_exchange_info()
|
|
40
|
+
else:
|
|
41
|
+
raise ValueError("Only 'FUTURES | UM', 'COINSFUTURES | CM' or 'SPOT' are supported for market_type")
|
|
42
|
+
|
|
43
|
+
return infos
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def fetch_file(url, local_file_storage, chunk_size=1024*1024):
|
|
47
|
+
"""
|
|
48
|
+
Load file from url and store it to specified storage
|
|
49
|
+
"""
|
|
50
|
+
file = split(url)[-1]
|
|
51
|
+
response = requests.get(url, stream=True)
|
|
52
|
+
with open(join(local_file_storage, file), "wb") as handle:
|
|
53
|
+
for data in tqdm(response.iter_content(chunk_size=chunk_size)):
|
|
54
|
+
handle.write(data)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def get_trades_files(symbol: str, instr_type: str, instr_subtype: str):
|
|
58
|
+
"""
|
|
59
|
+
Get list of trades files for specified instrument from Binance datastorage
|
|
60
|
+
"""
|
|
61
|
+
if instr_type.lower() == 'spot':
|
|
62
|
+
instr_subtype = ''
|
|
63
|
+
filter_str = join("data", instr_type.lower(), instr_subtype.lower(), "monthly", "trades", symbol.upper())
|
|
64
|
+
pg = requests.get(f"{BINANCE_DATA_STORAGE}/data.binance.vision?prefix={filter_str}/")
|
|
65
|
+
info = pd.read_xml(pg.text)
|
|
66
|
+
return [k for k in info.Key.dropna() if k.endswith('.zip')]
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def load_trades_for(symbol, instr_type='futures', instr_subtype='um', local_file_storage=DEFALT_LOCAL_FILE_STORAGE):
|
|
70
|
+
"""
|
|
71
|
+
Load trades from Binance data storage
|
|
72
|
+
>>> load_trades_for('ETHUSDT', 'futures', 'um')
|
|
73
|
+
"""
|
|
74
|
+
local_file_storage = makedirs(local_file_storage)
|
|
75
|
+
|
|
76
|
+
f_list = get_trades_files(symbol, instr_type, instr_subtype)
|
|
77
|
+
for r_file in tqdm(f_list):
|
|
78
|
+
dest_dir = join(local_file_storage, symbol.upper())
|
|
79
|
+
dest_file = join(dest_dir, basename(r_file))
|
|
80
|
+
if not exists(dest_file):
|
|
81
|
+
fetch_file(join(BINANCE_DATA_URL, r_file), dest_dir)
|
|
82
|
+
else:
|
|
83
|
+
logger.info(f"{dest_file} already loaded, skipping ...")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
# class BinanceHist:
|
|
87
|
+
# START_SPOT_HIST = pd.Timestamp('2017-01-01')
|
|
88
|
+
# START_FUT_HIST = pd.Timestamp('2019-09-08')
|
|
89
|
+
|
|
90
|
+
# def __init__(self, api_key, secret_key):
|
|
91
|
+
# self.client = Client(api_key=api_key, api_secret=secret_key)
|
|
92
|
+
|
|
93
|
+
# @staticmethod
|
|
94
|
+
# def from_env(path):
|
|
95
|
+
# if exists(path):
|
|
96
|
+
# return BinanceHist(**get_env_data_as_dict(path))
|
|
97
|
+
# else:
|
|
98
|
+
# raise ValueError(f"Can't find env file at {path}")
|
|
99
|
+
|
|
100
|
+
# @staticmethod
|
|
101
|
+
# def _t2s(time):
|
|
102
|
+
# _t = pd.Timestamp(time) if not isinstance(time, pd.Timestamp) else time
|
|
103
|
+
# return _t.strftime('%d %b %Y %H:%M:%S')
|
|
104
|
+
|
|
105
|
+
# def load_hist_spot_data(self, symbol, timeframe, start: pd.Timestamp, stop: pd.Timestamp, step='4W', timeout_sec=2):
|
|
106
|
+
# return self.load_hist_data(symbol, timeframe, start, stop, HistoricalKlinesType.SPOT, step, timeout_sec)
|
|
107
|
+
|
|
108
|
+
# def load_hist_futures_data(self, symbol, timeframe, start: pd.Timestamp, stop: pd.Timestamp, step='4W', timeout_sec=2):
|
|
109
|
+
# return self.load_hist_data(symbol, timeframe, start, stop, HistoricalKlinesType.FUTURES, step, timeout_sec)
|
|
110
|
+
|
|
111
|
+
# def update_hist_data(self, symbol, stype, timeframe, exchange_id, drop_prev=False, step='4W', timeout_sec=2):
|
|
112
|
+
# """
|
|
113
|
+
# Update loaded data in z db:
|
|
114
|
+
|
|
115
|
+
# >>> bh = BinanceHist.from_env('.env')
|
|
116
|
+
# >>> bh.update_hist_data('BTCUSDT', 'FUTURES', '1Min', 'BINANCEF')
|
|
117
|
+
# >>> bh.update_hist_data('BTCUSDT', 'SPOT', '1Min', 'BINANCE')
|
|
118
|
+
# """
|
|
119
|
+
# if isinstance(symbol, (tuple, list)):
|
|
120
|
+
# for s in symbol:
|
|
121
|
+
# print(f"Working on {symbol.index(s)}th from {len(symbol)} symbols...")
|
|
122
|
+
# self.update_hist_data(s, stype, timeframe, exchange_id, drop_prev=drop_prev, step=step, timeout_sec=timeout_sec)
|
|
123
|
+
# return
|
|
124
|
+
|
|
125
|
+
# klt = {
|
|
126
|
+
# 'futures': [HistoricalKlinesType.FUTURES, BinanceHist.START_FUT_HIST],
|
|
127
|
+
# 'spot': [HistoricalKlinesType.SPOT, BinanceHist.START_SPOT_HIST]
|
|
128
|
+
# }.get(stype.lower())
|
|
129
|
+
|
|
130
|
+
# if klt is None:
|
|
131
|
+
# raise ValueError(f"Unknown instrument type '{stype}' !")
|
|
132
|
+
|
|
133
|
+
# tD = pd.Timedelta(timeframe)
|
|
134
|
+
# now = self._t2s(pd.Timestamp(datetime.datetime.now(pytz.UTC).replace(second=0)) - tD)
|
|
135
|
+
|
|
136
|
+
# path = f'm1/{exchange_id}:{symbol}'
|
|
137
|
+
# if drop_prev:
|
|
138
|
+
# print(f' > Deleting data at {path} ...')
|
|
139
|
+
# z_del(path, dbname=_DEFAULT_MARKET_DATA_DB)
|
|
140
|
+
|
|
141
|
+
# mc = MongoController(dbname=_DEFAULT_MARKET_DATA_DB)
|
|
142
|
+
|
|
143
|
+
# hist = z_ld(path, dbname=_DEFAULT_MARKET_DATA_DB)
|
|
144
|
+
# if hist is None:
|
|
145
|
+
# last_known_date = klt[1]
|
|
146
|
+
# else:
|
|
147
|
+
# last_known_date = hist.index[-1]
|
|
148
|
+
|
|
149
|
+
# # 're-serialize' data to be accessible through gridfs
|
|
150
|
+
# if mc.get_count(path) == 0:
|
|
151
|
+
# z_save(path, hist, is_serialize=False, dbname=_DEFAULT_MARKET_DATA_DB)
|
|
152
|
+
|
|
153
|
+
# s_time = last_known_date.round(tD)
|
|
154
|
+
|
|
155
|
+
# # pull up recent data
|
|
156
|
+
# def __cb(data):
|
|
157
|
+
# # print(">>>>>> new data: ", len(data))
|
|
158
|
+
# mc.append_data(path, data)
|
|
159
|
+
|
|
160
|
+
# self.load_hist_data(symbol, timeframe, s_time, now, klt[0], step=step, timeout_sec=timeout_sec, new_data_callback=__cb)
|
|
161
|
+
# mc.close()
|
|
162
|
+
|
|
163
|
+
# # drop _id (after update through MongoController)
|
|
164
|
+
# pdata = z_ld(path, dbname=_DEFAULT_MARKET_DATA_DB)
|
|
165
|
+
# if pdata is not None:
|
|
166
|
+
# pdata = pdata.drop(columns='_id')
|
|
167
|
+
# z_save(path, pdata, dbname=_DEFAULT_MARKET_DATA_DB)
|
|
168
|
+
|
|
169
|
+
# def load_hist_data(self, symbol, timeframe, start: pd.Timestamp, stop: pd.Timestamp, ktype, step='4W', timeout_sec=2,
|
|
170
|
+
# new_data_callback: Callable[[pd.DataFrame], None]=None):
|
|
171
|
+
# start = pd.Timestamp(start) if not isinstance(start, pd.Timestamp) else start
|
|
172
|
+
# stop = pd.Timestamp(stop) if not isinstance(stop, pd.Timestamp) else stop
|
|
173
|
+
# df = pd.DataFrame()
|
|
174
|
+
|
|
175
|
+
# _tf_str = timeframe[:2].lower()
|
|
176
|
+
# tD = pd.Timedelta(timeframe)
|
|
177
|
+
# now = self._t2s(stop)
|
|
178
|
+
# tlr = pd.DatetimeIndex([start]).append(pd.date_range(start, now, freq=step).append(pd.DatetimeIndex([now])))
|
|
179
|
+
|
|
180
|
+
# print(f' >> Loading {green(symbol)} {yellow(timeframe)} for [{red(start)} -> {red(now)}]')
|
|
181
|
+
# s = tlr[0]
|
|
182
|
+
# for e in tqdm(tlr[1:]):
|
|
183
|
+
# if s + tD < e:
|
|
184
|
+
# _start, _stop = self._t2s(s + tD), self._t2s(e)
|
|
185
|
+
# # print(_start, _stop)
|
|
186
|
+
# chunk = None
|
|
187
|
+
# nerr = 0
|
|
188
|
+
# while nerr < 3:
|
|
189
|
+
# try:
|
|
190
|
+
# chunk = self.client.get_historical_klines(symbol, _tf_str, _start, _stop, klines_type=ktype, limit=1000)
|
|
191
|
+
# nerr = 100
|
|
192
|
+
# except BinanceAPIException as a:
|
|
193
|
+
# print('.', end='')
|
|
194
|
+
# break
|
|
195
|
+
# except Exception as err:
|
|
196
|
+
# nerr +=1
|
|
197
|
+
# print(red(str(err)))
|
|
198
|
+
# time.sleep(10)
|
|
199
|
+
|
|
200
|
+
# if chunk:
|
|
201
|
+
# data = pd.DataFrame(chunk, columns = ['timestamp', 'open', 'high', 'low', 'close', 'volume', 'close_time', 'quote_av', 'trades', 'tb_base_av', 'tb_quote_av', 'ignore' ])
|
|
202
|
+
# data.index = pd.to_datetime(data['timestamp'].rename('time'), unit='ms')
|
|
203
|
+
# data = data.drop(columns=['timestamp', 'close_time', 'quote_av', 'trades', 'tb_base_av', 'tb_quote_av', 'ignore' ]).astype(float)
|
|
204
|
+
# df = df.append(data)
|
|
205
|
+
# # callback on new data
|
|
206
|
+
# if new_data_callback is not None:
|
|
207
|
+
# new_data_callback(data)
|
|
208
|
+
|
|
209
|
+
# s = e
|
|
210
|
+
# time.sleep(timeout_sec)
|
|
211
|
+
# return df
|
|
212
|
+
|
qubx/utils/misc.py
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
import glob, os
|
|
2
|
+
from collections import OrderedDict, namedtuple
|
|
3
|
+
from os.path import basename, exists, dirname, join, expanduser
|
|
4
|
+
from typing import Optional, Union
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def version() -> str:
|
|
9
|
+
# - check current version
|
|
10
|
+
version = 'Dev'
|
|
11
|
+
try:
|
|
12
|
+
import importlib_metadata
|
|
13
|
+
version = importlib_metadata.version('qube2')
|
|
14
|
+
except:
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
return version
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
from ._pyxreloader import pyx_install_loader
|
|
21
|
+
|
|
22
|
+
def install_pyx_recompiler_for_dev():
|
|
23
|
+
if version().lower() == 'dev':
|
|
24
|
+
print(f" > [{green('dev')}] {red('installed cython rebuilding hook')}")
|
|
25
|
+
pyx_install_loader(['qubx.core', 'qubx.ta', 'qubx.data', 'qubx.strategies'])
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def runtime_env():
|
|
29
|
+
"""
|
|
30
|
+
Check what environment this script is being run under
|
|
31
|
+
:return: environment name, possible values:
|
|
32
|
+
- 'notebook' jupyter notebook
|
|
33
|
+
- 'shell' any interactive shell (ipython, PyCharm's console etc)
|
|
34
|
+
- 'python' standard python interpreter
|
|
35
|
+
- 'unknown' can't recognize environment
|
|
36
|
+
"""
|
|
37
|
+
try:
|
|
38
|
+
from IPython import get_ipython
|
|
39
|
+
shell = get_ipython().__class__.__name__
|
|
40
|
+
|
|
41
|
+
if shell == 'ZMQInteractiveShell': # Jupyter notebook or qtconsole
|
|
42
|
+
return 'notebook'
|
|
43
|
+
elif shell.endswith('TerminalInteractiveShell'): # Terminal running IPython
|
|
44
|
+
return 'shell'
|
|
45
|
+
else:
|
|
46
|
+
return 'unknown' # Other type (?)
|
|
47
|
+
except (NameError, ImportError):
|
|
48
|
+
return 'python' # Probably standard Python interpreter
|
|
49
|
+
|
|
50
|
+
_QUBX_FLDR = None
|
|
51
|
+
|
|
52
|
+
def get_local_qubx_folder() -> str:
|
|
53
|
+
global _QUBX_FLDR
|
|
54
|
+
|
|
55
|
+
if _QUBX_FLDR is None:
|
|
56
|
+
_QUBX_FLDR = makedirs(os.getenv('QUBXSTORAGE', os.path.expanduser('~/.qubx')))
|
|
57
|
+
|
|
58
|
+
return _QUBX_FLDR
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def add_project_to_system_path(project_folder:str = '~/projects'):
|
|
62
|
+
"""
|
|
63
|
+
Add path to projects folder to system python path to be able importing any modules from project
|
|
64
|
+
from test.Models.handy_utils import some_module
|
|
65
|
+
"""
|
|
66
|
+
import sys
|
|
67
|
+
from os.path import expanduser, relpath
|
|
68
|
+
from pathlib import Path
|
|
69
|
+
|
|
70
|
+
# we want to track folders with these files as separate paths
|
|
71
|
+
toml = Path('pyproject.toml')
|
|
72
|
+
src = Path('src')
|
|
73
|
+
|
|
74
|
+
try:
|
|
75
|
+
prj = Path(relpath(expanduser(project_folder)))
|
|
76
|
+
except ValueError as e:
|
|
77
|
+
# This error can occur on Windows if user folder and python file are on different drives
|
|
78
|
+
print(f"Qube> Error during get path to projects folder:\n{e}")
|
|
79
|
+
else:
|
|
80
|
+
insert_path_iff = lambda p: sys.path.insert(0, p.as_posix()) if p.as_posix() not in sys.path else None
|
|
81
|
+
if prj.exists():
|
|
82
|
+
insert_path_iff(prj)
|
|
83
|
+
|
|
84
|
+
for di in prj.iterdir():
|
|
85
|
+
_src = di / src
|
|
86
|
+
if (di / toml).exists():
|
|
87
|
+
# when we have src/
|
|
88
|
+
if _src.exists() and _src.is_dir():
|
|
89
|
+
insert_path_iff(_src)
|
|
90
|
+
else:
|
|
91
|
+
insert_path_iff(di)
|
|
92
|
+
else:
|
|
93
|
+
print(f'Qube> Cant find {project_folder} folder for adding to python path !')
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def is_localhost(host):
|
|
97
|
+
return host.lower() == 'localhost' or host == '127.0.0.1'
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def __wrap_with_color(code):
|
|
101
|
+
def inner(text, bold=False):
|
|
102
|
+
c = code
|
|
103
|
+
if bold:
|
|
104
|
+
c = "1;%s" % c
|
|
105
|
+
return "\033[%sm%s\033[0m" % (c, text)
|
|
106
|
+
|
|
107
|
+
return inner
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
red, green, yellow, blue, magenta, cyan, white = (
|
|
111
|
+
__wrap_with_color('31'),
|
|
112
|
+
__wrap_with_color('32'),
|
|
113
|
+
__wrap_with_color('33'),
|
|
114
|
+
__wrap_with_color('34'),
|
|
115
|
+
__wrap_with_color('35'),
|
|
116
|
+
__wrap_with_color('36'),
|
|
117
|
+
__wrap_with_color('37'),
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class Struct:
|
|
122
|
+
"""
|
|
123
|
+
Dynamic structure (similar to matlab's struct it allows to add new properties dynamically)
|
|
124
|
+
|
|
125
|
+
>>> a = Struct(x=1, y=2)
|
|
126
|
+
>>> a.z = 'Hello'
|
|
127
|
+
>>> print(a)
|
|
128
|
+
|
|
129
|
+
Struct(x=1, y=2, z='Hello')
|
|
130
|
+
|
|
131
|
+
>>> Struct(a=234, b=Struct(c=222)).to_dict()
|
|
132
|
+
|
|
133
|
+
{'a': 234, 'b': {'c': 222}}
|
|
134
|
+
|
|
135
|
+
>>> Struct({'a': 555}, a=123, b=Struct(c=222)).to_dict()
|
|
136
|
+
|
|
137
|
+
{'a': 123, 'b': {'c': 222}}
|
|
138
|
+
"""
|
|
139
|
+
|
|
140
|
+
def __init__(self, *args, **kwargs):
|
|
141
|
+
_odw = OrderedDict(**kwargs)
|
|
142
|
+
if args:
|
|
143
|
+
if isinstance(args[0], dict):
|
|
144
|
+
_odw = OrderedDict(Struct.dict2struct(args[0]).to_dict()) | _odw
|
|
145
|
+
elif isinstance(args[0], Struct):
|
|
146
|
+
_odw = args[0].to_dict() | _odw
|
|
147
|
+
self.__initialize(_odw.keys(), _odw.values())
|
|
148
|
+
|
|
149
|
+
def __initialize(self, fields, values):
|
|
150
|
+
self._fields = list(fields)
|
|
151
|
+
self._meta = namedtuple('Struct', ' '.join(fields))
|
|
152
|
+
self._inst = self._meta(*values)
|
|
153
|
+
|
|
154
|
+
def fields(self) -> list:
|
|
155
|
+
return self._fields
|
|
156
|
+
|
|
157
|
+
def __getitem__(self, idx: int):
|
|
158
|
+
return getattr(self._inst, self._fields[idx])
|
|
159
|
+
|
|
160
|
+
def __getattr__(self, k):
|
|
161
|
+
return getattr(self._inst, k)
|
|
162
|
+
|
|
163
|
+
def __or__(self, other: Union[dict, 'Struct']):
|
|
164
|
+
if isinstance(other, dict):
|
|
165
|
+
other = Struct.dict2struct(other)
|
|
166
|
+
elif not isinstance(other, Struct):
|
|
167
|
+
raise ValueError(f"Can't union with object of {type(other)} type ")
|
|
168
|
+
for f in other.fields():
|
|
169
|
+
self.__setattr__(f, other.__getattr__(f))
|
|
170
|
+
return self
|
|
171
|
+
|
|
172
|
+
def __dir__(self):
|
|
173
|
+
return self._fields
|
|
174
|
+
|
|
175
|
+
def __repr__(self):
|
|
176
|
+
return self._inst.__repr__()
|
|
177
|
+
|
|
178
|
+
def __setattr__(self, k, v):
|
|
179
|
+
if k not in ['_inst', '_meta', '_fields']:
|
|
180
|
+
new_vals = {**self._inst._asdict(), **{k: v}}
|
|
181
|
+
self.__initialize(new_vals.keys(), new_vals.values())
|
|
182
|
+
else:
|
|
183
|
+
super().__setattr__(k, v)
|
|
184
|
+
|
|
185
|
+
def __getstate__(self):
|
|
186
|
+
return self._inst._asdict()
|
|
187
|
+
|
|
188
|
+
def __setstate__(self, state):
|
|
189
|
+
self.__init__(**state)
|
|
190
|
+
|
|
191
|
+
def __ms2d(self, m) -> dict:
|
|
192
|
+
r = {}
|
|
193
|
+
for f in m._fields:
|
|
194
|
+
v = m.__getattr__(f)
|
|
195
|
+
r[f] = self.__ms2d(v) if isinstance(v, Struct) else v
|
|
196
|
+
return r
|
|
197
|
+
|
|
198
|
+
def to_dict(self) -> dict:
|
|
199
|
+
"""
|
|
200
|
+
Return this structure as dictionary
|
|
201
|
+
"""
|
|
202
|
+
return self.__ms2d(self)
|
|
203
|
+
|
|
204
|
+
def copy(self) -> 'Struct':
|
|
205
|
+
"""
|
|
206
|
+
Returns copy of this structure
|
|
207
|
+
"""
|
|
208
|
+
return Struct(self.to_dict())
|
|
209
|
+
|
|
210
|
+
@staticmethod
|
|
211
|
+
def dict2struct(d: dict) -> 'Struct':
|
|
212
|
+
"""
|
|
213
|
+
Convert dictionary to structure
|
|
214
|
+
>>> s = dict2struct({'f_1_0': 1, 'z': {'x': 1, 'y': 2}})
|
|
215
|
+
>>> print(s.z.x)
|
|
216
|
+
1
|
|
217
|
+
"""
|
|
218
|
+
m = Struct()
|
|
219
|
+
for k, v in d.items():
|
|
220
|
+
# skip if key is not valid identifier
|
|
221
|
+
if not k.isidentifier():
|
|
222
|
+
print(f"Struct> {k} doesn't look like as identifier - skip it")
|
|
223
|
+
continue
|
|
224
|
+
if isinstance(v, dict):
|
|
225
|
+
v = Struct.dict2struct(v)
|
|
226
|
+
m.__setattr__(k, v)
|
|
227
|
+
return m
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def makedirs(path: str, *args) -> str:
|
|
231
|
+
path = os.path.expanduser(os.path.join(*[path, *args]))
|
|
232
|
+
if not exists(path):
|
|
233
|
+
os.makedirs(path)
|
|
234
|
+
return path
|