Qubx 0.1.5__cp311-cp311-manylinux_2_35_x86_64.whl → 0.1.7__cp311-cp311-manylinux_2_35_x86_64.whl

qubx/data/readers.py

@@ -29,6 +29,13 @@ def _recognize_t(t: Union[int, str], defaultvalue, timeunit) -> int:
     return defaultvalue
 
 
+def _time(t, timestamp_units: str) -> int:
+    t = int(t) if isinstance(t, float) else t
+    if timestamp_units == 'ns':
+        return np.datetime64(t, 'ns').item() 
+    return np.datetime64(t, timestamp_units).astype('datetime64[ns]').item()
+
+
 def _find_column_index_in_list(xs, *args):
     xs = [x.lower() for x in xs]
     for a in args:
@@ -38,6 +45,9 @@ def _find_column_index_in_list(xs, *args):
     raise IndexError(f"Can't find any from {args} in list: {xs}")
 
 
+_FIND_TIME_COL_IDX = lambda column_names: _find_column_index_in_list(column_names, 'time', 'timestamp', 'datetime', 'date', 'open_time')
+
+
 class DataTransformer:
 
     def __init__(self) -> None:
@@ -121,9 +131,9 @@ class CsvStorageDataReader(DataReader):
         # - try to find range to load  
         start_idx, stop_idx = 0, table.num_rows
         try:
-            _time_field_idx = _find_column_index_in_list(fieldnames, 'time', 'timestamp', 'datetime', 'date')
+            _time_field_idx = _FIND_TIME_COL_IDX(fieldnames)
             _time_type = table.field(_time_field_idx).type
-            _time_unit = _time_type.unit if hasattr(_time_type, 'unit') else 's'
+            _time_unit = _time_type.unit if hasattr(_time_type, 'unit') else 'ms'
             _time_data = table[_time_field_idx]
 
             # - check if need convert time to primitive types (i.e. Date32 -> timestamp[x])
@@ -181,9 +191,11 @@ class AsPandasFrame(DataTransformer):
     """
     List of records to pandas dataframe transformer
     """
+    def __init__(self, timestamp_units=None) -> None:
+        self.timestamp_units = timestamp_units
 
     def start_transform(self, name: str, column_names: List[str]):
-        self._time_idx = _find_column_index_in_list(column_names, 'time', 'timestamp', 'datetime', 'date')
+        self._time_idx = _FIND_TIME_COL_IDX(column_names)
         self._column_names = column_names
         self._frame = pd.DataFrame()
     
@@ -191,6 +203,8 @@ class AsPandasFrame(DataTransformer):
         self._frame
         p = pd.DataFrame.from_records(rows_data, columns=self._column_names)
         p.set_index(self._column_names[self._time_idx], drop=True, inplace=True)
+        p.index = pd.to_datetime(p.index, unit=self.timestamp_units) if self.timestamp_units else p.index
+        p.index.rename('timestamp', inplace=True)
         p.sort_index(inplace=True)
         self._frame = pd.concat((self._frame, p), axis=0, sort=True)
         return p
@@ -200,6 +214,17 @@ class AsPandasFrame(DataTransformer):
 
 
 class AsOhlcvSeries(DataTransformer):
+    """
+    Convert incoming data into OHLCV series.
+
+    Incoming data may have one of the following structures: 
+
+        ```
+        ohlcv:        time,open,high,low,close,volume|quote_volume,(buy_volume)
+        quotes:       time,bid,ask,bidsize,asksize
+        trades (TAS): time,price,size,(is_taker)
+        ```
+    """
 
     def __init__(self, timeframe: str | None = None, timestamp_units='ns') -> None:
         super().__init__()
@@ -209,7 +234,7 @@ class AsOhlcvSeries(DataTransformer):
         self.timestamp_units = timestamp_units
 
     def start_transform(self, name: str, column_names: List[str]):
-        self._time_idx = _find_column_index_in_list(column_names, 'time', 'timestamp', 'datetime', 'date')
+        self._time_idx = _FIND_TIME_COL_IDX(column_names)
         self._volume_idx = None
         self._b_volume_idx = None
         try:
@@ -251,15 +276,10 @@ class AsOhlcvSeries(DataTransformer):
         if self.timeframe:
             self._series = OHLCV(self._name, self.timeframe)
 
-    def _time(self, t) -> int:
-        if self.timestamp_units == 'ns':
-            return np.datetime64(t, 'ns').item() 
-        return np.datetime64(t, self.timestamp_units).astype('datetime64[ns]').item()
-
     def _proc_ohlc(self, rows_data: List[List]):
         for d in rows_data:
             self._series.update_by_bar(
-                self._time(d[self._time_idx]),
+                _time(d[self._time_idx], self.timestamp_units),
                 d[self._open_idx], d[self._high_idx], d[self._low_idx], d[self._close_idx], 
                 d[self._volume_idx] if self._volume_idx else 0,
                 d[self._b_volume_idx] if self._b_volume_idx else 0
@@ -268,7 +288,7 @@ class AsOhlcvSeries(DataTransformer):
     def _proc_quotes(self, rows_data: List[List]):
         for d in rows_data:
             self._series.update(
-                self._time(d[self._time_idx]),
+                _time(d[self._time_idx], self.timestamp_units),
                 (d[self._ask_idx] + d[self._bid_idx])/2
             )
 
@@ -277,7 +297,7 @@ class AsOhlcvSeries(DataTransformer):
             a = d[self._taker_idx] if self._taker_idx else 0
             s = d[self._size_idx]
             b = s if a else 0 
-            self._series.update(self._time(d[self._time_idx]), d[self._price_idx], s, b)
+            self._series.update(_time(d[self._time_idx], self.timestamp_units), d[self._price_idx], s, b)
 
     def process_data(self, rows_data: List[List]) -> Any:
         if self._series is None:
@@ -302,10 +322,14 @@ class AsOhlcvSeries(DataTransformer):
 
 
 class AsQuotes(DataTransformer):
+    """
+    Tries to convert incoming data to list of Quote's
+    Data must have appropriate structure: bid, ask, bidsize, asksize and time
+    """
 
     def start_transform(self, name: str, column_names: List[str]):
         self.buffer = list()
-        self._time_idx = _find_column_index_in_list(column_names, 'time', 'timestamp', 'datetime')
+        self._time_idx = _FIND_TIME_COL_IDX(column_names)
         self._bid_idx = _find_column_index_in_list(column_names, 'bid')
         self._ask_idx = _find_column_index_in_list(column_names, 'ask')
         self._bidvol_idx = _find_column_index_in_list(column_names, 'bidvol', 'bid_vol', 'bidsize', 'bid_size')
@@ -320,6 +344,48 @@ class AsQuotes(DataTransformer):
                 bv = d[self._bidvol_idx]
                 av = d[self._askvol_idx]
                 self.buffer.append(Quote(t.as_unit('ns').asm8.item(), b, a, bv, av))
+                
+
+class AsTimestampedRecords(DataTransformer):
+    """
+    Convert incoming data to list or dictionaries with preprocessed timestamps ('timestamp_ns' and 'timestamp')
+    ```
+    [
+        {
+            'open_time': 1711944240000.0,
+            'open': 203.219,
+            'high': 203.33,
+            'low': 203.134,
+            'close': 203.175,
+            'volume': 10060.0,
+            ....
+            'timestamp_ns': 1711944240000000000,
+            'timestamp': Timestamp('2024-04-01 04:04:00')
+        },
+        ...
+    ] ```
+    """
+
+    def __init__(self, timestamp_units: str | None=None) -> None:
+        self.timestamp_units = timestamp_units
+
+    def start_transform(self, name: str, column_names: List[str]):
+        self.buffer = list()
+        self._time_idx = _FIND_TIME_COL_IDX(column_names)
+        self._column_names = column_names
+
+    def process_data(self, rows_data: Iterable) -> Any:
+        self.buffer.extend(rows_data)
+
+    def collect(self) -> Any:
+        res = []
+        for r in self.buffer:
+            t = r[self._time_idx]
+            if self.timestamp_units:
+                t = _time(t, self.timestamp_units)
+            di = dict(zip(self._column_names, r)) | { 'timestamp_ns': t, 'timestamp': pd.Timestamp(t) }
+            res.append(di)
+        return res
 
 
 class RestoreTicksFromOHLC(DataTransformer):
@@ -344,7 +410,7 @@ class RestoreTicksFromOHLC(DataTransformer):
     def start_transform(self, name: str, column_names: List[str]):
         self.buffer = []
         # - it will fail if receive data doesn't look as ohlcv 
-        self._time_idx = _find_column_index_in_list(column_names, 'time', 'timestamp', 'datetime', 'date')
+        self._time_idx = _FIND_TIME_COL_IDX(column_names)
         self._open_idx = _find_column_index_in_list(column_names, 'open')
         self._high_idx = _find_column_index_in_list(column_names, 'high')
         self._low_idx = _find_column_index_in_list(column_names, 'low')
@@ -465,15 +531,26 @@ class QuestDBConnector(DataReader):
              chunksize=0,  # TODO: use self._cursor.fetchmany in this case !!!!
              timeframe: str='1m') -> Any:
         start, end = handle_start_stop(start, stop)
-        w0 = f"timestamp >= '{start}'" if start else ''
-        w1 = f"timestamp <= '{end}'" if end else ''
-        where = f'where {w0} and {w1}' if (w0 and w1) else f"where {(w0 or w1)}"
+        _req = self._prepare_data_sql(data_id, start, end, timeframe)
+
+        self._cursor.execute(_req) # type: ignore
+        records = self._cursor.fetchall() # TODO: for chunksize > 0 use fetchmany etc
+
+        names = [d.name for d in self._cursor.description] # type: ignore
+        transform.start_transform(data_id, names)
+
+        transform.process_data(records)
+        return transform.collect()
 
+    def _prepare_data_sql(self, data_id: str, start: str|None, end: str|None, resample: str) -> str:
         # just a temp hack - actually we need to discuss symbology etc
         symbol = data_id#.split('.')[-1]
 
-        self._cursor.execute(
-            f"""
+        w0 = f"timestamp >= '{start}'" if start else ''
+        w1 = f"timestamp <= '{end}'" if end else ''
+        where = f'where {w0} and {w1}' if (w0 and w1) else f"where {(w0 or w1)}"
+
+        return f"""
                 select timestamp, 
                 first(open) as open, 
                 max(high) as high,
@@ -485,21 +562,15 @@ class QuestDBConnector(DataReader):
                 sum(taker_buy_volume) as taker_buy_volume,
                 sum(taker_buy_quote_volume) as taker_buy_quote_volume
                 from "{symbol.upper()}" {where}
-                SAMPLE by {timeframe};
-            """ # type: ignore
-        )
-        records = self._cursor.fetchall() # TODO: for chunksize > 0 use fetchmany etc
-        names = [d.name for d in self._cursor.description]
+                SAMPLE by {resample};
+            """ 
 
-        transform.start_transform(data_id, names)
-        
-        # d = np.array(records)
-        transform.process_data(records)
-        return transform.collect()
+    def _prepare_names_sql(self) -> str:
+        return "select table_name from tables()"
 
     @_retry
     def get_names(self) -> List[str] :
-        self._cursor.execute("select table_name from tables()")
+        self._cursor.execute(self._prepare_names_sql()) # type: ignore
         records = self._cursor.fetchall()
         return [r[0] for r in records]
 
@@ -511,3 +582,48 @@ class QuestDBConnector(DataReader):
             except:
                 pass
 
+
+class SnapshotsBuilder(DataTransformer):
+    """
+    Snapshots assembler from OB updates
+    """
+    def __init__(self, 
+                 levels: int=-1,     # how many levels restore, 1 - TOB, -1 - all
+                 as_frame=False      # result is dataframe
+    ):
+        self.buffer = []
+        self.levels = levels
+        self.as_frame = as_frame
+
+    def start_transform(self, name: str, column_names: List[str]):
+        # initialize buffer / series etc
+        # let's keep restored snapshots into some buffer etc
+        self.buffer = []
+
+        # do additional init stuff here
+
+    def process_data(self, rows_data:List[List]) -> Any:
+        for r in rows_data:
+            # restore snapshots and put into buffer or series
+            pass
+
+    def collect(self) -> Any:
+        # - may be convert it to pandas DataFrame ?
+        if self.as_frame:
+            return pd.DataFrame.from_records(self.buffer) # or custom transform
+
+        # - or just returns as plain list
+        return self.buffer
+
+
+class QuestDBOrderBookConnector(QuestDBConnector):
+    """
+    Example of custom OrderBook data connector 
+    """
+
+    def _prepare_data_sql(self, data_id: str, start: str|None, end: str|None, resample: str|None) -> str:
+        raise NotImplemented("TODO")
+
+    def _prepare_names_sql(self) -> str:
+        # return "select table_name from tables() where ..."
+        raise NotImplemented("TODO")

qubx/utils/_pyxreloader.py

@@ -5,10 +5,6 @@ from importlib.util import spec_from_file_location
 from importlib.machinery import ExtensionFileLoader, SourceFileLoader
 from typing import List
 
-# - disable warn about deprecation of imp module: after dev stage _pyxreloader will be removed
-import warnings
-warnings.filterwarnings("ignore", category=DeprecationWarning)
-import imp
 
 PYX_EXT = ".pyx"
 PYXDEP_EXT = ".pyxdep"
@@ -49,6 +45,10 @@ def handle_dependencies(pyxfilename):
 
 
 def handle_special_build(modname, pyxfilename):
+    try:
+        import imp
+    except:
+        return None, None
     special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT
     ext = None
     setup_args={}
@@ -135,6 +135,10 @@ def build_module(name, pyxfilename, user_setup_args, pyxbuild_dir=None, inplace=
 
 
 def load_module(name, pyxfilename, pyxbuild_dir=None, is_package=False, build_inplace=False, language_level=None, so_path=None):
+    try:
+        import imp
+    except:
+        return None
     try:
         if so_path is None:
             if is_package:

qubx/utils/time.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import List, Optional, Union
+from typing import List, Optional, Tuple, Union
 import numpy as np
 import re
 
@@ -115,7 +115,7 @@ def infer_series_frequency(series: Union[List, pd.DataFrame, pd.Series, pd.Datet
     return np.timedelta64(max(freqs, key=freqs.get))
 
 
-def handle_start_stop(s: Optional[str], e: Optional[str], convert=str) -> tuple:
+def handle_start_stop(s: Optional[str], e: Optional[str], convert=str) -> Tuple[str|None, str|None]:
     """
     Process start/stop times
 

{qubx-0.1.5.dist-info → qubx-0.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: Qubx
-Version: 0.1.5
+Version: 0.1.7
 Summary: Qubx - quantitative trading framework
 Home-page: https://github.com/dmarienko/Qubx
 Author: Dmitry Marienko
@@ -15,17 +15,25 @@ Requires-Dist: croniter (>=2.0.5,<3.0.0)
 Requires-Dist: cython (==3.0.8)
 Requires-Dist: importlib-metadata
 Requires-Dist: loguru (>=0.7.2,<0.8.0)
+Requires-Dist: matplotlib (>=3.8.4,<4.0.0)
 Requires-Dist: ntplib (>=0.4.0,<0.5.0)
+Requires-Dist: numba (>=0.59.1,<0.60.0)
 Requires-Dist: numpy (>=1.26.3,<2.0.0)
+Requires-Dist: pandas (>=2.2.2,<3.0.0)
+Requires-Dist: plotly (>=5.22.0,<6.0.0)
 Requires-Dist: psycopg (>=3.1.18,<4.0.0)
+Requires-Dist: psycopg-binary (>=3.1.19,<4.0.0)
+Requires-Dist: psycopg-pool (>=3.2.2,<4.0.0)
 Requires-Dist: pyarrow (>=15.0.0,<16.0.0)
 Requires-Dist: pydantic (>=1.10.2,<2.0.0)
 Requires-Dist: pymongo (>=4.6.1,<5.0.0)
 Requires-Dist: pytest[lazyfixture] (>=7.2.0,<8.0.0)
 Requires-Dist: python-binance (>=1.0.19,<2.0.0)
 Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
+Requires-Dist: scikit-learn (>=1.4.2,<2.0.0)
 Requires-Dist: scipy (>=1.12.0,<2.0.0)
 Requires-Dist: stackprinter (>=0.2.10,<0.3.0)
+Requires-Dist: statsmodels (>=0.14.2,<0.15.0)
 Requires-Dist: tqdm
 Project-URL: Repository, https://github.com/dmarienko/Qubx
 Description-Content-Type: text/markdown

{qubx-0.1.5.dist-info → qubx-0.1.7.dist-info}/RECORD

@@ -6,13 +6,13 @@ qubx/core/basics.py,sha256=2u7WV5KX-RbTmzoKfi1yT4HNLDPfQcFMCUZ1pVsM_VE,14777
 qubx/core/helpers.py,sha256=gPE78dO718NBY0-JbfqNGCzIvr4BVatFntNIy2RUrEY,11559
 qubx/core/loggers.py,sha256=HpgavBZegoDv9ssihtqX0pitXKULVAPHUpoE_volJw0,11910
 qubx/core/lookups.py,sha256=4aEC7b2AyEXFqHHGDenex3Z1FZGrpDSb8IwzBZrSqIA,13688
-qubx/core/series.cpython-311-x86_64-linux-gnu.so,sha256=6Thl2grZwLGSVJSe27Y4HgwIZ1HRvrLttx2QDY-bhag,698952
+qubx/core/series.cpython-311-x86_64-linux-gnu.so,sha256=Wpec9yDlhedIs11mCYl8pVf30bghhwL4UHBxgMesiak,698952
 qubx/core/series.pxd,sha256=IS89NQ5FYp3T0YIHe1lELKZIAKrNvX8K6WlLyac44I4,2847
 qubx/core/series.pyx,sha256=WEAjn4j3zn540Cxx68X5gRXilvwa7NGdbki6myzZbIM,28108
 qubx/core/strategy.py,sha256=Fs4fFyHaEGYuz7mBeQHBWFu3Ipg0yFzcxXhskgsPxJE,30330
-qubx/core/utils.cpython-311-x86_64-linux-gnu.so,sha256=zjjop5Q7v5O7NTg4YJoweL662Z0oez2KAYJA_-WhPMw,74216
+qubx/core/utils.cpython-311-x86_64-linux-gnu.so,sha256=nLncQM5F9XMpCda8IWifgSYWtoL0MCrQg27fPxK9Z2I,74216
 qubx/core/utils.pyx,sha256=6dQ8R02bl8V3f-W3Wk9-e86D9OvDz-5-4NA_dlF_xwc,1368
-qubx/data/readers.py,sha256=r5_DhzyaTMNGHr9sDjbIgK2kMcSC8fHYeDrb2ep1NLU,19648
+qubx/data/readers.py,sha256=wQc3tsLoNcvO9c2YFDxrN8jyXtPMbR3XrA_NBA38zp8,23350
 qubx/impl/ccxt_connector.py,sha256=NqF-tgxfTATnmVqKUonNXCAzECrDU8YrgqM3Nq06fw8,9150
 qubx/impl/ccxt_customizations.py,sha256=kK_4KmOyKvDVgd4MTkVg4CyqdjE-6r41siZIvLj-A-Q,3488
 qubx/impl/ccxt_trading.py,sha256=cmg4P-zd78w-V8j3-IGS2LFxikGhxFPgmCvz3sr065Q,9097
@@ -23,17 +23,17 @@ qubx/pandaz/__init__.py,sha256=Iw5uzicYGSC3FEKZ-W1O5-7cXq_P0kH11-EcXV0zZhs,175
 qubx/pandaz/ta.py,sha256=TUvjrvmk4EQvDcXoRp6Os08-HUap-ZvpSDGawhViOgg,85271
 qubx/pandaz/utils.py,sha256=FyLKQy8spkqxhBij_nPFC_ZzI_L3-IgB9O53MqWKmq0,19109
 qubx/ta/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-qubx/ta/indicators.cpython-311-x86_64-linux-gnu.so,sha256=1d34LKbFiANJPGpRtPuOluaLjhEqzv-Xr9GfSXzJ24g,284552
+qubx/ta/indicators.cpython-311-x86_64-linux-gnu.so,sha256=HFcrtnfX2BiiqymRwZ5lftBsqrF-TEmZKchPVzhbUbM,284552
 qubx/ta/indicators.pyx,sha256=P-GEYUks2lSHo6hbtUFAB7TWE1AunjLR4jIjwqPHrwU,7708
 qubx/trackers/__init__.py,sha256=1y_yvIy0OQwBqfhAW_EY33NxFzFSWvI0qNAPU6zchYc,60
 qubx/trackers/rebalancers.py,sha256=QCzANCooZBi2VMCBjjCPMq_Dt1h1zrBelATnfmVve74,5522
 qubx/utils/__init__.py,sha256=XJFje4jP69pnPTp7fpTUmqwXz9PKzGYtJf8-kBofum0,273
-qubx/utils/_pyxreloader.py,sha256=arPR_JQsxgSju0iqI9EG_u8TfeS3BgGMBGVRXaBj-UY,12076
+qubx/utils/_pyxreloader.py,sha256=FyqGzfSpZGYziB8JYS5AP3cLRAvJSIPAKgwQn0E4YQ0,12017
 qubx/utils/charting/mpl_helpers.py,sha256=ZaBrF0yOBOoVEk4TCBFi9KPyL3O5GPoteDglIzL8uSs,35158
 qubx/utils/marketdata/binance.py,sha256=36dl4rxOAGTeY3uoONmiPanj8BkP0oBdDiH-URJJo9A,10993
 qubx/utils/misc.py,sha256=z5rdz5hbRu9-F2QgF47OCkMvhfIkRKs-PHR8L5DWkBM,9831
 qubx/utils/runner.py,sha256=OY7SoRfxHwzn0rKTGB_lbg5zNASEL_49hQXWqs-LiMk,9306
-qubx/utils/time.py,sha256=mdQ02PGoUBm9iH_wvFIhAhOkBoJOpO24ZanWcGU8oms,4884
-qubx-0.1.5.dist-info/METADATA,sha256=UtgLrgWsZ3bVQiJo3Xe6sfOhPQubitebJwekMpuXLY0,2144
-qubx-0.1.5.dist-info/WHEEL,sha256=MLOa6LysROdjgj4FVxsHitAnIh8Be2D_c9ZSBHKrz2M,110
-qubx-0.1.5.dist-info/RECORD,,
+qubx/utils/time.py,sha256=_DjCdQditzZwMy_8rvPdWyw5tjw__2p24LMPgXdZ8i0,4911
+qubx-0.1.7.dist-info/METADATA,sha256=gjq_-njDfPiu_li6ID5hClbIvZL3NsVARvM6NRNZd9U,2490
+qubx-0.1.7.dist-info/WHEEL,sha256=MLOa6LysROdjgj4FVxsHitAnIh8Be2D_c9ZSBHKrz2M,110
+qubx-0.1.7.dist-info/RECORD,,