siglab-py 0.5.69__py3-none-any.whl → 0.5.71__py3-none-any.whl

siglab_py/tests/integration/market_data_util_tests.py

@@ -118,6 +118,40 @@ class MarketDataUtilTests(unittest.TestCase):
             assert set(pd_candles.columns) >= expected_columns, "Missing expected columns."
             assert pd_candles['timestamp_ms'].notna().all(), "timestamp_ms column contains NaN values."
             assert pd_candles['timestamp_ms'].is_monotonic_increasing, "Timestamps are not in ascending order."
+
+    def test_aggregate_candles(self):
+        end_date : datetime = datetime.today()
+        start_date : datetime = end_date + timedelta(hours=-8)
+
+        param = {
+            'apiKey' : None,
+            'secret' : None,
+            'password' : None,
+            'subaccount' : None,
+            'rateLimit' : 100,    # In ms
+            'options' : {
+                'defaultType': 'swap'            }
+        }
+
+        exchange : Exchange = okx(param) # type: ignore
+        normalized_symbols = [ 'BTC/USDT:USDT' ]
+        pd_candles: Union[pd.DataFrame, None] = fetch_candles(
+            start_ts=start_date.timestamp(),
+            end_ts=end_date.timestamp(),
+            exchange=exchange,
+            normalized_symbols=normalized_symbols,
+            candle_size='15m' # <---- aggregate 1m into 15m candles
+        )[normalized_symbols[0]]
+
+        assert pd_candles is not None
+        pd_candles['timestamp_ms_gap'] = pd_candles['timestamp_ms'].diff()
+        timestamp_ms_gap_median = pd_candles['timestamp_ms_gap'].median()
+        NUM_MS_IN_1HR = 60*60*1000
+        expected_15m_gap_ms = NUM_MS_IN_1HR/4
+        assert(timestamp_ms_gap_median==expected_15m_gap_ms)
+        total_num_rows = pd_candles.shape[0]
+        num_rows_with_15min_gaps = pd_candles[pd_candles.timestamp_ms_gap!=timestamp_ms_gap_median].shape[0]
+        assert(num_rows_with_15min_gaps/total_num_rows <= 0.4) # Why not 100% match? minute bars may have gaps (Also depends on what ticker)
         
     def test_fetch_candles_futubull(self):
         # You need Futu OpenD running and you need entitlements

siglab_py/util/market_data_util.py

@@ -220,8 +220,13 @@ def timestamp_to_datetime_cols(pd_candles : pd.DataFrame):
     )
 
     pd_candles['timestamp_ms_gap'] = pd_candles['timestamp_ms'] - pd_candles['timestamp_ms'].shift(1)
-    timestamp_ms_gap = pd_candles.iloc[-1]['timestamp_ms_gap']
-    assert(pd_candles[~pd_candles.timestamp_ms_gap.isna()][pd_candles.timestamp_ms_gap!=timestamp_ms_gap].shape[0]==0)
+    
+    # Depending on asset, minutes bar may have gaps
+    timestamp_ms_gap_median = pd_candles['timestamp_ms_gap'].median()
+    NUM_MS_IN_1HR = 60*60*1000
+    if timestamp_ms_gap_median>=NUM_MS_IN_1HR:
+        num_rows_with_expected_gap = pd_candles[~pd_candles.timestamp_ms_gap.isna()][pd_candles.timestamp_ms_gap==timestamp_ms_gap_median].shape[0]
+        assert(num_rows_with_expected_gap/pd_candles.shape[0]>0.9)
     pd_candles.drop(columns=['timestamp_ms_gap'], inplace=True)
 
 def timestamp_to_active_trading_regions(
@@ -426,6 +431,45 @@ class YahooExchange:
 
         return exchange_candles
 
+def aggregate_candles(
+    interval : str,
+    pd_candles : pd.DataFrame
+) -> pd.DataFrame:
+    if interval[-1]=='m':
+        # 'm' for pandas means months!
+        interval = interval.replace('m','min')
+    pd_candles.set_index('datetime', inplace=True)
+    pd_candles_aggregated = pd_candles.resample(interval).agg({
+        'exchange' : 'first',
+        'symbol' : 'first',
+        'timestamp_ms' : 'first',
+        
+        'open': 'first',
+        'high': 'max',
+        'low': 'min',
+        'close': 'last',
+        'volume': 'sum',
+
+        'datetime_utc' : 'first',
+        'year' : 'first',
+        'month' : 'first',
+        'day' : 'first',
+        'hour' : 'first',
+        'minute' : 'first',
+        'dayofweek' : 'first',
+        'week_of_month' : 'first',
+
+        'apac_trading_hr' : 'first',
+        'emea_trading_hr' : 'first',
+        'amer_trading_hr' : 'first',
+
+        'pct_chg_on_close' : 'sum',
+
+    })
+    pd_candles.reset_index(inplace=True)
+    pd_candles_aggregated.reset_index(inplace=True)
+    return pd_candles_aggregated
+    
 def fetch_historical_price(
     exchange,
     normalized_symbol : str,
@@ -472,19 +516,21 @@ def fetch_candles(
     validation_max_gaps : int = 10,
     validation_max_end_date_intervals : int = 1
 ) -> Dict[str, Union[pd.DataFrame, None]]:
-    
+    exchange_candles = { '' : None }
+    num_intervals = int(candle_size.replace(candle_size[-1],''))
+
     if end_ts>datetime.now().timestamp():
         end_ts = int(datetime.now().timestamp())
 
     if type(exchange) is YahooExchange:
-        return exchange.fetch_candles(
+        exchange_candles = exchange.fetch_candles(
                             start_ts=start_ts,
                             end_ts=end_ts,
                             symbols=normalized_symbols,
                             candle_size=candle_size
                         )
     elif type(exchange) is NASDAQExchange:
-        return exchange.fetch_candles(
+        exchange_candles = exchange.fetch_candles(
                             start_ts=start_ts,
                             end_ts=end_ts,
                             symbols=normalized_symbols,
@@ -501,9 +547,9 @@ def fetch_candles(
             pd_candles = exchange_candles[symbol]
             if not pd_candles is None:
                 fix_column_types(pd_candles) # You don't want to do this from Futubull as you'd need import Futubull from there: Circular references
-        return exchange_candles
+                
     elif issubclass(exchange.__class__, CcxtExchange):
-        return _fetch_candles_ccxt(
+        exchange_candles = _fetch_candles_ccxt(
             start_ts=start_ts,
             end_ts=end_ts,
             exchange=exchange,
@@ -511,7 +557,12 @@ def fetch_candles(
             candle_size=candle_size,
             num_candles_limit=num_candles_limit
         )
-    return { '' : None }
+    if num_intervals!=1:
+        for symbol in exchange_candles:
+            if not exchange_candles[symbol] is None:
+                exchange_candles[symbol] = aggregate_candles(candle_size, exchange_candles[symbol]) #  type: ignore
+
+    return exchange_candles # type: ignore
 
 '''
 Find listing date https://gist.github.com/mr-easy/5185b1dcdd5f9f908ff196446f092e9b
@@ -563,7 +614,7 @@ def _fetch_candles_ccxt(
             
         def _calc_increment(candle_size):
             increment = 1
-            num_intervals = int(candle_size[0])
+            num_intervals = int(candle_size.replace(candle_size[-1],''))
             interval_type = candle_size[-1]
             if interval_type == "m":
                 increment = 60

{siglab_py-0.5.69.dist-info → siglab_py-0.5.71.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: siglab_py
-Version: 0.5.69
+Version: 0.5.71
 Summary: Market data fetches, TA calculations and generic order gateway.
 Author: r0bbarh00d
 Author-email: r0bbarh00d <r0bbarh00d@gmail.com>

{siglab_py-0.5.69.dist-info → siglab_py-0.5.71.dist-info}/RECORD

@@ -20,7 +20,7 @@ siglab_py/ordergateway/gateway.py,sha256=Z-BQ-Z9gXoNrKQHzRIy9R1mnCybf9QwWhHpqkSI
 siglab_py/ordergateway/test_ordergateway.py,sha256=4PE2flp_soGcD3DrI7zJOzZndjkb6I5XaDrFNNq4Huo,4009
 siglab_py/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 siglab_py/tests/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-siglab_py/tests/integration/market_data_util_tests.py,sha256=p-RWIJZLyj0lAdfi4QTIeAttCm_e8mEVWFKh4OWuogU,7189
+siglab_py/tests/integration/market_data_util_tests.py,sha256=XKO8CX9AF7xRjRvt4lb938v_s89d2IBLAXKfZDdUxdY,8705
 siglab_py/tests/unit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 siglab_py/tests/unit/analytic_util_tests.py,sha256=wbasww0ZN9Kb-6e8V_q4o4hpstzVBvQ6Bop_0HZQUtw,5926
 siglab_py/tests/unit/market_data_util_tests.py,sha256=A1y83itISmMJdn6wLpfwcr4tGola8wTf1D1xbelMvgw,2026
@@ -29,13 +29,13 @@ siglab_py/tests/unit/trading_util_tests.py,sha256=LiflZrduWXyLMbpSFQCaydA7jdJx3v
 siglab_py/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 siglab_py/util/analytic_util.py,sha256=ywp-VI8UlmoYVej2SaJMrOyheFwyh9KVjsnfw55dpMU,63785
 siglab_py/util/aws_util.py,sha256=KGmjHrr1rpnnxr33nXHNzTul4tvyyxl9p6gpwNv0Ygc,2557
-siglab_py/util/market_data_util.py,sha256=mUXg4uaiX3b6_klgJWIEgnUQU4IUd6CwTOqKLiQWRlU,31307
+siglab_py/util/market_data_util.py,sha256=QPI7ZvtYzRllz6REBkQRouMwEG_iHPuDMsbt-4n5FSs,33054
 siglab_py/util/notification_util.py,sha256=tNZMUkkjz4q1CKqcQq62oEmZgHgNIwz2Iw9J22V22Zw,2668
 siglab_py/util/retry_util.py,sha256=g-UU6pkPouWZZRZEqP99R2Z0lX5xzckYkzjwqqSDpVQ,922
 siglab_py/util/simple_math.py,sha256=F7vGj0O2Y9EAGcMFR6SN1tTjBWO_a7YZeiTzk3eHaVI,8518
 siglab_py/util/slack_notification_util.py,sha256=G27n-adbT3Q6oaHSMvu_Nom794rrda5PprSF-zvmzkM,1912
 siglab_py/util/trading_util.py,sha256=dlIOzoMGnddLSFODcJ61EBH1Aeruq4IT2MsxIdFkV9I,5252
-siglab_py-0.5.69.dist-info/METADATA,sha256=QaGfEu-83Dyrx9WYb2z4dQzJaziY40smJpgcA2HCZPI,829
-siglab_py-0.5.69.dist-info/WHEEL,sha256=lTU6B6eIfYoiQJTZNc-fyaR6BpL6ehTzU3xGYxn2n8k,91
-siglab_py-0.5.69.dist-info/top_level.txt,sha256=AbD4VR9OqmMOGlMJLkAVPGQMtUPIQv0t1BF5xmcLJSk,10
-siglab_py-0.5.69.dist-info/RECORD,,
+siglab_py-0.5.71.dist-info/METADATA,sha256=6AfqxAvgCiB0Vob4DwZfjv2n9ALiHt2vqAlYAjHRoMU,829
+siglab_py-0.5.71.dist-info/WHEEL,sha256=lTU6B6eIfYoiQJTZNc-fyaR6BpL6ehTzU3xGYxn2n8k,91
+siglab_py-0.5.71.dist-info/top_level.txt,sha256=AbD4VR9OqmMOGlMJLkAVPGQMtUPIQv0t1BF5xmcLJSk,10
+siglab_py-0.5.71.dist-info/RECORD,,