Qubx 0.6.48__tar.gz → 0.6.49__tar.gz

{qubx-0.6.48 → qubx-0.6.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: Qubx
-Version: 0.6.48
+Version: 0.6.49
 Summary: Qubx - Quantitative Trading Framework
 Author: Dmitry Marienko
 Author-email: dmitry.marienko@xlydian.com

{qubx-0.6.48 → qubx-0.6.49}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "Qubx"
-version = "0.6.48"
+version = "0.6.49"
 description = "Qubx - Quantitative Trading Framework"
 authors = [ "Dmitry Marienko <dmitry.marienko@xlydian.com>", "Yuriy Arabskyy <yuriy.arabskyy@xlydian.com>",]
 readme = "README.md"

{qubx-0.6.48 → qubx-0.6.49}/src/qubx/core/mixins/universe.py

@@ -2,8 +2,6 @@ from qubx.core.basics import DataType, Instrument, TargetPosition
 from qubx.core.helpers import CachedMarketDataHolder
 from qubx.core.interfaces import (
     IAccountProcessor,
-    IBroker,
-    IDataProvider,
     IPositionGathering,
     IStrategy,
     IStrategyContext,
@@ -14,7 +12,6 @@ from qubx.core.interfaces import (
     RemovalPolicy,
 )
 from qubx.core.loggers import StrategyLogging
-from qubx.core.lookups import lookup
 
 
 class UniverseManager(IUniverseManager):
@@ -50,7 +47,7 @@ class UniverseManager(IUniverseManager):
         self._time_provider = time_provider
         self._account = account
         self._position_gathering = position_gathering
-        self._instruments = []
+        self._instruments = set()
         self._removal_queue = {}
 
     def _has_position(self, instrument: Instrument) -> bool:
@@ -72,7 +69,7 @@ class UniverseManager(IUniverseManager):
         ), "Invalid if_has_position_then policy"
 
         new_set = set(instruments)
-        prev_set = set(self._instruments)
+        prev_set = self._instruments.copy()
 
         # - determine instruments to remove depending on if_has_position_then policy
         may_be_removed = list(prev_set - new_set)
@@ -93,9 +90,7 @@ class UniverseManager(IUniverseManager):
         self._subscription_manager.commit()  # apply pending changes
 
         # set new instruments
-        self._instruments.clear()
-        self._instruments.extend(instruments)
-        self._instruments.extend(to_keep)
+        self._instruments = new_set | set(to_keep)
 
     def _get_what_can_be_removed_or_kept(
         self, may_be_removed: list[Instrument], skip_callback: bool, if_has_position_then: RemovalPolicy
@@ -105,12 +100,11 @@ class UniverseManager(IUniverseManager):
         for instr in may_be_removed:
             if immediately_close:
                 to_remove.append(instr)
+            elif self._has_position(instr):
+                self._removal_queue[instr] = (if_has_position_then, skip_callback)
+                to_keep.append(instr)
             else:
-                if self._has_position(instr):
-                    self._removal_queue[instr] = (if_has_position_then, skip_callback)
-                    to_keep.append(instr)
-                else: 
-                    to_remove.append(instr)
+                to_remove.append(instr)
         return to_remove, to_keep
 
     def __cleanup_removal_queue(self, instruments: list[Instrument]):
@@ -124,7 +118,7 @@ class UniverseManager(IUniverseManager):
         self.__cleanup_removal_queue(instruments)
         self._strategy.on_universe_change(self._context, instruments, [])
         self._subscription_manager.commit()
-        self._instruments.extend(instruments)
+        self._instruments.update(instruments)
 
     def remove_instruments(
         self,
@@ -146,12 +140,11 @@ class UniverseManager(IUniverseManager):
         self._subscription_manager.commit()
 
         # - update instruments list
-        self._instruments = list(set(self._instruments) - set(to_remove))
-        self._instruments.extend(to_keep)
+        self._instruments = (self._instruments - set(to_remove)) | set(to_keep)
 
     @property
     def instruments(self) -> list[Instrument]:
-        return self._instruments
+        return list(self._instruments)
 
     def __do_remove_instruments(self, instruments: list[Instrument]):
         """
@@ -227,7 +220,7 @@ class UniverseManager(IUniverseManager):
             # if aux is not None:
             #     instrument._aux_instrument = aux
             #     instruments.append(aux)
-            #     _ = self._trading_service.get_position(aux)
+            #     _ = self._account.get_position(aux)
 
     def on_alter_position(self, instrument: Instrument) -> None:
         """
@@ -247,7 +240,7 @@ class UniverseManager(IUniverseManager):
 
                 # - commit changes and remove instrument from the universe
                 self._subscription_manager.commit()
-                self._instruments.remove(instrument)
+                self._instruments.discard(instrument)
 
     def is_trading_allowed(self, instrument: Instrument) -> bool:
         if instrument in self._removal_queue:
@@ -261,7 +254,7 @@ class UniverseManager(IUniverseManager):
 
                 # - commit changes and remove instrument from the universe
                 self._subscription_manager.commit()
-                self._instruments.remove(instrument)
+                self._instruments.discard(instrument)
                 return False
 
         return True

{qubx-0.6.48 → qubx-0.6.49}/src/qubx/data/readers.py

@@ -1054,7 +1054,7 @@ def _retry(fn):
             # print(x, cls._reconnect_tries)
             try:
                 return fn(*args, **kw)
-            except (pg.InterfaceError, pg.OperationalError, AttributeError) as e:
+            except (pg.InterfaceError, pg.OperationalError, AttributeError):
                 logger.debug("Database Connection [InterfaceError or OperationalError]")
                 # print ("Idle for %s seconds" % (cls._reconnect_idle))
                 # time.sleep(cls._reconnect_idle)
@@ -1063,6 +1063,75 @@ def _retry(fn):
     return wrapper
 
 
+def _calculate_max_candles_chunksize(timeframe: str | None) -> int:
+    """
+    Calculate maximum chunksize for candles data based on timeframe.
+    Limits to at most 1 week of data for candles_1m data type.
+
+    Args:
+        timeframe: Timeframe string (e.g., "1m", "5m", "1h", "1d")
+
+    Returns:
+        Maximum number of candles representing 1 week of data
+    """
+    if not timeframe:
+        return 10080  # Default to 1 week of 1-minute candles
+
+    try:
+        # Convert timeframe to pandas Timedelta
+        tf_delta = pd.Timedelta(timeframe)
+
+        # Calculate how many candles fit in 1 week
+        one_week = pd.Timedelta("7d")
+        max_candles = int(one_week / tf_delta)
+
+        # Ensure we don't return 0 or negative values
+        return max(1, max_candles)
+    except (ValueError, TypeError):
+        # If timeframe can't be parsed, default to 1 week of 1-minute candles
+        return 10080
+
+
+def _calculate_time_windows_for_chunking(
+    start: str | None, end: str | None, timeframe: str, chunksize: int
+) -> list[tuple[pd.Timestamp, pd.Timestamp]]:
+    """
+    Calculate time windows for efficient chunking based on timeframe and chunksize.
+
+    Args:
+        start: Start time string
+        end: End time string
+        timeframe: Timeframe string (e.g., "1m", "5m", "1h")
+        chunksize: Number of candles per chunk
+
+    Returns:
+        List of (start_time, end_time) tuples for each chunk
+    """
+    if not start or not end:
+        return []
+
+    start_dt = pd.Timestamp(start)
+    end_dt = pd.Timestamp(end)
+
+    try:
+        # Calculate time period per chunk based on timeframe and chunksize
+        tf_delta = pd.Timedelta(timeframe)
+        chunk_duration = tf_delta * chunksize
+
+        windows = []
+        current_start = start_dt
+
+        while current_start < end_dt:
+            current_end = min(current_start + chunk_duration, end_dt)
+            windows.append((current_start, current_end))
+            current_start = current_end
+
+        return windows
+    except (ValueError, TypeError):
+        # If timeframe can't be parsed, fall back to single window
+        return [(start_dt, end_dt)]
+
+
 class QuestDBSqlBuilder:
     """
     Generic sql builder for QuestDB data
@@ -1387,28 +1456,46 @@ class QuestDBConnector(DataReader):
         data_type: str,
         builder: QuestDBSqlBuilder,
     ) -> Any:
-        start, end = handle_start_stop(start, stop)
-        _req = builder.prepare_data_sql(data_id, start, end, timeframe, data_type)
+        # Apply maximum chunksize limits for candles_1m data type when timeframe is provided
+        if chunksize > 0 and data_type == "candles_1m" and timeframe:
+            max_chunksize = _calculate_max_candles_chunksize(timeframe)
+            chunksize = min(chunksize, max_chunksize)
 
-        _cursor = self._connection.cursor()  # type: ignore
-        _cursor.execute(_req)  # type: ignore
-        names = [d.name for d in _cursor.description]  # type: ignore
+        start, end = handle_start_stop(start, stop)
+        # If timeframe is not specified, assume 1 minute as default
+        effective_timeframe = timeframe or "1m"
 
         if chunksize > 0:
+            # Use efficient chunking with multiple smaller queries
+            def _iter_efficient_chunks():
+                time_windows = _calculate_time_windows_for_chunking(start, end, effective_timeframe, chunksize)
+                _cursor = self._connection.cursor()  # type: ignore
 
-            def _iter_chunks():
-                while True:
-                    records = _cursor.fetchmany(chunksize)
-                    if not records:
-                        _cursor.close()
-                        break
-                    transform.start_transform(data_id, names, start=start, stop=stop)
-                    transform.process_data(records)
-                    yield transform.collect()
+                try:
+                    for window_start, window_end in time_windows:
+                        _req = builder.prepare_data_sql(
+                            data_id, str(window_start), str(window_end), effective_timeframe, data_type
+                        )
 
-            return _iter_chunks()
+                        _cursor.execute(_req)  # type: ignore
+                        names = [d.name for d in _cursor.description]  # type: ignore
+                        records = _cursor.fetchall()
+
+                        if records:
+                            transform.start_transform(data_id, names, start=start, stop=stop)
+                            transform.process_data(records)
+                            yield transform.collect()
+                finally:
+                    _cursor.close()
 
+            return _iter_efficient_chunks()
+
+        # No chunking requested - return all data at once
+        _req = builder.prepare_data_sql(data_id, start, end, effective_timeframe, data_type)
+        _cursor = self._connection.cursor()  # type: ignore
         try:
+            _cursor.execute(_req)  # type: ignore
+            names = [d.name for d in _cursor.description]  # type: ignore
             records = _cursor.fetchall()
             if not records:
                 return None
@@ -1481,7 +1568,6 @@ class QuestDBSqlOrderBookBuilder(QuestDBSqlCandlesBuilder):
         if not start or not end:
             raise ValueError("Start and end dates must be provided for orderbook data!")
         start_dt, end_dt = pd.Timestamp(start), pd.Timestamp(end)
-        delta = end_dt - start_dt
 
         raw_start_dt = start_dt.floor(self.SNAPSHOT_DELTA) - self.MIN_DELTA
 

{qubx-0.6.48 → qubx-0.6.49}/src/qubx/utils/runner/runner.py

@@ -681,11 +681,15 @@ def _run_warmup(
     # - create a restored state based on warmup runner context
     warmup_account = warmup_runner.ctx.account
 
+    # - get the instruments from the warmup runner context
+    _instruments = warmup_runner.ctx.instruments
     _positions = warmup_account.get_positions()
+    _positions = {k: v for k, v in _positions.items() if k in _instruments}
     _orders = warmup_account.get_orders()
     instrument_to_orders = defaultdict(list)
     for o in _orders.values():
-        instrument_to_orders[o.instrument].append(o)
+        if o.instrument in _instruments:
+            instrument_to_orders[o.instrument].append(o)
 
     # - set the warmup positions and orders
     ctx.set_warmup_positions(_positions)