PyAlgoEngine 0.8.0a13__tar.gz → 0.8.0a16__tar.gz

{pyalgoengine-0.8.0a13 → pyalgoengine-0.8.0a16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyAlgoEngine
-Version: 0.8.0a13
+Version: 0.8.0a16
 Summary: Basic algo engine
 Home-page: https://github.com/BolunHan/PyAlgoEngine
 Author: Bolun.Han

{pyalgoengine-0.8.0a13 → pyalgoengine-0.8.0a16}/PyAlgoEngine.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyAlgoEngine
-Version: 0.8.0a13
+Version: 0.8.0a16
 Summary: Basic algo engine
 Home-page: https://github.com/BolunHan/PyAlgoEngine
 Author: Bolun.Han

{pyalgoengine-0.8.0a13 → pyalgoengine-0.8.0a16}/algo_engine/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.8.0.alpha13"
+__version__ = "0.8.0.alpha16"
 
 import logging
 import os

pyalgoengine-0.8.0a16/algo_engine/backtest/__init__.py

@@ -0,0 +1,19 @@
+import logging
+
+from .. import LOGGER
+
+LOGGER = LOGGER.getChild('BackTest')
+
+
+def set_logger(logger: logging.Logger):
+    global LOGGER
+    LOGGER = logger
+
+    replay.LOGGER = LOGGER.getChild('Replay')
+    sim_match.LOGGER = LOGGER.getChild('SimMatch')
+
+
+from .replay import MarketDateCallable, MarketDataLoader, MarketDataBulkLoader, Replay, SimpleReplay, ProgressReplay, ProgressiveReplay
+from .sim_match import SimMatch
+
+__all__ = ['MarketDateCallable', 'MarketDataLoader', 'MarketDataBulkLoader', 'Replay', 'SimpleReplay', 'ProgressReplay', 'ProgressiveReplay', 'SimMatch']

pyalgoengine-0.8.0a16/algo_engine/backtest/replay.py

@@ -0,0 +1,439 @@
+import abc
+import datetime
+import inspect
+import operator
+import warnings
+from collections.abc import Sequence, Mapping, Iterable, Callable
+from typing import Literal, Protocol, runtime_checkable, get_type_hints
+
+from . import LOGGER
+from ..base import MarketData, DataType, MarketDataBuffer
+
+LOGGER = LOGGER.getChild('Replay')
+__all__ = ['MarketDateCallable', 'MarketDataLoader', 'MarketDataBulkLoader', 'Replay', 'SimpleReplay', 'ProgressReplay', 'ProgressiveReplay']
+
+
+@runtime_checkable
+class MarketDateCallable(Protocol):
+    def __call__(self, market_date: datetime.date) -> None:
+        ...
+
+
+@runtime_checkable
+class MarketDataLoader(Protocol):
+    def __call__(self, market_date: datetime.date, ticker: str, dtype: str | DataType) -> Sequence[MarketData] | Mapping[float, MarketData]:
+        pass
+
+
+@runtime_checkable
+class MarketDataBulkLoader(Protocol):
+    def __call__(self, market_date: datetime.date, tickers: Sequence[str], dtypes: Sequence[str | DataType]) -> Sequence[MarketData] | Mapping[float, MarketData] | MarketDataBuffer:
+        pass
+
+
+def check_protocol_signature(func: Callable, protocol: type) -> bool:
+    if not callable(func):
+        raise TypeError(f"{func} is not callable")
+
+    proto_sig = inspect.signature(protocol.__call__)
+    func_sig = inspect.signature(func)
+
+    proto_params = list(proto_sig.parameters.values())[1:]  # Skip 'self'
+    func_params = list(func_sig.parameters.values())
+
+    # Check for *args (VAR_POSITIONAL) — not allowed
+    for p in func_params:
+        if p.kind == inspect.Parameter.VAR_POSITIONAL:
+            raise TypeError(f"{func.__name__} uses *args, which is not allowed")
+
+    # Check number of required positional args (not counting **kwargs)
+    proto_arg_names = [p.name for p in proto_params if p.kind in (
+        inspect.Parameter.POSITIONAL_ONLY,
+        inspect.Parameter.POSITIONAL_OR_KEYWORD
+    )]
+
+    func_arg_names = [p.name for p in func_params if p.kind in (
+        inspect.Parameter.POSITIONAL_ONLY,
+        inspect.Parameter.POSITIONAL_OR_KEYWORD
+    )]
+
+    if proto_arg_names != func_arg_names:
+        raise TypeError(f"{func.__name__} argument names {func_arg_names} do not match protocol {proto_arg_names}")
+
+    # Type hint comparison (warn if mismatched, but allow)
+    proto_hints = get_type_hints(protocol.__call__)
+    func_hints = get_type_hints(func)
+
+    for pname in proto_arg_names:
+        expected = proto_hints.get(pname)
+        actual = func_hints.get(pname)
+        if expected and actual and expected != actual:
+            warnings.warn(
+                f"Type hint mismatch for parameter '{pname}': expected {expected}, got {actual}",
+                stacklevel=2
+            )
+
+    # Optional: check return type
+    expected_ret = proto_hints.get("return")
+    actual_ret = func_hints.get("return")
+    if expected_ret and actual_ret and expected_ret != actual_ret:
+        warnings.warn(
+            f"Return type mismatch: expected {expected_ret}, got {actual_ret}",
+            stacklevel=2
+        )
+
+    return True
+
+
+class Replay(object, metaclass=abc.ABCMeta):
+    # __slots__ = ('start_date', 'end_date', 'market_date', 'calendar', 'bod', 'eod', 'subscription', '_calendar', '_market_date', '_status', '_progress')
+
+    def __init__(self, start_date: datetime.date = None, end_date: datetime.date = None, market_date: datetime.date = None, calendar: Sequence[datetime.date] = None, bod: MarketDateCallable = None, eod: MarketDateCallable = None) -> None:
+        self.start_date = start_date or market_date or calendar[0]
+        self.end_date = end_date or calendar[-1]
+        self.market_date = market_date or start_date
+        self.calendar = calendar or []
+
+        self.bod = []
+        self.eod = []
+        self.subscription = {}
+
+        if bod is not None:
+            self.add_bod(bod)
+
+        if eod is not None:
+            self.add_eod(eod)
+
+    def add_bod(self, func: MarketDateCallable):
+        self.bod.append(func)
+
+    def add_eod(self, func: MarketDateCallable):
+        self.eod.append(func)
+
+    @classmethod
+    def get_dtype(cls, dtype: DataType | str) -> str | Literal['TickData', 'TickDataLite', 'OrderData', 'TransactionData']:
+        match dtype:
+            case 'TickData' | 'TickDataLite' | 'OrderData' | 'TransactionData':
+                return str(dtype)
+            case DataType.DTYPE_TICK | DataType.DTYPE_ORDER | DataType.DTYPE_TRANSACTION:
+                return DataType(dtype).name.removeprefix('DTYPE_').capitalize() + 'Data'
+            case DataType.DTYPE_TICK_LITE:
+                return 'Data'.join(_.capitalize() for _ in DataType(dtype).name.removeprefix('DTYPE_').split('_'))
+            case _:
+                raise ValueError(f'Invalid dtype {dtype}, expect str or int.')
+
+    def add_subscription(self, ticker: str, dtype: DataType | str):
+        dtype = self.get_dtype(dtype)
+        topic = f'{ticker}.{dtype}'
+
+        self.subscription[topic] = (ticker, dtype)
+
+    def remove_subscription(self, ticker: str, dtype: DataType | str):
+        dtype = self.get_dtype(dtype)
+        topic = f'{ticker}.{dtype}'
+
+        try:
+            self.subscription.pop(topic)
+        except KeyError as _:
+            LOGGER.info(f'{topic} not in {self.subscription}')
+
+    @abc.abstractmethod
+    def __next__(self):
+        ...
+
+    @abc.abstractmethod
+    def __iter__(self):
+        ...
+
+
+class SimpleReplay(Replay):
+    def __init__(
+            self,
+            loader: MarketDataBulkLoader | MarketDataLoader = None,
+            market_date: datetime.date = None,
+            start_date: datetime.date = None,
+            end_date: datetime.date = None,
+            calendar: Sequence[datetime.date] = None,
+            bod: MarketDateCallable = None,
+            eod: MarketDateCallable = None
+    ):
+        super().__init__(market_date=market_date, start_date=start_date, end_date=end_date, calendar=calendar, bod=bod, eod=eod)
+        self.loader = loader
+
+    def __iter__(self):
+        self._calendar = self.calendar or [self.start_date + datetime.timedelta(days=i) for i in range((self.end_date - self.start_date).days + 1)]
+        self._market_date = self.market_date or sorted(_ for _ in self._calendar if _ >= self.market_date)[0]
+        self._status = {market_date: 'skipped' if market_date < self.market_date else 'idle' for market_date in self._calendar}
+        self._idx_buffer = 0
+        self._idx_date = sum([1 for _ in self._calendar if _ < self.market_date])
+
+        for func in self.bod:
+            func(self._market_date)
+
+        self._safe_load()
+
+        return self
+
+    def __next__(self) -> MarketData:
+        if self._idx_buffer < self._buffer_size:
+            self._idx_buffer += 1
+            return next(self._buffer)
+
+        for func in self.eod:
+            func(self._market_date)
+
+        self._idx_buffer = 0
+        self._idx_date += 1
+
+        if self._idx_date >= len(self._calendar):
+            self._calendar.clear()
+            del self._calendar
+            del self._market_date
+            del self._status
+            del self._idx_buffer
+            del self._idx_date
+            del self._buffer
+            del self._buffer_size
+            raise StopIteration()
+
+        self._market_date = self._calendar[self._idx_date]
+
+        for func in self.bod:
+            func(self._market_date)
+
+        self._safe_load()
+        return self.__next__()
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}{{id={id(self)}, from={self.start_date}, to={self.end_date}}}'
+
+    def _bulk_load_protocol(self):
+        buffer = self.loader(market_date=self._market_date, tickers=self.tickers, dtypes=self.dtypes)
+        buffer.sort()
+
+        if isinstance(buffer, MarketDataBuffer):
+            self._buffer = buffer
+            self._buffer_size = len(self._buffer)
+        elif isinstance(buffer, Sequence):
+            self._buffer = iter(buffer)
+            self._buffer_size = len(buffer)
+        elif isinstance(buffer, Mapping):
+            self._buffer = iter(buffer.values())
+            self._buffer_size = len(buffer)
+
+    def _individual_load_protocol(self):
+        buffer = []
+        for topic, (_ticker, _dtype) in self.subscription.items():
+            LOGGER.info(f'{self} loading {self._market_date} {_ticker} {_dtype}')
+            data = self.loader(market_date=self._market_date, ticker=_ticker, dtype=_dtype)
+            if isinstance(data, Mapping):
+                buffer.extend(list(data.values()))
+            elif isinstance(data, Sequence):
+                buffer.extend(data)
+            else:
+                raise TypeError(f'The loader {self.loader} returned {type(data)}. Expect a sequence or mapping of MarketData')
+        buffer.sort(key=operator.attrgetter('timestamp', 'ticker', '_dtype'))
+        self._buffer = iter(buffer)
+        self._buffer_size = len(buffer)
+
+    def _safe_load(self):
+        if self.loader is None:
+            assert hasattr(self, '_buffer') and isinstance(self._buffer, Iterable), f'Without assigning a data loader, the _buffer of {self.__class__.__name__} should be set in bod process.'
+            return
+
+        is_bulk_loader = check_protocol_signature(self.loader, MarketDataBulkLoader)
+        is_individual_loader = check_protocol_signature(self.loader, MarketDataLoader)
+
+        if (is_bulk_loader and is_individual_loader) or (not is_bulk_loader and not is_individual_loader):
+            try:
+                return self._bulk_load_protocol()
+            except Exception as e:
+                LOGGER.info('Failed to load data using MarketDataBulkLoader protocol!')
+
+            try:
+                return self._individual_load_protocol()
+            except Exception as e:
+                LOGGER.info('Failed to load data using MarketDataLoader protocol!')
+                raise
+
+        if is_bulk_loader:
+            return self._bulk_load_protocol()
+
+        return self._individual_load_protocol()
+
+    @property
+    def progress(self) -> float:
+        if not hasattr(self, '_buffer'):
+            raise RuntimeError(f'{self.__class__.__name__} not started yet.')
+
+        return (self._idx_date + (self._idx_buffer / self._buffer_size - 1)) / len(self._calendar)
+
+    @property
+    def tickers(self) -> list[str]:
+        tickers = set()
+        for _, (ticker, dtype) in self.subscription.items():
+            tickers.add(ticker)
+        return list(tickers)
+
+    @property
+    def dtypes(self) -> list[str]:
+        dtypes = set()
+        for _, (ticker, dtype) in self.subscription.items():
+            dtypes.add(dtype)
+        return list(dtypes)
+
+    @property
+    def status(self) -> dict[datetime.date, str]:
+        if not hasattr(self, '_status'):
+            raise RuntimeError(f'{self.__class__.__name__} not started yet.')
+
+        return self._status
+
+
+class ProgressReplay(SimpleReplay):
+    def __init__(
+            self,
+            loader: MarketDataBulkLoader | MarketDataLoader = None,
+            market_date: datetime.date = None,
+            start_date: datetime.date = None,
+            end_date: datetime.date = None,
+            calendar: Sequence[datetime.date] = None,
+            bod: MarketDateCallable = None,
+            eod: MarketDateCallable = None,
+            **tqdm_kwargs
+    ):
+        super().__init__(
+            loader=loader,
+            market_date=market_date,
+            start_date=start_date,
+            end_date=end_date,
+            calendar=calendar,
+            bod=bod,
+            eod=eod
+        )
+
+        self._tqdm_kwargs = {
+            'total': 1,
+            'unit_scale': True,
+            'unit': 'percent',
+            'mininterval': 0.1,
+            'miniters': 0.001,
+            **tqdm_kwargs
+        }
+        self.add_bod(self._update_progress_bar)
+
+    def __iter__(self):
+        from tqdm.auto import tqdm
+        self._pbar = tqdm(**self._tqdm_kwargs)
+        iterator = super().__iter__()
+
+        try:
+            while True:
+                try:
+                    result = next(iterator)
+                    if self._pbar:
+                        self._pbar.update(self.progress)
+                    yield result
+                except StopIteration:
+                    break
+        finally:
+            if self._pbar is not None:
+                self._pbar.close()
+                self._pbar = None
+
+    def __next__(self) -> MarketData:
+        raise RuntimeError("MarketDataBufferReplay should be used as an iterator context")
+
+    def _update_progress_bar(self, market_date: datetime.date):
+        if self._pbar:
+            self._pbar.set_description(f'Replay {market_date:%Y-%m-%d} ({self._idx_date + 1} / {len(self._calendar)})')
+            self._pbar.refresh()
+
+
+class ProgressiveReplay(SimpleReplay):
+    """
+    progressively loading and replaying market data
+
+    requires arguments
+    loader: a data loading function. Expect loader = Callable(market_date: datetime.date, ticker: str, dtype: str| type) -> dict[any, MarketData]
+    start_date & end_date: the given replay period
+    or calendar: the given replay calendar.
+
+    accepts kwargs:
+    ticker / tickers: the given symbols to replay, expect a str| list[str]
+    dtype / dtypes: the given dtype(s) of symbol to replay, expect a str | type, list[str | type]. default = all, which is (TradeData, TickData, OrderBook)
+    subscription / subscribe: the given ticker-dtype pair to replay, expect a list[dict[str, str | type]]
+    """
+
+    def __init__(
+            self,
+            loader: MarketDataLoader,
+            tickers: str | Sequence[str] = None,
+            dtypes: str | DataType | Sequence[str] | Sequence[DataType] = None,
+            market_date: datetime.date = None,
+            start_date: datetime.date = None,
+            end_date: datetime.date = None,
+            calendar: Sequence[datetime.date] = None,
+            bod: MarketDateCallable = None,
+            eod: MarketDateCallable = None,
+            **progress_config
+    ) -> None:
+        warnings.deprecated('User ProgressReplay instead!')
+        self.loader = loader
+        super().__init__(loader=loader, market_date=market_date, start_date=start_date, end_date=end_date, calendar=calendar, bod=bod, eod=eod)
+
+        tickers = tickers or []
+        dtypes = dtypes or ['TransactionData', 'TickData', 'OrderData']
+
+        if not isinstance(loader, MarketDataLoader):
+            raise TypeError('loader function has 3 requires args, market_date, ticker and dtype.')
+
+        if isinstance(tickers, str):
+            tickers = [tickers]
+        elif isinstance(tickers, Iterable):
+            tickers = list(tickers)
+        else:
+            raise TypeError(f'Invalid ticker {tickers}, expect str or list[str]')
+
+        if isinstance(dtypes, (str, int, DataType)):
+            dtypes = [dtypes]
+        elif isinstance(dtypes, Iterable):
+            dtypes = list(dtypes)
+        else:
+            raise TypeError(f'Invalid dtype {dtypes}, expect str or list[str]')
+
+        for ticker in tickers:
+            for dtype in dtypes:
+                self.add_subscription(ticker=ticker, dtype=dtype)
+
+        self.progress_config = dict(
+            tasks=1,
+            **progress_config
+        )
+
+    def __iter__(self):
+        from ..base import Progress
+        self._pbar = Progress(**self.progress_config)
+        return super().__iter__()
+
+    def __next__(self) -> MarketData:
+        try:
+            result = super().__next__()
+            self._pbar.done_tasks = self.progress
+
+            if (not self._pbar.tick_size) \
+                    or self._pbar.progress >= self._pbar.tick_size + self._pbar.last_output \
+                    or self._pbar.is_done:
+                self._pbar.output()
+
+            return result
+        except StopIteration:
+            if not self._pbar.is_done:
+                self.progress.done_tasks = 1
+                self._pbar.output()
+            raise
+
+    def _update_progress_bar(self, market_date: datetime.date):
+        if self._pbar:
+            self.progress.prompt = f'Replay {market_date:%Y-%m-%d} ({self._idx_date + 1} / {len(self._calendar)}):'
+            self._pbar.output()

{pyalgoengine-0.8.0a13 → pyalgoengine-0.8.0a16}/algo_engine/base/market_data_buffer.pyi

@@ -33,6 +33,8 @@ class MarketDataBuffer:
 
     def update(self, dtype: int, **kwargs: dict[str, Any]) -> None: ...
 
+    def __getitem__(self, idx: int) -> MarketData | TransactionData | OrderData | TickDataLite | TickData | BarData: ...
+
     def __iter__(self) -> MarketDataBuffer: ...
 
     def __next__(self) -> MarketData | TransactionData | OrderData | TickDataLite | TickData | BarData: ...

pyalgoengine-0.8.0a13/algo_engine/backtest/__init__.py

@@ -1,19 +0,0 @@
-import logging
-
-from .. import LOGGER
-
-LOGGER = LOGGER.getChild('BackTest')
-
-
-def set_logger(logger: logging.Logger):
-    global LOGGER
-    LOGGER = logger
-
-    replay.LOGGER = LOGGER.getChild('Replay')
-    sim_match.LOGGER = LOGGER.getChild('SimMatch')
-
-
-from .replay import Replay, SimpleReplay, ProgressiveReplay
-from .sim_match import SimMatch
-
-__all__ = ['Replay', 'SimpleReplay', 'ProgressiveReplay', 'SimMatch']

pyalgoengine-0.8.0a13/algo_engine/backtest/replay.py

@@ -1,292 +0,0 @@
-import abc
-import datetime
-import inspect
-import operator
-from collections.abc import Mapping, Sequence, Iterator
-from typing import Iterable, Protocol
-
-from . import LOGGER
-from ..base import Progress, TickData, TransactionData, TradeData, OrderData, MarketData, MarketDataBuffer
-
-LOGGER = LOGGER.getChild('Replay')
-
-
-class Replay(object, metaclass=abc.ABCMeta):
-    @abc.abstractmethod
-    def __next__(self): ...
-
-    @abc.abstractmethod
-    def __iter__(self): ...
-
-
-class SimpleReplay(Replay):
-    def __init__(self, **kwargs):
-        self.eod = kwargs.pop('eod', None)
-        self.bod = kwargs.pop('bod', None)
-
-        self.replay_task = []
-        self.task_progress = 0
-        self.task_date = None
-        self.progress = Progress(tasks=1, **kwargs)
-
-    def load(self, data):
-        if isinstance(data, dict):
-            self.replay_task.extend(list(data.values()))
-        else:
-            self.replay_task.extend(data)
-
-    def reset(self):
-        self.replay_task.clear()
-        self.task_progress = 0
-        self.task_date = None
-        self.progress.reset()
-
-    def next_task(self):
-        if self.task_progress < len(self.replay_task):
-            market_data = self.replay_task[self.task_progress]
-            market_time = market_data.market_time
-
-            if isinstance(market_time, datetime.datetime):
-                market_date = market_time.date()
-            else:
-                market_date = market_time
-
-            if market_date != self.task_date:
-                if callable(self.eod) and self.task_date:
-                    self.eod(self.task_date)
-
-                self.task_date = market_date
-                self.progress.prompt = f'Replay {market_date:%Y-%m-%d}:'
-
-                if callable(self.bod):
-                    self.bod(market_date)
-
-            self.progress.done_tasks = self.task_progress / len(self.replay_task)
-
-            if (not self.progress.tick_size) or self.progress.progress >= self.progress.tick_size + self.progress.last_output:
-                self.progress.output()
-
-            self.task_progress += 1
-        else:
-            raise StopIteration()
-
-        return market_data
-
-    def __next__(self):
-        try:
-            return self.next_task()
-        except StopIteration:
-            if not self.progress.is_done:
-                self.progress.done_tasks = 1
-                self.progress.output()
-
-            self.reset()
-            raise StopIteration()
-
-    def __iter__(self):
-        return self
-
-
-class DataLoader(Protocol):
-    def __call__(self, market_date: datetime.date, ticker: str, dtype: str) -> Mapping[float, MarketData] | Sequence[MarketData] | MarketDataBuffer:
-        ...
-
-
-class ProgressiveReplay(Replay):
-    """
-    progressively loading and replaying market data
-
-    requires arguments
-    loader: a data loading function. Expect loader = Callable(market_date: datetime.date, ticker: str, dtype: str| type) -> dict[any, MarketData]
-    start_date & end_date: the given replay period
-    or calendar: the given replay calendar.
-
-    accepts kwargs:
-    ticker / tickers: the given symbols to replay, expect a str| list[str]
-    dtype / dtypes: the given dtype(s) of symbol to replay, expect a str | type, list[str | type]. default = all, which is (TradeData, TickData, OrderBook)
-    subscription / subscribe: the given ticker-dtype pair to replay, expect a list[dict[str, str | type]]
-    """
-
-    def __init__(
-            self,
-            loader: DataLoader,
-            **kwargs
-    ):
-        self.loader = loader
-        self.market_date: datetime.date | None = kwargs.pop('market_date', None)
-        self.start_date: datetime.date | None = kwargs.pop('start_date', None)
-        self.end_date: datetime.date | None = kwargs.pop('end_date', None)
-        self.calendar: list[datetime.date] | None = kwargs.pop('calendar', None)
-
-        self.eod = kwargs.pop('eod', None)
-        self.bod = kwargs.pop('bod', None)
-
-        self.replay_subscription = {}
-        self.replay_calendar = []
-        self.replay_task: Iterator | None = None
-        self.replay_task_length: int = 0
-        self.replay_status = {}
-
-        self.date_progress = 0
-        self.task_progress = 0
-        self.progress = Progress(tasks=1, **kwargs)
-
-        tickers: list[str] = kwargs.pop('ticker', kwargs.pop('tickers', []))
-        dtypes: list[str | type] = kwargs.pop('dtype', kwargs.pop('dtypes', [TradeData, TransactionData, OrderData, TickData]))
-
-        if not all([arg_name in inspect.getfullargspec(loader).args for arg_name in ['market_date', 'ticker', 'dtype']]):
-            raise TypeError('loader function has 3 requires args, market_date, ticker and dtype.')
-
-        if isinstance(tickers, str):
-            tickers = [tickers]
-        elif isinstance(tickers, Iterable):
-            tickers = list(tickers)
-        else:
-            raise TypeError(f'Invalid ticker {tickers}, expect str or list[str]')
-
-        if isinstance(dtypes, str) or inspect.isclass(dtypes):
-            dtypes = [dtypes]
-        elif isinstance(dtypes, Iterable):
-            dtypes = list(dtypes)
-        else:
-            raise TypeError(f'Invalid dtype {dtypes}, expect str or list[str]')
-
-        for ticker in tickers:
-            for dtype in dtypes:
-                self.add_subscription(ticker=ticker, dtype=dtype)
-
-        subscription = kwargs.pop('subscription', kwargs.pop('subscribe', []))
-
-        if isinstance(subscription, dict):
-            subscription = [subscription]
-
-        for sub in subscription:
-            self.add_subscription(**sub)
-
-        self.reset()
-
-    def add_subscription(self, ticker: str, dtype: type | str):
-        if isinstance(dtype, str):
-            pass
-        elif inspect.isclass(dtype):
-            dtype = dtype.__name__
-        else:
-            raise ValueError(f'Invalid dtype {dtype}, expect str or class.')
-
-        topic = f'{ticker}.{dtype}'
-        self.replay_subscription[topic] = (ticker, dtype)
-
-    def remove_subscription(self, ticker: str, dtype: type | str):
-        if isinstance(dtype, str):
-            pass
-        else:
-            dtype = dtype.__name__
-
-        topic = f'{ticker}.{dtype}'
-        self.replay_subscription.pop(topic, None)
-
-    def reset(self):
-        if self.calendar is None:
-            self.replay_calendar = [self.start_date + datetime.timedelta(days=i) for i in range((self.end_date - self.start_date).days + 1)]
-        else:
-            self.replay_calendar = self.calendar
-
-        if self.market_date is None:
-            self.market_date = self.replay_calendar[0] if self.replay_calendar else self.start_date
-        else:
-            date_to_replay = [_ for _ in self.replay_calendar if _ >= self.market_date]
-            self.market_date = date_to_replay[0] if date_to_replay else self.end_date
-
-        self.replay_status = {market_date: 'skipped' if market_date < self.market_date else 'idle' for market_date in self.replay_calendar}
-
-        self.task_progress = 0
-        self.replay_task_length = 0
-        self.replay_task = None
-        self.date_progress = sum([1 for _ in self.replay_calendar if _ < self.market_date])
-        self.progress.reset()
-
-        if self.date_progress:
-            self.progress.done_tasks = self.date_progress / len(self.replay_calendar)
-
-    def next_trade_day(self):
-        if self.date_progress >= len(self.replay_calendar):
-            raise StopIteration()
-
-        self.market_date = market_date = self.replay_calendar[self.date_progress]
-        self.replay_status[market_date] = 'started'
-        self.progress.prompt = f'Replay {market_date:%Y-%m-%d} ({self.date_progress + 1} / {len(self.replay_calendar)}):'
-
-        for topic in self.replay_subscription:
-            ticker, dtype = self.replay_subscription[topic]
-            LOGGER.info(f'{self} loading {market_date} {ticker} {dtype}...')
-            data = self.loader(market_date=market_date, ticker=ticker, dtype=dtype)
-            if isinstance(data, Mapping):
-                data = [data[ts] for ts in sorted(data)]  # expect to be a mapping of ts and data
-                self.replay_task = iter(data)
-                self.replay_task_length = len(data)
-            elif isinstance(data, Sequence):
-                data = sorted(data, key=operator.attrgetter('timestamp', 'ticker', '__class__.__name__'))
-                self.replay_task = iter(data)
-                self.replay_task_length = len(data)
-            elif isinstance(data, MarketDataBuffer):
-                data.sort()
-                self.replay_task = iter(data)
-                self.replay_task_length = len(data)
-            else:
-                raise TypeError(f'Invalid return type of dataloader, expect list, tuple, dict or MarketDataBuffer, got {type(data)}.')
-
-        LOGGER.info(f'{market_date} data loaded! {self.replay_task_length:,} entries.')
-        self.date_progress += 1
-
-    def next_task(self):
-        try:
-            data = next(self.replay_task)
-            self.task_progress += 1
-        except StopIteration:
-            if self.eod is not None and self.replay_status[self.market_date] == 'started':
-                self.eod(market_date=self.market_date, replay=self)
-                self.replay_status[self.market_date] = 'done'
-
-            self.replay_task = None
-            self.task_progress = 0
-
-            if self.bod is not None and self.date_progress < len(self.replay_calendar):
-                self.bod(market_date=self.replay_calendar[self.date_progress], replay=self)
-
-            # this is by designed, to load the new data after the bod is done.
-            self.next_trade_day()
-
-            #  the bod process should be moved here!
-
-            data = self.next_task()
-
-        if self.replay_task_length and self.replay_calendar:
-            current_progress = (self.date_progress - 1 + (self.task_progress / self.replay_task_length)) / len(self.replay_calendar)
-            self.progress.done_tasks = current_progress
-        else:
-            self.progress.done_tasks = 1
-
-        if (not self.progress.tick_size) \
-                or self.progress.progress >= self.progress.tick_size + self.progress.last_output \
-                or self.progress.is_done:
-            self.progress.output()
-
-        return data
-
-    def __next__(self) -> MarketData:
-        try:
-            return self.next_task()
-        except StopIteration:
-            if not self.progress.is_done:
-                self.progress.done_tasks = 1
-                self.progress.output()
-
-            self.reset()
-            raise StopIteration()
-
-    def __iter__(self):
-        self.reset()
-        return self
-
-    def __repr__(self):
-        return f'{self.__class__.__name__}{{id={id(self)}, from={self.start_date}, to={self.end_date}}}'