Qubx 1.0.0.dev2__tar.gz → 1.0.0.dev3__tar.gz

{qubx-1.0.0.dev2 → qubx-1.0.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Qubx
-Version: 1.0.0.dev2
+Version: 1.0.0.dev3
 Summary: Qubx - Quantitative Trading Framework
 Project-URL: homepage, https://xlydian.com
 Project-URL: repository, https://github.com/xLydianSoftware/Qubx
@@ -57,6 +57,8 @@ Requires-Dist: uvloop<1,>=0.22.1; sys_platform != 'win32'
 Requires-Dist: websockets==15.0.1
 Provides-Extra: k8
 Requires-Dist: prometheus-client<1,>=0.21.1; extra == 'k8'
+Provides-Extra: storage
+Requires-Dist: duckdb>=1.0.0; extra == 'storage'
 Description-Content-Type: text/markdown
 
 # Qubx - Quantitative Trading Framework

{qubx-1.0.0.dev2 → qubx-1.0.0.dev3}/pyproject.toml

@@ -66,6 +66,8 @@ docs = "https://xlydiansoftware.github.io/Qubx"
 [project.optional-dependencies]
 # Runtime optional features only (shipped with package)
 k8 = ["prometheus-client>=0.21.1,<1"]
+# Parquet-based backtest storage with DuckDB search and cloud (S3/GCS/Azure) support
+storage = ["duckdb>=1.0.0"]
 
 [project.scripts]
 qubx = "qubx.cli.commands:main"
@@ -177,4 +179,5 @@ dev = [
     "mongomock>=4.3.0,<5",
     "pytest-textual-snapshot>=1.1.0,<2",
     "git-cliff>=2.0.0",
+    "duckdb>=1.0.0"
 ]

{qubx-1.0.0.dev2 → qubx-1.0.0.dev3}/src/qubx/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '1.0.0.dev2'
-__version_tuple__ = version_tuple = (1, 0, 0, 'dev2')
+__version__ = version = '1.0.0.dev3'
+__version_tuple__ = version_tuple = (1, 0, 0, 'dev3')
 
 __commit_id__ = commit_id = None

qubx-1.0.0.dev3/src/qubx/backtester/__init__.py

@@ -0,0 +1,4 @@
+__all__ = ["BacktestStorage", "variate"]
+
+from .management import BacktestStorage
+from .optimization import variate

qubx-1.0.0.dev3/src/qubx/backtester/management.py

@@ -0,0 +1,531 @@
+"""
+Parquet-based backtest storage utilities — schemas, constants, and write helpers.
+
+Used by:
+    - qubx.core.metrics.TradingSessionResult  (result model)
+    - qubx.backtester.management.BacktestStorage (query interface)
+    - qubx.utils.results.SimulationResultsSaver (save / load)
+    - qubx.utils.runner.runner.simulate_strategy (cloud detection, tag helpers)
+
+Storage layout (single run)::
+
+    {base_path}/
+    └── {yaml.name}/                  # from cfg.name field (required)
+        └── {ShortClass}/             # short strategy class name(s), multi joined with '+'
+            └── YYYYMMDD_HHMMSS/      # unique per run
+                ├── _status.parquet   # written first, updated live during simulation
+                ├── _metadata.parquet # written on completion (all perf metrics)
+                ├── portfolio.parquet
+                ├── executions.parquet
+                ├── signals.parquet
+                ├── targets.parquet
+                └── config.yaml       # attached config file
+
+Storage layout (variation set)::
+
+    {base_path}/
+    └── {yaml.name}/
+        └── {ShortClass}/
+            └── YYYYMMDD_HHMMSS/
+                ├── _status.parquet
+                ├── _metadata.parquet  # N rows, one per variation — searchable by DuckDB
+                ├── var_000/
+                │   ├── portfolio.parquet
+                │   ├── executions.parquet
+                │   ├── signals.parquet
+                │   └── targets.parquet
+                ├── var_001/
+                │   └── ...
+                └── config.yaml
+
+DuckDB examples (via BacktestStorage)::
+
+    storage.search("sharpe > 2 AND mdd_pct < 25 AND list_contains(tags, 'momentum')")
+    storage.status("running")
+    storage.get_portfolio("my_strat/Nimble/20240301_120000", symbol="BTCUSDT", start="2024-01-01")
+"""
+
+import pandas as pd
+
+from qubx.core.metrics import TradingSessionResult
+from qubx.utils.misc import blue, cyan, green, magenta, red, yellow
+from qubx.utils.results import SimulationResultsSaver, is_cloud_path, resolve_s3_storage_options
+
+
+class BacktestStorage:
+    """
+    Query interface for parquet-based backtest storage.
+    Supports local directories and cloud paths (S3, GCS, Azure).
+
+    Uses DuckDB for fast metadata search and data queries across all stored backtests.
+
+    Storage layout (single run)::
+
+        {base_path}/
+        └── {yaml.name}/                  # from cfg.name field (required)
+            └── {ShortClass}/             # short strategy class name(s)
+                └── YYYYMMDD_HHMMSS/
+                    ├── _status.parquet   # live progress, written by SimulationResultsSaver
+                    ├── _metadata.parquet # completion metrics
+                    ├── portfolio.parquet
+                    ├── executions.parquet
+                    ├── signals.parquet
+                    ├── targets.parquet
+                    ├── emitter_data.parquet
+                    ├── transfers.parquet
+                    └── config.yaml
+
+    Examples::
+
+        # - local storage
+        storage = BacktestStorage("/backtests/")
+
+        # - S3 storage (creds from env: QUBX_S3_KEY / AWS_ACCESS_KEY_ID)
+        storage = BacktestStorage("s3://my-bucket/backtests/")
+
+        # - search: full DuckDB SQL WHERE clause
+        df = storage.search("sharpe > 2 AND mdd_pct < 25")
+        df = storage.search("list_contains(tags, 'momentum') AND cagr > 0.3")
+        df = storage.search("json_extract(parameters, '$.fast_period')::int > 10")
+        df = storage.search()  # - all results
+
+        # - live status dashboard
+        df = storage.status("running")
+
+        # - load result
+        result = storage.load("my_strat/Nimble/20240301_120000")
+
+        # - best variation from a variation set
+        result = storage.load_best_variation("my_strat/Nimble/20240301_130000", by="sharpe")
+    """
+
+    def __init__(self, base_path: str, storage_options: dict | None = None):
+        """
+        Initialize BacktestStorage.
+
+        Args:
+            base_path: Root path for backtest storage (local dir or cloud URI)
+            storage_options: Cloud storage credentials. None = auto-detect from:
+                             QUBX_S3_KEY / AWS_ACCESS_KEY_ID
+                             QUBX_S3_SECRET / AWS_SECRET_ACCESS_KEY
+                             QUBX_S3_REGION / AWS_DEFAULT_REGION
+                             QUBX_S3_ENDPOINT / AWS_ENDPOINT_URL
+        """
+        try:
+            import duckdb
+
+            self._duckdb = duckdb
+        except ImportError:
+            raise ImportError(
+                "duckdb is required for BacktestStorage. "
+                "Install with: pip install 'qubx[storage]' or pip install duckdb"
+            )
+        self.base_path = base_path.rstrip("/") + "/"
+        self._is_cloud = is_cloud_path(base_path)
+
+        # - for cloud paths: resolve credentials once (env vars → explicit dict)
+        self._storage_options: dict | None = resolve_s3_storage_options(storage_options) if self._is_cloud else None
+        self._conn = self._duckdb.connect()
+
+        if self._is_cloud:
+            self._setup_cloud_duckdb()
+
+    def _setup_cloud_duckdb(self) -> None:
+        """Configure DuckDB httpfs extension for cloud storage access."""
+        self._conn.execute("INSTALL httpfs; LOAD httpfs;")
+
+        # - _storage_options is already resolved at __init__ for cloud paths
+        opts = self._storage_options or {}
+        if "key" in opts:
+            self._conn.execute(f"SET s3_access_key_id='{opts['key']}';")
+        if "secret" in opts:
+            self._conn.execute(f"SET s3_secret_access_key='{opts['secret']}';")
+        if "endpoint_url" in opts:
+            # - strip protocol prefix — DuckDB expects hostname only
+            endpoint = opts["endpoint_url"].removeprefix("https://").removeprefix("http://")
+            self._conn.execute(f"SET s3_endpoint='{endpoint}';")
+        if "client_kwargs" in opts:
+            region = opts["client_kwargs"].get("region_name")
+            if region:
+                self._conn.execute(f"SET s3_region='{region}';")
+
+    def _glob(self, filename: str) -> str:
+        """
+        Build recursive glob pattern for a filename within base_path.
+        """
+        return f"{self.base_path}**/{filename}"
+
+    def search(
+        self,
+        where: str | None = None,
+        order_by: str = "sharpe DESC",
+        limit: int | None = None,
+    ) -> pd.DataFrame:
+        """
+        Search backtest metadata across all stored results using DuckDB SQL.
+
+        The WHERE clause has full DuckDB SQL power — no restrictions::
+
+            "sharpe > 2 AND mdd_pct < 25 AND author = 'alice'"
+            "list_contains(tags, 'momentum') AND cagr > 0.3"
+            "json_extract(parameters, '$.fast_period')::int > 10"
+            "is_variation = false"
+            "strategy_class LIKE '%Nimble%'"
+            "start >= '2024-01-01' AND sharpe BETWEEN 1.5 AND 4.0"
+
+        Regular backtests: one row per run.
+        Variation sets: N rows per set (one per variation), all with is_variation=true.
+
+        Args:
+            where: DuckDB SQL WHERE clause, or None to return all results
+            order_by: ORDER BY clause (default: "sharpe DESC")
+            limit: Maximum rows to return
+
+        Returns:
+            pd.DataFrame with matching metadata rows
+        """
+        glob = self._glob(SimulationResultsSaver.METADATA_FILE)
+        sql = f"SELECT * FROM read_parquet('{glob}', union_by_name=true)"
+        if where:
+            sql += f" WHERE {where}"
+        if order_by:
+            sql += f" ORDER BY {order_by}"
+        if limit is not None:
+            sql += f" LIMIT {limit}"
+        return self._conn.execute(sql).df()
+
+    def status(self, filter_status: str | None = None) -> pd.DataFrame:
+        """
+        Get status of all simulations (running, completed, failed, pending).
+
+        Reads _status.parquet files written by SimulationResultsSaver.
+        Works in real-time — running simulations update their status every 1%.
+
+        Args:
+            filter_status: Filter by status value ('running', 'completed', 'failed', 'pending'),
+                          or None to return all simulations
+
+        Returns:
+            pd.DataFrame with status rows, ordered by started_at DESC
+        """
+        glob = self._glob(SimulationResultsSaver.STATUS_FILE)
+        sql = f"SELECT * FROM read_parquet('{glob}', union_by_name=true)"
+        if filter_status:
+            sql += f" WHERE status = '{filter_status}'"
+        sql += " ORDER BY started_at DESC"
+        return self._conn.execute(sql).df()
+
+    def _load_from_path(self, run_path: str) -> TradingSessionResult:
+        """
+        Load a TradingSessionResult using the already-configured DuckDB connection.
+
+        All parquet reads go through ``self._conn`` (httpfs for S3, local for disk)
+        so no s3fs / aiobotocore dependency is needed and connection pooling is
+        handled by DuckDB internally.
+        """
+
+        def _read(filename: str) -> pd.DataFrame:
+            p = f"{run_path.rstrip('/')}/{filename}"
+            try:
+                return self._conn.execute(f"SELECT * FROM read_parquet('{p}')").df()
+            except Exception:
+                return pd.DataFrame()
+
+        meta_df = _read(SimulationResultsSaver.METADATA_FILE)
+        if meta_df.empty:
+            raise FileNotFoundError(f"Metadata not found at '{run_path}'")
+
+        return SimulationResultsSaver._from_dfs(
+            meta=meta_df.iloc[0].to_dict(),
+            portfolio=_read(SimulationResultsSaver.DATA_FILES["portfolio"]),
+            executions=_read(SimulationResultsSaver.DATA_FILES["executions"]),
+            signals=_read(SimulationResultsSaver.DATA_FILES["signals"]),
+            targets=_read(SimulationResultsSaver.DATA_FILES["targets"]),
+            transfers=_read(SimulationResultsSaver.DATA_FILES["transfers"]),
+            emitter=_read(SimulationResultsSaver.DATA_FILES["emitter"]),
+        )
+
+    def load(self, backtest_id: str) -> TradingSessionResult:
+        """
+        Load a TradingSessionResult by backtest_id.
+
+        Args:
+            backtest_id: Relative path within base_path,
+                        e.g. "my_strategy/Nimble/20240301_120000"
+
+        Returns:
+            TradingSessionResult with all data loaded from parquet
+        """
+        return self._load_from_path(f"{self.base_path}{backtest_id.strip('/')}/")
+
+    def load_best_variation(
+        self,
+        variation_set_id: str,
+        by: str = "sharpe",
+        ascending: bool = False,
+    ) -> TradingSessionResult:
+        """
+        Load the best-performing variation from a variation set.
+
+        The variation set _metadata.parquet has one row per variation.
+        Finds the best row by the given metric, then loads its data.
+
+        Args:
+            variation_set_id: Relative path to variation set root,
+                             e.g. "my_strategy/Nimble/20240301_130000"
+            by: Metric column to rank by (default: "sharpe")
+            ascending: If True, load minimum instead of maximum (default: False)
+
+        Returns:
+            TradingSessionResult of the best variation
+        """
+        meta_path = f"{self.base_path}{variation_set_id.strip('/')}/{SimulationResultsSaver.METADATA_FILE}"
+        order = "ASC" if ascending else "DESC"
+        row = self._conn.execute(f"SELECT * FROM read_parquet('{meta_path}') ORDER BY {by} {order} LIMIT 1").df()
+
+        if row.empty:
+            raise ValueError(f"No variations found at '{variation_set_id}'")
+
+        var_id = row["variation_id"].iloc[0]
+        run_path = f"{self.base_path}{variation_set_id.strip('/')}/{var_id}/"
+        return self._load_from_path(run_path)
+
+    def get_portfolio(
+        self,
+        backtest_id: str,
+        symbol: str | None = None,
+        start: str | None = None,
+        stop: str | None = None,
+    ) -> pd.DataFrame:
+        """
+        Get portfolio log data for a backtest.
+
+        Portfolio is stored in wide format: one column per symbol metric
+        (e.g. "BINANCE.UM:BTCUSDT_PnL", "BINANCE.UM:BTCUSDT_Commission").
+
+        Args:
+            backtest_id: Relative path within base_path
+            symbol: If set, returns only columns containing this symbol name (case-insensitive)
+            start: Start timestamp filter (inclusive)
+            stop: Stop timestamp filter (inclusive)
+
+        Returns:
+            pd.DataFrame with portfolio data
+        """
+        path = f"{self.base_path}{backtest_id.strip('/')}/{SimulationResultsSaver.DATA_FILES['portfolio']}"
+        return self._query_wide(path, symbol=symbol, start=start, stop=stop)
+
+    def get_executions(
+        self,
+        backtest_id: str,
+        symbol: str | None = None,
+        start: str | None = None,
+        stop: str | None = None,
+    ) -> pd.DataFrame:
+        """
+        Get execution log data for a backtest.
+
+        Args:
+            backtest_id: Relative path within base_path
+            symbol: Filter rows by instrument column (case-insensitive match)
+            start: Start timestamp filter (inclusive)
+            stop: Stop timestamp filter (inclusive)
+
+        Returns:
+            pd.DataFrame with execution data
+        """
+        path = f"{self.base_path}{backtest_id.strip('/')}/{SimulationResultsSaver.DATA_FILES['executions']}"
+        return self._query_long(path, symbol=symbol, start=start, stop=stop)
+
+    def get_signals(
+        self,
+        backtest_id: str,
+        symbol: str | None = None,
+        start: str | None = None,
+        stop: str | None = None,
+    ) -> pd.DataFrame:
+        """
+        Get signals log data for a backtest.
+
+        Args:
+            backtest_id: Relative path within base_path
+            symbol: Filter rows by instrument column (case-insensitive match)
+            start: Start timestamp filter (inclusive)
+            stop: Stop timestamp filter (inclusive)
+
+        Returns:
+            pd.DataFrame with signals data
+        """
+        path = f"{self.base_path}{backtest_id.strip('/')}/{SimulationResultsSaver.DATA_FILES['signals']}"
+        return self._query_long(path, symbol=symbol, start=start, stop=stop)
+
+    def _query_wide(
+        self,
+        path: str,
+        symbol: str | None = None,
+        start: str | None = None,
+        stop: str | None = None,
+    ) -> pd.DataFrame:
+        """
+        Query a wide-format parquet (portfolio log) with optional column/time filtering.
+        Symbol filtering selects columns containing the symbol string using DuckDB COLUMNS().
+        """
+        conditions = []
+        if start:
+            conditions.append(f"timestamp >= '{start}'")
+        if stop:
+            conditions.append(f"timestamp <= '{stop}'")
+        where_clause = f" WHERE {' AND '.join(conditions)}" if conditions else ""
+
+        if symbol:
+            sym = symbol.upper()
+            # - DuckDB COLUMNS() lambda: select timestamp + _backtest_id + symbol columns
+            sql = f"""
+                SELECT COLUMNS(c -> c = 'timestamp' OR c = '_backtest_id'
+                               OR contains(upper(c), '{sym}'))
+                FROM read_parquet('{path}'){where_clause}
+            """
+        else:
+            sql = f"SELECT * FROM read_parquet('{path}'){where_clause}"
+
+        return self._conn.execute(sql).df()
+
+    def _query_long(
+        self,
+        path: str,
+        symbol: str | None = None,
+        start: str | None = None,
+        stop: str | None = None,
+        symbol_col: str = "symbol",
+    ) -> pd.DataFrame:
+        """
+        Query a long-format parquet (executions, signals) with optional row filtering.
+        Symbol filtering matches rows where symbol_col contains the symbol string.
+        """
+        conditions = []
+        if start:
+            conditions.append(f"timestamp >= '{start}'")
+        if stop:
+            conditions.append(f"timestamp <= '{stop}'")
+        if symbol:
+            conditions.append(f"contains(upper({symbol_col}), '{symbol.upper()}')")
+        where_clause = f" WHERE {' AND '.join(conditions)}" if conditions else ""
+        sql = f"SELECT * FROM read_parquet('{path}'){where_clause}"
+        return self._conn.execute(sql).df()
+
+    def print(
+        self,
+        where: str | None = None,
+        order_by: str = "creation_time DESC",
+        limit: int | None = None,
+        params: bool = False,
+    ) -> None:
+        """
+        Pretty-print a colored list of backtests stored at base_path.
+
+        Matches the style of the old BacktestsResultsManager.list() — header line,
+        description, strategy / interval / capital / instruments, full metrics table.
+
+        Args:
+            where:    DuckDB WHERE clause to filter (e.g. ``"sharpe > 2"``).  None = all.
+            order_by: ORDER BY clause (default: ``"creation_time DESC"``).
+            limit:    Maximum number of results to display.
+            params:   If True, print strategy parameters below the metrics table.
+        """
+        df = self.search(where=where, order_by=order_by, limit=limit)
+
+        if df.empty:
+            print("No backtests found.")
+            return
+
+        _l = lambda v: [] if v is None else list(v)  # noqa: E731 — numpy array → Python list
+        _METRIC_COLS = ["gain", "cagr", "sharpe", "qr", "mdd_pct", "mdd_usd", "fees", "execs"]
+
+        for _, row in df.iterrows():
+            _id = row.get("backtest_id", "")
+            _name = row.get("name", "")
+            _cls = str(row.get("strategy_class", "")).split(".")[-1]
+            _created = pd.Timestamp(row.get("creation_time")).strftime("%Y-%m-%d %H:%M:%S")
+            _author = row.get("author", "")
+            _start = pd.Timestamp(row.get("start")).strftime("%Y-%m-%d")
+            _stop = pd.Timestamp(row.get("stop")).strftime("%Y-%m-%d")
+            _capital = row.get("capital", "")
+            _ccy = row.get("base_currency", "")
+            _comm = row.get("commissions", "")
+            _dscr = row.get("description", "") or ""
+            _tags = _l(row.get("tags"))
+            _symbols = ", ".join(_l(row.get("symbols")))
+            _is_var = row.get("is_variation", False)
+
+            # - header: id :: name ::: created by author
+            _s = f"{yellow(_id)} :: {red(_name)}"
+            if _is_var:
+                _var_id = row.get("variation_id", "")
+                _var_params = row.get("variation_params", "") or ""
+                _s += f" [{cyan(_var_id)}] {magenta(_var_params)}"
+            _s += f" ::: {magenta(_created)} by {cyan(_author)}"
+
+            # - description lines
+            if _dscr:
+                for _d in _dscr.split("\n"):
+                    if _d.strip():
+                        _s += f"\n\t{magenta('# ' + _d)}"
+
+            _s += f"\n\tstrategy:    {green(_cls)}"
+            _s += f"\n\tinterval:    {blue(_start)} - {blue(_stop)}"
+            _s += f"\n\tcapital:     {blue(str(_capital))} {_ccy} ({_comm})"
+            _s += f"\n\tinstruments: {blue(_symbols)}"
+            if _tags:
+                _s += f"\n\ttags:        {cyan(str(_tags))}"
+
+            print(_s)
+
+            # - performance metrics table (red header, cyan values — same as old manager)
+            _metrics = {
+                c: (int(row.get(c) or 0) if c == "execs" else round(float(row.get(c) or 0.0), 3))
+                for c in _METRIC_COLS
+                if c in row
+            }
+            _m_df = pd.DataFrame([_metrics])
+            _m_str = _m_df.to_string(index=False)
+            _h, _v = _m_str.split("\n")
+            print("\t " + red(_h))
+            print("\t " + cyan(_v))
+
+            # - optional parameters
+            if params:
+                import json as _json
+
+                _p = _json.loads(row.get("parameters") or "{}")
+                if _p:
+                    for k, v in _p.items():
+                        print(f"\t   {yellow(k)}: {cyan(str(v))}")
+
+            print()
+
+    def list(
+        self,
+        where: str | None = None,
+        order_by: str = "creation_time DESC",
+        limit: int | None = None,
+    ) -> list[str]:
+        """
+        Return a list of backtest IDs matching the given filter.
+
+        Args:
+            where:    Optional SQL WHERE clause to filter results.
+            order_by: SQL ORDER BY clause (default: ``creation_time DESC``).
+            limit:    Maximum number of IDs to return.
+
+        Returns:
+            List of backtest_id strings, e.g.
+            ``["my_strategy/Nimble/20240301_120000", ...]``
+        """
+        df = self.search(where=where, order_by=order_by, limit=limit)
+        if df.empty or "backtest_id" not in df.columns:
+            return []
+        return df["backtest_id"].tolist()
+
+    def export_backtests_to_markdown(self, backtest_id: str, path: str, tags: tuple[str] | None = None):
+        if tsr := self.load(backtest_id):
+            tsr.to_markdown(path, list(tags) if tags else None)