ebt-backtest 0.1.0__tar.gz

ebt_backtest-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ebt contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ebt_backtest-0.1.0/MANIFEST.in

@@ -0,0 +1,4 @@
+include LICENSE README.md
+include ebt_rust/Cargo.toml ebt_rust/Cargo.lock
+recursive-include ebt_rust/src *.rs
+recursive-include tests *.py

ebt_backtest-0.1.0/PKG-INFO

@@ -0,0 +1,208 @@
+Metadata-Version: 2.4
+Name: ebt-backtest
+Version: 0.1.0
+Summary: Backtesting and evaluation toolkit for position-based trading strategies
+License: MIT
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Rust
+Classifier: Intended Audience :: Financial and Insurance Industry
+Classifier: Topic :: Office/Business :: Financial :: Investment
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: scipy
+Requires-Dist: scikit-learn
+Requires-Dist: numba
+Requires-Dist: matplotlib
+Requires-Dist: seaborn
+Requires-Dist: Pillow
+Requires-Dist: tqdm
+Requires-Dist: psutil
+Requires-Dist: pyarrow
+Requires-Dist: dask[distributed]
+Requires-Dist: polars
+Requires-Dist: torch
+Requires-Dist: torcheval
+Requires-Dist: torch-lr-finder
+Requires-Dist: tensorboard
+Provides-Extra: nn
+Provides-Extra: notebook
+Requires-Dist: ipython; extra == "notebook"
+Requires-Dist: ipywidgets; extra == "notebook"
+Provides-Extra: optimize
+Requires-Dist: cvxpy; extra == "optimize"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Dynamic: license-file
+
+# ebt
+
+Backtesting and evaluation toolkit for position-based trading strategies.
+
+You bring a price frame (DatetimeIndex × assets) and a position frame on the same assets.
+ebt fits models walk-forward, turns predictions into positions, backtests them with an
+execution lag, reports key figures, and sweeps strategy variants into sub-portfolios.
+The hot loops (backtest, execution simulation, position combination, outlier detection)
+are Rust kernels built into the package as `ebt_rust`.
+
+## Features
+
+- **Walk-forward prediction** (`ebt.predict`): retrain any `fit`/`predict` model on a
+  rolling window with an embargo gap, for sklearn-style linear models and PyTorch nets.
+- **Feature and label helpers** (`ebt.features`): multi-scale EMA filter banks, forward
+  labels, OHLC resampling with causal timestamps, cross-sectional ranks.
+- **Backtest and key figures** (`ebt.simulate.kf`): PnL from positions and prices with an
+  integer or timedelta execution lag, Sharpe decomposition, turnover, holding time,
+  Sharpe-by-lag, trend correlation. Wide (one column per asset) and stacked/long input.
+- **One-pager and ensemble sweeps** (`ebt.simulate.plot`): `one_pager` evaluates one book
+  and draws the standard figure; `plot_ensemble` evaluates a list of variants, splits them
+  into sub-portfolios by config variable and summarises the sweep.
+- **Risk sizing and sanity checks**: `risk_adjust` to a target annualised risk,
+  `risk_to_lots`, cost-aware sizing (`ebt.simulate.costaware`), and a
+  `position_sanity_gate` that flags broken books before anyone reads their Sharpe.
+- **Execution models**: VWAP-ramped fills, spread-aware timing, trade suppression below
+  a risk threshold; effective-spread estimators from daily OHLC.
+- **Futures tooling** (`ebt.futures`): contract selection, rolls, Panama adjustment,
+  blended rolls, interpolated and open-interest-weighted continuous series, position
+  decomposition back into contracts.
+- **Streaming outlier detection** (`ebt.data.outlier`): point-in-time detectors for
+  trades, quotes and OHLCV bars.
+- **Parallel map** (`ebt.parallel`): fork or lean-subprocess workers with zero-copy
+  shared frames.
+
+## Install
+
+Requires Python 3.9+ and a Rust toolchain (https://rustup.rs) to build the extension.
+
+```bash
+pip install ebt-backtest
+```
+
+Wheels are published for Linux x86_64 and Windows (Python 3.9+); other platforms, including macOS, build from the sdist with a Rust toolchain. From a checkout:
+
+```bash
+pip install .
+# editable install (the top-level ebt_rust extension needs compat mode):
+pip install -e . --config-settings editable_mode=compat
+```
+
+Extras: `ebt[notebook]` (IPython widgets for the tabbed sweep views), `ebt[optimize]`
+(cvxpy for the allocation optimiser), `ebt[test]`.
+
+## Quickstart
+
+```python
+import numpy as np, pandas as pd
+import ebt
+from ebt.simulate.linear import Ridge
+from ebt.simulate.plot import one_pager, plot_ensemble
+from ebt.simulate.tricks import risk_adjust
+
+# synthetic 5-minute prices for four assets
+rng = np.random.default_rng(0)
+idx = pd.date_range('2022-01-03', '2023-12-29', freq='5min')
+rets = 1e-3 * rng.standard_normal((len(idx), 4))
+prices = pd.DataFrame(100 * np.exp(np.cumsum(rets, axis=0)), index=idx, columns=list('ABCD'))
+
+# daily features and forward labels; a bar's timestamp is when its price is known
+daily_ret = prices.resample('1D', label='right').last().dropna().pct_change()
+features = ebt.apply_filters(daily_ret, lengths=[4, 16, 64]).replace([np.inf, -np.inf], np.nan).dropna()
+labels = ebt.make_forward_labels(daily_ret, smoothing_length=5).reindex(features.index)
+
+# walk-forward fit: retrain every 60 days on an expanding window, 10-day embargo
+def walk_forward(model):
+    out = ebt.predict(features, labels, models=[model], min_lookback='120D',
+                      retrain_interval='60D', embargo='10D', progressbar=False)
+    return risk_adjust(out[model], prices, shift='2min')   # scale to annualised risk 1.0
+
+pos = walk_forward(Ridge(alpha=1.0))
+
+# evaluate against the raw prices with a 2-minute execution lag
+res = one_pager(pos, prices, shift='2min')
+print(res['kf'])            # sharpe, sharpe exbias, turnover, holding(days), lptp, ...
+
+# sweep variants and compare sub-portfolios per config variable
+configs = [{'alpha': a} for a in (0.1, 1.0, 10.0)]
+positions = [walk_forward(Ridge(alpha=c['alpha'])) for c in configs]
+ens = plot_ensemble(positions, prices, configs_df=pd.DataFrame(configs), shift='2min')
+print(ens['kfs_df'])        # one row per variant: config columns + key figures
+```
+
+Neural-net models work the same way: `ebt.simulate.nn.NeuralNetModel(create_net=FeedForwardNet,
+n_hidden=[16], ...)` trains with a masked MSE loss by default; pass `create_loss_f` for your own
+objective.
+
+## Conventions
+
+- **Timestamps mark when a value is known.** Resample prices with `label='right'`; the bar
+  stamped `T` holds prints in `(T - freq, T]`. `make_ohlc` and `make_forward_labels` follow this.
+- **Positions** are frames on the same assets as the prices. `NaN` means "carry the previous
+  position", `0` means flat. After `risk_adjust` a book has annualised risk 1.0, so key figures
+  across books are comparable.
+- **Execution lag** `shift` is an integer number of price bars or a timedelta (`'2min'`,
+  `pd.Timedelta('5m')`). A timedelta fills at the first price print at or after
+  `stamp + shift`, so pass raw prices at a resolution finer than the signal.
+- **Stacked (long) input**: `backtest_stacked` and friends accept one row per `(time, contract)`
+  in pandas or Polars. Column names are auto-detected (`ts` / `time` / `timestamp` for time,
+  `series` / `symbol` / `contract` for the contract, `price` / `close` / `return` for the value)
+  and can be given explicitly with `time_col=`, `contract_col=`, `pos_value_col=`, `price_value_col=`.
+- **Futures tools** take a `columns={...}` mapping (`time`, `contract`, `expiry`, `price`, `weight`)
+  so any source schema works without renaming; outputs add a `series` column naming the
+  continuous series (`'FUT(front)'`, `'FUT(30d)'`, `'FUT(top3)'`, ...).
+
+## Key figures
+
+Computed by `one_pager` and `plot_ensemble` ("kf" = key figure). Sharpe and risk use daily PnL.
+
+| Key figure | Meaning |
+|---|---|
+| `sharpe` | Annualised Sharpe of the book's daily PnL. |
+| `sharpe exbias` / `sharpe bias` | Sharpe of the timing component and of the static component. Each asset's position (in risk units) is split into its time-mean and the remainder; `bias` is the PnL of the constant mean position, `exbias` the rest. Slow books can earn most of their raw Sharpe from `bias`. |
+| `sharpe long` / `sharpe short` | Sharpe of the long and short legs. |
+| `sharpe w costs` | Sharpe after spread costs when `spreads=` is given. |
+| `turnover` | Daily `sum(|Δpos|) / mean(|pos|)`, frequency-independent. |
+| `holding(days)` | Average holding period implied by the turnover. |
+| `lptp` | Long-position-to-total-position: `0.5 + sum(pos) / (2 sum(|pos|))`. 1 long-only, 0 short-only, 0.5 dollar-neutral. |
+| `up_days` | Fraction of days with positive PnL. |
+| `risk` | Annualised std of daily PnL (1.0 after `risk_adjust`). |
+| `ncvar` | Normalised covariance ratio: the book's variance under an EWMA correlation model divided by its variance if the assets were uncorrelated. 1 for uncorrelated positions, above 1 when positions load on correlated assets, below 1 when they hedge each other. |
+| `sharpe_decay` | Log-log slope of Sharpe against holding time across smoothed versions of the book; near 0 the signal persists, around -0.5 it decays like noise. |
+| `skew`, `kurtosis` | Of the daily PnL. |
+
+## Package map
+
+| Module | Contents |
+|---|---|
+| `ebt.features.core` | `apply_filters`, `make_forward_labels`, `make_ohlc`, `make_daily_from_intraday`, `normalized_rank`, `ffill_timeseries` |
+| `ebt.simulate.predict` | `predict` walk-forward driver, `generator`, `enough_data` |
+| `ebt.simulate.model`, `.linear`, `.nn` | `Model` base class, sklearn wrappers (`Ridge`, `LassoLars`, `SparseMarkowitz`, ...), `NeuralNetModel` / `LSTMNeuralNetModel` |
+| `ebt.simulate.kf` | `backtest`, `backtest_stacked`, `sharpe`, `turnover`, `holding_time`, `lptp`, `ncvar`, `sharpe_per_lag`, `holding_curve`, `sharpe_decay`, `correlation_to_trend` |
+| `ebt.simulate.plot` | `one_pager`, `plot_ensemble`, `per_year_table`, figure building blocks |
+| `ebt.simulate.postprocessing`, `.tricks` | `risk_adjust`, `risk_to_lots`, `combine_positions`, `split_portfolios`, `nanagg` |
+| `ebt.simulate.gate` | `position_sanity_gate` |
+| `ebt.simulate.grid` | `ReducedGrid`: evaluate sparse signals on the price rows that matter |
+| `ebt.simulate.execution`, `.spread`, `.costaware` | execution simulators, spread estimators, cost-aware sizing and turnover budgets |
+| `ebt.simulate.optimize` | ensemble weight optimisation |
+| `ebt.futures` | contract schedules, rolls, continuous prices and weights, position decomposition |
+| `ebt.data.outlier` | `detect_outliers` for trades, quotes and bars |
+| `ebt.parallel` | `pmap` |
+| `ebt_rust` | the compiled kernels (not a public API) |
+
+## Tests
+
+```bash
+pip install -e . --config-settings editable_mode=compat
+python -m pytest tests
+cd ebt_rust && cargo test --profile perf
+```
+
+## Releasing
+
+`scripts/release.sh X.Y.Z` tags a release and the GitHub workflow publishes it; see `RELEASING.md`.
+
+## License
+
+MIT, see `LICENSE`.

ebt_backtest-0.1.0/README.md

@@ -0,0 +1,168 @@
+# ebt
+
+Backtesting and evaluation toolkit for position-based trading strategies.
+
+You bring a price frame (DatetimeIndex × assets) and a position frame on the same assets.
+ebt fits models walk-forward, turns predictions into positions, backtests them with an
+execution lag, reports key figures, and sweeps strategy variants into sub-portfolios.
+The hot loops (backtest, execution simulation, position combination, outlier detection)
+are Rust kernels built into the package as `ebt_rust`.
+
+## Features
+
+- **Walk-forward prediction** (`ebt.predict`): retrain any `fit`/`predict` model on a
+  rolling window with an embargo gap, for sklearn-style linear models and PyTorch nets.
+- **Feature and label helpers** (`ebt.features`): multi-scale EMA filter banks, forward
+  labels, OHLC resampling with causal timestamps, cross-sectional ranks.
+- **Backtest and key figures** (`ebt.simulate.kf`): PnL from positions and prices with an
+  integer or timedelta execution lag, Sharpe decomposition, turnover, holding time,
+  Sharpe-by-lag, trend correlation. Wide (one column per asset) and stacked/long input.
+- **One-pager and ensemble sweeps** (`ebt.simulate.plot`): `one_pager` evaluates one book
+  and draws the standard figure; `plot_ensemble` evaluates a list of variants, splits them
+  into sub-portfolios by config variable and summarises the sweep.
+- **Risk sizing and sanity checks**: `risk_adjust` to a target annualised risk,
+  `risk_to_lots`, cost-aware sizing (`ebt.simulate.costaware`), and a
+  `position_sanity_gate` that flags broken books before anyone reads their Sharpe.
+- **Execution models**: VWAP-ramped fills, spread-aware timing, trade suppression below
+  a risk threshold; effective-spread estimators from daily OHLC.
+- **Futures tooling** (`ebt.futures`): contract selection, rolls, Panama adjustment,
+  blended rolls, interpolated and open-interest-weighted continuous series, position
+  decomposition back into contracts.
+- **Streaming outlier detection** (`ebt.data.outlier`): point-in-time detectors for
+  trades, quotes and OHLCV bars.
+- **Parallel map** (`ebt.parallel`): fork or lean-subprocess workers with zero-copy
+  shared frames.
+
+## Install
+
+Requires Python 3.9+ and a Rust toolchain (https://rustup.rs) to build the extension.
+
+```bash
+pip install ebt-backtest
+```
+
+Wheels are published for Linux x86_64 and Windows (Python 3.9+); other platforms, including macOS, build from the sdist with a Rust toolchain. From a checkout:
+
+```bash
+pip install .
+# editable install (the top-level ebt_rust extension needs compat mode):
+pip install -e . --config-settings editable_mode=compat
+```
+
+Extras: `ebt[notebook]` (IPython widgets for the tabbed sweep views), `ebt[optimize]`
+(cvxpy for the allocation optimiser), `ebt[test]`.
+
+## Quickstart
+
+```python
+import numpy as np, pandas as pd
+import ebt
+from ebt.simulate.linear import Ridge
+from ebt.simulate.plot import one_pager, plot_ensemble
+from ebt.simulate.tricks import risk_adjust
+
+# synthetic 5-minute prices for four assets
+rng = np.random.default_rng(0)
+idx = pd.date_range('2022-01-03', '2023-12-29', freq='5min')
+rets = 1e-3 * rng.standard_normal((len(idx), 4))
+prices = pd.DataFrame(100 * np.exp(np.cumsum(rets, axis=0)), index=idx, columns=list('ABCD'))
+
+# daily features and forward labels; a bar's timestamp is when its price is known
+daily_ret = prices.resample('1D', label='right').last().dropna().pct_change()
+features = ebt.apply_filters(daily_ret, lengths=[4, 16, 64]).replace([np.inf, -np.inf], np.nan).dropna()
+labels = ebt.make_forward_labels(daily_ret, smoothing_length=5).reindex(features.index)
+
+# walk-forward fit: retrain every 60 days on an expanding window, 10-day embargo
+def walk_forward(model):
+    out = ebt.predict(features, labels, models=[model], min_lookback='120D',
+                      retrain_interval='60D', embargo='10D', progressbar=False)
+    return risk_adjust(out[model], prices, shift='2min')   # scale to annualised risk 1.0
+
+pos = walk_forward(Ridge(alpha=1.0))
+
+# evaluate against the raw prices with a 2-minute execution lag
+res = one_pager(pos, prices, shift='2min')
+print(res['kf'])            # sharpe, sharpe exbias, turnover, holding(days), lptp, ...
+
+# sweep variants and compare sub-portfolios per config variable
+configs = [{'alpha': a} for a in (0.1, 1.0, 10.0)]
+positions = [walk_forward(Ridge(alpha=c['alpha'])) for c in configs]
+ens = plot_ensemble(positions, prices, configs_df=pd.DataFrame(configs), shift='2min')
+print(ens['kfs_df'])        # one row per variant: config columns + key figures
+```
+
+Neural-net models work the same way: `ebt.simulate.nn.NeuralNetModel(create_net=FeedForwardNet,
+n_hidden=[16], ...)` trains with a masked MSE loss by default; pass `create_loss_f` for your own
+objective.
+
+## Conventions
+
+- **Timestamps mark when a value is known.** Resample prices with `label='right'`; the bar
+  stamped `T` holds prints in `(T - freq, T]`. `make_ohlc` and `make_forward_labels` follow this.
+- **Positions** are frames on the same assets as the prices. `NaN` means "carry the previous
+  position", `0` means flat. After `risk_adjust` a book has annualised risk 1.0, so key figures
+  across books are comparable.
+- **Execution lag** `shift` is an integer number of price bars or a timedelta (`'2min'`,
+  `pd.Timedelta('5m')`). A timedelta fills at the first price print at or after
+  `stamp + shift`, so pass raw prices at a resolution finer than the signal.
+- **Stacked (long) input**: `backtest_stacked` and friends accept one row per `(time, contract)`
+  in pandas or Polars. Column names are auto-detected (`ts` / `time` / `timestamp` for time,
+  `series` / `symbol` / `contract` for the contract, `price` / `close` / `return` for the value)
+  and can be given explicitly with `time_col=`, `contract_col=`, `pos_value_col=`, `price_value_col=`.
+- **Futures tools** take a `columns={...}` mapping (`time`, `contract`, `expiry`, `price`, `weight`)
+  so any source schema works without renaming; outputs add a `series` column naming the
+  continuous series (`'FUT(front)'`, `'FUT(30d)'`, `'FUT(top3)'`, ...).
+
+## Key figures
+
+Computed by `one_pager` and `plot_ensemble` ("kf" = key figure). Sharpe and risk use daily PnL.
+
+| Key figure | Meaning |
+|---|---|
+| `sharpe` | Annualised Sharpe of the book's daily PnL. |
+| `sharpe exbias` / `sharpe bias` | Sharpe of the timing component and of the static component. Each asset's position (in risk units) is split into its time-mean and the remainder; `bias` is the PnL of the constant mean position, `exbias` the rest. Slow books can earn most of their raw Sharpe from `bias`. |
+| `sharpe long` / `sharpe short` | Sharpe of the long and short legs. |
+| `sharpe w costs` | Sharpe after spread costs when `spreads=` is given. |
+| `turnover` | Daily `sum(|Δpos|) / mean(|pos|)`, frequency-independent. |
+| `holding(days)` | Average holding period implied by the turnover. |
+| `lptp` | Long-position-to-total-position: `0.5 + sum(pos) / (2 sum(|pos|))`. 1 long-only, 0 short-only, 0.5 dollar-neutral. |
+| `up_days` | Fraction of days with positive PnL. |
+| `risk` | Annualised std of daily PnL (1.0 after `risk_adjust`). |
+| `ncvar` | Normalised covariance ratio: the book's variance under an EWMA correlation model divided by its variance if the assets were uncorrelated. 1 for uncorrelated positions, above 1 when positions load on correlated assets, below 1 when they hedge each other. |
+| `sharpe_decay` | Log-log slope of Sharpe against holding time across smoothed versions of the book; near 0 the signal persists, around -0.5 it decays like noise. |
+| `skew`, `kurtosis` | Of the daily PnL. |
+
+## Package map
+
+| Module | Contents |
+|---|---|
+| `ebt.features.core` | `apply_filters`, `make_forward_labels`, `make_ohlc`, `make_daily_from_intraday`, `normalized_rank`, `ffill_timeseries` |
+| `ebt.simulate.predict` | `predict` walk-forward driver, `generator`, `enough_data` |
+| `ebt.simulate.model`, `.linear`, `.nn` | `Model` base class, sklearn wrappers (`Ridge`, `LassoLars`, `SparseMarkowitz`, ...), `NeuralNetModel` / `LSTMNeuralNetModel` |
+| `ebt.simulate.kf` | `backtest`, `backtest_stacked`, `sharpe`, `turnover`, `holding_time`, `lptp`, `ncvar`, `sharpe_per_lag`, `holding_curve`, `sharpe_decay`, `correlation_to_trend` |
+| `ebt.simulate.plot` | `one_pager`, `plot_ensemble`, `per_year_table`, figure building blocks |
+| `ebt.simulate.postprocessing`, `.tricks` | `risk_adjust`, `risk_to_lots`, `combine_positions`, `split_portfolios`, `nanagg` |
+| `ebt.simulate.gate` | `position_sanity_gate` |
+| `ebt.simulate.grid` | `ReducedGrid`: evaluate sparse signals on the price rows that matter |
+| `ebt.simulate.execution`, `.spread`, `.costaware` | execution simulators, spread estimators, cost-aware sizing and turnover budgets |
+| `ebt.simulate.optimize` | ensemble weight optimisation |
+| `ebt.futures` | contract schedules, rolls, continuous prices and weights, position decomposition |
+| `ebt.data.outlier` | `detect_outliers` for trades, quotes and bars |
+| `ebt.parallel` | `pmap` |
+| `ebt_rust` | the compiled kernels (not a public API) |
+
+## Tests
+
+```bash
+pip install -e . --config-settings editable_mode=compat
+python -m pytest tests
+cd ebt_rust && cargo test --profile perf
+```
+
+## Releasing
+
+`scripts/release.sh X.Y.Z` tags a release and the GitHub workflow publishes it; see `RELEASING.md`.
+
+## License
+
+MIT, see `LICENSE`.

ebt_backtest-0.1.0/ebt/__init__.py

@@ -0,0 +1,10 @@
+from ebt.simulate.timeseries import *
+from ebt.simulate.stats import *
+from ebt.simulate.predict import *
+from ebt.simulate.kf import *
+from ebt.simulate.plot import *
+from ebt.data import *
+from ebt.simulate.features import *
+from ebt.simulate.tricks import *
+from ebt import simulate
+from ebt.simulate.optimize import *

ebt_backtest-0.1.0/ebt/data/__init__.py

@@ -0,0 +1,2 @@
+from ebt.data.types import *
+from ebt.data.outlier import *

ebt_backtest-0.1.0/ebt/data/outlier.py

@@ -0,0 +1,146 @@
+"""Streaming outlier detection for trades, quotes and OHLCV bars (Rust kernels).
+
+The detectors are sequential and point-in-time: the decision for a row depends only
+on the rows before it. See ``detect_outliers``.
+"""
+
+from __future__ import annotations
+
+from typing import Optional, Dict, Literal
+
+import pandas as pd
+import numpy as np
+
+
+Kind = Literal["auto", "bars", "trades", "quotes"]
+
+
+def detect_kind(df: pd.DataFrame) -> Kind:
+    cols = {c.lower() for c in df.columns}
+    if {"open", "high", "low", "close"}.issubset(cols):
+        return "bars"
+    if {"bid", "ask"}.issubset(cols) or {"bid_price", "ask_price"}.issubset(cols):
+        return "quotes"
+    if "price" in cols:
+        return "trades"
+    raise ValueError("Could not detect kind. Provide kind='bars'|'trades'|'quotes'.")
+
+
+_RUST_DEFAULT_COLUMNS = {
+    "trades": {"time": None, "price": "price", "volume": "volume"},
+    "quotes": {"time": None, "bid": "bid_price", "ask": "ask_price"},
+    "bars": {"time": None, "open": "open", "high": "high", "low": "low", "close": "close", "volume": "volume"},
+}
+
+
+def detect_outliers(df, kind: Kind = "auto", columns: Optional[Dict[str, str]] = None,
+                    return_debug: bool = False, **config):
+    """Streaming outlier detection with the Rust detectors.
+
+    trades: ensemble of an adaptive level filter, rolling z-score, IQR, price
+        jump, time gap and volume checks; rejected when enough methods vote against.
+    quotes: mid-price level filter cascaded with a spread upper-bound filter.
+    bars: bar consistency checks (non-finite prices, high < low), then the trade
+        ensemble on the close.
+
+    Parameters
+    ----------
+    df : pandas or polars DataFrame, sorted by time. A pandas DatetimeIndex is used
+        as the time column when ``columns`` doesn't name one.
+    kind : "auto" | "trades" | "quotes" | "bars"
+    columns : maps roles to column names, e.g. ``{"time": "ts", "price": "px"}``.
+        Roles: trades — time, price, volume; quotes — time, bid, ask;
+        bars — time, open, high, low, close, volume. Without a time column the
+        frame's single datetime column is used. A missing volume column counts as 0.
+    return_debug : return a DataFrame (same library as the input) of
+        ``accepted`` plus the detector's internal state per row.
+    **config : detector settings.
+        trades/bars: ``zscore_threshold``, ``iqr_multiplier``, ``min_votes_to_reject``,
+        ``window_size``, ``merge_stat_votes``, ``gap_widen_threshold_sec``,
+        ``warmup_threshold``, ``min_scale``, ``min_scale_ticks``, ``n_switch``,
+        ``min_switch_seconds``, ``warmup_hard_limit``.
+        quotes: ``mid_tau_min``, ``mid_tau_max``, ``mid_n_switch``,
+        ``spread_max_consecutive_reject``, ``max_spread``, ``warmup_count``,
+        ``allow_crossed``, ``mid_min_scale``, ``mid_min_scale_ticks``,
+        ``mid_min_switch_seconds``, ``spread_min_scale``, ``spread_min_scale_ticks``,
+        ``warmup_hard_limit``.
+        The level filter's acceptance band is the learned variance floored at
+        ``min_scale_ticks`` learned ticks (not a fixed percentage); a regime switch needs
+        ``n_switch`` confirmations spanning ``min_switch_seconds``; during warm-up a
+        price further than ``warmup_hard_limit`` from the last accepted one is rejected
+        outright; crossed quotes are invalid unless ``allow_crossed``.
+
+    Returns
+    -------
+    numpy bool array (True = accepted), or the debug DataFrame. Rows with a missing
+    value are not accepted and don't update the detector.
+    """
+    import ebt_rust
+
+    is_polars = type(df).__module__.startswith("polars")
+    pdf = df.to_pandas() if is_polars else df
+    if kind == "auto":
+        kind = detect_kind(pdf)
+        if kind == "quotes" and "bid_price" not in pdf.columns:
+            kind_cols = {"bid": "bid", "ask": "ask"}
+        else:
+            kind_cols = {}
+    else:
+        kind_cols = {}
+    if kind not in _RUST_DEFAULT_COLUMNS:
+        raise ValueError(f"Unknown kind: {kind}")
+    cols = {**_RUST_DEFAULT_COLUMNS[kind], **kind_cols, **(columns or {})}
+
+    if cols["time"] is not None:
+        t = pdf[cols["time"]]
+    elif pd.api.types.is_datetime64_any_dtype(pdf.index):
+        t = pdf.index.to_series()
+    else:
+        candidates = [c for c in pdf.columns if pd.api.types.is_datetime64_any_dtype(pdf[c])]
+        if len(candidates) != 1:
+            raise ValueError(f"Pass columns={{'time': ...}}: found {len(candidates)} datetime columns {candidates}")
+        t = pdf[candidates[0]]
+    if pd.api.types.is_datetime64_any_dtype(t):
+        t = t.dt.tz_convert("UTC").dt.tz_localize(None) if getattr(t.dt, "tz", None) is not None else t
+        time_ns = t.to_numpy(dtype="datetime64[ns]").view("int64")
+        valid = ~t.isna().to_numpy()
+    else:
+        valid = ~t.isna().to_numpy()
+        time_ns = t.fillna(0).to_numpy(dtype="int64")
+
+    def col(role, default_zero=False):
+        name = cols[role]
+        if name not in pdf.columns:
+            if default_zero:
+                return np.zeros(len(pdf))
+            raise KeyError(f"Column {name!r} for {role!r} not found; pass columns={{'{role}': ...}}")
+        s = pdf[name]
+        return s.to_numpy(dtype="float64", na_value=np.nan), ~s.isna().to_numpy()
+
+    def values(roles, zero_ok=()):
+        nonlocal valid
+        out = []
+        for r in roles:
+            v = col(r, default_zero=r in zero_ok)
+            if isinstance(v, tuple):
+                v, ok = v
+                valid = valid & ok
+            out.append(np.ascontiguousarray(v))
+        return out
+
+    if kind == "trades":
+        args = values(["price", "volume"], zero_ok=("volume",))
+        res = ebt_rust.detect_outliers(time_ns, *args, valid=valid, return_debug=return_debug, **config)
+    elif kind == "quotes":
+        args = values(["bid", "ask"])
+        res = ebt_rust.detect_outliers_quotes(time_ns, *args, valid=valid, return_debug=return_debug, **config)
+    else:
+        args = values(["open", "high", "low", "close", "volume"], zero_ok=("volume",))
+        res = ebt_rust.detect_outliers_ohlcv(time_ns, *args, valid=valid, return_debug=return_debug, **config)
+
+    if not return_debug:
+        return res
+    if is_polars:
+        import polars as pl
+        return pl.DataFrame(res)
+    return pd.DataFrame(res, index=pdf.index)

ebt_backtest-0.1.0/ebt/data/types.py

@@ -0,0 +1,69 @@
+import os
+import pickle
+import shutil
+import tempfile
+import uuid
+from collections.abc import MutableMapping
+from urllib.parse import quote
+from ebt.simulate.tricks import is_dask_process
+
+
+class LazyDiskDict(MutableMapping):
+    """A dict whose values live on disk (one pickle per key) and are loaded on access.
+
+    Each instance owns a private directory ``<folder>/lazy_disk_dict_<uuid>/`` that is
+    removed when the instance is garbage collected (unless ``monitor_dask`` is set and the
+    instance lives inside a dask worker, where the files may be shared). Keys are
+    ``str()``-ed and percent-encoded for the file name, so any key is safe ('a/b' does not
+    become a sub-directory) and distinct keys never collide. The directory name is a uuid,
+    not ``id(self)``: ``id`` is recycled by CPython, so a later instance could otherwise
+    read or overwrite the files an earlier (undeleted) one left behind.
+    """
+
+    def __init__(self, d=None, folder=None, monitor_dask=False):
+        self._folder = folder or tempfile.gettempdir()
+        self._dir = os.path.join(self._folder, f'lazy_disk_dict_{uuid.uuid4().hex}')
+        os.makedirs(self._dir, exist_ok=True)
+        self._keys = set()  # Keep track of keys explicitly
+        self.is_dask = is_dask_process() if monitor_dask else False
+        if d is not None:
+            for k, v in d.items():
+                self[k] = v
+
+    def _filepath(self, key):
+        """A safe file name for a given key (percent-encoded, no path separators)."""
+        return os.path.join(self._dir, f'{quote(str(key), safe="")}.pkl')
+
+    def __getitem__(self, key):
+        if key not in self._keys:
+            raise KeyError(key)
+        with open(self._filepath(key), "rb") as f:
+            return pickle.load(f)
+
+    def __setitem__(self, key, value):
+        os.makedirs(self._dir, exist_ok=True)
+        with open(self._filepath(key), "wb") as f:
+            pickle.dump(value, f, protocol=pickle.HIGHEST_PROTOCOL)
+        self._keys.add(key)
+
+    def __delitem__(self, key):
+        if key not in self._keys:
+            raise KeyError(key)
+        os.remove(self._filepath(key))
+        self._keys.remove(key)
+
+    def __del__(self):
+        if getattr(self, 'is_dask', False):
+            return  # files may be shared with dask workers; leave them
+        d = getattr(self, '_dir', None)
+        if d is not None:
+            shutil.rmtree(d, ignore_errors=True)
+
+    def __iter__(self):
+        return iter(self._keys)
+
+    def __len__(self):
+        return len(self._keys)
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({list(self._keys)})"

ebt_backtest-0.1.0/ebt/features/__init__.py

@@ -0,0 +1 @@
+from ebt.features.core import *