backtest-bias 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- backtest_bias-0.1.0/LICENSE +21 -0
- backtest_bias-0.1.0/PKG-INFO +92 -0
- backtest_bias-0.1.0/README.md +72 -0
- backtest_bias-0.1.0/backtest_bias/__init__.py +17 -0
- backtest_bias-0.1.0/backtest_bias/core.py +157 -0
- backtest_bias-0.1.0/backtest_bias.egg-info/PKG-INFO +92 -0
- backtest_bias-0.1.0/backtest_bias.egg-info/SOURCES.txt +11 -0
- backtest_bias-0.1.0/backtest_bias.egg-info/dependency_links.txt +1 -0
- backtest_bias-0.1.0/backtest_bias.egg-info/requires.txt +2 -0
- backtest_bias-0.1.0/backtest_bias.egg-info/top_level.txt +1 -0
- backtest_bias-0.1.0/pyproject.toml +27 -0
- backtest_bias-0.1.0/setup.cfg +4 -0
- backtest_bias-0.1.0/tests/test_core.py +88 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ayan Jain
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: backtest-bias
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Checks whether your backtest data is lying to you: survivorship, dead-name coverage, integrity gates for financial price panels.
|
|
5
|
+
Author: Ayan Jain
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Finance-broski/backtest-bias
|
|
8
|
+
Keywords: backtesting,survivorship-bias,quant,data-quality,point-in-time
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Office/Business :: Financial :: Investment
|
|
14
|
+
Requires-Python: >=3.9
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: pandas>=1.5
|
|
18
|
+
Requires-Dist: numpy>=1.23
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# backtest-bias
|
|
22
|
+
|
|
23
|
+
**Checks whether your backtest data is lying to you.**
|
|
24
|
+
|
|
25
|
+
Most backtests don't fail loudly. They flatter you quietly, because the data underneath them is
|
|
26
|
+
missing the stocks that died. This library tests your price panel for that — in one line — and
|
|
27
|
+
tells you roughly what it costs when it finds it.
|
|
28
|
+
|
|
29
|
+
## The measured numbers this library is built on
|
|
30
|
+
|
|
31
|
+
These are not estimates. I measured them on real Indian market data and published the write-ups:
|
|
32
|
+
|
|
33
|
+
- **23%** of the top-500 Indian stocks (as of 2015) are invisible to yfinance today — delisted,
|
|
34
|
+
merged, suspended. Any backtest built on it runs on survivors only.
|
|
35
|
+
- Survivor-only universes inflated equal-weight returns by **+0.8 to +2.5 pp/yr** depending on
|
|
36
|
+
universe vintage — same market, same method, 3x difference. Anyone quoting one number is guessing.
|
|
37
|
+
- On the most widely used Kaggle NSE dataset, index-membership look-ahead added **+10%** terminal
|
|
38
|
+
wealth cap-weighted and **+43%** equal-weighted over 2010-2021. The bias depends on construction.
|
|
39
|
+
- Comparable mid/large-cap universes lose **10-30%** of their names over multi-year windows.
|
|
40
|
+
If your panel lost zero, your panel is the problem.
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
|
|
44
|
+
```
|
|
45
|
+
pip install backtest-bias
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## 30 seconds to a verdict
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
import pandas as pd
|
|
52
|
+
from backtest_bias import check_survivorship
|
|
53
|
+
|
|
54
|
+
prices = pd.read_csv("my_panel.csv") # wide (date x symbols) or long (date/symbol/close)
|
|
55
|
+
report = check_survivorship(prices)
|
|
56
|
+
print(report.summary())
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
survivorship check: 412 symbols over 9.2y, 0 died in-window (0%)
|
|
61
|
+
verdict: SEVERE - 412 names over 9.2y with zero deaths is the survivor-only signature;
|
|
62
|
+
comparable universes lose 10%-30% of names over such windows
|
|
63
|
+
expect EW returns inflated roughly +0.8-2.5 pp/yr vs an honest universe (measured,
|
|
64
|
+
vintage-dependent; see backtest_bias.REFERENCES)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## What v0.1 ships
|
|
68
|
+
|
|
69
|
+
| function | what it answers |
|
|
70
|
+
|---|---|
|
|
71
|
+
| `check_survivorship(prices)` | does my universe contain the stocks that died, or only the winners? Full report with severity and a measured bias estimate |
|
|
72
|
+
| `dead_name_ratio(prices)` | one number: what fraction of my names end before the panel does. `0.0` = pure survivor panel |
|
|
73
|
+
| `assert_integrity(prices)` | CI gate: raise if the panel smells survivor-only, so a silent re-download of bad data fails your pipeline instead of flattering your backtest |
|
|
74
|
+
|
|
75
|
+
Input handling is forgiving: wide panels, long frames, sniffed column names, NaN-padded
|
|
76
|
+
histories. Anything the library cannot judge honestly, it raises instead of guessing.
|
|
77
|
+
|
|
78
|
+
## Roadmap
|
|
79
|
+
|
|
80
|
+
- **v0.2** — look-ahead / point-in-time violations: fundamentals dated by period instead of
|
|
81
|
+
announcement, index membership applied backwards, same-bar signal fills
|
|
82
|
+
- **v0.3** — rename-continuity and corporate-action gap detection
|
|
83
|
+
|
|
84
|
+
## Who
|
|
85
|
+
|
|
86
|
+
I'm [Ayan Jain](https://www.linkedin.com/in/ayanjain259). I build point-in-time Indian
|
|
87
|
+
equity data and audit backtests and datasets for bias — the measured numbers above come from
|
|
88
|
+
those audits. If you want this class of check run on your own backtest by a person instead of a
|
|
89
|
+
library, that's my [Bias Check](https://forms.gle/sAvosfHnitCBm9FD7): fixed price, 48h, written
|
|
90
|
+
verdict.
|
|
91
|
+
|
|
92
|
+
MIT licensed. Issues and war stories welcome — especially datasets that fooled you.
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# backtest-bias
|
|
2
|
+
|
|
3
|
+
**Checks whether your backtest data is lying to you.**
|
|
4
|
+
|
|
5
|
+
Most backtests don't fail loudly. They flatter you quietly, because the data underneath them is
|
|
6
|
+
missing the stocks that died. This library tests your price panel for that — in one line — and
|
|
7
|
+
tells you roughly what it costs when it finds it.
|
|
8
|
+
|
|
9
|
+
## The measured numbers this library is built on
|
|
10
|
+
|
|
11
|
+
These are not estimates. I measured them on real Indian market data and published the write-ups:
|
|
12
|
+
|
|
13
|
+
- **23%** of the top-500 Indian stocks (as of 2015) are invisible to yfinance today — delisted,
|
|
14
|
+
merged, suspended. Any backtest built on it runs on survivors only.
|
|
15
|
+
- Survivor-only universes inflated equal-weight returns by **+0.8 to +2.5 pp/yr** depending on
|
|
16
|
+
universe vintage — same market, same method, 3x difference. Anyone quoting one number is guessing.
|
|
17
|
+
- On the most widely used Kaggle NSE dataset, index-membership look-ahead added **+10%** terminal
|
|
18
|
+
wealth cap-weighted and **+43%** equal-weighted over 2010-2021. The bias depends on construction.
|
|
19
|
+
- Comparable mid/large-cap universes lose **10-30%** of their names over multi-year windows.
|
|
20
|
+
If your panel lost zero, your panel is the problem.
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
pip install backtest-bias
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## 30 seconds to a verdict
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
import pandas as pd
|
|
32
|
+
from backtest_bias import check_survivorship
|
|
33
|
+
|
|
34
|
+
prices = pd.read_csv("my_panel.csv") # wide (date x symbols) or long (date/symbol/close)
|
|
35
|
+
report = check_survivorship(prices)
|
|
36
|
+
print(report.summary())
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
survivorship check: 412 symbols over 9.2y, 0 died in-window (0%)
|
|
41
|
+
verdict: SEVERE - 412 names over 9.2y with zero deaths is the survivor-only signature;
|
|
42
|
+
comparable universes lose 10%-30% of names over such windows
|
|
43
|
+
expect EW returns inflated roughly +0.8-2.5 pp/yr vs an honest universe (measured,
|
|
44
|
+
vintage-dependent; see backtest_bias.REFERENCES)
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## What v0.1 ships
|
|
48
|
+
|
|
49
|
+
| function | what it answers |
|
|
50
|
+
|---|---|
|
|
51
|
+
| `check_survivorship(prices)` | does my universe contain the stocks that died, or only the winners? Full report with severity and a measured bias estimate |
|
|
52
|
+
| `dead_name_ratio(prices)` | one number: what fraction of my names end before the panel does. `0.0` = pure survivor panel |
|
|
53
|
+
| `assert_integrity(prices)` | CI gate: raise if the panel smells survivor-only, so a silent re-download of bad data fails your pipeline instead of flattering your backtest |
|
|
54
|
+
|
|
55
|
+
Input handling is forgiving: wide panels, long frames, sniffed column names, NaN-padded
|
|
56
|
+
histories. Anything the library cannot judge honestly, it raises instead of guessing.
|
|
57
|
+
|
|
58
|
+
## Roadmap
|
|
59
|
+
|
|
60
|
+
- **v0.2** — look-ahead / point-in-time violations: fundamentals dated by period instead of
|
|
61
|
+
announcement, index membership applied backwards, same-bar signal fills
|
|
62
|
+
- **v0.3** — rename-continuity and corporate-action gap detection
|
|
63
|
+
|
|
64
|
+
## Who
|
|
65
|
+
|
|
66
|
+
I'm [Ayan Jain](https://www.linkedin.com/in/ayanjain259). I build point-in-time Indian
|
|
67
|
+
equity data and audit backtests and datasets for bias — the measured numbers above come from
|
|
68
|
+
those audits. If you want this class of check run on your own backtest by a person instead of a
|
|
69
|
+
library, that's my [Bias Check](https://forms.gle/sAvosfHnitCBm9FD7): fixed price, 48h, written
|
|
70
|
+
verdict.
|
|
71
|
+
|
|
72
|
+
MIT licensed. Issues and war stories welcome — especially datasets that fooled you.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""backtest-bias: checks whether your backtest data is lying to you.
|
|
2
|
+
|
|
3
|
+
v0.1 ships one killer check done properly - survivorship - plus a CI gate.
|
|
4
|
+
Roadmap: look-ahead/PIT violations (v0.2), rename-continuity & corporate-action gaps (v0.3).
|
|
5
|
+
"""
|
|
6
|
+
from .core import (
|
|
7
|
+
REFERENCES,
|
|
8
|
+
SurvivorshipReport,
|
|
9
|
+
assert_integrity,
|
|
10
|
+
check_survivorship,
|
|
11
|
+
dead_name_ratio,
|
|
12
|
+
to_wide,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
__version__ = "0.1.0"
|
|
16
|
+
__all__ = ["check_survivorship", "dead_name_ratio", "assert_integrity", "to_wide",
|
|
17
|
+
"SurvivorshipReport", "REFERENCES", "__version__"]
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""backtest-bias core: survivorship checks for financial price panels.
|
|
2
|
+
|
|
3
|
+
Design rules:
|
|
4
|
+
- The library checks YOUR data. It ships no market data of its own.
|
|
5
|
+
- The packaged benchmarks are published, measured numbers (sources in REFERENCES) so a finding
|
|
6
|
+
can be quantified, not just flagged.
|
|
7
|
+
- Anything the library cannot judge honestly, it says so instead of guessing.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import math
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
import pandas as pd
|
|
16
|
+
|
|
17
|
+
# Published, measured benchmarks that motivated this library (see README for the write-ups).
|
|
18
|
+
# These quantify what a survivor-only panel typically does to a backtest.
|
|
19
|
+
REFERENCES = {
|
|
20
|
+
"IN": {
|
|
21
|
+
"death_rate_range": (0.10, 0.30), # fraction of a mid/large-cap universe that stops
|
|
22
|
+
# trading over a multi-year window (measured 2013 &
|
|
23
|
+
# 2015 vintages, Indian top-500)
|
|
24
|
+
"bias_pp_per_year": (0.8, 2.5), # measured EW return inflation from survivor-only
|
|
25
|
+
# universes, vintage-dependent
|
|
26
|
+
"construction_dependence": "+10% terminal wealth (cap-weighted) to +43% (equal-weight) "
|
|
27
|
+
"measured on one widely used dataset, 2010-2021",
|
|
28
|
+
},
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def to_wide(prices: pd.DataFrame,
|
|
33
|
+
date_col: str = None, symbol_col: str = None, value_col: str = None) -> pd.DataFrame:
|
|
34
|
+
"""Accept a wide panel (DatetimeIndex x symbols) or a long frame (date/symbol/price columns)
|
|
35
|
+
and return wide. Column names are sniffed case-insensitively when not given."""
|
|
36
|
+
if isinstance(prices.index, pd.DatetimeIndex) and prices.shape[1] > 1 and \
|
|
37
|
+
not any(str(c).lower() in ("symbol", "ticker") for c in prices.columns):
|
|
38
|
+
return prices.sort_index()
|
|
39
|
+
df = prices.copy()
|
|
40
|
+
cols = {str(c).lower(): c for c in df.columns}
|
|
41
|
+
date_col = date_col or next((cols[k] for k in ("date", "dt", "day", "time") if k in cols), None)
|
|
42
|
+
symbol_col = symbol_col or next((cols[k] for k in ("symbol", "ticker", "scrip", "name") if k in cols), None)
|
|
43
|
+
value_col = value_col or next((cols[k] for k in ("close", "adj_close", "price", "px", "nav") if k in cols), None)
|
|
44
|
+
if not (date_col and symbol_col and value_col):
|
|
45
|
+
raise ValueError("could not sniff long-format columns; pass date_col/symbol_col/value_col")
|
|
46
|
+
df[date_col] = pd.to_datetime(df[date_col])
|
|
47
|
+
return df.pivot_table(index=date_col, columns=symbol_col, values=value_col, aggfunc="last").sort_index()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def dead_name_ratio(prices: pd.DataFrame, gap_days: int = 45, min_obs: int = 60,
|
|
51
|
+
as_of=None, **to_wide_kw) -> float:
|
|
52
|
+
"""Fraction of symbols whose data ENDS well before the panel does (i.e. names that died and
|
|
53
|
+
were kept — the thing survivor-only panels don't have). 0.0 = your panel only contains
|
|
54
|
+
the living."""
|
|
55
|
+
w = to_wide(prices, **to_wide_kw)
|
|
56
|
+
end = pd.Timestamp(as_of) if as_of is not None else w.index.max()
|
|
57
|
+
counted = dead = 0
|
|
58
|
+
for c in w.columns:
|
|
59
|
+
s = w[c].dropna()
|
|
60
|
+
if len(s) < min_obs:
|
|
61
|
+
continue
|
|
62
|
+
counted += 1
|
|
63
|
+
if s.index.max() < end - pd.Timedelta(days=gap_days):
|
|
64
|
+
dead += 1
|
|
65
|
+
if counted == 0:
|
|
66
|
+
raise ValueError(f"no symbol has >= {min_obs} observations; nothing to judge")
|
|
67
|
+
return dead / counted
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass
|
|
71
|
+
class SurvivorshipReport:
|
|
72
|
+
n_symbols: int
|
|
73
|
+
n_dead_in_window: int
|
|
74
|
+
dead_ratio: float
|
|
75
|
+
window_years: float
|
|
76
|
+
survivor_only_suspected: bool
|
|
77
|
+
severity: str # "clean" | "warn" | "severe"
|
|
78
|
+
detail: str
|
|
79
|
+
market: str = "IN"
|
|
80
|
+
dead_symbols: list = field(default_factory=list)
|
|
81
|
+
|
|
82
|
+
def estimated_bias_pp_per_year(self):
|
|
83
|
+
"""Published measured band for what a survivor-only panel adds to EW returns."""
|
|
84
|
+
ref = REFERENCES.get(self.market)
|
|
85
|
+
return ref["bias_pp_per_year"] if (ref and self.survivor_only_suspected) else (0.0, 0.0)
|
|
86
|
+
|
|
87
|
+
def summary(self) -> str:
|
|
88
|
+
lines = [f"survivorship check: {self.n_symbols} symbols over {self.window_years:.1f}y, "
|
|
89
|
+
f"{self.n_dead_in_window} died in-window ({self.dead_ratio:.0%})",
|
|
90
|
+
f"verdict: {self.severity.upper()} - {self.detail}"]
|
|
91
|
+
lo, hi = self.estimated_bias_pp_per_year()
|
|
92
|
+
if hi > 0:
|
|
93
|
+
lines.append(f"expect EW returns inflated roughly +{lo}-{hi} pp/yr vs an honest "
|
|
94
|
+
f"universe (measured, vintage-dependent; see backtest_bias.REFERENCES)")
|
|
95
|
+
return "\n".join(lines)
|
|
96
|
+
|
|
97
|
+
def __repr__(self):
|
|
98
|
+
return f"<SurvivorshipReport {self.severity}: {self.n_dead_in_window}/{self.n_symbols} dead, {self.window_years:.1f}y>"
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def check_survivorship(prices: pd.DataFrame, gap_days: int = 45, min_obs: int = 60,
|
|
102
|
+
min_names: int = 20, min_years: float = 3.0, market: str = "IN",
|
|
103
|
+
**to_wide_kw) -> SurvivorshipReport:
|
|
104
|
+
"""THE core question: does your universe contain names that died inside your test window?
|
|
105
|
+
|
|
106
|
+
A multi-year panel of 20+ names with ZERO deaths carries the survivor-only signature:
|
|
107
|
+
every stock in it is a stock that made it to the end, and your backtest literally cannot
|
|
108
|
+
buy the ones that didn't. Reference universes lose 10-30% of names over such windows."""
|
|
109
|
+
w = to_wide(prices, **to_wide_kw)
|
|
110
|
+
end = w.index.max()
|
|
111
|
+
years = max((end - w.index.min()).days / 365.25, 0.01)
|
|
112
|
+
firsts, lasts, dead = {}, {}, []
|
|
113
|
+
for c in w.columns:
|
|
114
|
+
s = w[c].dropna()
|
|
115
|
+
if len(s) < min_obs:
|
|
116
|
+
continue
|
|
117
|
+
firsts[c], lasts[c] = s.index.min(), s.index.max()
|
|
118
|
+
if lasts[c] < end - pd.Timedelta(days=gap_days):
|
|
119
|
+
dead.append(c)
|
|
120
|
+
n = len(firsts)
|
|
121
|
+
if n == 0:
|
|
122
|
+
raise ValueError(f"no symbol has >= {min_obs} observations; nothing to judge")
|
|
123
|
+
ratio = len(dead) / n
|
|
124
|
+
|
|
125
|
+
if ratio == 0 and n >= min_names and years >= min_years:
|
|
126
|
+
sev, suspect = "severe", True
|
|
127
|
+
detail = (f"{n} names over {years:.1f}y with zero deaths is the survivor-only signature; "
|
|
128
|
+
f"comparable universes lose "
|
|
129
|
+
f"{REFERENCES[market]['death_rate_range'][0]:.0%}-{REFERENCES[market]['death_rate_range'][1]:.0%} "
|
|
130
|
+
f"of names over such windows" if market in REFERENCES else
|
|
131
|
+
f"{n} names over {years:.1f}y with zero deaths is the survivor-only signature")
|
|
132
|
+
elif ratio == 0 and n >= min_names:
|
|
133
|
+
sev, suspect = "warn", False
|
|
134
|
+
detail = f"zero deaths but only {years:.1f}y of window - re-run on your full research span"
|
|
135
|
+
elif ratio < 0.05 and years >= 5:
|
|
136
|
+
sev, suspect = "warn", False
|
|
137
|
+
detail = f"suspiciously few deaths ({ratio:.1%}) for a {years:.1f}y window - check how the universe was built"
|
|
138
|
+
else:
|
|
139
|
+
sev, suspect = "clean", False
|
|
140
|
+
detail = "dead names present in the panel; survivor-only construction not indicated"
|
|
141
|
+
|
|
142
|
+
return SurvivorshipReport(n_symbols=n, n_dead_in_window=len(dead), dead_ratio=ratio,
|
|
143
|
+
window_years=years, survivor_only_suspected=suspect,
|
|
144
|
+
severity=sev, detail=detail, market=market,
|
|
145
|
+
dead_symbols=sorted(dead)[:50])
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def assert_integrity(prices: pd.DataFrame, min_dead_ratio: float = 0.05,
|
|
149
|
+
gap_days: int = 45, min_obs: int = 60, **to_wide_kw) -> None:
|
|
150
|
+
"""CI gate: raise if the panel smells survivor-only. Wire it into your data pipeline so a
|
|
151
|
+
silent re-download of survivor-only data fails the build instead of flattering the backtest."""
|
|
152
|
+
r = dead_name_ratio(prices, gap_days=gap_days, min_obs=min_obs, **to_wide_kw)
|
|
153
|
+
if r < min_dead_ratio:
|
|
154
|
+
raise AssertionError(
|
|
155
|
+
f"survivorship gate failed: dead-name ratio {r:.1%} < required {min_dead_ratio:.0%}. "
|
|
156
|
+
f"This panel likely only contains stocks that survived to the end; backtests on it "
|
|
157
|
+
f"will overstate returns (measured +0.8-2.5 pp/yr EW on Indian data).")
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: backtest-bias
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Checks whether your backtest data is lying to you: survivorship, dead-name coverage, integrity gates for financial price panels.
|
|
5
|
+
Author: Ayan Jain
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Finance-broski/backtest-bias
|
|
8
|
+
Keywords: backtesting,survivorship-bias,quant,data-quality,point-in-time
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Office/Business :: Financial :: Investment
|
|
14
|
+
Requires-Python: >=3.9
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Requires-Dist: pandas>=1.5
|
|
18
|
+
Requires-Dist: numpy>=1.23
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# backtest-bias
|
|
22
|
+
|
|
23
|
+
**Checks whether your backtest data is lying to you.**
|
|
24
|
+
|
|
25
|
+
Most backtests don't fail loudly. They flatter you quietly, because the data underneath them is
|
|
26
|
+
missing the stocks that died. This library tests your price panel for that — in one line — and
|
|
27
|
+
tells you roughly what it costs when it finds it.
|
|
28
|
+
|
|
29
|
+
## The measured numbers this library is built on
|
|
30
|
+
|
|
31
|
+
These are not estimates. I measured them on real Indian market data and published the write-ups:
|
|
32
|
+
|
|
33
|
+
- **23%** of the top-500 Indian stocks (as of 2015) are invisible to yfinance today — delisted,
|
|
34
|
+
merged, suspended. Any backtest built on it runs on survivors only.
|
|
35
|
+
- Survivor-only universes inflated equal-weight returns by **+0.8 to +2.5 pp/yr** depending on
|
|
36
|
+
universe vintage — same market, same method, 3x difference. Anyone quoting one number is guessing.
|
|
37
|
+
- On the most widely used Kaggle NSE dataset, index-membership look-ahead added **+10%** terminal
|
|
38
|
+
wealth cap-weighted and **+43%** equal-weighted over 2010-2021. The bias depends on construction.
|
|
39
|
+
- Comparable mid/large-cap universes lose **10-30%** of their names over multi-year windows.
|
|
40
|
+
If your panel lost zero, your panel is the problem.
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
|
|
44
|
+
```
|
|
45
|
+
pip install backtest-bias
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## 30 seconds to a verdict
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
import pandas as pd
|
|
52
|
+
from backtest_bias import check_survivorship
|
|
53
|
+
|
|
54
|
+
prices = pd.read_csv("my_panel.csv") # wide (date x symbols) or long (date/symbol/close)
|
|
55
|
+
report = check_survivorship(prices)
|
|
56
|
+
print(report.summary())
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
survivorship check: 412 symbols over 9.2y, 0 died in-window (0%)
|
|
61
|
+
verdict: SEVERE - 412 names over 9.2y with zero deaths is the survivor-only signature;
|
|
62
|
+
comparable universes lose 10%-30% of names over such windows
|
|
63
|
+
expect EW returns inflated roughly +0.8-2.5 pp/yr vs an honest universe (measured,
|
|
64
|
+
vintage-dependent; see backtest_bias.REFERENCES)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## What v0.1 ships
|
|
68
|
+
|
|
69
|
+
| function | what it answers |
|
|
70
|
+
|---|---|
|
|
71
|
+
| `check_survivorship(prices)` | does my universe contain the stocks that died, or only the winners? Full report with severity and a measured bias estimate |
|
|
72
|
+
| `dead_name_ratio(prices)` | one number: what fraction of my names end before the panel does. `0.0` = pure survivor panel |
|
|
73
|
+
| `assert_integrity(prices)` | CI gate: raise if the panel smells survivor-only, so a silent re-download of bad data fails your pipeline instead of flattering your backtest |
|
|
74
|
+
|
|
75
|
+
Input handling is forgiving: wide panels, long frames, sniffed column names, NaN-padded
|
|
76
|
+
histories. Anything the library cannot judge honestly, it raises instead of guessing.
|
|
77
|
+
|
|
78
|
+
## Roadmap
|
|
79
|
+
|
|
80
|
+
- **v0.2** — look-ahead / point-in-time violations: fundamentals dated by period instead of
|
|
81
|
+
announcement, index membership applied backwards, same-bar signal fills
|
|
82
|
+
- **v0.3** — rename-continuity and corporate-action gap detection
|
|
83
|
+
|
|
84
|
+
## Who
|
|
85
|
+
|
|
86
|
+
I'm [Ayan Jain](https://www.linkedin.com/in/ayanjain259). I build point-in-time Indian
|
|
87
|
+
equity data and audit backtests and datasets for bias — the measured numbers above come from
|
|
88
|
+
those audits. If you want this class of check run on your own backtest by a person instead of a
|
|
89
|
+
library, that's my [Bias Check](https://forms.gle/sAvosfHnitCBm9FD7): fixed price, 48h, written
|
|
90
|
+
verdict.
|
|
91
|
+
|
|
92
|
+
MIT licensed. Issues and war stories welcome — especially datasets that fooled you.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
backtest_bias/__init__.py
|
|
5
|
+
backtest_bias/core.py
|
|
6
|
+
backtest_bias.egg-info/PKG-INFO
|
|
7
|
+
backtest_bias.egg-info/SOURCES.txt
|
|
8
|
+
backtest_bias.egg-info/dependency_links.txt
|
|
9
|
+
backtest_bias.egg-info/requires.txt
|
|
10
|
+
backtest_bias.egg-info/top_level.txt
|
|
11
|
+
tests/test_core.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
backtest_bias
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "backtest-bias"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Checks whether your backtest data is lying to you: survivorship, dead-name coverage, integrity gates for financial price panels."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
authors = [{ name = "Ayan Jain" }]
|
|
12
|
+
requires-python = ">=3.9"
|
|
13
|
+
dependencies = ["pandas>=1.5", "numpy>=1.23"]
|
|
14
|
+
keywords = ["backtesting", "survivorship-bias", "quant", "data-quality", "point-in-time"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Financial and Insurance Industry",
|
|
18
|
+
"License :: OSI Approved :: MIT License",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Topic :: Office/Business :: Financial :: Investment",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.urls]
|
|
24
|
+
Homepage = "https://github.com/Finance-broski/backtest-bias"
|
|
25
|
+
|
|
26
|
+
[tool.setuptools.packages.find]
|
|
27
|
+
include = ["backtest_bias*"]
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""backtest-bias v0.1 regression: synthetic panels with known truth; every verdict must match.
|
|
2
|
+
Run: python -m pytest tests/ -q (or python tests/test_core.py)"""
|
|
3
|
+
import sys, os
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
9
|
+
from backtest_bias import (assert_integrity, check_survivorship, dead_name_ratio, to_wide)
|
|
10
|
+
|
|
11
|
+
RNG = np.random.default_rng(7)
|
|
12
|
+
DAYS = pd.bdate_range("2015-01-01", "2024-12-31")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _walk(n):
|
|
16
|
+
return 100 * np.exp(np.cumsum(RNG.normal(0.0003, 0.02, n)))
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def survivor_panel(n_names=30):
|
|
20
|
+
"""Everyone lives to the end: the survivor-only signature."""
|
|
21
|
+
return pd.DataFrame({f"S{i:03d}": _walk(len(DAYS)) for i in range(n_names)}, index=DAYS)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def honest_panel(n_names=30, death_frac=0.3):
|
|
25
|
+
"""30% of names die somewhere in the middle: what real history looks like."""
|
|
26
|
+
df = survivor_panel(n_names)
|
|
27
|
+
dead = list(df.columns[: int(n_names * death_frac)])
|
|
28
|
+
for i, c in enumerate(dead):
|
|
29
|
+
cut = DAYS[len(DAYS) // 3 + i * 40]
|
|
30
|
+
df.loc[df.index > cut, c] = np.nan
|
|
31
|
+
return df, dead
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_survivor_only_flagged_severe():
|
|
35
|
+
rep = check_survivorship(survivor_panel())
|
|
36
|
+
assert rep.severity == "severe" and rep.survivor_only_suspected
|
|
37
|
+
assert rep.n_dead_in_window == 0
|
|
38
|
+
lo, hi = rep.estimated_bias_pp_per_year()
|
|
39
|
+
assert hi == 2.5 and lo == 0.8
|
|
40
|
+
assert "survivor-only signature" in rep.summary()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_honest_panel_clean():
|
|
44
|
+
df, dead = honest_panel()
|
|
45
|
+
rep = check_survivorship(df)
|
|
46
|
+
assert rep.severity == "clean" and not rep.survivor_only_suspected
|
|
47
|
+
assert rep.n_dead_in_window == len(dead)
|
|
48
|
+
assert set(rep.dead_symbols) == set(dead)
|
|
49
|
+
assert rep.estimated_bias_pp_per_year() == (0.0, 0.0)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_dead_name_ratio_numbers():
|
|
53
|
+
df, dead = honest_panel(30, 0.3)
|
|
54
|
+
r = dead_name_ratio(df)
|
|
55
|
+
assert abs(r - len(dead) / 30) < 1e-9
|
|
56
|
+
assert dead_name_ratio(survivor_panel()) == 0.0
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_assert_integrity_gate():
|
|
60
|
+
with pytest.raises(AssertionError, match="survivorship gate failed"):
|
|
61
|
+
assert_integrity(survivor_panel())
|
|
62
|
+
df, _ = honest_panel()
|
|
63
|
+
assert_integrity(df) # should not raise
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_long_format_sniffing():
|
|
67
|
+
df, dead = honest_panel(25)
|
|
68
|
+
long = df.stack().rename("close").reset_index()
|
|
69
|
+
long.columns = ["Date", "Ticker", "Close"]
|
|
70
|
+
rep = check_survivorship(long)
|
|
71
|
+
assert rep.n_symbols == 25 and rep.n_dead_in_window == len(dead)
|
|
72
|
+
w = to_wide(long)
|
|
73
|
+
assert w.shape[1] == 25 and isinstance(w.index, pd.DatetimeIndex)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_short_window_only_warns():
|
|
77
|
+
short = survivor_panel().iloc[:300]
|
|
78
|
+
rep = check_survivorship(short)
|
|
79
|
+
assert rep.severity == "warn" and not rep.survivor_only_suspected
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
if __name__ == "__main__":
|
|
83
|
+
for fn in [test_survivor_only_flagged_severe, test_honest_panel_clean,
|
|
84
|
+
test_dead_name_ratio_numbers, test_assert_integrity_gate,
|
|
85
|
+
test_long_format_sniffing, test_short_window_only_warns]:
|
|
86
|
+
fn()
|
|
87
|
+
print(f" PASS {fn.__name__}")
|
|
88
|
+
print("backtest-bias v0.1: GREEN")
|