factor-qc 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- factor_qc-0.1.0/LICENSE +21 -0
- factor_qc-0.1.0/PKG-INFO +168 -0
- factor_qc-0.1.0/README.md +145 -0
- factor_qc-0.1.0/pyproject.toml +36 -0
- factor_qc-0.1.0/setup.cfg +4 -0
- factor_qc-0.1.0/src/factor_qc/__init__.py +34 -0
- factor_qc-0.1.0/src/factor_qc/__main__.py +4 -0
- factor_qc-0.1.0/src/factor_qc/cli.py +114 -0
- factor_qc-0.1.0/src/factor_qc/gate.py +230 -0
- factor_qc-0.1.0/src/factor_qc/stats.py +493 -0
- factor_qc-0.1.0/src/factor_qc.egg-info/PKG-INFO +168 -0
- factor_qc-0.1.0/src/factor_qc.egg-info/SOURCES.txt +17 -0
- factor_qc-0.1.0/src/factor_qc.egg-info/dependency_links.txt +1 -0
- factor_qc-0.1.0/src/factor_qc.egg-info/entry_points.txt +3 -0
- factor_qc-0.1.0/src/factor_qc.egg-info/requires.txt +1 -0
- factor_qc-0.1.0/src/factor_qc.egg-info/top_level.txt +1 -0
- factor_qc-0.1.0/tests/test_cli.py +74 -0
- factor_qc-0.1.0/tests/test_gate.py +57 -0
- factor_qc-0.1.0/tests/test_stats.py +102 -0
factor_qc-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Factor QC contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
factor_qc-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: factor-qc
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Fail-closed quality gate for backtests: DSR, PBO, Harvey-Liu haircut and MinTRL in one numpy-only engine, with P0/P1/P2 severity grading.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Keywords: backtest,overfitting,deflated-sharpe,pbo,mintrl,factor,qc,quant
|
|
7
|
+
Classifier: Development Status :: 3 - Alpha
|
|
8
|
+
Classifier: Environment :: Console
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Office/Business :: Financial :: Investment
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: numpy>=1.24
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# factor-qc
|
|
25
|
+
|
|
26
|
+
A **fail-closed quality gate** for backtests: one numpy-only engine covering
|
|
27
|
+
Deflated Sharpe Ratio, Probability of Backtest Overfitting (CSCV), the
|
|
28
|
+
Harvey-Liu multiple-testing haircut and Minimum Track Record Length —
|
|
29
|
+
graded P0/P1/P2, and **it refuses to judge a backtest that will not declare
|
|
30
|
+
how many configurations were tried**. Python 3.11+, one dependency
|
|
31
|
+
(`numpy`), Windows / Linux / macOS.
|
|
32
|
+
|
|
33
|
+
**Status:** v0.1 — alpha. The statistics are battle-tested inside a
|
|
34
|
+
production research pipeline and validated against published reference
|
|
35
|
+
values, but this standalone package is new: expect the CLI to shift before
|
|
36
|
+
v1.0.
|
|
37
|
+
|
|
38
|
+
## Why this exists
|
|
39
|
+
|
|
40
|
+
The standard story: you try 200 factor configurations, the best one shows a
|
|
41
|
+
Sharpe of 1.65, you feel great. The honest story: with 200 trials of pure
|
|
42
|
+
noise, *someone* is going to show a Sharpe of 1.65 — the expected maximum of
|
|
43
|
+
200 zero-true-SR trials — and it will not be your skill, it will be your
|
|
44
|
+
selection bias.
|
|
45
|
+
|
|
46
|
+
Most backtest tooling computes statistics and prints reports. `factor-qc`
|
|
47
|
+
is a **gate**: it decides, with graded severity, whether a candidate may
|
|
48
|
+
pass — and its default answer is *no*:
|
|
49
|
+
|
|
50
|
+
- **P0 (fatal)** — DSR below threshold, PBO above threshold, haircut Sharpe
|
|
51
|
+
below floor, MinTRL longer than the sample → the candidate must not pass.
|
|
52
|
+
- **P1 (warning)** — weak PSR vs zero, short sample, aggressive trial count
|
|
53
|
+
→ proceed with eyes open.
|
|
54
|
+
- **P2 (info)** — non-normal moments, tiny trial count → recorded, no action.
|
|
55
|
+
|
|
56
|
+
## Philosophy
|
|
57
|
+
|
|
58
|
+
**Honesty is the default; the gate is fail-closed.**
|
|
59
|
+
|
|
60
|
+
The one non-negotiable input is `n_trials`: the honest count of
|
|
61
|
+
configurations you tried. Without it there is no deflation benchmark
|
|
62
|
+
([Bailey & López de Prado 2014](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2460551)),
|
|
63
|
+
no haircut ([Harvey, Liu & Zhu 2016, RFS](https://doi.org/10.1093/rfs/hhv059)),
|
|
64
|
+
no track-record floor
|
|
65
|
+
([Bailey & López de Prado 2018, JPM](https://ideas.repec.org/a/rsk/journl/0journalpm-v44n5.html))
|
|
66
|
+
and no overfitting probability
|
|
67
|
+
([Bailey, Borwein, López de Prado & Zhu 2017, JCF](https://escholarship.org/uc/item/4w1110bb)).
|
|
68
|
+
Refuse to declare, and the gate refuses to judge — that asymmetry is the
|
|
69
|
+
point. `qc check` exits non-zero on any P0 failure, so it drops into CI,
|
|
70
|
+
pre-commit hooks and research gates as a hard blocker, not a suggestion.
|
|
71
|
+
|
|
72
|
+
Two design commitments that keep it honest:
|
|
73
|
+
|
|
74
|
+
1. **numpy only, no scipy** — the standard-normal inverse CDF is Acklam's
|
|
75
|
+
rational approximation with one Newton refinement; every number in the
|
|
76
|
+
report is reproducible from the code in this repo, no hidden black box.
|
|
77
|
+
2. **PBO is optional but explicit** — without a trials matrix the gate says
|
|
78
|
+
PBO *was not computed*, and the DSR cross-trial variance degrades to a
|
|
79
|
+
conservative single-trial estimate. Absence of evidence is reported as
|
|
80
|
+
absence, never as evidence.
|
|
81
|
+
|
|
82
|
+
## Quick start
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
# install from PyPI (once published)
|
|
86
|
+
pip install factor-qc
|
|
87
|
+
|
|
88
|
+
# or run without installing anything:
|
|
89
|
+
# PYTHONPATH=src python -m factor_qc --help
|
|
90
|
+
|
|
91
|
+
python examples/demo.py # try it on reproducible synthetic cases
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Your own backtest:
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
# returns.json = JSON list of per-period returns of the selected candidate
|
|
98
|
+
# trials.json = JSON 2D matrix (T x N) of every configuration you tried
|
|
99
|
+
|
|
100
|
+
qc check --returns returns.json --trials trials.json --n-trials 200
|
|
101
|
+
# -> FAIL - P0 blocker(s): dsr: 0.63 vs 0.95; mintrl: 250.8 vs <= 1000; ...
|
|
102
|
+
|
|
103
|
+
qc check --returns returns.json --n-trials 5 --json # machine-readable
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Exit codes: `0` = no P0 failures (P1/P2 may still be failing), `1` = at
|
|
107
|
+
least one P0 failure (or missing `n_trials`), `2` = usage error. Wire it
|
|
108
|
+
into CI as a hard gate.
|
|
109
|
+
|
|
110
|
+
## Commands
|
|
111
|
+
|
|
112
|
+
| Command | What it does |
|
|
113
|
+
| --- | --- |
|
|
114
|
+
| `check` | Run the gate: DSR, PBO (when `--trials` given), haircut Sharpe, MinTRL as P0; PSR-vs-zero, sample length, trial aggression as P1; moments and trial count as P2. Human-readable or `--json` output |
|
|
115
|
+
| `version` | Print version |
|
|
116
|
+
|
|
117
|
+
Flags: `--returns` (required), `--trials` (optional), `--n-trials`
|
|
118
|
+
(required unless `require_declared_trials` is disabled in code),
|
|
119
|
+
`--periods-per-year` (default 252), `--n-blocks` (CSCV granularity, default
|
|
120
|
+
16).
|
|
121
|
+
|
|
122
|
+
## The checks
|
|
123
|
+
|
|
124
|
+
| Check | Severity | Method | Reference |
|
|
125
|
+
| --- | --- | --- | --- |
|
|
126
|
+
| `dsr` | P0 | Deflated Sharpe Ratio: P(SR > E[max SR of N trials]) under non-normal moments | [Bailey & López de Prado (2014), JPM 40(5)](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2460551) |
|
|
127
|
+
| `pbo` | P0 | Probability of Backtest Overfitting via Combinatorially-Symmetric Cross-Validation (12,870 splits at n_blocks=16) | [Bailey, Borwein, López de Prado & Zhu (2017), JCF](https://escholarship.org/uc/item/4w1110bb) |
|
|
128
|
+
| `haircut_sharpe` | P0 | Multiple-testing haircut of the Sharpe ratio (Bonferroni/Holm/BHY) | [Harvey & Liu (2015)](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2528780) |
|
|
129
|
+
| `mintrl` | P0 | Minimum Track Record Length: observations needed before SR is significant | [Bailey & López de Prado (2018), JPM 44(5)](https://ideas.repec.org/a/rsk/journl/0journalpm-v44n5.html) |
|
|
130
|
+
| `psr_vs_zero` | P1 | Probabilistic Sharpe vs zero | [Bailey & López de Prado (2012)](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2168747) |
|
|
131
|
+
| `sample_length` | P1 | ≥ 252 observations | — |
|
|
132
|
+
| `trial_aggression` | P1 | n_trials ≤ n_obs / 5 | — |
|
|
133
|
+
| `return_moments` | P2 | skew ≈ 0, kurtosis ≈ 3 | — |
|
|
134
|
+
| `trial_count` | P2 | n_trials ≥ 5 | — |
|
|
135
|
+
|
|
136
|
+
The P0 set mirrors the spirit of [Harvey, Liu & Zhu (2016),
|
|
137
|
+
"…and the Cross-Section of Expected Returns"](https://doi.org/10.1093/rfs/hhv059):
|
|
138
|
+
a factor must survive multiple-testing correction to earn the right to be
|
|
139
|
+
called a factor. The gate is the machine version of that editorial stance.
|
|
140
|
+
|
|
141
|
+
## Performance note
|
|
142
|
+
|
|
143
|
+
CSCV enumerates C(n_blocks, n_blocks/2) splits — 12,870 at the default 16.
|
|
144
|
+
On large trial matrices (T=1000, N=200) that takes minutes; use
|
|
145
|
+
`--n-blocks 8` (70 splits) or `10` (252 splits) for interactive speed at
|
|
146
|
+
slightly coarser granularity.
|
|
147
|
+
|
|
148
|
+
## Development
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
python -m pip install -e . pytest
|
|
152
|
+
python -m pytest
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
CI runs the full test suite on Ubuntu, Windows and macOS with Python 3.11
|
|
156
|
+
and 3.12. Issues are handled on weekends; pull requests are welcome.
|
|
157
|
+
|
|
158
|
+
## Related work
|
|
159
|
+
|
|
160
|
+
- [Bailey & López de Prado (2014), The Deflated Sharpe Ratio](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2460551)
|
|
161
|
+
- [Bailey, Borwein, López de Prado & Zhu (2017), The Probability of Backtest Overfitting](https://escholarship.org/uc/item/4w1110bb)
|
|
162
|
+
- [Harvey, Liu & Zhu (2016), …and the Cross-Section of Expected Returns (RFS)](https://doi.org/10.1093/rfs/hhv059)
|
|
163
|
+
- [Harvey & Liu (2021), Lucky Factors (JFE)](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2528780)
|
|
164
|
+
- [Mobarekeh & López de Prado (2024), Backtest Overfitting in the Machine Learning Era (SSRN 4778909)](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=4778909) — why OOS methods still need honest trial accounting
|
|
165
|
+
|
|
166
|
+
## License
|
|
167
|
+
|
|
168
|
+
MIT
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# factor-qc
|
|
2
|
+
|
|
3
|
+
A **fail-closed quality gate** for backtests: one numpy-only engine covering
|
|
4
|
+
Deflated Sharpe Ratio, Probability of Backtest Overfitting (CSCV), the
|
|
5
|
+
Harvey-Liu multiple-testing haircut and Minimum Track Record Length —
|
|
6
|
+
graded P0/P1/P2, and **it refuses to judge a backtest that will not declare
|
|
7
|
+
how many configurations were tried**. Python 3.11+, one dependency
|
|
8
|
+
(`numpy`), Windows / Linux / macOS.
|
|
9
|
+
|
|
10
|
+
**Status:** v0.1 — alpha. The statistics are battle-tested inside a
|
|
11
|
+
production research pipeline and validated against published reference
|
|
12
|
+
values, but this standalone package is new: expect the CLI to shift before
|
|
13
|
+
v1.0.
|
|
14
|
+
|
|
15
|
+
## Why this exists
|
|
16
|
+
|
|
17
|
+
The standard story: you try 200 factor configurations, the best one shows a
|
|
18
|
+
Sharpe of 1.65, you feel great. The honest story: with 200 trials of pure
|
|
19
|
+
noise, *someone* is going to show a Sharpe of 1.65 — the expected maximum of
|
|
20
|
+
200 zero-true-SR trials — and it will not be your skill, it will be your
|
|
21
|
+
selection bias.
|
|
22
|
+
|
|
23
|
+
Most backtest tooling computes statistics and prints reports. `factor-qc`
|
|
24
|
+
is a **gate**: it decides, with graded severity, whether a candidate may
|
|
25
|
+
pass — and its default answer is *no*:
|
|
26
|
+
|
|
27
|
+
- **P0 (fatal)** — DSR below threshold, PBO above threshold, haircut Sharpe
|
|
28
|
+
below floor, MinTRL longer than the sample → the candidate must not pass.
|
|
29
|
+
- **P1 (warning)** — weak PSR vs zero, short sample, aggressive trial count
|
|
30
|
+
→ proceed with eyes open.
|
|
31
|
+
- **P2 (info)** — non-normal moments, tiny trial count → recorded, no action.
|
|
32
|
+
|
|
33
|
+
## Philosophy
|
|
34
|
+
|
|
35
|
+
**Honesty is the default; the gate is fail-closed.**
|
|
36
|
+
|
|
37
|
+
The one non-negotiable input is `n_trials`: the honest count of
|
|
38
|
+
configurations you tried. Without it there is no deflation benchmark
|
|
39
|
+
([Bailey & López de Prado 2014](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2460551)),
|
|
40
|
+
no haircut ([Harvey, Liu & Zhu 2016, RFS](https://doi.org/10.1093/rfs/hhv059)),
|
|
41
|
+
no track-record floor
|
|
42
|
+
([Bailey & López de Prado 2018, JPM](https://ideas.repec.org/a/rsk/journl/0journalpm-v44n5.html))
|
|
43
|
+
and no overfitting probability
|
|
44
|
+
([Bailey, Borwein, López de Prado & Zhu 2017, JCF](https://escholarship.org/uc/item/4w1110bb)).
|
|
45
|
+
Refuse to declare, and the gate refuses to judge — that asymmetry is the
|
|
46
|
+
point. `qc check` exits non-zero on any P0 failure, so it drops into CI,
|
|
47
|
+
pre-commit hooks and research gates as a hard blocker, not a suggestion.
|
|
48
|
+
|
|
49
|
+
Two design commitments that keep it honest:
|
|
50
|
+
|
|
51
|
+
1. **numpy only, no scipy** — the standard-normal inverse CDF is Acklam's
|
|
52
|
+
rational approximation with one Newton refinement; every number in the
|
|
53
|
+
report is reproducible from the code in this repo, no hidden black box.
|
|
54
|
+
2. **PBO is optional but explicit** — without a trials matrix the gate says
|
|
55
|
+
PBO *was not computed*, and the DSR cross-trial variance degrades to a
|
|
56
|
+
conservative single-trial estimate. Absence of evidence is reported as
|
|
57
|
+
absence, never as evidence.
|
|
58
|
+
|
|
59
|
+
## Quick start
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
# install from PyPI (once published)
|
|
63
|
+
pip install factor-qc
|
|
64
|
+
|
|
65
|
+
# or run without installing anything:
|
|
66
|
+
# PYTHONPATH=src python -m factor_qc --help
|
|
67
|
+
|
|
68
|
+
python examples/demo.py # try it on reproducible synthetic cases
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Your own backtest:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
# returns.json = JSON list of per-period returns of the selected candidate
|
|
75
|
+
# trials.json = JSON 2D matrix (T x N) of every configuration you tried
|
|
76
|
+
|
|
77
|
+
qc check --returns returns.json --trials trials.json --n-trials 200
|
|
78
|
+
# -> FAIL - P0 blocker(s): dsr: 0.63 vs 0.95; mintrl: 250.8 vs <= 1000; ...
|
|
79
|
+
|
|
80
|
+
qc check --returns returns.json --n-trials 5 --json # machine-readable
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Exit codes: `0` = no P0 failures (P1/P2 may still be failing), `1` = at
|
|
84
|
+
least one P0 failure (or missing `n_trials`), `2` = usage error. Wire it
|
|
85
|
+
into CI as a hard gate.
|
|
86
|
+
|
|
87
|
+
## Commands
|
|
88
|
+
|
|
89
|
+
| Command | What it does |
|
|
90
|
+
| --- | --- |
|
|
91
|
+
| `check` | Run the gate: DSR, PBO (when `--trials` given), haircut Sharpe, MinTRL as P0; PSR-vs-zero, sample length, trial aggression as P1; moments and trial count as P2. Human-readable or `--json` output |
|
|
92
|
+
| `version` | Print version |
|
|
93
|
+
|
|
94
|
+
Flags: `--returns` (required), `--trials` (optional), `--n-trials`
|
|
95
|
+
(required unless `require_declared_trials` is disabled in code),
|
|
96
|
+
`--periods-per-year` (default 252), `--n-blocks` (CSCV granularity, default
|
|
97
|
+
16).
|
|
98
|
+
|
|
99
|
+
## The checks
|
|
100
|
+
|
|
101
|
+
| Check | Severity | Method | Reference |
|
|
102
|
+
| --- | --- | --- | --- |
|
|
103
|
+
| `dsr` | P0 | Deflated Sharpe Ratio: P(SR > E[max SR of N trials]) under non-normal moments | [Bailey & López de Prado (2014), JPM 40(5)](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2460551) |
|
|
104
|
+
| `pbo` | P0 | Probability of Backtest Overfitting via Combinatorially-Symmetric Cross-Validation (12,870 splits at n_blocks=16) | [Bailey, Borwein, López de Prado & Zhu (2017), JCF](https://escholarship.org/uc/item/4w1110bb) |
|
|
105
|
+
| `haircut_sharpe` | P0 | Multiple-testing haircut of the Sharpe ratio (Bonferroni/Holm/BHY) | [Harvey & Liu (2015)](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2528780) |
|
|
106
|
+
| `mintrl` | P0 | Minimum Track Record Length: observations needed before SR is significant | [Bailey & López de Prado (2018), JPM 44(5)](https://ideas.repec.org/a/rsk/journl/0journalpm-v44n5.html) |
|
|
107
|
+
| `psr_vs_zero` | P1 | Probabilistic Sharpe vs zero | [Bailey & López de Prado (2012)](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2168747) |
|
|
108
|
+
| `sample_length` | P1 | ≥ 252 observations | — |
|
|
109
|
+
| `trial_aggression` | P1 | n_trials ≤ n_obs / 5 | — |
|
|
110
|
+
| `return_moments` | P2 | skew ≈ 0, kurtosis ≈ 3 | — |
|
|
111
|
+
| `trial_count` | P2 | n_trials ≥ 5 | — |
|
|
112
|
+
|
|
113
|
+
The P0 set mirrors the spirit of [Harvey, Liu & Zhu (2016),
|
|
114
|
+
"…and the Cross-Section of Expected Returns"](https://doi.org/10.1093/rfs/hhv059):
|
|
115
|
+
a factor must survive multiple-testing correction to earn the right to be
|
|
116
|
+
called a factor. The gate is the machine version of that editorial stance.
|
|
117
|
+
|
|
118
|
+
## Performance note
|
|
119
|
+
|
|
120
|
+
CSCV enumerates C(n_blocks, n_blocks/2) splits — 12,870 at the default 16.
|
|
121
|
+
On large trial matrices (T=1000, N=200) that takes minutes; use
|
|
122
|
+
`--n-blocks 8` (70 splits) or `10` (252 splits) for interactive speed at
|
|
123
|
+
slightly coarser granularity.
|
|
124
|
+
|
|
125
|
+
## Development
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
python -m pip install -e . pytest
|
|
129
|
+
python -m pytest
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
CI runs the full test suite on Ubuntu, Windows and macOS with Python 3.11
|
|
133
|
+
and 3.12. Issues are handled on weekends; pull requests are welcome.
|
|
134
|
+
|
|
135
|
+
## Related work
|
|
136
|
+
|
|
137
|
+
- [Bailey & López de Prado (2014), The Deflated Sharpe Ratio](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2460551)
|
|
138
|
+
- [Bailey, Borwein, López de Prado & Zhu (2017), The Probability of Backtest Overfitting](https://escholarship.org/uc/item/4w1110bb)
|
|
139
|
+
- [Harvey, Liu & Zhu (2016), …and the Cross-Section of Expected Returns (RFS)](https://doi.org/10.1093/rfs/hhv059)
|
|
140
|
+
- [Harvey & Liu (2021), Lucky Factors (JFE)](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2528780)
|
|
141
|
+
- [Mobarekeh & López de Prado (2024), Backtest Overfitting in the Machine Learning Era (SSRN 4778909)](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=4778909) — why OOS methods still need honest trial accounting
|
|
142
|
+
|
|
143
|
+
## License
|
|
144
|
+
|
|
145
|
+
MIT
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "factor-qc"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Fail-closed quality gate for backtests: DSR, PBO, Harvey-Liu haircut and MinTRL in one numpy-only engine, with P0/P1/P2 severity grading."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
dependencies = ["numpy>=1.24"]
|
|
13
|
+
keywords = ["backtest", "overfitting", "deflated-sharpe", "pbo", "mintrl", "factor", "qc", "quant"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Intended Audience :: Financial and Insurance Industry",
|
|
19
|
+
"Intended Audience :: Science/Research",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Topic :: Office/Business :: Financial :: Investment",
|
|
25
|
+
"Topic :: Scientific/Engineering",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.scripts]
|
|
29
|
+
qc = "factor_qc.cli:main"
|
|
30
|
+
factor-qc = "factor_qc.cli:main"
|
|
31
|
+
|
|
32
|
+
[tool.setuptools.packages.find]
|
|
33
|
+
where = ["src"]
|
|
34
|
+
|
|
35
|
+
[tool.pytest.ini_options]
|
|
36
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""factor-qc: fail-closed quality gate for backtests.
|
|
2
|
+
|
|
3
|
+
One numpy-only engine for the standard backtest-overfit statistics —
|
|
4
|
+
Deflated Sharpe Ratio, Probability of Backtest Overfitting (CSCV),
|
|
5
|
+
Harvey-Liu multiple-testing haircut, Minimum Track Record Length — wrapped
|
|
6
|
+
in a gate that refuses to judge a backtest that does not declare how many
|
|
7
|
+
configurations were tried.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from .gate import run_gate
|
|
11
|
+
from .stats import (
|
|
12
|
+
build_check_artifact,
|
|
13
|
+
build_overfit_report,
|
|
14
|
+
deflated_sharpe_ratio,
|
|
15
|
+
haircut_sharpe,
|
|
16
|
+
minimum_track_record_length,
|
|
17
|
+
probability_of_backtest_overfitting,
|
|
18
|
+
sharpe_ratio,
|
|
19
|
+
skew_kurt,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
__version__ = "0.1.0"
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"build_check_artifact",
|
|
26
|
+
"build_overfit_report",
|
|
27
|
+
"deflated_sharpe_ratio",
|
|
28
|
+
"haircut_sharpe",
|
|
29
|
+
"minimum_track_record_length",
|
|
30
|
+
"probability_of_backtest_overfitting",
|
|
31
|
+
"run_gate",
|
|
32
|
+
"sharpe_ratio",
|
|
33
|
+
"skew_kurt",
|
|
34
|
+
]
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Command-line interface for factor-qc.
|
|
2
|
+
|
|
3
|
+
Subcommands:
|
|
4
|
+
|
|
5
|
+
- ``check`` run the fail-closed gate on a returns series
|
|
6
|
+
- ``version`` print version
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import json
|
|
13
|
+
import sys
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
|
|
19
|
+
from . import __version__
|
|
20
|
+
from .gate import run_gate
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _load_returns(path: str) -> np.ndarray:
|
|
24
|
+
value = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
25
|
+
if not isinstance(value, list) or not all(isinstance(x, (int, float)) for x in value):
|
|
26
|
+
raise ValueError(f"returns must be a JSON list of numbers: {path}")
|
|
27
|
+
return np.asarray(value, dtype=float)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _load_trials(path: str) -> np.ndarray:
|
|
31
|
+
value = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
32
|
+
if not isinstance(value, list) or not value:
|
|
33
|
+
raise ValueError(f"trials must be a non-empty JSON list of lists: {path}")
|
|
34
|
+
rows = []
|
|
35
|
+
for row in value:
|
|
36
|
+
if not isinstance(row, list):
|
|
37
|
+
raise ValueError(f"trials rows must be lists: {path}")
|
|
38
|
+
rows.append([float(x) for x in row])
|
|
39
|
+
matrix = np.asarray(rows, dtype=float)
|
|
40
|
+
if matrix.ndim != 2:
|
|
41
|
+
raise ValueError(f"trials must be 2D (T x N): {path}")
|
|
42
|
+
return matrix
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
46
|
+
parser = argparse.ArgumentParser(
|
|
47
|
+
prog="qc",
|
|
48
|
+
description="Fail-closed quality gate for backtests.",
|
|
49
|
+
)
|
|
50
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
51
|
+
|
|
52
|
+
check = sub.add_parser("check", help="run the fail-closed gate")
|
|
53
|
+
check.add_argument("--returns", required=True, help="JSON list of per-period returns")
|
|
54
|
+
check.add_argument("--trials", default=None, help="JSON 2D matrix (T x N) of trial returns")
|
|
55
|
+
check.add_argument(
|
|
56
|
+
"--n-trials",
|
|
57
|
+
type=int,
|
|
58
|
+
default=None,
|
|
59
|
+
help="honest number of configurations tried (required, fail-closed)",
|
|
60
|
+
)
|
|
61
|
+
check.add_argument("--periods-per-year", type=int, default=252)
|
|
62
|
+
check.add_argument(
|
|
63
|
+
"--n-blocks",
|
|
64
|
+
type=int,
|
|
65
|
+
default=16,
|
|
66
|
+
help="CSCV blocks for PBO (default 16 = 12,870 splits; use 8 or 10 for speed)",
|
|
67
|
+
)
|
|
68
|
+
check.add_argument("--json", action="store_true", help="machine-readable output")
|
|
69
|
+
|
|
70
|
+
sub.add_parser("version", help="print version")
|
|
71
|
+
return parser
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def main(argv: list[str] | None = None) -> int:
|
|
75
|
+
parser = build_parser()
|
|
76
|
+
args = parser.parse_args(argv)
|
|
77
|
+
|
|
78
|
+
if args.command == "version":
|
|
79
|
+
print(__version__)
|
|
80
|
+
return 0
|
|
81
|
+
|
|
82
|
+
if args.command == "check":
|
|
83
|
+
returns = _load_returns(args.returns)
|
|
84
|
+
trials = _load_trials(args.trials) if args.trials else None
|
|
85
|
+
if args.n_trials is not None and args.n_trials < 1:
|
|
86
|
+
parser.error("--n-trials must be >= 1")
|
|
87
|
+
body: dict[str, Any] = run_gate(
|
|
88
|
+
returns,
|
|
89
|
+
args.n_trials,
|
|
90
|
+
trials_matrix=trials,
|
|
91
|
+
periods_per_year=args.periods_per_year,
|
|
92
|
+
n_blocks=args.n_blocks,
|
|
93
|
+
)
|
|
94
|
+
if args.json:
|
|
95
|
+
print(json.dumps(body, ensure_ascii=False, indent=2))
|
|
96
|
+
else:
|
|
97
|
+
print(body["verdict"])
|
|
98
|
+
if body["report"] is not None:
|
|
99
|
+
print(body["report_text"])
|
|
100
|
+
for check in body["checks"]:
|
|
101
|
+
marker = "PASS" if check["passed"] else "FAIL"
|
|
102
|
+
print(
|
|
103
|
+
f" [{marker}] {check['severity']} {check['check_id']}: "
|
|
104
|
+
f"{check['title']} (value={check['value']}, "
|
|
105
|
+
f"threshold={check['threshold']})"
|
|
106
|
+
)
|
|
107
|
+
return 0 if body["passed"] else 1
|
|
108
|
+
|
|
109
|
+
parser.error(f"unknown command: {args.command}")
|
|
110
|
+
return 2
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
if __name__ == "__main__":
|
|
114
|
+
sys.exit(main())
|