trialgate 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trialgate-0.1.0/LICENSE +21 -0
- trialgate-0.1.0/PKG-INFO +187 -0
- trialgate-0.1.0/README.md +158 -0
- trialgate-0.1.0/pyproject.toml +69 -0
- trialgate-0.1.0/setup.cfg +4 -0
- trialgate-0.1.0/src/trialgate/__init__.py +72 -0
- trialgate-0.1.0/src/trialgate/gates.py +99 -0
- trialgate-0.1.0/src/trialgate/holdout.py +154 -0
- trialgate-0.1.0/src/trialgate/py.typed +0 -0
- trialgate-0.1.0/src/trialgate/registry.py +174 -0
- trialgate-0.1.0/src/trialgate/validation.py +238 -0
- trialgate-0.1.0/src/trialgate.egg-info/PKG-INFO +187 -0
- trialgate-0.1.0/src/trialgate.egg-info/SOURCES.txt +18 -0
- trialgate-0.1.0/src/trialgate.egg-info/dependency_links.txt +1 -0
- trialgate-0.1.0/src/trialgate.egg-info/requires.txt +5 -0
- trialgate-0.1.0/src/trialgate.egg-info/top_level.txt +1 -0
- trialgate-0.1.0/tests/test_gates.py +54 -0
- trialgate-0.1.0/tests/test_holdout.py +69 -0
- trialgate-0.1.0/tests/test_registry.py +127 -0
- trialgate-0.1.0/tests/test_validation.py +107 -0
trialgate-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Tsai Chih-Chun (0Smallcat0)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
trialgate-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: trialgate
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Anti-overfitting validation gate for backtested trading strategies: append-only trial registry, CSCV/PBO, Deflated Sharpe Ratio, single-use holdout lock.
|
|
5
|
+
Author: Tsai Chih-Chun (0Smallcat0)
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Repository, https://github.com/0Smallcat0/trialgate
|
|
8
|
+
Project-URL: Issues, https://github.com/0Smallcat0/trialgate/issues
|
|
9
|
+
Project-URL: Reference deployment, https://github.com/0Smallcat0/crypto-quant-signal
|
|
10
|
+
Keywords: backtesting,overfitting,deflated-sharpe-ratio,probability-of-backtest-overfitting,cscv,quantitative-finance,validation,holdout
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Office/Business :: Financial :: Investment
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest; extra == "dev"
|
|
26
|
+
Requires-Dist: ruff; extra == "dev"
|
|
27
|
+
Requires-Dist: mypy; extra == "dev"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
30
|
+
# trialgate
|
|
31
|
+
|
|
32
|
+
[](https://github.com/0Smallcat0/trialgate/actions/workflows/ci.yml)
|
|
33
|
+
[](https://www.python.org/downloads/)
|
|
34
|
+
[](https://mypy.readthedocs.io/)
|
|
35
|
+
[](LICENSE)
|
|
36
|
+
|
|
37
|
+
An **anti-overfitting validation gate** for backtested trading strategies.
|
|
38
|
+
Zero dependencies, pure standard library, fully typed.
|
|
39
|
+
|
|
40
|
+
> A strategy earns belief by surviving verification, not by looking good
|
|
41
|
+
> in-sample. Optimizing four parameters on a **pure random walk** can produce
|
|
42
|
+
> an in-sample Sharpe of 1.27. A single backtest — however pretty — is not
|
|
43
|
+
> evidence.
|
|
44
|
+
|
|
45
|
+
`trialgate` packages the mechanical parts of a six-gate qualification
|
|
46
|
+
standard, built from the anti-overfitting literature (Bailey & López de
|
|
47
|
+
Prado, 2014; Bailey, Borwein, López de Prado & Zhu, 2015; Arnott, Harvey &
|
|
48
|
+
Markowitz, 2019):
|
|
49
|
+
|
|
50
|
+
| Gate | What it enforces | Module |
|
|
51
|
+
|------|------------------|--------|
|
|
52
|
+
| 1. Trial registry | Every backtest of every variant is recorded append-only; unregistered results are void. The registry maintains the honest trial count N. | `trialgate.registry` |
|
|
53
|
+
| 2. Data floor | Enough observations (default ≥ 1,000 daily) spanning real regimes. | `trialgate.gates` |
|
|
54
|
+
| 3. PBO ≤ 0.05 | Probability of Backtest Overfitting via CSCV — all C(S, S/2) symmetric train/test splits. | `trialgate.validation` |
|
|
55
|
+
| 4. DSR ≥ 0.95 | Deflated Sharpe Ratio — the observed Sharpe must beat the expected maximum of N noise trials, adjusted for skew, kurtosis, and sample length. | `trialgate.validation` |
|
|
56
|
+
| 5. Single-use holdout | The most recent data slice is locked at first backtest; spending it is a one-way, logged event. Iterated out-of-sample is not out-of-sample. | `trialgate.holdout` |
|
|
57
|
+
| 6. Paper trading | ≥ 3 months live paper execution with measured real costs. Procedural — deliberately **not** in this library: it is a measurement, not a computation. | — |
|
|
58
|
+
|
|
59
|
+
Passing all six is a **necessary condition, not a profit guarantee**: in a
|
|
60
|
+
verified 2022 case study, the least-overfit strategy still lost ~35% in two
|
|
61
|
+
months.
|
|
62
|
+
|
|
63
|
+
## Install
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
pip install trialgate
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Quickstart
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
from datetime import UTC, datetime
|
|
73
|
+
|
|
74
|
+
from trialgate import (
|
|
75
|
+
append_trial, config_hash_for, load_trials, trial_count,
|
|
76
|
+
initialize_holdout, require_data_allowed,
|
|
77
|
+
deflated_sharpe_ratio, probability_of_backtest_overfitting,
|
|
78
|
+
non_annualized_sharpe_variance, evaluate_gates,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
NOW = datetime.now(tz=UTC)
|
|
82
|
+
|
|
83
|
+
# Gate 5 — lock the most recent ~12 months BEFORE the first backtest.
|
|
84
|
+
initialize_holdout(
|
|
85
|
+
"state/holdout.json",
|
|
86
|
+
holdout_start=datetime(2025, 7, 1, tzinfo=UTC),
|
|
87
|
+
locked_at=NOW,
|
|
88
|
+
)
|
|
89
|
+
# Every regular run must prove it stays out of the holdout.
|
|
90
|
+
require_data_allowed("state/holdout.json", data_end=datetime(2025, 6, 30, tzinfo=UTC))
|
|
91
|
+
|
|
92
|
+
# Gate 1 — register EVERY backtest execution, including the quick ones.
|
|
93
|
+
append_trial(
|
|
94
|
+
"state/registry.jsonl",
|
|
95
|
+
recorded_at=NOW,
|
|
96
|
+
config_hash=config_hash_for({"lookbacks": [20, 65, 150, 200]}),
|
|
97
|
+
code_version="8b91661",
|
|
98
|
+
strategy_id="daily_trend_ensemble",
|
|
99
|
+
parameters={"lookbacks": "20,65,150,200"},
|
|
100
|
+
universe=("BTCUSDT", "ETHUSDT"),
|
|
101
|
+
data_start=datetime(2019, 1, 1, tzinfo=UTC),
|
|
102
|
+
data_end=datetime(2025, 6, 30, tzinfo=UTC),
|
|
103
|
+
cost_assumptions={"round_trip_bps": "25"},
|
|
104
|
+
metrics={"annualized_sharpe": "1.10"},
|
|
105
|
+
operator_note="baseline",
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
# Gates 3 + 4 — computed over ALL registered trials, never a flattering subset.
|
|
109
|
+
trials = load_trials("state/registry.jsonl")
|
|
110
|
+
annualized = [float(t.metrics["annualized_sharpe"]) for t in trials]
|
|
111
|
+
|
|
112
|
+
pbo = probability_of_backtest_overfitting(performance_matrix, block_count=16)
|
|
113
|
+
dsr = deflated_sharpe_ratio(
|
|
114
|
+
candidate_daily_returns,
|
|
115
|
+
trial_sharpe_variance=non_annualized_sharpe_variance(annualized),
|
|
116
|
+
effective_trials=trial_count("state/registry.jsonl"),
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
# Gates 2-4 combined verdict.
|
|
120
|
+
report = evaluate_gates(
|
|
121
|
+
observations=len(candidate_daily_returns),
|
|
122
|
+
pbo=pbo.pbo,
|
|
123
|
+
deflated_sharpe=dsr.deflated_sharpe_ratio,
|
|
124
|
+
)
|
|
125
|
+
print(report.summary())
|
|
126
|
+
# PASS data floor (gate 2): 1848 >= 1000
|
|
127
|
+
# PASS PBO (gate 3): 0.02 <= 0.05
|
|
128
|
+
# PASS DSR (gate 4): 0.97 >= 0.95
|
|
129
|
+
# QUALIFIED
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
`performance_matrix` is a T×N list of per-period returns, one column per
|
|
133
|
+
registered configuration; `candidate_daily_returns` is the candidate's daily
|
|
134
|
+
return series. Both come from your backtest engine — `trialgate` judges, it
|
|
135
|
+
does not backtest.
|
|
136
|
+
|
|
137
|
+
## The gate's job is to say no
|
|
138
|
+
|
|
139
|
+
This library is extracted from a live system and has one verdict of each kind
|
|
140
|
+
on record:
|
|
141
|
+
|
|
142
|
+
- **Reference deployment (in progress)** —
|
|
143
|
+
[crypto-quant-signal](https://github.com/0Smallcat0/crypto-quant-signal):
|
|
144
|
+
a daily crypto trend-signal system currently spending its locked holdout
|
|
145
|
+
and 90-day paper period against these exact gates.
|
|
146
|
+
- **A registered FAIL** —
|
|
147
|
+
[tw-stock-trading](https://github.com/0Smallcat0/tw-stock-trading): the
|
|
148
|
+
same strategy family adapted to the Taiwan 0050 ETF **failed its
|
|
149
|
+
pre-registered claim** (CAGR trailed dividend-included buy-and-hold by
|
|
150
|
+
5.3 pp/year over 21 years; even the zero-cost upper bound barely reached
|
|
151
|
+
the tolerance line). Per the pre-registered rule, the runtime was never
|
|
152
|
+
built. The FAIL report *is* the product.
|
|
153
|
+
Full story: [docs/rejecting-my-own-strategy.md](docs/rejecting-my-own-strategy.md).
|
|
154
|
+
|
|
155
|
+
## Design notes
|
|
156
|
+
|
|
157
|
+
- **Zero dependencies.** Standard library only; frozen, slotted dataclasses;
|
|
158
|
+
ships `py.typed` and passes `mypy --strict`.
|
|
159
|
+
- **UTC or it didn't happen.** Naive datetimes are rejected everywhere.
|
|
160
|
+
- **Raises instead of clamping.** When the DSR variance approximation breaks
|
|
161
|
+
down, you get an exception, not a confident 0.0/1.0.
|
|
162
|
+
- **Append-only state.** The registry is a JSONL log; the holdout lock is a
|
|
163
|
+
one-way JSON file. Both are plain text you can audit in any editor.
|
|
164
|
+
- **Judgement, not orchestration.** Bring your own backtest engine and data;
|
|
165
|
+
the library never touches an exchange or a price feed.
|
|
166
|
+
|
|
167
|
+
## Provenance
|
|
168
|
+
|
|
169
|
+
The gate specification, thresholds, and doctrine were defined and verified by
|
|
170
|
+
the author from the primary literature; implementation was AI-assisted
|
|
171
|
+
(spec-driven development with Claude), with every module reviewed against the
|
|
172
|
+
spec and covered by tests before acceptance. This mirrors the library's own
|
|
173
|
+
thesis: artifacts earn trust through verification, not through how they were
|
|
174
|
+
produced.
|
|
175
|
+
|
|
176
|
+
## References
|
|
177
|
+
|
|
178
|
+
- Bailey & López de Prado — *The Deflated Sharpe Ratio* (Journal of Portfolio
|
|
179
|
+
Management, 2014)
|
|
180
|
+
- Bailey, Borwein, López de Prado & Zhu — *The Probability of Backtest
|
|
181
|
+
Overfitting* (Journal of Computational Finance, 2015)
|
|
182
|
+
- Arnott, Harvey & Markowitz — *A Backtesting Protocol in the Era of Machine
|
|
183
|
+
Learning* (Journal of Financial Data Science, 2019)
|
|
184
|
+
|
|
185
|
+
## License
|
|
186
|
+
|
|
187
|
+
[MIT](LICENSE) © Tsai Chih-Chun (0Smallcat0)
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# trialgate
|
|
2
|
+
|
|
3
|
+
[](https://github.com/0Smallcat0/trialgate/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.python.org/downloads/)
|
|
5
|
+
[](https://mypy.readthedocs.io/)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+
An **anti-overfitting validation gate** for backtested trading strategies.
|
|
9
|
+
Zero dependencies, pure standard library, fully typed.
|
|
10
|
+
|
|
11
|
+
> A strategy earns belief by surviving verification, not by looking good
|
|
12
|
+
> in-sample. Optimizing four parameters on a **pure random walk** can produce
|
|
13
|
+
> an in-sample Sharpe of 1.27. A single backtest — however pretty — is not
|
|
14
|
+
> evidence.
|
|
15
|
+
|
|
16
|
+
`trialgate` packages the mechanical parts of a six-gate qualification
|
|
17
|
+
standard, built from the anti-overfitting literature (Bailey & López de
|
|
18
|
+
Prado, 2014; Bailey, Borwein, López de Prado & Zhu, 2015; Arnott, Harvey &
|
|
19
|
+
Markowitz, 2019):
|
|
20
|
+
|
|
21
|
+
| Gate | What it enforces | Module |
|
|
22
|
+
|------|------------------|--------|
|
|
23
|
+
| 1. Trial registry | Every backtest of every variant is recorded append-only; unregistered results are void. The registry maintains the honest trial count N. | `trialgate.registry` |
|
|
24
|
+
| 2. Data floor | Enough observations (default ≥ 1,000 daily) spanning real regimes. | `trialgate.gates` |
|
|
25
|
+
| 3. PBO ≤ 0.05 | Probability of Backtest Overfitting via CSCV — all C(S, S/2) symmetric train/test splits. | `trialgate.validation` |
|
|
26
|
+
| 4. DSR ≥ 0.95 | Deflated Sharpe Ratio — the observed Sharpe must beat the expected maximum of N noise trials, adjusted for skew, kurtosis, and sample length. | `trialgate.validation` |
|
|
27
|
+
| 5. Single-use holdout | The most recent data slice is locked at first backtest; spending it is a one-way, logged event. Iterated out-of-sample is not out-of-sample. | `trialgate.holdout` |
|
|
28
|
+
| 6. Paper trading | ≥ 3 months live paper execution with measured real costs. Procedural — deliberately **not** in this library: it is a measurement, not a computation. | — |
|
|
29
|
+
|
|
30
|
+
Passing all six is a **necessary condition, not a profit guarantee**: in a
|
|
31
|
+
verified 2022 case study, the least-overfit strategy still lost ~35% in two
|
|
32
|
+
months.
|
|
33
|
+
|
|
34
|
+
## Install
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install trialgate
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Quickstart
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from datetime import UTC, datetime
|
|
44
|
+
|
|
45
|
+
from trialgate import (
|
|
46
|
+
append_trial, config_hash_for, load_trials, trial_count,
|
|
47
|
+
initialize_holdout, require_data_allowed,
|
|
48
|
+
deflated_sharpe_ratio, probability_of_backtest_overfitting,
|
|
49
|
+
non_annualized_sharpe_variance, evaluate_gates,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
NOW = datetime.now(tz=UTC)
|
|
53
|
+
|
|
54
|
+
# Gate 5 — lock the most recent ~12 months BEFORE the first backtest.
|
|
55
|
+
initialize_holdout(
|
|
56
|
+
"state/holdout.json",
|
|
57
|
+
holdout_start=datetime(2025, 7, 1, tzinfo=UTC),
|
|
58
|
+
locked_at=NOW,
|
|
59
|
+
)
|
|
60
|
+
# Every regular run must prove it stays out of the holdout.
|
|
61
|
+
require_data_allowed("state/holdout.json", data_end=datetime(2025, 6, 30, tzinfo=UTC))
|
|
62
|
+
|
|
63
|
+
# Gate 1 — register EVERY backtest execution, including the quick ones.
|
|
64
|
+
append_trial(
|
|
65
|
+
"state/registry.jsonl",
|
|
66
|
+
recorded_at=NOW,
|
|
67
|
+
config_hash=config_hash_for({"lookbacks": [20, 65, 150, 200]}),
|
|
68
|
+
code_version="8b91661",
|
|
69
|
+
strategy_id="daily_trend_ensemble",
|
|
70
|
+
parameters={"lookbacks": "20,65,150,200"},
|
|
71
|
+
universe=("BTCUSDT", "ETHUSDT"),
|
|
72
|
+
data_start=datetime(2019, 1, 1, tzinfo=UTC),
|
|
73
|
+
data_end=datetime(2025, 6, 30, tzinfo=UTC),
|
|
74
|
+
cost_assumptions={"round_trip_bps": "25"},
|
|
75
|
+
metrics={"annualized_sharpe": "1.10"},
|
|
76
|
+
operator_note="baseline",
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
# Gates 3 + 4 — computed over ALL registered trials, never a flattering subset.
|
|
80
|
+
trials = load_trials("state/registry.jsonl")
|
|
81
|
+
annualized = [float(t.metrics["annualized_sharpe"]) for t in trials]
|
|
82
|
+
|
|
83
|
+
pbo = probability_of_backtest_overfitting(performance_matrix, block_count=16)
|
|
84
|
+
dsr = deflated_sharpe_ratio(
|
|
85
|
+
candidate_daily_returns,
|
|
86
|
+
trial_sharpe_variance=non_annualized_sharpe_variance(annualized),
|
|
87
|
+
effective_trials=trial_count("state/registry.jsonl"),
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
# Gates 2-4 combined verdict.
|
|
91
|
+
report = evaluate_gates(
|
|
92
|
+
observations=len(candidate_daily_returns),
|
|
93
|
+
pbo=pbo.pbo,
|
|
94
|
+
deflated_sharpe=dsr.deflated_sharpe_ratio,
|
|
95
|
+
)
|
|
96
|
+
print(report.summary())
|
|
97
|
+
# PASS data floor (gate 2): 1848 >= 1000
|
|
98
|
+
# PASS PBO (gate 3): 0.02 <= 0.05
|
|
99
|
+
# PASS DSR (gate 4): 0.97 >= 0.95
|
|
100
|
+
# QUALIFIED
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
`performance_matrix` is a T×N list of per-period returns, one column per
|
|
104
|
+
registered configuration; `candidate_daily_returns` is the candidate's daily
|
|
105
|
+
return series. Both come from your backtest engine — `trialgate` judges, it
|
|
106
|
+
does not backtest.
|
|
107
|
+
|
|
108
|
+
## The gate's job is to say no
|
|
109
|
+
|
|
110
|
+
This library is extracted from a live system and has one verdict of each kind
|
|
111
|
+
on record:
|
|
112
|
+
|
|
113
|
+
- **Reference deployment (in progress)** —
|
|
114
|
+
[crypto-quant-signal](https://github.com/0Smallcat0/crypto-quant-signal):
|
|
115
|
+
a daily crypto trend-signal system currently spending its locked holdout
|
|
116
|
+
and 90-day paper period against these exact gates.
|
|
117
|
+
- **A registered FAIL** —
|
|
118
|
+
[tw-stock-trading](https://github.com/0Smallcat0/tw-stock-trading): the
|
|
119
|
+
same strategy family adapted to the Taiwan 0050 ETF **failed its
|
|
120
|
+
pre-registered claim** (CAGR trailed dividend-included buy-and-hold by
|
|
121
|
+
5.3 pp/year over 21 years; even the zero-cost upper bound barely reached
|
|
122
|
+
the tolerance line). Per the pre-registered rule, the runtime was never
|
|
123
|
+
built. The FAIL report *is* the product.
|
|
124
|
+
Full story: [docs/rejecting-my-own-strategy.md](docs/rejecting-my-own-strategy.md).
|
|
125
|
+
|
|
126
|
+
## Design notes
|
|
127
|
+
|
|
128
|
+
- **Zero dependencies.** Standard library only; frozen, slotted dataclasses;
|
|
129
|
+
ships `py.typed` and passes `mypy --strict`.
|
|
130
|
+
- **UTC or it didn't happen.** Naive datetimes are rejected everywhere.
|
|
131
|
+
- **Raises instead of clamping.** When the DSR variance approximation breaks
|
|
132
|
+
down, you get an exception, not a confident 0.0/1.0.
|
|
133
|
+
- **Append-only state.** The registry is a JSONL log; the holdout lock is a
|
|
134
|
+
one-way JSON file. Both are plain text you can audit in any editor.
|
|
135
|
+
- **Judgement, not orchestration.** Bring your own backtest engine and data;
|
|
136
|
+
the library never touches an exchange or a price feed.
|
|
137
|
+
|
|
138
|
+
## Provenance
|
|
139
|
+
|
|
140
|
+
The gate specification, thresholds, and doctrine were defined and verified by
|
|
141
|
+
the author from the primary literature; implementation was AI-assisted
|
|
142
|
+
(spec-driven development with Claude), with every module reviewed against the
|
|
143
|
+
spec and covered by tests before acceptance. This mirrors the library's own
|
|
144
|
+
thesis: artifacts earn trust through verification, not through how they were
|
|
145
|
+
produced.
|
|
146
|
+
|
|
147
|
+
## References
|
|
148
|
+
|
|
149
|
+
- Bailey & López de Prado — *The Deflated Sharpe Ratio* (Journal of Portfolio
|
|
150
|
+
Management, 2014)
|
|
151
|
+
- Bailey, Borwein, López de Prado & Zhu — *The Probability of Backtest
|
|
152
|
+
Overfitting* (Journal of Computational Finance, 2015)
|
|
153
|
+
- Arnott, Harvey & Markowitz — *A Backtesting Protocol in the Era of Machine
|
|
154
|
+
Learning* (Journal of Financial Data Science, 2019)
|
|
155
|
+
|
|
156
|
+
## License
|
|
157
|
+
|
|
158
|
+
[MIT](LICENSE) © Tsai Chih-Chun (0Smallcat0)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=69", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "trialgate"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Anti-overfitting validation gate for backtested trading strategies: append-only trial registry, CSCV/PBO, Deflated Sharpe Ratio, single-use holdout lock."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Tsai Chih-Chun (0Smallcat0)" }]
|
|
13
|
+
keywords = [
|
|
14
|
+
"backtesting",
|
|
15
|
+
"overfitting",
|
|
16
|
+
"deflated-sharpe-ratio",
|
|
17
|
+
"probability-of-backtest-overfitting",
|
|
18
|
+
"cscv",
|
|
19
|
+
"quantitative-finance",
|
|
20
|
+
"validation",
|
|
21
|
+
"holdout",
|
|
22
|
+
]
|
|
23
|
+
classifiers = [
|
|
24
|
+
"Development Status :: 4 - Beta",
|
|
25
|
+
"Intended Audience :: Financial and Insurance Industry",
|
|
26
|
+
"Intended Audience :: Science/Research",
|
|
27
|
+
"License :: OSI Approved :: MIT License",
|
|
28
|
+
"Operating System :: OS Independent",
|
|
29
|
+
"Programming Language :: Python :: 3.11",
|
|
30
|
+
"Programming Language :: Python :: 3.12",
|
|
31
|
+
"Programming Language :: Python :: 3.13",
|
|
32
|
+
"Topic :: Office/Business :: Financial :: Investment",
|
|
33
|
+
"Typing :: Typed",
|
|
34
|
+
]
|
|
35
|
+
dependencies = []
|
|
36
|
+
|
|
37
|
+
[project.urls]
|
|
38
|
+
Repository = "https://github.com/0Smallcat0/trialgate"
|
|
39
|
+
Issues = "https://github.com/0Smallcat0/trialgate/issues"
|
|
40
|
+
"Reference deployment" = "https://github.com/0Smallcat0/crypto-quant-signal"
|
|
41
|
+
|
|
42
|
+
[project.optional-dependencies]
|
|
43
|
+
dev = [
|
|
44
|
+
"pytest",
|
|
45
|
+
"ruff",
|
|
46
|
+
"mypy",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
[tool.setuptools.packages.find]
|
|
50
|
+
where = ["src"]
|
|
51
|
+
|
|
52
|
+
[tool.setuptools.package-data]
|
|
53
|
+
trialgate = ["py.typed"]
|
|
54
|
+
|
|
55
|
+
[tool.ruff]
|
|
56
|
+
target-version = "py311"
|
|
57
|
+
line-length = 100
|
|
58
|
+
src = ["src", "tests"]
|
|
59
|
+
|
|
60
|
+
[tool.ruff.lint]
|
|
61
|
+
select = ["E", "F", "I", "UP"]
|
|
62
|
+
|
|
63
|
+
[tool.mypy]
|
|
64
|
+
python_version = "3.11"
|
|
65
|
+
strict = true
|
|
66
|
+
warn_unused_configs = true
|
|
67
|
+
|
|
68
|
+
[tool.pytest.ini_options]
|
|
69
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""trialgate: an anti-overfitting validation gate for backtested strategies.
|
|
2
|
+
|
|
3
|
+
A strategy earns belief by surviving verification, not by looking good
|
|
4
|
+
in-sample. This package provides the mechanical parts of a six-gate
|
|
5
|
+
qualification standard: an append-only trial registry, CSCV/PBO and
|
|
6
|
+
Deflated Sharpe Ratio statistics, a single-use holdout lock, and threshold
|
|
7
|
+
gates that combine them into a verdict.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from trialgate.gates import (
|
|
11
|
+
DEFAULT_MAX_PBO,
|
|
12
|
+
DEFAULT_MIN_DSR,
|
|
13
|
+
DEFAULT_MIN_OBSERVATIONS,
|
|
14
|
+
GateCheck,
|
|
15
|
+
GateReport,
|
|
16
|
+
evaluate_gates,
|
|
17
|
+
)
|
|
18
|
+
from trialgate.holdout import (
|
|
19
|
+
HoldoutState,
|
|
20
|
+
HoldoutViolationError,
|
|
21
|
+
initialize_holdout,
|
|
22
|
+
load_holdout,
|
|
23
|
+
require_data_allowed,
|
|
24
|
+
spend_holdout,
|
|
25
|
+
)
|
|
26
|
+
from trialgate.registry import (
|
|
27
|
+
TrialRecord,
|
|
28
|
+
TrialRegistryError,
|
|
29
|
+
append_trial,
|
|
30
|
+
config_hash_for,
|
|
31
|
+
load_trials,
|
|
32
|
+
trial_count,
|
|
33
|
+
)
|
|
34
|
+
from trialgate.validation import (
|
|
35
|
+
DeflatedSharpeResult,
|
|
36
|
+
PBOResult,
|
|
37
|
+
ValidationInputError,
|
|
38
|
+
deflated_sharpe_ratio,
|
|
39
|
+
non_annualized_sharpe_variance,
|
|
40
|
+
probability_of_backtest_overfitting,
|
|
41
|
+
sharpe_ratio,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
__version__ = "0.1.0"
|
|
45
|
+
|
|
46
|
+
__all__ = [
|
|
47
|
+
"DEFAULT_MAX_PBO",
|
|
48
|
+
"DEFAULT_MIN_DSR",
|
|
49
|
+
"DEFAULT_MIN_OBSERVATIONS",
|
|
50
|
+
"DeflatedSharpeResult",
|
|
51
|
+
"GateCheck",
|
|
52
|
+
"GateReport",
|
|
53
|
+
"HoldoutState",
|
|
54
|
+
"HoldoutViolationError",
|
|
55
|
+
"PBOResult",
|
|
56
|
+
"TrialRecord",
|
|
57
|
+
"TrialRegistryError",
|
|
58
|
+
"ValidationInputError",
|
|
59
|
+
"append_trial",
|
|
60
|
+
"config_hash_for",
|
|
61
|
+
"deflated_sharpe_ratio",
|
|
62
|
+
"evaluate_gates",
|
|
63
|
+
"initialize_holdout",
|
|
64
|
+
"load_holdout",
|
|
65
|
+
"load_trials",
|
|
66
|
+
"non_annualized_sharpe_variance",
|
|
67
|
+
"probability_of_backtest_overfitting",
|
|
68
|
+
"require_data_allowed",
|
|
69
|
+
"sharpe_ratio",
|
|
70
|
+
"spend_holdout",
|
|
71
|
+
"trial_count",
|
|
72
|
+
]
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Threshold gates and the combined qualification report.
|
|
2
|
+
|
|
3
|
+
The six-gate qualification standard (numbering follows the reference
|
|
4
|
+
deployment, a live crypto signal system):
|
|
5
|
+
|
|
6
|
+
1. Trial registry (structural) -> trialgate.registry
|
|
7
|
+
2. Data floor (statistical) -> judged here
|
|
8
|
+
3. PBO <= 0.05 (statistical) -> computed by trialgate.validation, judged here
|
|
9
|
+
4. DSR >= 0.95 (statistical) -> computed by trialgate.validation, judged here
|
|
10
|
+
5. Single-use holdout (structural) -> trialgate.holdout
|
|
11
|
+
6. Paper trading with measured real costs (procedural) -> out of library
|
|
12
|
+
scope on purpose: it is a live measurement over months, not a computation.
|
|
13
|
+
|
|
14
|
+
Passing every gate is a NECESSARY condition for trusting a backtest, not a
|
|
15
|
+
profit guarantee: in a verified 2022 case study the least-overfit strategy
|
|
16
|
+
still lost ~35% in two months.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
|
|
23
|
+
DEFAULT_MIN_OBSERVATIONS = 1000
|
|
24
|
+
DEFAULT_MAX_PBO = 0.05
|
|
25
|
+
DEFAULT_MIN_DSR = 0.95
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True, slots=True)
|
|
29
|
+
class GateCheck:
|
|
30
|
+
"""One threshold verdict."""
|
|
31
|
+
|
|
32
|
+
gate: str
|
|
33
|
+
passed: bool
|
|
34
|
+
observed: float
|
|
35
|
+
threshold: float
|
|
36
|
+
comparison: str
|
|
37
|
+
|
|
38
|
+
def describe(self) -> str:
|
|
39
|
+
status = "PASS" if self.passed else "FAIL"
|
|
40
|
+
return f"{status} {self.gate}: {self.observed:g} {self.comparison} {self.threshold:g}"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True, slots=True)
|
|
44
|
+
class GateReport:
|
|
45
|
+
"""Combined verdict over the statistical gates (2-4)."""
|
|
46
|
+
|
|
47
|
+
checks: tuple[GateCheck, ...]
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def passed(self) -> bool:
|
|
51
|
+
return all(check.passed for check in self.checks)
|
|
52
|
+
|
|
53
|
+
def summary(self) -> str:
|
|
54
|
+
lines = [check.describe() for check in self.checks]
|
|
55
|
+
lines.append("QUALIFIED" if self.passed else "NOT QUALIFIED")
|
|
56
|
+
return "\n".join(lines)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def evaluate_gates(
|
|
60
|
+
*,
|
|
61
|
+
observations: int,
|
|
62
|
+
pbo: float,
|
|
63
|
+
deflated_sharpe: float,
|
|
64
|
+
min_observations: int = DEFAULT_MIN_OBSERVATIONS,
|
|
65
|
+
max_pbo: float = DEFAULT_MAX_PBO,
|
|
66
|
+
min_dsr: float = DEFAULT_MIN_DSR,
|
|
67
|
+
) -> GateReport:
|
|
68
|
+
"""Judge the statistical gates (2-4) against their thresholds.
|
|
69
|
+
|
|
70
|
+
Feed the honest numbers: ``observations`` from the actual data span,
|
|
71
|
+
``pbo`` from probability_of_backtest_overfitting() over ALL registered
|
|
72
|
+
trials, ``deflated_sharpe`` from deflated_sharpe_ratio() using the
|
|
73
|
+
registry's full trial count. Flattering inputs void the report.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
checks = (
|
|
77
|
+
GateCheck(
|
|
78
|
+
gate="data floor (gate 2)",
|
|
79
|
+
passed=observations >= min_observations,
|
|
80
|
+
observed=float(observations),
|
|
81
|
+
threshold=float(min_observations),
|
|
82
|
+
comparison=">=",
|
|
83
|
+
),
|
|
84
|
+
GateCheck(
|
|
85
|
+
gate="PBO (gate 3)",
|
|
86
|
+
passed=pbo <= max_pbo,
|
|
87
|
+
observed=pbo,
|
|
88
|
+
threshold=max_pbo,
|
|
89
|
+
comparison="<=",
|
|
90
|
+
),
|
|
91
|
+
GateCheck(
|
|
92
|
+
gate="DSR (gate 4)",
|
|
93
|
+
passed=deflated_sharpe >= min_dsr,
|
|
94
|
+
observed=deflated_sharpe,
|
|
95
|
+
threshold=min_dsr,
|
|
96
|
+
comparison=">=",
|
|
97
|
+
),
|
|
98
|
+
)
|
|
99
|
+
return GateReport(checks=checks)
|