trialgate 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Tsai Chih-Chun (0Smallcat0)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,187 @@
1
+ Metadata-Version: 2.4
2
+ Name: trialgate
3
+ Version: 0.1.0
4
+ Summary: Anti-overfitting validation gate for backtested trading strategies: append-only trial registry, CSCV/PBO, Deflated Sharpe Ratio, single-use holdout lock.
5
+ Author: Tsai Chih-Chun (0Smallcat0)
6
+ License: MIT
7
+ Project-URL: Repository, https://github.com/0Smallcat0/trialgate
8
+ Project-URL: Issues, https://github.com/0Smallcat0/trialgate/issues
9
+ Project-URL: Reference deployment, https://github.com/0Smallcat0/crypto-quant-signal
10
+ Keywords: backtesting,overfitting,deflated-sharpe-ratio,probability-of-backtest-overfitting,cscv,quantitative-finance,validation,holdout
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Financial and Insurance Industry
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Office/Business :: Financial :: Investment
20
+ Classifier: Typing :: Typed
21
+ Requires-Python: >=3.11
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE
24
+ Provides-Extra: dev
25
+ Requires-Dist: pytest; extra == "dev"
26
+ Requires-Dist: ruff; extra == "dev"
27
+ Requires-Dist: mypy; extra == "dev"
28
+ Dynamic: license-file
29
+
30
+ # trialgate
31
+
32
+ [![CI](https://github.com/0Smallcat0/trialgate/actions/workflows/ci.yml/badge.svg)](https://github.com/0Smallcat0/trialgate/actions/workflows/ci.yml)
33
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/)
34
+ [![mypy: strict](https://img.shields.io/badge/mypy-strict-blue.svg)](https://mypy.readthedocs.io/)
35
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
36
+
37
+ An **anti-overfitting validation gate** for backtested trading strategies.
38
+ Zero dependencies, pure standard library, fully typed.
39
+
40
+ > A strategy earns belief by surviving verification, not by looking good
41
+ > in-sample. Optimizing four parameters on a **pure random walk** can produce
42
+ > an in-sample Sharpe of 1.27. A single backtest — however pretty — is not
43
+ > evidence.
44
+
45
+ `trialgate` packages the mechanical parts of a six-gate qualification
46
+ standard, built from the anti-overfitting literature (Bailey & López de
47
+ Prado, 2014; Bailey, Borwein, López de Prado & Zhu, 2015; Arnott, Harvey &
48
+ Markowitz, 2019):
49
+
50
+ | Gate | What it enforces | Module |
51
+ |------|------------------|--------|
52
+ | 1. Trial registry | Every backtest of every variant is recorded append-only; unregistered results are void. The registry maintains the honest trial count N. | `trialgate.registry` |
53
+ | 2. Data floor | Enough observations (default ≥ 1,000 daily) spanning real regimes. | `trialgate.gates` |
54
+ | 3. PBO ≤ 0.05 | Probability of Backtest Overfitting via CSCV — all C(S, S/2) symmetric train/test splits. | `trialgate.validation` |
55
+ | 4. DSR ≥ 0.95 | Deflated Sharpe Ratio — the observed Sharpe must beat the expected maximum of N noise trials, adjusted for skew, kurtosis, and sample length. | `trialgate.validation` |
56
+ | 5. Single-use holdout | The most recent data slice is locked at first backtest; spending it is a one-way, logged event. Iterated out-of-sample is not out-of-sample. | `trialgate.holdout` |
57
+ | 6. Paper trading | ≥ 3 months live paper execution with measured real costs. Procedural — deliberately **not** in this library: it is a measurement, not a computation. | — |
58
+
59
+ Passing all six is a **necessary condition, not a profit guarantee**: in a
60
+ verified 2022 case study, the least-overfit strategy still lost ~35% in two
61
+ months.
62
+
63
+ ## Install
64
+
65
+ ```bash
66
+ pip install trialgate
67
+ ```
68
+
69
+ ## Quickstart
70
+
71
+ ```python
72
+ from datetime import UTC, datetime
73
+
74
+ from trialgate import (
75
+ append_trial, config_hash_for, load_trials, trial_count,
76
+ initialize_holdout, require_data_allowed,
77
+ deflated_sharpe_ratio, probability_of_backtest_overfitting,
78
+ non_annualized_sharpe_variance, evaluate_gates,
79
+ )
80
+
81
+ NOW = datetime.now(tz=UTC)
82
+
83
+ # Gate 5 — lock the most recent ~12 months BEFORE the first backtest.
84
+ initialize_holdout(
85
+ "state/holdout.json",
86
+ holdout_start=datetime(2025, 7, 1, tzinfo=UTC),
87
+ locked_at=NOW,
88
+ )
89
+ # Every regular run must prove it stays out of the holdout.
90
+ require_data_allowed("state/holdout.json", data_end=datetime(2025, 6, 30, tzinfo=UTC))
91
+
92
+ # Gate 1 — register EVERY backtest execution, including the quick ones.
93
+ append_trial(
94
+ "state/registry.jsonl",
95
+ recorded_at=NOW,
96
+ config_hash=config_hash_for({"lookbacks": [20, 65, 150, 200]}),
97
+ code_version="8b91661",
98
+ strategy_id="daily_trend_ensemble",
99
+ parameters={"lookbacks": "20,65,150,200"},
100
+ universe=("BTCUSDT", "ETHUSDT"),
101
+ data_start=datetime(2019, 1, 1, tzinfo=UTC),
102
+ data_end=datetime(2025, 6, 30, tzinfo=UTC),
103
+ cost_assumptions={"round_trip_bps": "25"},
104
+ metrics={"annualized_sharpe": "1.10"},
105
+ operator_note="baseline",
106
+ )
107
+
108
+ # Gates 3 + 4 — computed over ALL registered trials, never a flattering subset.
109
+ trials = load_trials("state/registry.jsonl")
110
+ annualized = [float(t.metrics["annualized_sharpe"]) for t in trials]
111
+
112
+ pbo = probability_of_backtest_overfitting(performance_matrix, block_count=16)
113
+ dsr = deflated_sharpe_ratio(
114
+ candidate_daily_returns,
115
+ trial_sharpe_variance=non_annualized_sharpe_variance(annualized),
116
+ effective_trials=trial_count("state/registry.jsonl"),
117
+ )
118
+
119
+ # Gates 2-4 combined verdict.
120
+ report = evaluate_gates(
121
+ observations=len(candidate_daily_returns),
122
+ pbo=pbo.pbo,
123
+ deflated_sharpe=dsr.deflated_sharpe_ratio,
124
+ )
125
+ print(report.summary())
126
+ # PASS data floor (gate 2): 1848 >= 1000
127
+ # PASS PBO (gate 3): 0.02 <= 0.05
128
+ # PASS DSR (gate 4): 0.97 >= 0.95
129
+ # QUALIFIED
130
+ ```
131
+
132
+ `performance_matrix` is a T×N list of per-period returns, one column per
133
+ registered configuration; `candidate_daily_returns` is the candidate's daily
134
+ return series. Both come from your backtest engine — `trialgate` judges, it
135
+ does not backtest.
136
+
137
+ ## The gate's job is to say no
138
+
139
+ This library is extracted from a live system and has one verdict of each kind
140
+ on record:
141
+
142
+ - **Reference deployment (in progress)** —
143
+ [crypto-quant-signal](https://github.com/0Smallcat0/crypto-quant-signal):
144
+ a daily crypto trend-signal system currently spending its locked holdout
145
+ and 90-day paper period against these exact gates.
146
+ - **A registered FAIL** —
147
+ [tw-stock-trading](https://github.com/0Smallcat0/tw-stock-trading): the
148
+ same strategy family adapted to the Taiwan 0050 ETF **failed its
149
+ pre-registered claim** (CAGR trailed dividend-included buy-and-hold by
150
+ 5.3 pp/year over 21 years; even the zero-cost upper bound barely reached
151
+ the tolerance line). Per the pre-registered rule, the runtime was never
152
+ built. The FAIL report *is* the product.
153
+ Full story: [docs/rejecting-my-own-strategy.md](docs/rejecting-my-own-strategy.md).
154
+
155
+ ## Design notes
156
+
157
+ - **Zero dependencies.** Standard library only; frozen, slotted dataclasses;
158
+ ships `py.typed` and passes `mypy --strict`.
159
+ - **UTC or it didn't happen.** Naive datetimes are rejected everywhere.
160
+ - **Raises instead of clamping.** When the DSR variance approximation breaks
161
+ down, you get an exception, not a confident 0.0/1.0.
162
+ - **Append-only state.** The registry is a JSONL log; the holdout lock is a
163
+ one-way JSON file. Both are plain text you can audit in any editor.
164
+ - **Judgement, not orchestration.** Bring your own backtest engine and data;
165
+ the library never touches an exchange or a price feed.
166
+
167
+ ## Provenance
168
+
169
+ The gate specification, thresholds, and doctrine were defined and verified by
170
+ the author from the primary literature; implementation was AI-assisted
171
+ (spec-driven development with Claude), with every module reviewed against the
172
+ spec and covered by tests before acceptance. This mirrors the library's own
173
+ thesis: artifacts earn trust through verification, not through how they were
174
+ produced.
175
+
176
+ ## References
177
+
178
+ - Bailey & López de Prado — *The Deflated Sharpe Ratio* (Journal of Portfolio
179
+ Management, 2014)
180
+ - Bailey, Borwein, López de Prado & Zhu — *The Probability of Backtest
181
+ Overfitting* (Journal of Computational Finance, 2015)
182
+ - Arnott, Harvey & Markowitz — *A Backtesting Protocol in the Era of Machine
183
+ Learning* (Journal of Financial Data Science, 2019)
184
+
185
+ ## License
186
+
187
+ [MIT](LICENSE) © Tsai Chih-Chun (0Smallcat0)
@@ -0,0 +1,158 @@
1
+ # trialgate
2
+
3
+ [![CI](https://github.com/0Smallcat0/trialgate/actions/workflows/ci.yml/badge.svg)](https://github.com/0Smallcat0/trialgate/actions/workflows/ci.yml)
4
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/)
5
+ [![mypy: strict](https://img.shields.io/badge/mypy-strict-blue.svg)](https://mypy.readthedocs.io/)
6
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
7
+
8
+ An **anti-overfitting validation gate** for backtested trading strategies.
9
+ Zero dependencies, pure standard library, fully typed.
10
+
11
+ > A strategy earns belief by surviving verification, not by looking good
12
+ > in-sample. Optimizing four parameters on a **pure random walk** can produce
13
+ > an in-sample Sharpe of 1.27. A single backtest — however pretty — is not
14
+ > evidence.
15
+
16
+ `trialgate` packages the mechanical parts of a six-gate qualification
17
+ standard, built from the anti-overfitting literature (Bailey & López de
18
+ Prado, 2014; Bailey, Borwein, López de Prado & Zhu, 2015; Arnott, Harvey &
19
+ Markowitz, 2019):
20
+
21
+ | Gate | What it enforces | Module |
22
+ |------|------------------|--------|
23
+ | 1. Trial registry | Every backtest of every variant is recorded append-only; unregistered results are void. The registry maintains the honest trial count N. | `trialgate.registry` |
24
+ | 2. Data floor | Enough observations (default ≥ 1,000 daily) spanning real regimes. | `trialgate.gates` |
25
+ | 3. PBO ≤ 0.05 | Probability of Backtest Overfitting via CSCV — all C(S, S/2) symmetric train/test splits. | `trialgate.validation` |
26
+ | 4. DSR ≥ 0.95 | Deflated Sharpe Ratio — the observed Sharpe must beat the expected maximum of N noise trials, adjusted for skew, kurtosis, and sample length. | `trialgate.validation` |
27
+ | 5. Single-use holdout | The most recent data slice is locked at first backtest; spending it is a one-way, logged event. Iterated out-of-sample is not out-of-sample. | `trialgate.holdout` |
28
+ | 6. Paper trading | ≥ 3 months live paper execution with measured real costs. Procedural — deliberately **not** in this library: it is a measurement, not a computation. | — |
29
+
30
+ Passing all six is a **necessary condition, not a profit guarantee**: in a
31
+ verified 2022 case study, the least-overfit strategy still lost ~35% in two
32
+ months.
33
+
34
+ ## Install
35
+
36
+ ```bash
37
+ pip install trialgate
38
+ ```
39
+
40
+ ## Quickstart
41
+
42
+ ```python
43
+ from datetime import UTC, datetime
44
+
45
+ from trialgate import (
46
+ append_trial, config_hash_for, load_trials, trial_count,
47
+ initialize_holdout, require_data_allowed,
48
+ deflated_sharpe_ratio, probability_of_backtest_overfitting,
49
+ non_annualized_sharpe_variance, evaluate_gates,
50
+ )
51
+
52
+ NOW = datetime.now(tz=UTC)
53
+
54
+ # Gate 5 — lock the most recent ~12 months BEFORE the first backtest.
55
+ initialize_holdout(
56
+ "state/holdout.json",
57
+ holdout_start=datetime(2025, 7, 1, tzinfo=UTC),
58
+ locked_at=NOW,
59
+ )
60
+ # Every regular run must prove it stays out of the holdout.
61
+ require_data_allowed("state/holdout.json", data_end=datetime(2025, 6, 30, tzinfo=UTC))
62
+
63
+ # Gate 1 — register EVERY backtest execution, including the quick ones.
64
+ append_trial(
65
+ "state/registry.jsonl",
66
+ recorded_at=NOW,
67
+ config_hash=config_hash_for({"lookbacks": [20, 65, 150, 200]}),
68
+ code_version="8b91661",
69
+ strategy_id="daily_trend_ensemble",
70
+ parameters={"lookbacks": "20,65,150,200"},
71
+ universe=("BTCUSDT", "ETHUSDT"),
72
+ data_start=datetime(2019, 1, 1, tzinfo=UTC),
73
+ data_end=datetime(2025, 6, 30, tzinfo=UTC),
74
+ cost_assumptions={"round_trip_bps": "25"},
75
+ metrics={"annualized_sharpe": "1.10"},
76
+ operator_note="baseline",
77
+ )
78
+
79
+ # Gates 3 + 4 — computed over ALL registered trials, never a flattering subset.
80
+ trials = load_trials("state/registry.jsonl")
81
+ annualized = [float(t.metrics["annualized_sharpe"]) for t in trials]
82
+
83
+ pbo = probability_of_backtest_overfitting(performance_matrix, block_count=16)
84
+ dsr = deflated_sharpe_ratio(
85
+ candidate_daily_returns,
86
+ trial_sharpe_variance=non_annualized_sharpe_variance(annualized),
87
+ effective_trials=trial_count("state/registry.jsonl"),
88
+ )
89
+
90
+ # Gates 2-4 combined verdict.
91
+ report = evaluate_gates(
92
+ observations=len(candidate_daily_returns),
93
+ pbo=pbo.pbo,
94
+ deflated_sharpe=dsr.deflated_sharpe_ratio,
95
+ )
96
+ print(report.summary())
97
+ # PASS data floor (gate 2): 1848 >= 1000
98
+ # PASS PBO (gate 3): 0.02 <= 0.05
99
+ # PASS DSR (gate 4): 0.97 >= 0.95
100
+ # QUALIFIED
101
+ ```
102
+
103
+ `performance_matrix` is a T×N list of per-period returns, one column per
104
+ registered configuration; `candidate_daily_returns` is the candidate's daily
105
+ return series. Both come from your backtest engine — `trialgate` judges, it
106
+ does not backtest.
107
+
108
+ ## The gate's job is to say no
109
+
110
+ This library is extracted from a live system and has one verdict of each kind
111
+ on record:
112
+
113
+ - **Reference deployment (in progress)** —
114
+ [crypto-quant-signal](https://github.com/0Smallcat0/crypto-quant-signal):
115
+ a daily crypto trend-signal system currently spending its locked holdout
116
+ and 90-day paper period against these exact gates.
117
+ - **A registered FAIL** —
118
+ [tw-stock-trading](https://github.com/0Smallcat0/tw-stock-trading): the
119
+ same strategy family adapted to the Taiwan 0050 ETF **failed its
120
+ pre-registered claim** (CAGR trailed dividend-included buy-and-hold by
121
+ 5.3 pp/year over 21 years; even the zero-cost upper bound barely reached
122
+ the tolerance line). Per the pre-registered rule, the runtime was never
123
+ built. The FAIL report *is* the product.
124
+ Full story: [docs/rejecting-my-own-strategy.md](docs/rejecting-my-own-strategy.md).
125
+
126
+ ## Design notes
127
+
128
+ - **Zero dependencies.** Standard library only; frozen, slotted dataclasses;
129
+ ships `py.typed` and passes `mypy --strict`.
130
+ - **UTC or it didn't happen.** Naive datetimes are rejected everywhere.
131
+ - **Raises instead of clamping.** When the DSR variance approximation breaks
132
+ down, you get an exception, not a confident 0.0/1.0.
133
+ - **Append-only state.** The registry is a JSONL log; the holdout lock is a
134
+ one-way JSON file. Both are plain text you can audit in any editor.
135
+ - **Judgement, not orchestration.** Bring your own backtest engine and data;
136
+ the library never touches an exchange or a price feed.
137
+
138
+ ## Provenance
139
+
140
+ The gate specification, thresholds, and doctrine were defined and verified by
141
+ the author from the primary literature; implementation was AI-assisted
142
+ (spec-driven development with Claude), with every module reviewed against the
143
+ spec and covered by tests before acceptance. This mirrors the library's own
144
+ thesis: artifacts earn trust through verification, not through how they were
145
+ produced.
146
+
147
+ ## References
148
+
149
+ - Bailey & López de Prado — *The Deflated Sharpe Ratio* (Journal of Portfolio
150
+ Management, 2014)
151
+ - Bailey, Borwein, López de Prado & Zhu — *The Probability of Backtest
152
+ Overfitting* (Journal of Computational Finance, 2015)
153
+ - Arnott, Harvey & Markowitz — *A Backtesting Protocol in the Era of Machine
154
+ Learning* (Journal of Financial Data Science, 2019)
155
+
156
+ ## License
157
+
158
+ [MIT](LICENSE) © Tsai Chih-Chun (0Smallcat0)
@@ -0,0 +1,69 @@
1
+ [build-system]
2
+ requires = ["setuptools>=69", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "trialgate"
7
+ version = "0.1.0"
8
+ description = "Anti-overfitting validation gate for backtested trading strategies: append-only trial registry, CSCV/PBO, Deflated Sharpe Ratio, single-use holdout lock."
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ license = { text = "MIT" }
12
+ authors = [{ name = "Tsai Chih-Chun (0Smallcat0)" }]
13
+ keywords = [
14
+ "backtesting",
15
+ "overfitting",
16
+ "deflated-sharpe-ratio",
17
+ "probability-of-backtest-overfitting",
18
+ "cscv",
19
+ "quantitative-finance",
20
+ "validation",
21
+ "holdout",
22
+ ]
23
+ classifiers = [
24
+ "Development Status :: 4 - Beta",
25
+ "Intended Audience :: Financial and Insurance Industry",
26
+ "Intended Audience :: Science/Research",
27
+ "License :: OSI Approved :: MIT License",
28
+ "Operating System :: OS Independent",
29
+ "Programming Language :: Python :: 3.11",
30
+ "Programming Language :: Python :: 3.12",
31
+ "Programming Language :: Python :: 3.13",
32
+ "Topic :: Office/Business :: Financial :: Investment",
33
+ "Typing :: Typed",
34
+ ]
35
+ dependencies = []
36
+
37
+ [project.urls]
38
+ Repository = "https://github.com/0Smallcat0/trialgate"
39
+ Issues = "https://github.com/0Smallcat0/trialgate/issues"
40
+ "Reference deployment" = "https://github.com/0Smallcat0/crypto-quant-signal"
41
+
42
+ [project.optional-dependencies]
43
+ dev = [
44
+ "pytest",
45
+ "ruff",
46
+ "mypy",
47
+ ]
48
+
49
+ [tool.setuptools.packages.find]
50
+ where = ["src"]
51
+
52
+ [tool.setuptools.package-data]
53
+ trialgate = ["py.typed"]
54
+
55
+ [tool.ruff]
56
+ target-version = "py311"
57
+ line-length = 100
58
+ src = ["src", "tests"]
59
+
60
+ [tool.ruff.lint]
61
+ select = ["E", "F", "I", "UP"]
62
+
63
+ [tool.mypy]
64
+ python_version = "3.11"
65
+ strict = true
66
+ warn_unused_configs = true
67
+
68
+ [tool.pytest.ini_options]
69
+ testpaths = ["tests"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,72 @@
1
+ """trialgate: an anti-overfitting validation gate for backtested strategies.
2
+
3
+ A strategy earns belief by surviving verification, not by looking good
4
+ in-sample. This package provides the mechanical parts of a six-gate
5
+ qualification standard: an append-only trial registry, CSCV/PBO and
6
+ Deflated Sharpe Ratio statistics, a single-use holdout lock, and threshold
7
+ gates that combine them into a verdict.
8
+ """
9
+
10
+ from trialgate.gates import (
11
+ DEFAULT_MAX_PBO,
12
+ DEFAULT_MIN_DSR,
13
+ DEFAULT_MIN_OBSERVATIONS,
14
+ GateCheck,
15
+ GateReport,
16
+ evaluate_gates,
17
+ )
18
+ from trialgate.holdout import (
19
+ HoldoutState,
20
+ HoldoutViolationError,
21
+ initialize_holdout,
22
+ load_holdout,
23
+ require_data_allowed,
24
+ spend_holdout,
25
+ )
26
+ from trialgate.registry import (
27
+ TrialRecord,
28
+ TrialRegistryError,
29
+ append_trial,
30
+ config_hash_for,
31
+ load_trials,
32
+ trial_count,
33
+ )
34
+ from trialgate.validation import (
35
+ DeflatedSharpeResult,
36
+ PBOResult,
37
+ ValidationInputError,
38
+ deflated_sharpe_ratio,
39
+ non_annualized_sharpe_variance,
40
+ probability_of_backtest_overfitting,
41
+ sharpe_ratio,
42
+ )
43
+
44
+ __version__ = "0.1.0"
45
+
46
+ __all__ = [
47
+ "DEFAULT_MAX_PBO",
48
+ "DEFAULT_MIN_DSR",
49
+ "DEFAULT_MIN_OBSERVATIONS",
50
+ "DeflatedSharpeResult",
51
+ "GateCheck",
52
+ "GateReport",
53
+ "HoldoutState",
54
+ "HoldoutViolationError",
55
+ "PBOResult",
56
+ "TrialRecord",
57
+ "TrialRegistryError",
58
+ "ValidationInputError",
59
+ "append_trial",
60
+ "config_hash_for",
61
+ "deflated_sharpe_ratio",
62
+ "evaluate_gates",
63
+ "initialize_holdout",
64
+ "load_holdout",
65
+ "load_trials",
66
+ "non_annualized_sharpe_variance",
67
+ "probability_of_backtest_overfitting",
68
+ "require_data_allowed",
69
+ "sharpe_ratio",
70
+ "spend_holdout",
71
+ "trial_count",
72
+ ]
@@ -0,0 +1,99 @@
1
+ """Threshold gates and the combined qualification report.
2
+
3
+ The six-gate qualification standard (numbering follows the reference
4
+ deployment, a live crypto signal system):
5
+
6
+ 1. Trial registry (structural) -> trialgate.registry
7
+ 2. Data floor (statistical) -> judged here
8
+ 3. PBO <= 0.05 (statistical) -> computed by trialgate.validation, judged here
9
+ 4. DSR >= 0.95 (statistical) -> computed by trialgate.validation, judged here
10
+ 5. Single-use holdout (structural) -> trialgate.holdout
11
+ 6. Paper trading with measured real costs (procedural) -> out of library
12
+ scope on purpose: it is a live measurement over months, not a computation.
13
+
14
+ Passing every gate is a NECESSARY condition for trusting a backtest, not a
15
+ profit guarantee: in a verified 2022 case study the least-overfit strategy
16
+ still lost ~35% in two months.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ from dataclasses import dataclass
22
+
23
+ DEFAULT_MIN_OBSERVATIONS = 1000
24
+ DEFAULT_MAX_PBO = 0.05
25
+ DEFAULT_MIN_DSR = 0.95
26
+
27
+
28
+ @dataclass(frozen=True, slots=True)
29
+ class GateCheck:
30
+ """One threshold verdict."""
31
+
32
+ gate: str
33
+ passed: bool
34
+ observed: float
35
+ threshold: float
36
+ comparison: str
37
+
38
+ def describe(self) -> str:
39
+ status = "PASS" if self.passed else "FAIL"
40
+ return f"{status} {self.gate}: {self.observed:g} {self.comparison} {self.threshold:g}"
41
+
42
+
43
+ @dataclass(frozen=True, slots=True)
44
+ class GateReport:
45
+ """Combined verdict over the statistical gates (2-4)."""
46
+
47
+ checks: tuple[GateCheck, ...]
48
+
49
+ @property
50
+ def passed(self) -> bool:
51
+ return all(check.passed for check in self.checks)
52
+
53
+ def summary(self) -> str:
54
+ lines = [check.describe() for check in self.checks]
55
+ lines.append("QUALIFIED" if self.passed else "NOT QUALIFIED")
56
+ return "\n".join(lines)
57
+
58
+
59
+ def evaluate_gates(
60
+ *,
61
+ observations: int,
62
+ pbo: float,
63
+ deflated_sharpe: float,
64
+ min_observations: int = DEFAULT_MIN_OBSERVATIONS,
65
+ max_pbo: float = DEFAULT_MAX_PBO,
66
+ min_dsr: float = DEFAULT_MIN_DSR,
67
+ ) -> GateReport:
68
+ """Judge the statistical gates (2-4) against their thresholds.
69
+
70
+ Feed the honest numbers: ``observations`` from the actual data span,
71
+ ``pbo`` from probability_of_backtest_overfitting() over ALL registered
72
+ trials, ``deflated_sharpe`` from deflated_sharpe_ratio() using the
73
+ registry's full trial count. Flattering inputs void the report.
74
+ """
75
+
76
+ checks = (
77
+ GateCheck(
78
+ gate="data floor (gate 2)",
79
+ passed=observations >= min_observations,
80
+ observed=float(observations),
81
+ threshold=float(min_observations),
82
+ comparison=">=",
83
+ ),
84
+ GateCheck(
85
+ gate="PBO (gate 3)",
86
+ passed=pbo <= max_pbo,
87
+ observed=pbo,
88
+ threshold=max_pbo,
89
+ comparison="<=",
90
+ ),
91
+ GateCheck(
92
+ gate="DSR (gate 4)",
93
+ passed=deflated_sharpe >= min_dsr,
94
+ observed=deflated_sharpe,
95
+ threshold=min_dsr,
96
+ comparison=">=",
97
+ ),
98
+ )
99
+ return GateReport(checks=checks)