backtest-audit-kit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- backtest_audit_kit-0.1.0/LICENSE +21 -0
- backtest_audit_kit-0.1.0/PKG-INFO +167 -0
- backtest_audit_kit-0.1.0/README.md +146 -0
- backtest_audit_kit-0.1.0/auditkit/__init__.py +16 -0
- backtest_audit_kit-0.1.0/auditkit/checks/__init__.py +5 -0
- backtest_audit_kit-0.1.0/auditkit/checks/fees.py +184 -0
- backtest_audit_kit-0.1.0/auditkit/checks/invariants.py +183 -0
- backtest_audit_kit-0.1.0/auditkit/checks/lookahead.py +252 -0
- backtest_audit_kit-0.1.0/auditkit/checks/mutants.py +317 -0
- backtest_audit_kit-0.1.0/backtest_audit_kit.egg-info/PKG-INFO +167 -0
- backtest_audit_kit-0.1.0/backtest_audit_kit.egg-info/SOURCES.txt +18 -0
- backtest_audit_kit-0.1.0/backtest_audit_kit.egg-info/dependency_links.txt +1 -0
- backtest_audit_kit-0.1.0/backtest_audit_kit.egg-info/requires.txt +3 -0
- backtest_audit_kit-0.1.0/backtest_audit_kit.egg-info/top_level.txt +1 -0
- backtest_audit_kit-0.1.0/pyproject.toml +35 -0
- backtest_audit_kit-0.1.0/setup.cfg +4 -0
- backtest_audit_kit-0.1.0/tests/test_fees.py +56 -0
- backtest_audit_kit-0.1.0/tests/test_invariants.py +71 -0
- backtest_audit_kit-0.1.0/tests/test_lookahead.py +56 -0
- backtest_audit_kit-0.1.0/tests/test_mutants.py +67 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 rytjy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: backtest-audit-kit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Audit a backtest: look-ahead lint, ledger invariants, mutation coverage, cost checks. Pure standard library.
|
|
5
|
+
Author: rytjy
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/rytjy/backtest-audit-kit
|
|
8
|
+
Project-URL: Repository, https://github.com/rytjy/backtest-audit-kit
|
|
9
|
+
Keywords: backtesting,lookahead-bias,audit,trading,quant,mutation-testing,invariants,backtest-overfitting
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Office/Business :: Financial :: Investment
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: pytest; extra == "dev"
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# backtest-audit-kit
|
|
23
|
+
|
|
24
|
+
**"I verify whether your strategy is real — or whether the backtest lied to you."**
|
|
25
|
+
|
|
26
|
+
A tiny, dependency-free toolkit that audits a backtest the way an accountant
|
|
27
|
+
audits a ledger and a QA engineer audits a test suite. It does not sell you
|
|
28
|
+
returns; it finds the four ways a backtest most often lies.
|
|
29
|
+
|
|
30
|
+
> **回测审计 / 策略代码复核服务** — 一页说明与联系方式 → **[PORTFOLIO.md](PORTFOLIO.md)**
|
|
31
|
+
> **Backtest audits · strategy-code review** — one-page overview → **[PORTFOLIO.md](PORTFOLIO.md)**
|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
## English
|
|
36
|
+
|
|
37
|
+
**What it is.** A pure standard-library (Python 3.9+) kit of four independent
|
|
38
|
+
checkers for a trading backtest:
|
|
39
|
+
|
|
40
|
+
| Checker | Question it answers |
|
|
41
|
+
|---|---|
|
|
42
|
+
| `checks/lookahead.py` | Does any decision on bar `i` see bars `i+1...`? (AST-based future-function lint) |
|
|
43
|
+
| `checks/invariants.py` | Does the ledger obey accounting invariants? (equity identity, `wd_cum` carry, monotonic counters, non-negative balances) |
|
|
44
|
+
| `checks/mutants.py` | Which bugs would your test suite *not* catch? (mutation testing; survivors = coverage blind spots) |
|
|
45
|
+
| `checks/fees.py` | Are fees two-sided, funding charged per bar, and slippage counted once per fill? |
|
|
46
|
+
|
|
47
|
+
**Who it is for.** Quant developers, prop-desk researchers, and anyone who has
|
|
48
|
+
ever shipped a strategy that looked great in a notebook and died in
|
|
49
|
+
production. If you are reviewing someone else's backtest — or your own — this
|
|
50
|
+
gives you a checklist, reproduction scripts, and before/after numbers instead
|
|
51
|
+
of opinions.
|
|
52
|
+
|
|
53
|
+
**What I deliver.** ① A ranked findings list plus a runnable reproduction
|
|
54
|
+
script for each problem. ② A minimal fix patch. ③ A before/after reconciliation
|
|
55
|
+
of the headline numbers (win rate, net PnL, multiple, MDD, t-stat).
|
|
56
|
+
|
|
57
|
+
**Sample cases** (full write-ups in `cases/`):
|
|
58
|
+
|
|
59
|
+
1. **Look-ahead** — the exit scan ran on the entry bar, so every signal knew
|
|
60
|
+
its outcome. Fixed: 91 trades / 90.1% / **+$214.37** → 26 trades / 65.4% /
|
|
61
|
+
**-$0.12**; a tick-level re-run gives 27 trades / 66.7% / **+$0.37**. The
|
|
62
|
+
leak was worth **≈ $214** of imaginary PnL — the whole profit.
|
|
63
|
+
2. **Withdrawal accounting** — the bust path dropped `wd_cum`, so any strategy
|
|
64
|
+
with withdrawals was systematically understated. Fixed: same 2024–26 config
|
|
65
|
+
**7.82x → 215.27x** (a ≈ 27x gap; withdrawal rules "double-keep-half" vs
|
|
66
|
+
"double-keep-100" differ by ~27x).
|
|
67
|
+
3. **Drawdown metric** — the circuit-breaker reset the running peak, so the
|
|
68
|
+
"reported MDD" was only the local drawdown. Investor-view raw drawdown
|
|
69
|
+
**99.7%** vs reported **53.1%** — a **46 pp** gap. Conclusion: MDD must be
|
|
70
|
+
written twice (breaker-local *and* raw).
|
|
71
|
+
(*Bonus case 4:* 5-minute bar clustering inflated `n` ~11x and turned
|
|
72
|
+
t = 15.60 into t = 1.75 after a 24h cooldown.)
|
|
73
|
+
4. **Cost omission** (`cases/case5_cost_omission.md`) — a high-turnover 50x
|
|
74
|
+
book charged **no** fee, funding or slippage; its take-profit needed only a
|
|
75
|
+
**+0.08%** move while one round trip costs **0.09%** (the venue's real 4.5 bps
|
|
76
|
+
taker), so the target sat below the toll and every win is a net loss. At the
|
|
77
|
+
venue rate the omitted cost ran **~5.4x the starting capital** and **~17.8x the
|
|
78
|
+
reported profit**.
|
|
79
|
+
|
|
80
|
+
**Install.**
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
pip install backtest-audit-kit
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
**Use it in code** (four checkers, pure standard library):
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from auditkit.checks import lookahead
|
|
90
|
+
findings = lookahead.detect_file("strategy.py") # future-data lint
|
|
91
|
+
print(lookahead.format_findings(findings))
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
**How to run.**
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
cd portfolio/backtest-audit-kit
|
|
98
|
+
python3 -m unittest discover tests # stdlib only, no install
|
|
99
|
+
# or, if you have pytest:
|
|
100
|
+
python3 -m pytest tests/ -q
|
|
101
|
+
python3 examples/run_demo.py # see all four checkers in action
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
**Honesty note.** The look-ahead checker is a heuristic linter, not a proof
|
|
105
|
+
engine: it flags the *shapes* that leak, and a stand-alone forward-simulation
|
|
106
|
+
pass is reported `MEDIUM` for human review rather than being declared safe.
|
|
107
|
+
Mutation testing reports blind spots; it never claims your strategy is
|
|
108
|
+
profitable. No checker here predicts returns.
|
|
109
|
+
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
## 中文
|
|
113
|
+
|
|
114
|
+
**是什么。** 一个纯 Python 标准库(3.9+)实现的回测审计工具包,四个互相独立
|
|
115
|
+
的检查器:
|
|
116
|
+
|
|
117
|
+
| 检查器 | 回答的问题 |
|
|
118
|
+
|---|---|
|
|
119
|
+
| `checks/lookahead.py` | 在 bar `i` 上的决策,是否读到了 `i+1...` 的数据?(基于 AST 的前视/未来函数检测) |
|
|
120
|
+
| `checks/invariants.py` | 账本是否满足会计不变量?(权益恒等式、`wd_cum` 结转、计数器单调、余额非负) |
|
|
121
|
+
| `checks/mutants.py` | 你的测试套件**抓不到**哪些 bug?(变异测试;存活的变异体 = 覆盖盲区) |
|
|
122
|
+
| `checks/fees.py` | 手续费是否双边计、资金费是否逐 bar 计、滑点是否每笔只算一次? |
|
|
123
|
+
|
|
124
|
+
**给谁用。** 量化开发者、自营研究岗、以及任何"回测很美、实盘就死"过的
|
|
125
|
+
人。带看别人的回测(或自己的)时,它给你的是**问题清单 + 复现脚本 + 修复前后
|
|
126
|
+
的数字**,而不是一句"我觉得不太行"。
|
|
127
|
+
|
|
128
|
+
**交付物承诺。** ① 按严重度排序的问题清单,每个问题配一个可运行的复现脚本。
|
|
129
|
+
② 最小修复补丁。③ 修复前后的对账数字(胜率 / 净利 / 倍数 / 回撤 / t 值)。
|
|
130
|
+
|
|
131
|
+
**案例摘要**(完整写在 `cases/`):
|
|
132
|
+
|
|
133
|
+
1. **前视 bug** —— 出场扫描写在开仓 bar 的循环内,每个信号都提前知道结果。
|
|
134
|
+
修复后:91 笔 / 90.1% / **+$214.37** → 26 笔 / 65.4% / **-$0.12**;tick 级
|
|
135
|
+
复跑得 27 笔 / 66.7% / **+$0.37**。这个泄漏值 **约 $214 的虚假收益**,
|
|
136
|
+
即全部利润。
|
|
137
|
+
2. **提现会计 bug** —— 爆仓返程丢了 `wd_cum`,导致所有带提现的策略历史收益被
|
|
138
|
+
系统性低估。修复后:同一配置 2024–26 **7.82x → 215.27x**(相差约 27 倍;
|
|
139
|
+
提现规则"翻 2 留 1"与"翻 200 留 100"之间也差约 27 倍)。
|
|
140
|
+
3. **回撤口径 bug** —— 熔断触发时把 peak 重置为 cap,"报告 MDD"只是每段熔断
|
|
141
|
+
后的局部回撤。投资者口径的原始回撤 **99.7%** vs 报告 **53.1%**,**差 46
|
|
142
|
+
个百分点**。结论:MDD 必须双写(熔断口径 + 原始口径)。
|
|
143
|
+
(**附赠案例 4**:一段暴跌连打十几根 5min bar 让样本量虚增约 11 倍,t 值从
|
|
144
|
+
15.60 在 24h 冷却去重后掉到 1.75。)
|
|
145
|
+
4. **成本漏算**(`cases/case5_cost_omission.md`)—— 一个高换手 50x 账本**完全没算**手续费/资金费/滑点;止盈只需价格动 **+0.08%**,而一趟往返 **0.09%**(场馆真实 4.5bps taker)⇒ 目标在成本之下,每笔赢单净亏。按场馆真实费率,漏掉的成本约 **5.4× 本金**、**17.8× 账面收益**。
|
|
146
|
+
|
|
147
|
+
**怎么跑。**
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
cd portfolio/backtest-audit-kit
|
|
151
|
+
python3 -m unittest discover tests # 只用标准库,无需安装
|
|
152
|
+
# 若装了 pytest:
|
|
153
|
+
python3 -m pytest tests/ -q
|
|
154
|
+
python3 examples/run_demo.py # 四个检查器全跑一遍
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
**诚实声明。** 前视检查器是启发式 linter,不是证明器:它抓的是会泄漏的**代码
|
|
158
|
+
形态**,独立的"向前模拟"循环会被标为 `MEDIUM` 交人工复核,而不是被判安全。
|
|
159
|
+
变异测试只报告覆盖盲区,从不声称你的策略赚钱。本工具包不预测任何收益。
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
## Also in this account · 另见
|
|
164
|
+
|
|
165
|
+
- [`outpost/`](./outpost/) — 一个"只放能跑、能查、能被否证的东西"的据点:
|
|
166
|
+
单文件表格整理工具(零依赖、离线自检 14 项)+ 一份**被自己推翻**的回测(做空山寨无 edge)
|
|
167
|
+
+ 一次"两条腿都断了"的数据源修复。不写客户名、不写"我能做什么"。
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# backtest-audit-kit
|
|
2
|
+
|
|
3
|
+
**"I verify whether your strategy is real — or whether the backtest lied to you."**
|
|
4
|
+
|
|
5
|
+
A tiny, dependency-free toolkit that audits a backtest the way an accountant
|
|
6
|
+
audits a ledger and a QA engineer audits a test suite. It does not sell you
|
|
7
|
+
returns; it finds the four ways a backtest most often lies.
|
|
8
|
+
|
|
9
|
+
> **回测审计 / 策略代码复核服务** — 一页说明与联系方式 → **[PORTFOLIO.md](PORTFOLIO.md)**
|
|
10
|
+
> **Backtest audits · strategy-code review** — one-page overview → **[PORTFOLIO.md](PORTFOLIO.md)**
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
## English
|
|
15
|
+
|
|
16
|
+
**What it is.** A pure standard-library (Python 3.9+) kit of four independent
|
|
17
|
+
checkers for a trading backtest:
|
|
18
|
+
|
|
19
|
+
| Checker | Question it answers |
|
|
20
|
+
|---|---|
|
|
21
|
+
| `checks/lookahead.py` | Does any decision on bar `i` see bars `i+1...`? (AST-based future-function lint) |
|
|
22
|
+
| `checks/invariants.py` | Does the ledger obey accounting invariants? (equity identity, `wd_cum` carry, monotonic counters, non-negative balances) |
|
|
23
|
+
| `checks/mutants.py` | Which bugs would your test suite *not* catch? (mutation testing; survivors = coverage blind spots) |
|
|
24
|
+
| `checks/fees.py` | Are fees two-sided, funding charged per bar, and slippage counted once per fill? |
|
|
25
|
+
|
|
26
|
+
**Who it is for.** Quant developers, prop-desk researchers, and anyone who has
|
|
27
|
+
ever shipped a strategy that looked great in a notebook and died in
|
|
28
|
+
production. If you are reviewing someone else's backtest — or your own — this
|
|
29
|
+
gives you a checklist, reproduction scripts, and before/after numbers instead
|
|
30
|
+
of opinions.
|
|
31
|
+
|
|
32
|
+
**What I deliver.** ① A ranked findings list plus a runnable reproduction
|
|
33
|
+
script for each problem. ② A minimal fix patch. ③ A before/after reconciliation
|
|
34
|
+
of the headline numbers (win rate, net PnL, multiple, MDD, t-stat).
|
|
35
|
+
|
|
36
|
+
**Sample cases** (full write-ups in `cases/`):
|
|
37
|
+
|
|
38
|
+
1. **Look-ahead** — the exit scan ran on the entry bar, so every signal knew
|
|
39
|
+
its outcome. Fixed: 91 trades / 90.1% / **+$214.37** → 26 trades / 65.4% /
|
|
40
|
+
**-$0.12**; a tick-level re-run gives 27 trades / 66.7% / **+$0.37**. The
|
|
41
|
+
leak was worth **≈ $214** of imaginary PnL — the whole profit.
|
|
42
|
+
2. **Withdrawal accounting** — the bust path dropped `wd_cum`, so any strategy
|
|
43
|
+
with withdrawals was systematically understated. Fixed: same 2024–26 config
|
|
44
|
+
**7.82x → 215.27x** (a ≈ 27x gap; withdrawal rules "double-keep-half" vs
|
|
45
|
+
"double-keep-100" differ by ~27x).
|
|
46
|
+
3. **Drawdown metric** — the circuit-breaker reset the running peak, so the
|
|
47
|
+
"reported MDD" was only the local drawdown. Investor-view raw drawdown
|
|
48
|
+
**99.7%** vs reported **53.1%** — a **46 pp** gap. Conclusion: MDD must be
|
|
49
|
+
written twice (breaker-local *and* raw).
|
|
50
|
+
(*Bonus case 4:* 5-minute bar clustering inflated `n` ~11x and turned
|
|
51
|
+
t = 15.60 into t = 1.75 after a 24h cooldown.)
|
|
52
|
+
4. **Cost omission** (`cases/case5_cost_omission.md`) — a high-turnover 50x
|
|
53
|
+
book charged **no** fee, funding or slippage; its take-profit needed only a
|
|
54
|
+
**+0.08%** move while one round trip costs **0.09%** (the venue's real 4.5 bps
|
|
55
|
+
taker), so the target sat below the toll and every win is a net loss. At the
|
|
56
|
+
venue rate the omitted cost ran **~5.4x the starting capital** and **~17.8x the
|
|
57
|
+
reported profit**.
|
|
58
|
+
|
|
59
|
+
**Install.**
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
pip install backtest-audit-kit
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
**Use it in code** (four checkers, pure standard library):
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
from auditkit.checks import lookahead
|
|
69
|
+
findings = lookahead.detect_file("strategy.py") # future-data lint
|
|
70
|
+
print(lookahead.format_findings(findings))
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
**How to run.**
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
cd portfolio/backtest-audit-kit
|
|
77
|
+
python3 -m unittest discover tests # stdlib only, no install
|
|
78
|
+
# or, if you have pytest:
|
|
79
|
+
python3 -m pytest tests/ -q
|
|
80
|
+
python3 examples/run_demo.py # see all four checkers in action
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
**Honesty note.** The look-ahead checker is a heuristic linter, not a proof
|
|
84
|
+
engine: it flags the *shapes* that leak, and a stand-alone forward-simulation
|
|
85
|
+
pass is reported `MEDIUM` for human review rather than being declared safe.
|
|
86
|
+
Mutation testing reports blind spots; it never claims your strategy is
|
|
87
|
+
profitable. No checker here predicts returns.
|
|
88
|
+
|
|
89
|
+
---
|
|
90
|
+
|
|
91
|
+
## 中文
|
|
92
|
+
|
|
93
|
+
**是什么。** 一个纯 Python 标准库(3.9+)实现的回测审计工具包,四个互相独立
|
|
94
|
+
的检查器:
|
|
95
|
+
|
|
96
|
+
| 检查器 | 回答的问题 |
|
|
97
|
+
|---|---|
|
|
98
|
+
| `checks/lookahead.py` | 在 bar `i` 上的决策,是否读到了 `i+1...` 的数据?(基于 AST 的前视/未来函数检测) |
|
|
99
|
+
| `checks/invariants.py` | 账本是否满足会计不变量?(权益恒等式、`wd_cum` 结转、计数器单调、余额非负) |
|
|
100
|
+
| `checks/mutants.py` | 你的测试套件**抓不到**哪些 bug?(变异测试;存活的变异体 = 覆盖盲区) |
|
|
101
|
+
| `checks/fees.py` | 手续费是否双边计、资金费是否逐 bar 计、滑点是否每笔只算一次? |
|
|
102
|
+
|
|
103
|
+
**给谁用。** 量化开发者、自营研究岗、以及任何"回测很美、实盘就死"过的
|
|
104
|
+
人。带看别人的回测(或自己的)时,它给你的是**问题清单 + 复现脚本 + 修复前后
|
|
105
|
+
的数字**,而不是一句"我觉得不太行"。
|
|
106
|
+
|
|
107
|
+
**交付物承诺。** ① 按严重度排序的问题清单,每个问题配一个可运行的复现脚本。
|
|
108
|
+
② 最小修复补丁。③ 修复前后的对账数字(胜率 / 净利 / 倍数 / 回撤 / t 值)。
|
|
109
|
+
|
|
110
|
+
**案例摘要**(完整写在 `cases/`):
|
|
111
|
+
|
|
112
|
+
1. **前视 bug** —— 出场扫描写在开仓 bar 的循环内,每个信号都提前知道结果。
|
|
113
|
+
修复后:91 笔 / 90.1% / **+$214.37** → 26 笔 / 65.4% / **-$0.12**;tick 级
|
|
114
|
+
复跑得 27 笔 / 66.7% / **+$0.37**。这个泄漏值 **约 $214 的虚假收益**,
|
|
115
|
+
即全部利润。
|
|
116
|
+
2. **提现会计 bug** —— 爆仓返程丢了 `wd_cum`,导致所有带提现的策略历史收益被
|
|
117
|
+
系统性低估。修复后:同一配置 2024–26 **7.82x → 215.27x**(相差约 27 倍;
|
|
118
|
+
提现规则"翻 2 留 1"与"翻 200 留 100"之间也差约 27 倍)。
|
|
119
|
+
3. **回撤口径 bug** —— 熔断触发时把 peak 重置为 cap,"报告 MDD"只是每段熔断
|
|
120
|
+
后的局部回撤。投资者口径的原始回撤 **99.7%** vs 报告 **53.1%**,**差 46
|
|
121
|
+
个百分点**。结论:MDD 必须双写(熔断口径 + 原始口径)。
|
|
122
|
+
(**附赠案例 4**:一段暴跌连打十几根 5min bar 让样本量虚增约 11 倍,t 值从
|
|
123
|
+
15.60 在 24h 冷却去重后掉到 1.75。)
|
|
124
|
+
4. **成本漏算**(`cases/case5_cost_omission.md`)—— 一个高换手 50x 账本**完全没算**手续费/资金费/滑点;止盈只需价格动 **+0.08%**,而一趟往返 **0.09%**(场馆真实 4.5bps taker)⇒ 目标在成本之下,每笔赢单净亏。按场馆真实费率,漏掉的成本约 **5.4× 本金**、**17.8× 账面收益**。
|
|
125
|
+
|
|
126
|
+
**怎么跑。**
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
cd portfolio/backtest-audit-kit
|
|
130
|
+
python3 -m unittest discover tests # 只用标准库,无需安装
|
|
131
|
+
# 若装了 pytest:
|
|
132
|
+
python3 -m pytest tests/ -q
|
|
133
|
+
python3 examples/run_demo.py # 四个检查器全跑一遍
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
**诚实声明。** 前视检查器是启发式 linter,不是证明器:它抓的是会泄漏的**代码
|
|
137
|
+
形态**,独立的"向前模拟"循环会被标为 `MEDIUM` 交人工复核,而不是被判安全。
|
|
138
|
+
变异测试只报告覆盖盲区,从不声称你的策略赚钱。本工具包不预测任何收益。
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
## Also in this account · 另见
|
|
143
|
+
|
|
144
|
+
- [`outpost/`](./outpost/) — 一个"只放能跑、能查、能被否证的东西"的据点:
|
|
145
|
+
单文件表格整理工具(零依赖、离线自检 14 项)+ 一份**被自己推翻**的回测(做空山寨无 edge)
|
|
146
|
+
+ 一次"两条腿都断了"的数据源修复。不写客户名、不写"我能做什么"。
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""auditkit — a tiny, dependency-free toolkit for auditing backtests.
|
|
2
|
+
|
|
3
|
+
The toolkit answers one question: *is this strategy real, or did the backtest
|
|
4
|
+
lie to you?* It ships four independent checkers:
|
|
5
|
+
|
|
6
|
+
* ``auditkit.checks.lookahead`` — AST-based look-ahead / future-function lint.
|
|
7
|
+
* ``auditkit.checks.invariants`` — ledger invariants (equity identity, wd_cum, monotonic counters).
|
|
8
|
+
* ``auditkit.checks.mutants`` — mutation testing: which bugs would your tests *not* catch?
|
|
9
|
+
* ``auditkit.checks.fees`` — fee / funding / slippage accounting checks.
|
|
10
|
+
|
|
11
|
+
Everything is pure standard library (Python 3.9+). No pandas, no numpy,
|
|
12
|
+
no network access.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
__version__ = "0.1.0"
|
|
16
|
+
__all__ = ["checks"]
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""Fee / funding / slippage accounting.
|
|
2
|
+
|
|
3
|
+
Costs are where optimistic backtests hide. Three failure modes dominate:
|
|
4
|
+
|
|
5
|
+
* **one-sided fees** — the engine charges the entry fee but forgets the exit
|
|
6
|
+
(or vice-versa), so every trade is quietly cheaper than reality;
|
|
7
|
+
* **funding charged once** — a perp position held for ``k`` bars should pay
|
|
8
|
+
funding ``k`` times, but the engine deducts a single period;
|
|
9
|
+
* **double-counted slippage** — slippage is applied to both the raw fill *and*
|
|
10
|
+
the already-slippaged price, charging it twice per fill.
|
|
11
|
+
|
|
12
|
+
Each trade is a ``dict``::
|
|
13
|
+
|
|
14
|
+
{
|
|
15
|
+
"id": "t1",
|
|
16
|
+
"entry_notional": 10_000.0,
|
|
17
|
+
"exit_notional": 10_000.0, # optional, defaults to entry_notional
|
|
18
|
+
"holding_bars": 8,
|
|
19
|
+
"fee_paid": 10.0, # total round-trip fee actually charged
|
|
20
|
+
"funding_paid": 8.0, # total funding actually charged
|
|
21
|
+
"slippage_paid": 4.0, # total slippage cost actually charged
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
Rates are in **basis points** (1 bp = 0.01%).
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
from dataclasses import dataclass
|
|
30
|
+
from typing import List, Optional, Sequence
|
|
31
|
+
|
|
32
|
+
__all__ = ["FeeIssue", "check_trades", "check_trade", "DEFAULT_RATES"]
|
|
33
|
+
|
|
34
|
+
BPS = 10_000.0
|
|
35
|
+
DEFAULT_RATES = {
|
|
36
|
+
"fee_bps": 5.0, # per side (taker)
|
|
37
|
+
"funding_bps_per_bar": 1.0,
|
|
38
|
+
"slippage_bps": 2.0, # per fill
|
|
39
|
+
}
|
|
40
|
+
REL_TOL = 0.05 # 5% tolerance on each estimate
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class FeeIssue:
|
|
45
|
+
rule: str
|
|
46
|
+
trade_id: object
|
|
47
|
+
message: str
|
|
48
|
+
expected: Optional[float] = None
|
|
49
|
+
actual: Optional[float] = None
|
|
50
|
+
|
|
51
|
+
def as_dict(self) -> dict:
|
|
52
|
+
return {
|
|
53
|
+
"rule": self.rule,
|
|
54
|
+
"trade_id": self.trade_id,
|
|
55
|
+
"message": self.message,
|
|
56
|
+
"expected": self.expected,
|
|
57
|
+
"actual": self.actual,
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
def __str__(self) -> str: # pragma: no cover - cosmetic
|
|
61
|
+
return f"[{self.rule}] trade={self.trade_id}: {self.message}"
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _f(trade: dict, key: str, default: float = 0.0) -> float:
|
|
65
|
+
val = trade.get(key, default)
|
|
66
|
+
try:
|
|
67
|
+
return float(val)
|
|
68
|
+
except (TypeError, ValueError):
|
|
69
|
+
return default
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _outside(actual: float, expected: float, rel_tol: float) -> Optional[str]:
|
|
73
|
+
"""Return ``"low"``, ``"high"``, or ``None`` relative to ``expected``."""
|
|
74
|
+
if expected <= 0:
|
|
75
|
+
return "high" if actual > 1e-9 else None
|
|
76
|
+
ratio = actual / expected
|
|
77
|
+
if ratio < 1 - rel_tol:
|
|
78
|
+
return "low"
|
|
79
|
+
if ratio > 1 + rel_tol:
|
|
80
|
+
return "high"
|
|
81
|
+
return None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def check_trade(
|
|
85
|
+
trade: dict,
|
|
86
|
+
fee_bps: float = DEFAULT_RATES["fee_bps"],
|
|
87
|
+
funding_bps_per_bar: float = DEFAULT_RATES["funding_bps_per_bar"],
|
|
88
|
+
slippage_bps: float = DEFAULT_RATES["slippage_bps"],
|
|
89
|
+
rel_tol: float = REL_TOL,
|
|
90
|
+
) -> List[FeeIssue]:
|
|
91
|
+
issues: List[FeeIssue] = []
|
|
92
|
+
tid = trade.get("id", "?")
|
|
93
|
+
entry = _f(trade, "entry_notional")
|
|
94
|
+
exit_n = _f(trade, "exit_notional", entry) or entry
|
|
95
|
+
bars = int(_f(trade, "holding_bars", 1)) or 1
|
|
96
|
+
|
|
97
|
+
# --- round-trip fees (both sides) ------------------------------------- #
|
|
98
|
+
expected_fee = (entry + exit_n) * fee_bps / BPS
|
|
99
|
+
actual_fee = _f(trade, "fee_paid", float("nan"))
|
|
100
|
+
if actual_fee == actual_fee: # not NaN -> provided
|
|
101
|
+
side = _outside(actual_fee, expected_fee, rel_tol)
|
|
102
|
+
if side == "low":
|
|
103
|
+
issues.append(
|
|
104
|
+
FeeIssue(
|
|
105
|
+
"ONE_SIDED_FEE",
|
|
106
|
+
tid,
|
|
107
|
+
f"fee {actual_fee:.4f} < round-trip expectation "
|
|
108
|
+
f"{expected_fee:.4f} (entry+exit) -- likely only one side charged",
|
|
109
|
+
expected_fee,
|
|
110
|
+
actual_fee,
|
|
111
|
+
)
|
|
112
|
+
)
|
|
113
|
+
elif side == "high":
|
|
114
|
+
issues.append(
|
|
115
|
+
FeeIssue(
|
|
116
|
+
"FEE_OVERCHARGE",
|
|
117
|
+
tid,
|
|
118
|
+
f"fee {actual_fee:.4f} > round-trip expectation {expected_fee:.4f}",
|
|
119
|
+
expected_fee,
|
|
120
|
+
actual_fee,
|
|
121
|
+
)
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
# --- funding per bar -------------------------------------------------- #
|
|
125
|
+
expected_funding = entry * funding_bps_per_bar / BPS * bars
|
|
126
|
+
actual_funding = _f(trade, "funding_paid", float("nan"))
|
|
127
|
+
if actual_funding == actual_funding and bars > 1:
|
|
128
|
+
side = _outside(actual_funding, expected_funding, rel_tol)
|
|
129
|
+
if side == "low":
|
|
130
|
+
issues.append(
|
|
131
|
+
FeeIssue(
|
|
132
|
+
"FUNDING_ONCE",
|
|
133
|
+
tid,
|
|
134
|
+
f"funding {actual_funding:.4f} for {bars} bars < "
|
|
135
|
+
f"{expected_funding:.4f} -- charged once, not per bar",
|
|
136
|
+
expected_funding,
|
|
137
|
+
actual_funding,
|
|
138
|
+
)
|
|
139
|
+
)
|
|
140
|
+
elif side == "high":
|
|
141
|
+
issues.append(
|
|
142
|
+
FeeIssue(
|
|
143
|
+
"FUNDING_OVERCHARGE",
|
|
144
|
+
tid,
|
|
145
|
+
f"funding {actual_funding:.4f} > {expected_funding:.4f} "
|
|
146
|
+
f"for {bars} bars",
|
|
147
|
+
expected_funding,
|
|
148
|
+
actual_funding,
|
|
149
|
+
)
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
# --- slippage once per fill (round trip = two fills) ------------------ #
|
|
153
|
+
expected_slip = (entry + exit_n) * slippage_bps / BPS
|
|
154
|
+
actual_slip = _f(trade, "slippage_paid", float("nan"))
|
|
155
|
+
if actual_slip == actual_slip:
|
|
156
|
+
side = _outside(actual_slip, expected_slip, rel_tol)
|
|
157
|
+
if side == "high":
|
|
158
|
+
issues.append(
|
|
159
|
+
FeeIssue(
|
|
160
|
+
"SLIPPAGE_MULTIPLE",
|
|
161
|
+
tid,
|
|
162
|
+
f"slippage {actual_slip:.4f} > one-pass expectation "
|
|
163
|
+
f"{expected_slip:.4f} -- slippage looks double-counted",
|
|
164
|
+
expected_slip,
|
|
165
|
+
actual_slip,
|
|
166
|
+
)
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
return issues
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def check_trades(
|
|
173
|
+
trades: Sequence[dict],
|
|
174
|
+
fee_bps: float = DEFAULT_RATES["fee_bps"],
|
|
175
|
+
funding_bps_per_bar: float = DEFAULT_RATES["funding_bps_per_bar"],
|
|
176
|
+
slippage_bps: float = DEFAULT_RATES["slippage_bps"],
|
|
177
|
+
rel_tol: float = REL_TOL,
|
|
178
|
+
) -> List[FeeIssue]:
|
|
179
|
+
out: List[FeeIssue] = []
|
|
180
|
+
for trade in trades:
|
|
181
|
+
out.extend(
|
|
182
|
+
check_trade(trade, fee_bps, funding_bps_per_bar, slippage_bps, rel_tol)
|
|
183
|
+
)
|
|
184
|
+
return out
|