timeleak 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- timeleak-0.1.0/LICENSE +21 -0
- timeleak-0.1.0/PKG-INFO +165 -0
- timeleak-0.1.0/README.md +135 -0
- timeleak-0.1.0/pyproject.toml +56 -0
- timeleak-0.1.0/setup.cfg +4 -0
- timeleak-0.1.0/src/timeleak/__init__.py +20 -0
- timeleak-0.1.0/src/timeleak/__main__.py +6 -0
- timeleak-0.1.0/src/timeleak/analyzer.py +247 -0
- timeleak-0.1.0/src/timeleak/cli.py +142 -0
- timeleak-0.1.0/src/timeleak/rules.py +154 -0
- timeleak-0.1.0/src/timeleak.egg-info/PKG-INFO +165 -0
- timeleak-0.1.0/src/timeleak.egg-info/SOURCES.txt +15 -0
- timeleak-0.1.0/src/timeleak.egg-info/dependency_links.txt +1 -0
- timeleak-0.1.0/src/timeleak.egg-info/entry_points.txt +2 -0
- timeleak-0.1.0/src/timeleak.egg-info/requires.txt +3 -0
- timeleak-0.1.0/src/timeleak.egg-info/top_level.txt +1 -0
- timeleak-0.1.0/tests/test_analyzer.py +189 -0
timeleak-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Martex
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
timeleak-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: timeleak
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Static linter that finds data-leakage patterns in time-series machine learning code.
|
|
5
|
+
Author-email: Martex <mvnikolov2009@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/MartexHACK/timeleak
|
|
8
|
+
Project-URL: Issues, https://github.com/MartexHACK/timeleak/issues
|
|
9
|
+
Keywords: data-leakage,linter,static-analysis,time-series,machine-learning,backtesting,quantitative-finance,pre-commit
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.9
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
# timeleak
|
|
32
|
+
|
|
33
|
+
**A static linter that finds data leakage in time-series machine learning code.**
|
|
34
|
+
|
|
35
|
+
[](https://github.com/MartexHACK/timeleak/actions/workflows/ci.yml)
|
|
36
|
+
[](https://www.python.org/)
|
|
37
|
+
[](LICENSE)
|
|
38
|
+
[](pyproject.toml)
|
|
39
|
+
|
|
40
|
+
Your backtest shows a Sharpe of 2.4. Live, it is 0.1.
|
|
41
|
+
|
|
42
|
+
Usually nothing exotic went wrong — a scaler was fitted before the train/test split, a
|
|
43
|
+
window was centred, or a gap was filled backwards. Each of these quietly hands the model
|
|
44
|
+
information it could not have had at prediction time. None of them raise an error. None of
|
|
45
|
+
them fail a unit test. The model just looks brilliant until real money is on it.
|
|
46
|
+
|
|
47
|
+
`timeleak` reads your source with Python's `ast` module and points at the specific line.
|
|
48
|
+
|
|
49
|
+
```console
|
|
50
|
+
$ timeleak examples/leaky_pipeline.py
|
|
51
|
+
examples/leaky_pipeline.py:15:6: TL005 backward fill propagates future values into earlier rows [method='bfill']
|
|
52
|
+
fix: use forward fill (ffill), or drop the leading NaNs instead
|
|
53
|
+
examples/leaky_pipeline.py:18:15: TL004 rolling(center=True) centres the window, so each row sees future rows
|
|
54
|
+
fix: use the default center=False so the window only looks backwards
|
|
55
|
+
examples/leaky_pipeline.py:22:54: TL007 column rescaled with a statistic computed over the whole frame, including the test period [df.std()]
|
|
56
|
+
fix: compute the statistic on the training slice only, or use a Pipeline
|
|
57
|
+
examples/leaky_pipeline.py:25:17: TL003 negative .shift() pulls future values into the current row (look-ahead)
|
|
58
|
+
fix: if this builds a forward-looking label that is fine, but it must never become a feature
|
|
59
|
+
examples/leaky_pipeline.py:33:5: TL001 transformer fitted before train/test split leaks test statistics into training [StandardScaler.fit_transform()]
|
|
60
|
+
fix: move the transform inside a sklearn Pipeline so it is refit on each training fold
|
|
61
|
+
examples/leaky_pipeline.py:36:36: TL002 train_test_split shuffles by default, which destroys time order
|
|
62
|
+
fix: pass shuffle=False for time-ordered data, or split on an explicit date boundary
|
|
63
|
+
examples/leaky_pipeline.py:42:10: TL006 K-fold style cross-validation on time series trains on data after the test block [cross_val_score with default KFold]
|
|
64
|
+
fix: use TimeSeriesSplit, or a purged split with an embargo if labels span several bars
|
|
65
|
+
|
|
66
|
+
7 finding(s) in 1 file(s) (5 error, 2 warning)
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Install
|
|
70
|
+
|
|
71
|
+
Not on PyPI yet. Install straight from the repository:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
pip install git+https://github.com/MartexHACK/timeleak
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Once it is published, `pip install timeleak` will work too.
|
|
78
|
+
|
|
79
|
+
Zero runtime dependencies — it is `ast` and the standard library. It never imports or
|
|
80
|
+
executes the code it analyses, so it is safe to point at a repository you do not trust.
|
|
81
|
+
|
|
82
|
+
## Usage
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
timeleak # scan the current directory
|
|
86
|
+
timeleak src/ notebooks/ # scan specific paths
|
|
87
|
+
timeleak --select TL001,TL003 . # only these rules
|
|
88
|
+
timeleak --ignore TL002 . # everything except this one
|
|
89
|
+
timeleak --format json . # machine-readable, for CI
|
|
90
|
+
timeleak --list-rules # the catalogue
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Exit code is `1` when anything is found and `0` when clean, so it drops straight into CI.
|
|
94
|
+
Use `--exit-zero` if you want the report without failing the build.
|
|
95
|
+
|
|
96
|
+
## Rules
|
|
97
|
+
|
|
98
|
+
| Code | Severity | What it catches |
|
|
99
|
+
|---|---|---|
|
|
100
|
+
| `TL001` | error | A stateful transformer (`StandardScaler`, `SimpleImputer`, `PCA`, `SMOTE`, …) fitted **before** `train_test_split`. The fitted state carries the test period's statistics. |
|
|
101
|
+
| `TL002` | warning | `train_test_split` without `shuffle=False`. It shuffles by default, which destroys time order. |
|
|
102
|
+
| `TL003` | error | `.shift(-n)` — a negative shift pulls future values into the current row. Fine for building a label, fatal as a feature. |
|
|
103
|
+
| `TL004` | error | `rolling(center=True)` — a centred window averages bars either side of each row, so every value sees the future. |
|
|
104
|
+
| `TL005` | error | `bfill()` / `fillna(method='bfill')` — backward fill propagates future values into earlier rows. |
|
|
105
|
+
| `TL006` | warning | `KFold`, `StratifiedKFold`, or a `cv=<int>` helper on time series. Later folds train on data that follows the test block. |
|
|
106
|
+
| `TL007` | error | A column rescaled with a whole-frame statistic, e.g. `df['z'] = (df['x'] - df['x'].mean()) / df['x'].std()`. |
|
|
107
|
+
|
|
108
|
+
### Suppressing a finding
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
y = df["close"].shift(-1) # noqa: TL003 <- intentional label
|
|
112
|
+
y = df["close"].shift(-1) # noqa <- suppress everything on this line
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## pre-commit
|
|
116
|
+
|
|
117
|
+
```yaml
|
|
118
|
+
repos:
|
|
119
|
+
- repo: https://github.com/MartexHACK/timeleak
|
|
120
|
+
rev: v0.1.0
|
|
121
|
+
hooks:
|
|
122
|
+
- id: timeleak
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## GitHub Actions
|
|
126
|
+
|
|
127
|
+
```yaml
|
|
128
|
+
- run: pip install git+https://github.com/MartexHACK/timeleak
|
|
129
|
+
- run: timeleak src/
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## What it does not do
|
|
133
|
+
|
|
134
|
+
Being honest about the boundaries, because a linter that overpromises gets muted:
|
|
135
|
+
|
|
136
|
+
- **It is syntactic, not semantic.** It cannot follow a leak across function boundaries or
|
|
137
|
+
through a variable that changes meaning. It finds the common shapes, not every leak.
|
|
138
|
+
- **It does not know whether your data is a time series.** `TL002` and `TL006` are warnings
|
|
139
|
+
precisely because plain K-fold is correct on i.i.d. data. On a genuinely shuffled
|
|
140
|
+
cross-sectional dataset, silence them.
|
|
141
|
+
- **A clean run is not a guarantee.** It means the seven known shapes are absent. Walk-forward
|
|
142
|
+
validation on data the model has never touched is still the real test.
|
|
143
|
+
- **Notebooks are not parsed yet.** `.ipynb` support is planned; today you would export to
|
|
144
|
+
`.py` first.
|
|
145
|
+
|
|
146
|
+
If you hit a false positive, that is a bug worth reporting — a noisy linter is a useless one.
|
|
147
|
+
|
|
148
|
+
## Why these seven
|
|
149
|
+
|
|
150
|
+
They are the leaks that survive code review because they look like ordinary data preparation.
|
|
151
|
+
Nobody writes `model.fit(X_test)`; people write `scaler.fit_transform(df)` on line 12 and split
|
|
152
|
+
on line 30, and the two lines never appear on screen together.
|
|
153
|
+
|
|
154
|
+
## Development
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
git clone https://github.com/MartexHACK/timeleak
|
|
158
|
+
cd timeleak
|
|
159
|
+
pip install -e ".[dev]"
|
|
160
|
+
pytest
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
## License
|
|
164
|
+
|
|
165
|
+
MIT — see [LICENSE](LICENSE).
|
timeleak-0.1.0/README.md
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# timeleak
|
|
2
|
+
|
|
3
|
+
**A static linter that finds data leakage in time-series machine learning code.**
|
|
4
|
+
|
|
5
|
+
[](https://github.com/MartexHACK/timeleak/actions/workflows/ci.yml)
|
|
6
|
+
[](https://www.python.org/)
|
|
7
|
+
[](LICENSE)
|
|
8
|
+
[](pyproject.toml)
|
|
9
|
+
|
|
10
|
+
Your backtest shows a Sharpe of 2.4. Live, it is 0.1.
|
|
11
|
+
|
|
12
|
+
Usually nothing exotic went wrong — a scaler was fitted before the train/test split, a
|
|
13
|
+
window was centred, or a gap was filled backwards. Each of these quietly hands the model
|
|
14
|
+
information it could not have had at prediction time. None of them raise an error. None of
|
|
15
|
+
them fail a unit test. The model just looks brilliant until real money is on it.
|
|
16
|
+
|
|
17
|
+
`timeleak` reads your source with Python's `ast` module and points at the specific line.
|
|
18
|
+
|
|
19
|
+
```console
|
|
20
|
+
$ timeleak examples/leaky_pipeline.py
|
|
21
|
+
examples/leaky_pipeline.py:15:6: TL005 backward fill propagates future values into earlier rows [method='bfill']
|
|
22
|
+
fix: use forward fill (ffill), or drop the leading NaNs instead
|
|
23
|
+
examples/leaky_pipeline.py:18:15: TL004 rolling(center=True) centres the window, so each row sees future rows
|
|
24
|
+
fix: use the default center=False so the window only looks backwards
|
|
25
|
+
examples/leaky_pipeline.py:22:54: TL007 column rescaled with a statistic computed over the whole frame, including the test period [df.std()]
|
|
26
|
+
fix: compute the statistic on the training slice only, or use a Pipeline
|
|
27
|
+
examples/leaky_pipeline.py:25:17: TL003 negative .shift() pulls future values into the current row (look-ahead)
|
|
28
|
+
fix: if this builds a forward-looking label that is fine, but it must never become a feature
|
|
29
|
+
examples/leaky_pipeline.py:33:5: TL001 transformer fitted before train/test split leaks test statistics into training [StandardScaler.fit_transform()]
|
|
30
|
+
fix: move the transform inside a sklearn Pipeline so it is refit on each training fold
|
|
31
|
+
examples/leaky_pipeline.py:36:36: TL002 train_test_split shuffles by default, which destroys time order
|
|
32
|
+
fix: pass shuffle=False for time-ordered data, or split on an explicit date boundary
|
|
33
|
+
examples/leaky_pipeline.py:42:10: TL006 K-fold style cross-validation on time series trains on data after the test block [cross_val_score with default KFold]
|
|
34
|
+
fix: use TimeSeriesSplit, or a purged split with an embargo if labels span several bars
|
|
35
|
+
|
|
36
|
+
7 finding(s) in 1 file(s) (5 error, 2 warning)
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Install
|
|
40
|
+
|
|
41
|
+
Not on PyPI yet. Install straight from the repository:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install git+https://github.com/MartexHACK/timeleak
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Once it is published, `pip install timeleak` will work too.
|
|
48
|
+
|
|
49
|
+
Zero runtime dependencies — it is `ast` and the standard library. It never imports or
|
|
50
|
+
executes the code it analyses, so it is safe to point at a repository you do not trust.
|
|
51
|
+
|
|
52
|
+
## Usage
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
timeleak # scan the current directory
|
|
56
|
+
timeleak src/ notebooks/ # scan specific paths
|
|
57
|
+
timeleak --select TL001,TL003 . # only these rules
|
|
58
|
+
timeleak --ignore TL002 . # everything except this one
|
|
59
|
+
timeleak --format json . # machine-readable, for CI
|
|
60
|
+
timeleak --list-rules # the catalogue
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Exit code is `1` when anything is found and `0` when clean, so it drops straight into CI.
|
|
64
|
+
Use `--exit-zero` if you want the report without failing the build.
|
|
65
|
+
|
|
66
|
+
## Rules
|
|
67
|
+
|
|
68
|
+
| Code | Severity | What it catches |
|
|
69
|
+
|---|---|---|
|
|
70
|
+
| `TL001` | error | A stateful transformer (`StandardScaler`, `SimpleImputer`, `PCA`, `SMOTE`, …) fitted **before** `train_test_split`. The fitted state carries the test period's statistics. |
|
|
71
|
+
| `TL002` | warning | `train_test_split` without `shuffle=False`. It shuffles by default, which destroys time order. |
|
|
72
|
+
| `TL003` | error | `.shift(-n)` — a negative shift pulls future values into the current row. Fine for building a label, fatal as a feature. |
|
|
73
|
+
| `TL004` | error | `rolling(center=True)` — a centred window averages bars either side of each row, so every value sees the future. |
|
|
74
|
+
| `TL005` | error | `bfill()` / `fillna(method='bfill')` — backward fill propagates future values into earlier rows. |
|
|
75
|
+
| `TL006` | warning | `KFold`, `StratifiedKFold`, or a `cv=<int>` helper on time series. Later folds train on data that follows the test block. |
|
|
76
|
+
| `TL007` | error | A column rescaled with a whole-frame statistic, e.g. `df['z'] = (df['x'] - df['x'].mean()) / df['x'].std()`. |
|
|
77
|
+
|
|
78
|
+
### Suppressing a finding
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
y = df["close"].shift(-1) # noqa: TL003 <- intentional label
|
|
82
|
+
y = df["close"].shift(-1) # noqa <- suppress everything on this line
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## pre-commit
|
|
86
|
+
|
|
87
|
+
```yaml
|
|
88
|
+
repos:
|
|
89
|
+
- repo: https://github.com/MartexHACK/timeleak
|
|
90
|
+
rev: v0.1.0
|
|
91
|
+
hooks:
|
|
92
|
+
- id: timeleak
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## GitHub Actions
|
|
96
|
+
|
|
97
|
+
```yaml
|
|
98
|
+
- run: pip install git+https://github.com/MartexHACK/timeleak
|
|
99
|
+
- run: timeleak src/
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
## What it does not do
|
|
103
|
+
|
|
104
|
+
Being honest about the boundaries, because a linter that overpromises gets muted:
|
|
105
|
+
|
|
106
|
+
- **It is syntactic, not semantic.** It cannot follow a leak across function boundaries or
|
|
107
|
+
through a variable that changes meaning. It finds the common shapes, not every leak.
|
|
108
|
+
- **It does not know whether your data is a time series.** `TL002` and `TL006` are warnings
|
|
109
|
+
precisely because plain K-fold is correct on i.i.d. data. On a genuinely shuffled
|
|
110
|
+
cross-sectional dataset, silence them.
|
|
111
|
+
- **A clean run is not a guarantee.** It means the seven known shapes are absent. Walk-forward
|
|
112
|
+
validation on data the model has never touched is still the real test.
|
|
113
|
+
- **Notebooks are not parsed yet.** `.ipynb` support is planned; today you would export to
|
|
114
|
+
`.py` first.
|
|
115
|
+
|
|
116
|
+
If you hit a false positive, that is a bug worth reporting — a noisy linter is a useless one.
|
|
117
|
+
|
|
118
|
+
## Why these seven
|
|
119
|
+
|
|
120
|
+
They are the leaks that survive code review because they look like ordinary data preparation.
|
|
121
|
+
Nobody writes `model.fit(X_test)`; people write `scaler.fit_transform(df)` on line 12 and split
|
|
122
|
+
on line 30, and the two lines never appear on screen together.
|
|
123
|
+
|
|
124
|
+
## Development
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
git clone https://github.com/MartexHACK/timeleak
|
|
128
|
+
cd timeleak
|
|
129
|
+
pip install -e ".[dev]"
|
|
130
|
+
pytest
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## License
|
|
134
|
+
|
|
135
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "timeleak"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Static linter that finds data-leakage patterns in time-series machine learning code."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Martex", email = "mvnikolov2009@gmail.com" }]
|
|
13
|
+
keywords = [
|
|
14
|
+
"data-leakage",
|
|
15
|
+
"linter",
|
|
16
|
+
"static-analysis",
|
|
17
|
+
"time-series",
|
|
18
|
+
"machine-learning",
|
|
19
|
+
"backtesting",
|
|
20
|
+
"quantitative-finance",
|
|
21
|
+
"pre-commit",
|
|
22
|
+
]
|
|
23
|
+
classifiers = [
|
|
24
|
+
"Development Status :: 4 - Beta",
|
|
25
|
+
"Environment :: Console",
|
|
26
|
+
"Intended Audience :: Developers",
|
|
27
|
+
"Intended Audience :: Science/Research",
|
|
28
|
+
"License :: OSI Approved :: MIT License",
|
|
29
|
+
"Programming Language :: Python :: 3",
|
|
30
|
+
"Programming Language :: Python :: 3.9",
|
|
31
|
+
"Programming Language :: Python :: 3.10",
|
|
32
|
+
"Programming Language :: Python :: 3.11",
|
|
33
|
+
"Programming Language :: Python :: 3.12",
|
|
34
|
+
"Programming Language :: Python :: 3.13",
|
|
35
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
36
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
37
|
+
"Typing :: Typed",
|
|
38
|
+
]
|
|
39
|
+
dependencies = []
|
|
40
|
+
|
|
41
|
+
[project.optional-dependencies]
|
|
42
|
+
dev = ["pytest>=7.0"]
|
|
43
|
+
|
|
44
|
+
[project.urls]
|
|
45
|
+
Homepage = "https://github.com/MartexHACK/timeleak"
|
|
46
|
+
Issues = "https://github.com/MartexHACK/timeleak/issues"
|
|
47
|
+
|
|
48
|
+
[project.scripts]
|
|
49
|
+
timeleak = "timeleak.cli:main"
|
|
50
|
+
|
|
51
|
+
[tool.setuptools.packages.find]
|
|
52
|
+
where = ["src"]
|
|
53
|
+
|
|
54
|
+
[tool.pytest.ini_options]
|
|
55
|
+
testpaths = ["tests"]
|
|
56
|
+
pythonpath = ["src"]
|
timeleak-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""timeleak - a static linter for data leakage in time-series ML code."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.0"
|
|
6
|
+
|
|
7
|
+
from .analyzer import analyze_file, analyze_source, iter_python_files
|
|
8
|
+
from .rules import ALL_RULES, RULES_BY_CODE, Finding, Rule, Severity
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"__version__",
|
|
12
|
+
"ALL_RULES",
|
|
13
|
+
"RULES_BY_CODE",
|
|
14
|
+
"Finding",
|
|
15
|
+
"Rule",
|
|
16
|
+
"Severity",
|
|
17
|
+
"analyze_file",
|
|
18
|
+
"analyze_source",
|
|
19
|
+
"iter_python_files",
|
|
20
|
+
]
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""AST analysis that turns Python source into leakage findings.
|
|
2
|
+
|
|
3
|
+
The analyzer is deliberately syntactic. It never imports or executes the file
|
|
4
|
+
under test, so it is safe to run on untrusted code and inside pre-commit.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import ast
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
from typing import Iterator
|
|
13
|
+
|
|
14
|
+
from .rules import (
|
|
15
|
+
CV_CONSUMERS,
|
|
16
|
+
FULL_SAMPLE_REDUCERS,
|
|
17
|
+
TL001,
|
|
18
|
+
TL002,
|
|
19
|
+
TL003,
|
|
20
|
+
TL004,
|
|
21
|
+
TL005,
|
|
22
|
+
TL006,
|
|
23
|
+
TL007,
|
|
24
|
+
NON_TIME_AWARE_CV,
|
|
25
|
+
STATEFUL_TRANSFORMERS,
|
|
26
|
+
Finding,
|
|
27
|
+
Rule,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
_NOQA = re.compile(r"#\s*noqa(?::\s*(?P<codes>[A-Z0-9,\s]+))?", re.IGNORECASE)
|
|
31
|
+
|
|
32
|
+
# Reductions chained off one of these are windowed, not whole-sample, so they
|
|
33
|
+
# are legitimate: df["x"].rolling(20).mean() looks backwards by construction.
|
|
34
|
+
_WINDOWED = frozenset({"rolling", "expanding", "ewm", "groupby", "resample"})
|
|
35
|
+
|
|
36
|
+
_FIT_METHODS = frozenset({"fit", "fit_transform"})
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _root_name(node: ast.AST) -> str | None:
|
|
40
|
+
"""Return the identifier at the base of an attribute/subscript/call chain."""
|
|
41
|
+
while True:
|
|
42
|
+
if isinstance(node, ast.Name):
|
|
43
|
+
return node.id
|
|
44
|
+
if isinstance(node, ast.Attribute):
|
|
45
|
+
node = node.value
|
|
46
|
+
elif isinstance(node, ast.Subscript):
|
|
47
|
+
node = node.value
|
|
48
|
+
elif isinstance(node, ast.Call):
|
|
49
|
+
node = node.func
|
|
50
|
+
else:
|
|
51
|
+
return None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _called_name(call: ast.Call) -> str | None:
|
|
55
|
+
"""Return the final component of the callee, e.g. `fit` or `StandardScaler`."""
|
|
56
|
+
if isinstance(call.func, ast.Attribute):
|
|
57
|
+
return call.func.attr
|
|
58
|
+
if isinstance(call.func, ast.Name):
|
|
59
|
+
return call.func.id
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _keyword(call: ast.Call, name: str) -> ast.expr | None:
|
|
64
|
+
for kw in call.keywords:
|
|
65
|
+
if kw.arg == name:
|
|
66
|
+
return kw.value
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _is_true(node: ast.expr | None) -> bool:
|
|
71
|
+
return isinstance(node, ast.Constant) and node.value is True
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _negative_int(node: ast.expr | None) -> bool:
|
|
75
|
+
return (
|
|
76
|
+
isinstance(node, ast.UnaryOp)
|
|
77
|
+
and isinstance(node.op, ast.USub)
|
|
78
|
+
and isinstance(node.operand, ast.Constant)
|
|
79
|
+
and isinstance(node.operand.value, int)
|
|
80
|
+
and node.operand.value != 0
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _is_full_sample_reduction(call: ast.Call) -> bool:
|
|
85
|
+
"""True for df.mean() but not for df.rolling(20).mean()."""
|
|
86
|
+
if not isinstance(call.func, ast.Attribute):
|
|
87
|
+
return False
|
|
88
|
+
if call.func.attr not in FULL_SAMPLE_REDUCERS:
|
|
89
|
+
return False
|
|
90
|
+
receiver = call.func.value
|
|
91
|
+
if isinstance(receiver, ast.Call) and _called_name(receiver) in _WINDOWED:
|
|
92
|
+
return False
|
|
93
|
+
return True
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class _Analyzer(ast.NodeVisitor):
|
|
97
|
+
def __init__(self, path: str, lines: list[str]) -> None:
|
|
98
|
+
self.path = path
|
|
99
|
+
self.lines = lines
|
|
100
|
+
self.findings: list[Finding] = []
|
|
101
|
+
self.transformer_vars: dict[str, str] = {}
|
|
102
|
+
self.fit_events: list[tuple[ast.Call, str]] = []
|
|
103
|
+
self.split_lines: list[int] = []
|
|
104
|
+
|
|
105
|
+
# -- reporting ---------------------------------------------------------
|
|
106
|
+
def _suppressed(self, line: int, code: str) -> bool:
|
|
107
|
+
if not 1 <= line <= len(self.lines):
|
|
108
|
+
return False
|
|
109
|
+
match = _NOQA.search(self.lines[line - 1])
|
|
110
|
+
if match is None:
|
|
111
|
+
return False
|
|
112
|
+
codes = match.group("codes")
|
|
113
|
+
if codes is None:
|
|
114
|
+
return True
|
|
115
|
+
return code in {c.strip().upper() for c in codes.split(",") if c.strip()}
|
|
116
|
+
|
|
117
|
+
def _report(self, rule: Rule, node: ast.AST, context: str = "") -> None:
|
|
118
|
+
line = getattr(node, "lineno", 0)
|
|
119
|
+
if self._suppressed(line, rule.code):
|
|
120
|
+
return
|
|
121
|
+
self.findings.append(
|
|
122
|
+
Finding(
|
|
123
|
+
rule=rule,
|
|
124
|
+
path=self.path,
|
|
125
|
+
line=line,
|
|
126
|
+
col=getattr(node, "col_offset", 0) + 1,
|
|
127
|
+
context=context,
|
|
128
|
+
)
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
# -- passes ------------------------------------------------------------
|
|
132
|
+
def collect_assignments(self, tree: ast.AST) -> None:
|
|
133
|
+
"""Pre-pass: remember which locals hold a stateful transformer."""
|
|
134
|
+
for node in ast.walk(tree):
|
|
135
|
+
if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Call):
|
|
136
|
+
continue
|
|
137
|
+
ctor = _called_name(node.value)
|
|
138
|
+
if ctor not in STATEFUL_TRANSFORMERS:
|
|
139
|
+
continue
|
|
140
|
+
for target in node.targets:
|
|
141
|
+
if isinstance(target, ast.Name):
|
|
142
|
+
self.transformer_vars[target.id] = ctor
|
|
143
|
+
|
|
144
|
+
def visit_Assign(self, node: ast.Assign) -> None:
|
|
145
|
+
for target in node.targets:
|
|
146
|
+
if not isinstance(target, ast.Subscript):
|
|
147
|
+
continue
|
|
148
|
+
base = _root_name(target)
|
|
149
|
+
if base is None:
|
|
150
|
+
continue
|
|
151
|
+
for inner in ast.walk(node.value):
|
|
152
|
+
if (
|
|
153
|
+
isinstance(inner, ast.Call)
|
|
154
|
+
and _is_full_sample_reduction(inner)
|
|
155
|
+
and _root_name(inner) == base
|
|
156
|
+
):
|
|
157
|
+
self._report(TL007, inner, base + "." + inner.func.attr + "()")
|
|
158
|
+
break
|
|
159
|
+
self.generic_visit(node)
|
|
160
|
+
|
|
161
|
+
def visit_Call(self, node: ast.Call) -> None:
|
|
162
|
+
name = _called_name(node)
|
|
163
|
+
|
|
164
|
+
if name == "train_test_split":
|
|
165
|
+
self.split_lines.append(node.lineno)
|
|
166
|
+
shuffle = _keyword(node, "shuffle")
|
|
167
|
+
if shuffle is None or _is_true(shuffle):
|
|
168
|
+
self._report(TL002, node)
|
|
169
|
+
|
|
170
|
+
elif name in _FIT_METHODS and isinstance(node.func, ast.Attribute):
|
|
171
|
+
receiver = node.func.value
|
|
172
|
+
label: str | None = None
|
|
173
|
+
if isinstance(receiver, ast.Name) and receiver.id in self.transformer_vars:
|
|
174
|
+
label = self.transformer_vars[receiver.id]
|
|
175
|
+
elif isinstance(receiver, ast.Call) and _called_name(receiver) in STATEFUL_TRANSFORMERS:
|
|
176
|
+
label = _called_name(receiver)
|
|
177
|
+
if label is not None:
|
|
178
|
+
self.fit_events.append((node, label))
|
|
179
|
+
|
|
180
|
+
elif name == "shift":
|
|
181
|
+
periods = node.args[0] if node.args else _keyword(node, "periods")
|
|
182
|
+
if _negative_int(periods):
|
|
183
|
+
self._report(TL003, node)
|
|
184
|
+
|
|
185
|
+
elif name == "rolling":
|
|
186
|
+
if _is_true(_keyword(node, "center")):
|
|
187
|
+
self._report(TL004, node)
|
|
188
|
+
|
|
189
|
+
elif name in {"bfill", "backfill"}:
|
|
190
|
+
self._report(TL005, node, "." + name + "()")
|
|
191
|
+
|
|
192
|
+
elif name == "fillna":
|
|
193
|
+
method = _keyword(node, "method")
|
|
194
|
+
if isinstance(method, ast.Constant) and method.value in {"bfill", "backfill"}:
|
|
195
|
+
self._report(TL005, node, "method=" + repr(method.value))
|
|
196
|
+
|
|
197
|
+
elif name in NON_TIME_AWARE_CV:
|
|
198
|
+
self._report(TL006, node, name)
|
|
199
|
+
|
|
200
|
+
elif name in CV_CONSUMERS:
|
|
201
|
+
cv = _keyword(node, "cv")
|
|
202
|
+
if cv is None or (isinstance(cv, ast.Constant) and isinstance(cv.value, int)):
|
|
203
|
+
self._report(TL006, node, name + " with default KFold")
|
|
204
|
+
|
|
205
|
+
self.generic_visit(node)
|
|
206
|
+
|
|
207
|
+
def finalize(self) -> None:
|
|
208
|
+
"""TL001 needs ordering, so it runs once the whole module is walked."""
|
|
209
|
+
if not self.split_lines:
|
|
210
|
+
return
|
|
211
|
+
first_split = min(self.split_lines)
|
|
212
|
+
for call, label in self.fit_events:
|
|
213
|
+
if call.lineno < first_split:
|
|
214
|
+
self._report(TL001, call, label + "." + call.func.attr + "()")
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def analyze_source(source: str, path: str = "<string>") -> list[Finding]:
|
|
218
|
+
"""Analyze a source string and return findings sorted by position."""
|
|
219
|
+
try:
|
|
220
|
+
tree = ast.parse(source, filename=path)
|
|
221
|
+
except SyntaxError as exc:
|
|
222
|
+
raise ValueError(path + ": could not parse (" + str(exc.msg) + ")") from exc
|
|
223
|
+
|
|
224
|
+
analyzer = _Analyzer(path, source.splitlines())
|
|
225
|
+
analyzer.collect_assignments(tree)
|
|
226
|
+
analyzer.visit(tree)
|
|
227
|
+
analyzer.finalize()
|
|
228
|
+
return sorted(analyzer.findings, key=lambda f: (f.line, f.col, f.code))
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def analyze_file(path: str) -> list[Finding]:
|
|
232
|
+
with open(path, "r", encoding="utf-8") as handle:
|
|
233
|
+
return analyze_source(handle.read(), path)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def iter_python_files(paths: list[str], exclude: frozenset[str] = frozenset()) -> Iterator[str]:
|
|
237
|
+
"""Yield .py files under the given paths, skipping excluded directory names."""
|
|
238
|
+
for target in paths:
|
|
239
|
+
if os.path.isfile(target):
|
|
240
|
+
if target.endswith(".py"):
|
|
241
|
+
yield target
|
|
242
|
+
continue
|
|
243
|
+
for root, dirs, files in os.walk(target):
|
|
244
|
+
dirs[:] = [d for d in dirs if d not in exclude and not d.startswith(".")]
|
|
245
|
+
for filename in sorted(files):
|
|
246
|
+
if filename.endswith(".py"):
|
|
247
|
+
yield os.path.join(root, filename)
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""Command line interface for timeleak."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from typing import Sequence
|
|
9
|
+
|
|
10
|
+
from . import __version__
|
|
11
|
+
from .analyzer import analyze_file, iter_python_files
|
|
12
|
+
from .rules import ALL_RULES, RULES_BY_CODE, Finding, Severity
|
|
13
|
+
|
|
14
|
+
DEFAULT_EXCLUDES = frozenset(
|
|
15
|
+
{"__pycache__", "node_modules", "venv", "env", "build", "dist", "site-packages"}
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
_COLORS = {
|
|
19
|
+
Severity.ERROR: "\033[31m",
|
|
20
|
+
Severity.WARNING: "\033[33m",
|
|
21
|
+
}
|
|
22
|
+
_RESET = "\033[0m"
|
|
23
|
+
_DIM = "\033[2m"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _supports_color(stream) -> bool:
|
|
27
|
+
return hasattr(stream, "isatty") and stream.isatty()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _render_text(findings: list[Finding], color: bool, stream) -> None:
|
|
31
|
+
for finding in findings:
|
|
32
|
+
rule = finding.rule
|
|
33
|
+
location = f"{finding.path}:{finding.line}:{finding.col}"
|
|
34
|
+
code = rule.code
|
|
35
|
+
if color:
|
|
36
|
+
code = f"{_COLORS[rule.severity]}{code}{_RESET}"
|
|
37
|
+
context = f" [{finding.context}]" if finding.context else ""
|
|
38
|
+
print(f"{location}: {code} {rule.message}{context}", file=stream)
|
|
39
|
+
fix = f" fix: {rule.fix}"
|
|
40
|
+
print(f"{_DIM}{fix}{_RESET}" if color else fix, file=stream)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _render_json(findings: list[Finding], stream) -> None:
|
|
44
|
+
payload = [
|
|
45
|
+
{
|
|
46
|
+
"path": f.path,
|
|
47
|
+
"line": f.line,
|
|
48
|
+
"column": f.col,
|
|
49
|
+
"code": f.rule.code,
|
|
50
|
+
"name": f.rule.name,
|
|
51
|
+
"severity": str(f.rule.severity),
|
|
52
|
+
"message": f.rule.message,
|
|
53
|
+
"fix": f.rule.fix,
|
|
54
|
+
"context": f.context,
|
|
55
|
+
}
|
|
56
|
+
for f in findings
|
|
57
|
+
]
|
|
58
|
+
json.dump(payload, stream, indent=2)
|
|
59
|
+
stream.write("\n")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
63
|
+
parser = argparse.ArgumentParser(
|
|
64
|
+
prog="timeleak",
|
|
65
|
+
description="Find data-leakage patterns in time-series machine learning code.",
|
|
66
|
+
)
|
|
67
|
+
parser.add_argument("paths", nargs="*", default=["."], help="files or directories (default: .)")
|
|
68
|
+
parser.add_argument("--format", choices=("text", "json"), default="text")
|
|
69
|
+
parser.add_argument("--select", metavar="CODES", help="only report these codes, comma separated")
|
|
70
|
+
parser.add_argument("--ignore", metavar="CODES", help="suppress these codes, comma separated")
|
|
71
|
+
parser.add_argument(
|
|
72
|
+
"--exclude",
|
|
73
|
+
metavar="NAMES",
|
|
74
|
+
help="extra directory names to skip, comma separated",
|
|
75
|
+
)
|
|
76
|
+
parser.add_argument(
|
|
77
|
+
"--exit-zero",
|
|
78
|
+
action="store_true",
|
|
79
|
+
help="always exit 0, even when findings are reported",
|
|
80
|
+
)
|
|
81
|
+
parser.add_argument("--list-rules", action="store_true", help="print the rule catalogue and exit")
|
|
82
|
+
parser.add_argument("--version", action="version", version=f"timeleak {__version__}")
|
|
83
|
+
return parser
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _parse_codes(raw: str | None) -> set[str] | None:
|
|
87
|
+
if not raw:
|
|
88
|
+
return None
|
|
89
|
+
codes = {c.strip().upper() for c in raw.split(",") if c.strip()}
|
|
90
|
+
unknown = codes - set(RULES_BY_CODE)
|
|
91
|
+
if unknown:
|
|
92
|
+
raise SystemExit(f"timeleak: unknown rule code(s): {', '.join(sorted(unknown))}")
|
|
93
|
+
return codes
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
97
|
+
args = _build_parser().parse_args(argv)
|
|
98
|
+
|
|
99
|
+
if args.list_rules:
|
|
100
|
+
for rule in ALL_RULES:
|
|
101
|
+
print(f"{rule.code} {rule.severity.value:<7} {rule.name}")
|
|
102
|
+
print(f" {rule.message}")
|
|
103
|
+
return 0
|
|
104
|
+
|
|
105
|
+
select = _parse_codes(args.select)
|
|
106
|
+
ignore = _parse_codes(args.ignore) or set()
|
|
107
|
+
excludes = DEFAULT_EXCLUDES | {
|
|
108
|
+
name.strip() for name in (args.exclude or "").split(",") if name.strip()
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
findings: list[Finding] = []
|
|
112
|
+
scanned = 0
|
|
113
|
+
for path in iter_python_files(list(args.paths), excludes):
|
|
114
|
+
scanned += 1
|
|
115
|
+
try:
|
|
116
|
+
findings.extend(analyze_file(path))
|
|
117
|
+
except (ValueError, OSError) as exc:
|
|
118
|
+
print(f"timeleak: skipped {exc}", file=sys.stderr)
|
|
119
|
+
|
|
120
|
+
if select is not None:
|
|
121
|
+
findings = [f for f in findings if f.code in select]
|
|
122
|
+
findings = [f for f in findings if f.code not in ignore]
|
|
123
|
+
|
|
124
|
+
if args.format == "json":
|
|
125
|
+
_render_json(findings, sys.stdout)
|
|
126
|
+
else:
|
|
127
|
+
_render_text(findings, _supports_color(sys.stdout), sys.stdout)
|
|
128
|
+
errors = sum(1 for f in findings if f.rule.severity is Severity.ERROR)
|
|
129
|
+
warnings = len(findings) - errors
|
|
130
|
+
summary = (
|
|
131
|
+
f"\n{len(findings)} finding(s) in {scanned} file(s) "
|
|
132
|
+
f"({errors} error, {warnings} warning)"
|
|
133
|
+
)
|
|
134
|
+
print(summary if findings else f"clean: no leakage patterns in {scanned} file(s)")
|
|
135
|
+
|
|
136
|
+
if args.exit_zero:
|
|
137
|
+
return 0
|
|
138
|
+
return 1 if findings else 0
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
if __name__ == "__main__":
|
|
142
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""Rule catalogue and the Finding record produced by the analyzer."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from enum import Enum
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Severity(str, Enum):
|
|
10
|
+
"""How confident the rule is that it has found a real leak."""
|
|
11
|
+
|
|
12
|
+
ERROR = "error"
|
|
13
|
+
WARNING = "warning"
|
|
14
|
+
|
|
15
|
+
def __str__(self) -> str: # pragma: no cover - trivial
|
|
16
|
+
return self.value
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class Rule:
|
|
21
|
+
code: str
|
|
22
|
+
name: str
|
|
23
|
+
severity: Severity
|
|
24
|
+
message: str
|
|
25
|
+
fix: str
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class Finding:
|
|
30
|
+
"""One rule violation, anchored to a source location."""
|
|
31
|
+
|
|
32
|
+
rule: Rule
|
|
33
|
+
path: str
|
|
34
|
+
line: int
|
|
35
|
+
col: int
|
|
36
|
+
context: str = ""
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def code(self) -> str:
|
|
40
|
+
return self.rule.code
|
|
41
|
+
|
|
42
|
+
def format_text(self) -> str:
|
|
43
|
+
head = f"{self.path}:{self.line}:{self.col}: {self.rule.code} {self.rule.message}"
|
|
44
|
+
if self.context:
|
|
45
|
+
head += f" [{self.context}]"
|
|
46
|
+
return f"{head}\n fix: {self.rule.fix}"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
TL001 = Rule(
|
|
50
|
+
code="TL001",
|
|
51
|
+
name="fit-before-split",
|
|
52
|
+
severity=Severity.ERROR,
|
|
53
|
+
message="transformer fitted before train/test split leaks test statistics into training",
|
|
54
|
+
fix="move the transform inside a sklearn Pipeline so it is refit on each training fold",
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
TL002 = Rule(
|
|
58
|
+
code="TL002",
|
|
59
|
+
name="shuffled-split",
|
|
60
|
+
severity=Severity.WARNING,
|
|
61
|
+
message="train_test_split shuffles by default, which destroys time order",
|
|
62
|
+
fix="pass shuffle=False for time-ordered data, or split on an explicit date boundary",
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
TL003 = Rule(
|
|
66
|
+
code="TL003",
|
|
67
|
+
name="negative-shift",
|
|
68
|
+
severity=Severity.ERROR,
|
|
69
|
+
message="negative .shift() pulls future values into the current row (look-ahead)",
|
|
70
|
+
fix="if this builds a forward-looking label that is fine, but it must never become a feature",
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
TL004 = Rule(
|
|
74
|
+
code="TL004",
|
|
75
|
+
name="centered-window",
|
|
76
|
+
severity=Severity.ERROR,
|
|
77
|
+
message="rolling(center=True) centres the window, so each row sees future rows",
|
|
78
|
+
fix="use the default center=False so the window only looks backwards",
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
TL005 = Rule(
|
|
82
|
+
code="TL005",
|
|
83
|
+
name="backward-fill",
|
|
84
|
+
severity=Severity.ERROR,
|
|
85
|
+
message="backward fill propagates future values into earlier rows",
|
|
86
|
+
fix="use forward fill (ffill), or drop the leading NaNs instead",
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
TL006 = Rule(
|
|
90
|
+
code="TL006",
|
|
91
|
+
name="non-time-aware-cv",
|
|
92
|
+
severity=Severity.WARNING,
|
|
93
|
+
message="K-fold style cross-validation on time series trains on data after the test block",
|
|
94
|
+
fix="use TimeSeriesSplit, or a purged split with an embargo if labels span several bars",
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
TL007 = Rule(
|
|
98
|
+
code="TL007",
|
|
99
|
+
name="full-sample-statistic",
|
|
100
|
+
severity=Severity.ERROR,
|
|
101
|
+
message="column rescaled with a statistic computed over the whole frame, including the test period",
|
|
102
|
+
fix="compute the statistic on the training slice only, or use a Pipeline",
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
ALL_RULES: tuple[Rule, ...] = (TL001, TL002, TL003, TL004, TL005, TL006, TL007)
|
|
106
|
+
|
|
107
|
+
RULES_BY_CODE: dict[str, Rule] = {r.code: r for r in ALL_RULES}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
# Transformers whose fitted state carries dataset-wide statistics. Fitting any of
|
|
111
|
+
# these on the full frame before splitting is the classic silent leak.
|
|
112
|
+
STATEFUL_TRANSFORMERS: frozenset[str] = frozenset(
|
|
113
|
+
{
|
|
114
|
+
"StandardScaler",
|
|
115
|
+
"MinMaxScaler",
|
|
116
|
+
"MaxAbsScaler",
|
|
117
|
+
"RobustScaler",
|
|
118
|
+
"Normalizer",
|
|
119
|
+
"QuantileTransformer",
|
|
120
|
+
"PowerTransformer",
|
|
121
|
+
"SimpleImputer",
|
|
122
|
+
"KNNImputer",
|
|
123
|
+
"IterativeImputer",
|
|
124
|
+
"LabelEncoder",
|
|
125
|
+
"OrdinalEncoder",
|
|
126
|
+
"OneHotEncoder",
|
|
127
|
+
"TargetEncoder",
|
|
128
|
+
"PCA",
|
|
129
|
+
"TruncatedSVD",
|
|
130
|
+
"FactorAnalysis",
|
|
131
|
+
"SelectKBest",
|
|
132
|
+
"SelectPercentile",
|
|
133
|
+
"VarianceThreshold",
|
|
134
|
+
"SMOTE",
|
|
135
|
+
"ADASYN",
|
|
136
|
+
"RandomOverSampler",
|
|
137
|
+
"RandomUnderSampler",
|
|
138
|
+
}
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
# Cross-validator constructors that ignore time ordering entirely.
|
|
142
|
+
NON_TIME_AWARE_CV: frozenset[str] = frozenset(
|
|
143
|
+
{"KFold", "StratifiedKFold", "ShuffleSplit", "StratifiedShuffleSplit", "RepeatedKFold"}
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
# Helpers that silently default to KFold when `cv` is an int or omitted.
|
|
147
|
+
CV_CONSUMERS: frozenset[str] = frozenset(
|
|
148
|
+
{"cross_val_score", "cross_validate", "cross_val_predict", "GridSearchCV", "RandomizedSearchCV"}
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
# Whole-frame reductions that must not be computed before the split.
|
|
152
|
+
FULL_SAMPLE_REDUCERS: frozenset[str] = frozenset(
|
|
153
|
+
{"mean", "std", "var", "min", "max", "median", "quantile", "sum"}
|
|
154
|
+
)
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: timeleak
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Static linter that finds data-leakage patterns in time-series machine learning code.
|
|
5
|
+
Author-email: Martex <mvnikolov2009@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/MartexHACK/timeleak
|
|
8
|
+
Project-URL: Issues, https://github.com/MartexHACK/timeleak/issues
|
|
9
|
+
Keywords: data-leakage,linter,static-analysis,time-series,machine-learning,backtesting,quantitative-finance,pre-commit
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.9
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
# timeleak
|
|
32
|
+
|
|
33
|
+
**A static linter that finds data leakage in time-series machine learning code.**
|
|
34
|
+
|
|
35
|
+
[](https://github.com/MartexHACK/timeleak/actions/workflows/ci.yml)
|
|
36
|
+
[](https://www.python.org/)
|
|
37
|
+
[](LICENSE)
|
|
38
|
+
[](pyproject.toml)
|
|
39
|
+
|
|
40
|
+
Your backtest shows a Sharpe of 2.4. Live, it is 0.1.
|
|
41
|
+
|
|
42
|
+
Usually nothing exotic went wrong — a scaler was fitted before the train/test split, a
|
|
43
|
+
window was centred, or a gap was filled backwards. Each of these quietly hands the model
|
|
44
|
+
information it could not have had at prediction time. None of them raise an error. None of
|
|
45
|
+
them fail a unit test. The model just looks brilliant until real money is on it.
|
|
46
|
+
|
|
47
|
+
`timeleak` reads your source with Python's `ast` module and points at the specific line.
|
|
48
|
+
|
|
49
|
+
```console
|
|
50
|
+
$ timeleak examples/leaky_pipeline.py
|
|
51
|
+
examples/leaky_pipeline.py:15:6: TL005 backward fill propagates future values into earlier rows [method='bfill']
|
|
52
|
+
fix: use forward fill (ffill), or drop the leading NaNs instead
|
|
53
|
+
examples/leaky_pipeline.py:18:15: TL004 rolling(center=True) centres the window, so each row sees future rows
|
|
54
|
+
fix: use the default center=False so the window only looks backwards
|
|
55
|
+
examples/leaky_pipeline.py:22:54: TL007 column rescaled with a statistic computed over the whole frame, including the test period [df.std()]
|
|
56
|
+
fix: compute the statistic on the training slice only, or use a Pipeline
|
|
57
|
+
examples/leaky_pipeline.py:25:17: TL003 negative .shift() pulls future values into the current row (look-ahead)
|
|
58
|
+
fix: if this builds a forward-looking label that is fine, but it must never become a feature
|
|
59
|
+
examples/leaky_pipeline.py:33:5: TL001 transformer fitted before train/test split leaks test statistics into training [StandardScaler.fit_transform()]
|
|
60
|
+
fix: move the transform inside a sklearn Pipeline so it is refit on each training fold
|
|
61
|
+
examples/leaky_pipeline.py:36:36: TL002 train_test_split shuffles by default, which destroys time order
|
|
62
|
+
fix: pass shuffle=False for time-ordered data, or split on an explicit date boundary
|
|
63
|
+
examples/leaky_pipeline.py:42:10: TL006 K-fold style cross-validation on time series trains on data after the test block [cross_val_score with default KFold]
|
|
64
|
+
fix: use TimeSeriesSplit, or a purged split with an embargo if labels span several bars
|
|
65
|
+
|
|
66
|
+
7 finding(s) in 1 file(s) (5 error, 2 warning)
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Install
|
|
70
|
+
|
|
71
|
+
Not on PyPI yet. Install straight from the repository:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
pip install git+https://github.com/MartexHACK/timeleak
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Once it is published, `pip install timeleak` will work too.
|
|
78
|
+
|
|
79
|
+
Zero runtime dependencies — it is `ast` and the standard library. It never imports or
|
|
80
|
+
executes the code it analyses, so it is safe to point at a repository you do not trust.
|
|
81
|
+
|
|
82
|
+
## Usage
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
timeleak # scan the current directory
|
|
86
|
+
timeleak src/ notebooks/ # scan specific paths
|
|
87
|
+
timeleak --select TL001,TL003 . # only these rules
|
|
88
|
+
timeleak --ignore TL002 . # everything except this one
|
|
89
|
+
timeleak --format json . # machine-readable, for CI
|
|
90
|
+
timeleak --list-rules # the catalogue
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Exit code is `1` when anything is found and `0` when clean, so it drops straight into CI.
|
|
94
|
+
Use `--exit-zero` if you want the report without failing the build.
|
|
95
|
+
|
|
96
|
+
## Rules
|
|
97
|
+
|
|
98
|
+
| Code | Severity | What it catches |
|
|
99
|
+
|---|---|---|
|
|
100
|
+
| `TL001` | error | A stateful transformer (`StandardScaler`, `SimpleImputer`, `PCA`, `SMOTE`, …) fitted **before** `train_test_split`. The fitted state carries the test period's statistics. |
|
|
101
|
+
| `TL002` | warning | `train_test_split` without `shuffle=False`. It shuffles by default, which destroys time order. |
|
|
102
|
+
| `TL003` | error | `.shift(-n)` — a negative shift pulls future values into the current row. Fine for building a label, fatal as a feature. |
|
|
103
|
+
| `TL004` | error | `rolling(center=True)` — a centred window averages bars either side of each row, so every value sees the future. |
|
|
104
|
+
| `TL005` | error | `bfill()` / `fillna(method='bfill')` — backward fill propagates future values into earlier rows. |
|
|
105
|
+
| `TL006` | warning | `KFold`, `StratifiedKFold`, or a `cv=<int>` helper on time series. Later folds train on data that follows the test block. |
|
|
106
|
+
| `TL007` | error | A column rescaled with a whole-frame statistic, e.g. `df['z'] = (df['x'] - df['x'].mean()) / df['x'].std()`. |
|
|
107
|
+
|
|
108
|
+
### Suppressing a finding
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
y = df["close"].shift(-1) # noqa: TL003 <- intentional label
|
|
112
|
+
y = df["close"].shift(-1) # noqa <- suppress everything on this line
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## pre-commit
|
|
116
|
+
|
|
117
|
+
```yaml
|
|
118
|
+
repos:
|
|
119
|
+
- repo: https://github.com/MartexHACK/timeleak
|
|
120
|
+
rev: v0.1.0
|
|
121
|
+
hooks:
|
|
122
|
+
- id: timeleak
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## GitHub Actions
|
|
126
|
+
|
|
127
|
+
```yaml
|
|
128
|
+
- run: pip install git+https://github.com/MartexHACK/timeleak
|
|
129
|
+
- run: timeleak src/
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## What it does not do
|
|
133
|
+
|
|
134
|
+
Being honest about the boundaries, because a linter that overpromises gets muted:
|
|
135
|
+
|
|
136
|
+
- **It is syntactic, not semantic.** It cannot follow a leak across function boundaries or
|
|
137
|
+
through a variable that changes meaning. It finds the common shapes, not every leak.
|
|
138
|
+
- **It does not know whether your data is a time series.** `TL002` and `TL006` are warnings
|
|
139
|
+
precisely because plain K-fold is correct on i.i.d. data. On a genuinely shuffled
|
|
140
|
+
cross-sectional dataset, silence them.
|
|
141
|
+
- **A clean run is not a guarantee.** It means the seven known shapes are absent. Walk-forward
|
|
142
|
+
validation on data the model has never touched is still the real test.
|
|
143
|
+
- **Notebooks are not parsed yet.** `.ipynb` support is planned; today you would export to
|
|
144
|
+
`.py` first.
|
|
145
|
+
|
|
146
|
+
If you hit a false positive, that is a bug worth reporting — a noisy linter is a useless one.
|
|
147
|
+
|
|
148
|
+
## Why these seven
|
|
149
|
+
|
|
150
|
+
They are the leaks that survive code review because they look like ordinary data preparation.
|
|
151
|
+
Nobody writes `model.fit(X_test)`; people write `scaler.fit_transform(df)` on line 12 and split
|
|
152
|
+
on line 30, and the two lines never appear on screen together.
|
|
153
|
+
|
|
154
|
+
## Development
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
git clone https://github.com/MartexHACK/timeleak
|
|
158
|
+
cd timeleak
|
|
159
|
+
pip install -e ".[dev]"
|
|
160
|
+
pytest
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
## License
|
|
164
|
+
|
|
165
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/timeleak/__init__.py
|
|
5
|
+
src/timeleak/__main__.py
|
|
6
|
+
src/timeleak/analyzer.py
|
|
7
|
+
src/timeleak/cli.py
|
|
8
|
+
src/timeleak/rules.py
|
|
9
|
+
src/timeleak.egg-info/PKG-INFO
|
|
10
|
+
src/timeleak.egg-info/SOURCES.txt
|
|
11
|
+
src/timeleak.egg-info/dependency_links.txt
|
|
12
|
+
src/timeleak.egg-info/entry_points.txt
|
|
13
|
+
src/timeleak.egg-info/requires.txt
|
|
14
|
+
src/timeleak.egg-info/top_level.txt
|
|
15
|
+
tests/test_analyzer.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
timeleak
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from timeleak import analyze_source
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def codes(source: str) -> list[str]:
|
|
9
|
+
return [f.code for f in analyze_source(source)]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# --- TL001 fit before split -------------------------------------------------
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_scaler_fitted_before_split_is_flagged():
|
|
16
|
+
src = """
|
|
17
|
+
from sklearn.preprocessing import StandardScaler
|
|
18
|
+
from sklearn.model_selection import train_test_split
|
|
19
|
+
scaler = StandardScaler()
|
|
20
|
+
X = scaler.fit_transform(df)
|
|
21
|
+
X_tr, X_te = train_test_split(X, shuffle=False)
|
|
22
|
+
"""
|
|
23
|
+
assert "TL001" in codes(src)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_inline_constructor_fit_before_split_is_flagged():
|
|
27
|
+
src = """
|
|
28
|
+
from sklearn.model_selection import train_test_split
|
|
29
|
+
X = StandardScaler().fit_transform(df)
|
|
30
|
+
a, b = train_test_split(X, shuffle=False)
|
|
31
|
+
"""
|
|
32
|
+
assert "TL001" in codes(src)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_scaler_fitted_after_split_is_clean():
|
|
36
|
+
src = """
|
|
37
|
+
from sklearn.model_selection import train_test_split
|
|
38
|
+
X_tr, X_te = train_test_split(X, shuffle=False)
|
|
39
|
+
scaler = StandardScaler()
|
|
40
|
+
scaler.fit(X_tr)
|
|
41
|
+
"""
|
|
42
|
+
assert "TL001" not in codes(src)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_no_split_in_file_means_no_tl001():
|
|
46
|
+
"""Without a split there is nothing to leak across, so stay quiet."""
|
|
47
|
+
src = """
|
|
48
|
+
scaler = StandardScaler()
|
|
49
|
+
X = scaler.fit_transform(df)
|
|
50
|
+
"""
|
|
51
|
+
assert "TL001" not in codes(src)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_model_fit_is_not_a_transformer_fit():
|
|
55
|
+
src = """
|
|
56
|
+
from sklearn.model_selection import train_test_split
|
|
57
|
+
model = RandomForestClassifier()
|
|
58
|
+
model.fit(X, y)
|
|
59
|
+
a, b = train_test_split(X, shuffle=False)
|
|
60
|
+
"""
|
|
61
|
+
assert "TL001" not in codes(src)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# --- TL002 shuffled split ---------------------------------------------------
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@pytest.mark.parametrize(
|
|
68
|
+
"call,expected",
|
|
69
|
+
[
|
|
70
|
+
("train_test_split(X, y)", True),
|
|
71
|
+
("train_test_split(X, y, shuffle=True)", True),
|
|
72
|
+
("train_test_split(X, y, shuffle=False)", False),
|
|
73
|
+
],
|
|
74
|
+
)
|
|
75
|
+
def test_shuffle_detection(call, expected):
|
|
76
|
+
assert ("TL002" in codes(call)) is expected
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# --- TL003 negative shift ---------------------------------------------------
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_negative_shift_is_flagged():
|
|
83
|
+
assert "TL003" in codes("y = df['close'].shift(-1)")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def test_negative_shift_keyword_is_flagged():
|
|
87
|
+
assert "TL003" in codes("y = df['close'].shift(periods=-5)")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_positive_shift_is_clean():
|
|
91
|
+
assert "TL003" not in codes("x = df['close'].shift(1)")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# --- TL004 centered window --------------------------------------------------
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def test_centered_rolling_is_flagged():
|
|
98
|
+
assert "TL004" in codes("m = df['x'].rolling(20, center=True).mean()")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def test_default_rolling_is_clean():
|
|
102
|
+
assert "TL004" not in codes("m = df['x'].rolling(20).mean()")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
# --- TL005 backward fill ----------------------------------------------------
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def test_bfill_method_is_flagged():
|
|
109
|
+
assert "TL005" in codes("df = df.fillna(method='bfill')")
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_bfill_shorthand_is_flagged():
|
|
113
|
+
assert "TL005" in codes("df = df.bfill()")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def test_ffill_is_clean():
|
|
117
|
+
assert "TL005" not in codes("df = df.fillna(method='ffill')")
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
# --- TL006 non-time-aware CV ------------------------------------------------
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def test_kfold_is_flagged():
|
|
124
|
+
assert "TL006" in codes("cv = KFold(n_splits=5)")
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def test_cross_val_score_with_int_cv_is_flagged():
|
|
128
|
+
assert "TL006" in codes("s = cross_val_score(model, X, y, cv=5)")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def test_cross_val_score_with_timeseriessplit_is_clean():
|
|
132
|
+
assert "TL006" not in codes("s = cross_val_score(model, X, y, cv=TimeSeriesSplit(5))")
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
# --- TL007 full-sample statistic -------------------------------------------
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def test_global_zscore_is_flagged():
|
|
139
|
+
assert "TL007" in codes("df['z'] = (df['x'] - df['x'].mean()) / df['x'].std()")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def test_rolling_mean_assignment_is_clean():
|
|
143
|
+
"""A windowed mean looks backwards, so it must not trip TL007."""
|
|
144
|
+
assert "TL007" not in codes("df['z'] = df['x'].rolling(20).mean()")
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def test_groupby_mean_assignment_is_clean():
|
|
148
|
+
assert "TL007" not in codes("df['z'] = df.groupby('sym').mean()")
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def test_statistic_from_another_frame_is_clean():
|
|
152
|
+
assert "TL007" not in codes("df['z'] = train['x'].mean()")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
# --- suppression and plumbing ----------------------------------------------
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def test_bare_noqa_suppresses():
|
|
159
|
+
assert codes("y = df['c'].shift(-1) # noqa") == []
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def test_targeted_noqa_suppresses_only_that_code():
|
|
163
|
+
assert codes("y = df['c'].shift(-1) # noqa: TL003") == []
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def test_unrelated_noqa_does_not_suppress():
|
|
167
|
+
assert "TL003" in codes("y = df['c'].shift(-1) # noqa: TL004")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def test_findings_are_sorted_by_position():
|
|
171
|
+
src = "a = df.bfill()\nb = df['x'].shift(-1)\n"
|
|
172
|
+
found = analyze_source(src)
|
|
173
|
+
assert [f.line for f in found] == sorted(f.line for f in found)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def test_syntax_error_raises_value_error():
|
|
177
|
+
with pytest.raises(ValueError):
|
|
178
|
+
analyze_source("def broken(:\n")
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def test_clean_file_reports_nothing():
|
|
182
|
+
src = """
|
|
183
|
+
import pandas as pd
|
|
184
|
+
df = pd.read_csv('prices.csv')
|
|
185
|
+
df['ret'] = df['close'].pct_change()
|
|
186
|
+
df['ma'] = df['close'].rolling(20).mean()
|
|
187
|
+
df = df.fillna(method='ffill')
|
|
188
|
+
"""
|
|
189
|
+
assert codes(src) == []
|