drift-or-shift 0.1.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- drift_or_shift-0.1.0a1/LICENSE +21 -0
- drift_or_shift-0.1.0a1/PKG-INFO +218 -0
- drift_or_shift-0.1.0a1/README.md +174 -0
- drift_or_shift-0.1.0a1/pyproject.toml +192 -0
- drift_or_shift-0.1.0a1/setup.cfg +4 -0
- drift_or_shift-0.1.0a1/src/caliblab/__init__.py +159 -0
- drift_or_shift-0.1.0a1/src/caliblab/py.typed +0 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/__init__.py +113 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/calibration.py +92 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/data_synth.py +170 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/drift_monitor.py +124 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/drift_variants.py +66 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/ess.py +26 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/__init__.py +10 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/_common.py +106 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp10_credit_card_fraud.py +179 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp11_high_variance_medical.py +221 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp1_label_shift_synth.py +157 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp2_auc_pr_invariance.py +118 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp3_ess_vs_weight.py +80 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp4_concept_drift.py +170 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp5_realdata_breast_cancer.py +163 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp6_calibration_label_shift.py +184 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp7_drift_types.py +199 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp8_multimodal_label_shift.py +176 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp9_covtype_label_shift.py +171 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/experiments/__init__.py +31 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/io_utils.py +42 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/metrics.py +56 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/models.py +43 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/plotting.py +75 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/py.typed +0 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/reporting.py +230 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift/shift.py +53 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift.egg-info/PKG-INFO +218 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift.egg-info/SOURCES.txt +68 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift.egg-info/dependency_links.txt +1 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift.egg-info/entry_points.txt +12 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift.egg-info/requires.txt +19 -0
- drift_or_shift-0.1.0a1/src/drift_or_shift.egg-info/top_level.txt +3 -0
- drift_or_shift-0.1.0a1/src/drift_shift_pipeline/__init__.py +30 -0
- drift_or_shift-0.1.0a1/src/drift_shift_pipeline/py.typed +0 -0
- drift_or_shift-0.1.0a1/tests/test_aggregate_results_alerts.py +47 -0
- drift_or_shift-0.1.0a1/tests/test_array_like_api.py +203 -0
- drift_or_shift-0.1.0a1/tests/test_caliblab.py +192 -0
- drift_or_shift-0.1.0a1/tests/test_caliblab_leakage.py +112 -0
- drift_or_shift-0.1.0a1/tests/test_calibration.py +75 -0
- drift_or_shift-0.1.0a1/tests/test_data_synth.py +57 -0
- drift_or_shift-0.1.0a1/tests/test_documented_results.py +605 -0
- drift_or_shift-0.1.0a1/tests/test_drift_alerts.py +138 -0
- drift_or_shift-0.1.0a1/tests/test_drift_monitor.py +30 -0
- drift_or_shift-0.1.0a1/tests/test_drift_variants.py +47 -0
- drift_or_shift-0.1.0a1/tests/test_ess.py +35 -0
- drift_or_shift-0.1.0a1/tests/test_experiments_cli.py +234 -0
- drift_or_shift-0.1.0a1/tests/test_experiments_common.py +31 -0
- drift_or_shift-0.1.0a1/tests/test_io_utils.py +27 -0
- drift_or_shift-0.1.0a1/tests/test_models_metrics.py +47 -0
- drift_or_shift-0.1.0a1/tests/test_monitor_drift_script.py +63 -0
- drift_or_shift-0.1.0a1/tests/test_package_alias.py +49 -0
- drift_or_shift-0.1.0a1/tests/test_packaging.py +79 -0
- drift_or_shift-0.1.0a1/tests/test_release_tools.py +311 -0
- drift_or_shift-0.1.0a1/tests/test_reporting.py +97 -0
- drift_or_shift-0.1.0a1/tests/test_reporting_edge_cases.py +203 -0
- drift_or_shift-0.1.0a1/tests/test_reproduce_smoke.py +75 -0
- drift_or_shift-0.1.0a1/tests/test_script_entrypoints.py +432 -0
- drift_or_shift-0.1.0a1/tests/test_shift.py +55 -0
- drift_or_shift-0.1.0a1/tests/test_smoke_aggregate_drift_and_alerts.py +91 -0
- drift_or_shift-0.1.0a1/tests/test_smoke_exp1_script.py +24 -0
- drift_or_shift-0.1.0a1/tests/test_tooling_pins.py +140 -0
- drift_or_shift-0.1.0a1/tests/test_validation_contracts.py +328 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Diogo Ribeiro
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: drift-or-shift
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: Label shift vs. concept drift: Bayes-optimal offset correction, calibration, and drift diagnostics.
|
|
5
|
+
Author-email: Diogo Ribeiro <hansolo.dj@gmail.com>
|
|
6
|
+
Maintainer-email: Diogo Ribeiro <hansolo.dj@gmail.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/DiogoRibeiro7/drift-or-shift
|
|
9
|
+
Project-URL: Repository, https://github.com/DiogoRibeiro7/drift-or-shift
|
|
10
|
+
Project-URL: Issues, https://github.com/DiogoRibeiro7/drift-or-shift/issues
|
|
11
|
+
Project-URL: Changelog, https://github.com/DiogoRibeiro7/drift-or-shift/blob/main/CHANGELOG.md
|
|
12
|
+
Keywords: label-shift,concept-drift,prior-shift,calibration,machine-learning,reproducible-research
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Requires-Dist: numpy>=1.23
|
|
28
|
+
Requires-Dist: scikit-learn>=1.2
|
|
29
|
+
Requires-Dist: pandas>=1.5
|
|
30
|
+
Requires-Dist: matplotlib>=3.6
|
|
31
|
+
Requires-Dist: pyyaml>=6.0
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: black==26.5.1; extra == "dev"
|
|
34
|
+
Requires-Dist: build==1.6.1; extra == "dev"
|
|
35
|
+
Requires-Dist: mypy==2.3.1; extra == "dev"
|
|
36
|
+
Requires-Dist: pandas-stubs==2.3.3.260113; extra == "dev"
|
|
37
|
+
Requires-Dist: pre-commit==4.6.2; extra == "dev"
|
|
38
|
+
Requires-Dist: pytest==9.1.1; extra == "dev"
|
|
39
|
+
Requires-Dist: pytest-cov==7.1.0; extra == "dev"
|
|
40
|
+
Requires-Dist: ruff==0.16.8; extra == "dev"
|
|
41
|
+
Requires-Dist: tomli==2.4.1; python_version < "3.11" and extra == "dev"
|
|
42
|
+
Requires-Dist: types-PyYAML==6.0.12.20260906; extra == "dev"
|
|
43
|
+
Dynamic: license-file
|
|
44
|
+
|
|
45
|
+
# drift-or-shift
|
|
46
|
+
|
|
47
|
+
[](https://github.com/DiogoRibeiro7/drift-or-shift/actions/workflows/ci.yml)
|
|
48
|
+
[](https://www.python.org/)
|
|
49
|
+
[](LICENSE)
|
|
50
|
+
[](https://github.com/psf/black)
|
|
51
|
+
[](https://mypy-lang.org/)
|
|
52
|
+
|
|
53
|
+
A reproducible research pipeline for distinguishing **label shift** from **concept
|
|
54
|
+
drift**, and for showing exactly when a training-free logit offset is sufficient —
|
|
55
|
+
and when it is not.
|
|
56
|
+
|
|
57
|
+
The experiments reproduce and extend the analysis in
|
|
58
|
+
[SSRN 6052514](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=6052514): start from synthetic two-class Gaussian
|
|
59
|
+
data, apply the Bayes-optimal offset correction without retraining, then observe
|
|
60
|
+
which metrics are invariant (ROC AUC) and which are not (PR-AUC, effective sample
|
|
61
|
+
size). When the class-conditionals themselves move, the offset provably cannot
|
|
62
|
+
recover the new posterior, and retraining becomes necessary.
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
## Installation
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
# From source, for development
|
|
70
|
+
git clone https://github.com/DiogoRibeiro7/drift-or-shift.git
|
|
71
|
+
cd drift-or-shift
|
|
72
|
+
make install # editable install + dev extras + pre-commit hooks
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Without `make`:
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
python -m pip install -e ".[dev]"
|
|
79
|
+
python -m pre_commit install
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Runtime install only:
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
python -m pip install .
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Requires Python 3.10+. This installs the `drift_or_shift` package, the
|
|
89
|
+
`drift_shift_pipeline` compatibility alias, the `caliblab` calibration
|
|
90
|
+
benchmarking helpers, and the eleven `dos-expN` console scripts.
|
|
91
|
+
|
|
92
|
+
## Quickstart
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
# Fast end-to-end sanity check (seconds)
|
|
96
|
+
python scripts/smoke_exp1.py --results-dir results/smoke
|
|
97
|
+
|
|
98
|
+
# A full experiment
|
|
99
|
+
dos-exp1 --pi-train 0.2 --pi-tests 0.05 0.2 0.5 --results-dir results
|
|
100
|
+
|
|
101
|
+
# Aggregate every run into a reviewable dashboard
|
|
102
|
+
python scripts/aggregate_results.py --results-dir results \
|
|
103
|
+
--output reports/dashboard.md --figure reports/best_risk.png
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Each run writes `tables/`, `figures/`, and a `_summary.json` into
|
|
107
|
+
`results/<experiment>/<utc-timestamp>/`.
|
|
108
|
+
|
|
109
|
+
## Core concepts
|
|
110
|
+
|
|
111
|
+
### Label shift vs. concept drift
|
|
112
|
+
|
|
113
|
+
| | Label shift | Concept drift |
|
|
114
|
+
| --- | --- | --- |
|
|
115
|
+
| What moves | Class prior `π` only | The class-conditionals `p(x\|y)` |
|
|
116
|
+
| Posterior recoverable without retraining? | **Yes** — additive logit offset | **No** |
|
|
117
|
+
| Demonstrated by | Exp 1, 2, 5, 8–11 | Exp 4, 7 |
|
|
118
|
+
|
|
119
|
+
### Offset correction
|
|
120
|
+
|
|
121
|
+
Logits are log-odds, so shifting the prior from `π_train` to `π_test` is a
|
|
122
|
+
constant additive term:
|
|
123
|
+
|
|
124
|
+
```
|
|
125
|
+
offset = log( π_test · (1 − π_train) / (π_train · (1 − π_test)) )
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Apply the offset **before** comparing against the cost-derived threshold
|
|
129
|
+
`log(c10 / c01)`; with default costs (`c10 = c01 = 1`) that threshold is zero.
|
|
130
|
+
Because a constant offset preserves the ranking of scores, ROC AUC is invariant
|
|
131
|
+
under label shift — while PR-AUC, which depends on prevalence, is not.
|
|
132
|
+
|
|
133
|
+
## Experiments
|
|
134
|
+
|
|
135
|
+
| Command | What it demonstrates |
|
|
136
|
+
| --- | --- |
|
|
137
|
+
| `dos-exp1` | Synthetic label shift: offset correction vs. oracle risk |
|
|
138
|
+
| `dos-exp2` | ROC AUC invariance and PR-AUC prevalence dependence |
|
|
139
|
+
| `dos-exp3` | ESS fraction shrinking as class weight α grows |
|
|
140
|
+
| `dos-exp4` | Concept drift: standard vs. offset vs. retrained |
|
|
141
|
+
| `dos-exp5` | Breast cancer label shift replication with resampling |
|
|
142
|
+
| `dos-exp6` | Calibration (temperature, isotonic) vs. offset under label shift |
|
|
143
|
+
| `dos-exp7` | Drift-type sweep: covariance, feature, and label shift |
|
|
144
|
+
| `dos-exp8` | Multimodal label shift with mixture components |
|
|
145
|
+
| `dos-exp9` | Covertype: real, high-dimensional label shift |
|
|
146
|
+
| `dos-exp10` | Credit-card fraud benchmark with reweighted prevalences |
|
|
147
|
+
| `dos-exp11` | High-variance medical-style benchmark with nonlinear test shifts |
|
|
148
|
+
|
|
149
|
+
Sources live in [src/drift_or_shift/experiments/](src/drift_or_shift/experiments/).
|
|
150
|
+
Every script accepts `--help`.
|
|
151
|
+
|
|
152
|
+
## Reproducibility
|
|
153
|
+
|
|
154
|
+
- All generators take an explicit `seed`; summaries record seeds, sizes, costs,
|
|
155
|
+
and prevalence grids.
|
|
156
|
+
- Run-directory names and summary timestamps are **UTC** and timezone-aware.
|
|
157
|
+
- `ExperimentConfig` in `drift_or_shift.experiments._common` (or the CLI flags)
|
|
158
|
+
pins a run's configuration.
|
|
159
|
+
- `reproduce/` holds a separate calibration-benchmark replication driven by
|
|
160
|
+
[reproduce/config/default.yaml](reproduce/config/default.yaml).
|
|
161
|
+
|
|
162
|
+
## Tooling utilities
|
|
163
|
+
|
|
164
|
+
- `drift_or_shift.drift_variants` — inject covariance/feature shifts, label
|
|
165
|
+
noise, and density-ratio scoring without rebuilding the generators.
|
|
166
|
+
- `scripts/aggregate_results.py` — collect every `results/*/*_summary.json` into
|
|
167
|
+
a markdown dashboard; `--fail-on-alerts` exits non-zero when drift is detected.
|
|
168
|
+
- `scripts/drift_alerts.py` — flag `feature_max_*` statistics above threshold
|
|
169
|
+
into `reports/drift_alerts.csv`.
|
|
170
|
+
- `scripts/watch_results.py` — poll `results/` and refresh alerts + dashboard as
|
|
171
|
+
new summaries land (`--once` for a single pass).
|
|
172
|
+
|
|
173
|
+
See [docs/CLI_USAGE.md](docs/CLI_USAGE.md) for full flag documentation.
|
|
174
|
+
|
|
175
|
+
## Development
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
make check # ruff + black + mypy + pytest, exactly what CI runs
|
|
179
|
+
make test-cov # tests with an HTML coverage report
|
|
180
|
+
make build # sdist + wheel, validated with twine
|
|
181
|
+
make help # all targets
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
CI runs the suite on Python 3.10–3.13 on Linux, plus Windows and macOS spot
|
|
185
|
+
checks, and verifies that the built wheel installs and runs in a clean
|
|
186
|
+
environment.
|
|
187
|
+
|
|
188
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for branching and PR conventions, and
|
|
189
|
+
[CHANGELOG.md](CHANGELOG.md) for release history.
|
|
190
|
+
|
|
191
|
+
## Project documents
|
|
192
|
+
|
|
193
|
+
| File | Contents |
|
|
194
|
+
| --- | --- |
|
|
195
|
+
| [RESULTS_DIGEST.md](RESULTS_DIGEST.md) | Latest metrics and artifact paths |
|
|
196
|
+
| [REPORT.md](REPORT.md) | Reproduction steps and what each figure claims |
|
|
197
|
+
| [notes/lessons.md](notes/lessons.md) | Counter-intuitive findings worth revisiting |
|
|
198
|
+
| [ROADMAP.md](ROADMAP.md) | Milestones |
|
|
199
|
+
| [FUTURE.md](FUTURE.md) | Candidate extensions |
|
|
200
|
+
| [RELEASE.md](RELEASE.md) | Release checklist |
|
|
201
|
+
|
|
202
|
+
## Citation
|
|
203
|
+
|
|
204
|
+
If this code supports academic work, please cite the underlying paper
|
|
205
|
+
([SSRN 6052514](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=6052514)) and reference this repository:
|
|
206
|
+
|
|
207
|
+
```bibtex
|
|
208
|
+
@software{ribeiro_drift_shift_pipeline,
|
|
209
|
+
author = {Ribeiro, Diogo},
|
|
210
|
+
title = {drift-or-shift: label shift, offset correction and drift diagnostics},
|
|
211
|
+
url = {https://github.com/DiogoRibeiro7/drift-or-shift},
|
|
212
|
+
version = {0.1.0}
|
|
213
|
+
}
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
## License
|
|
217
|
+
|
|
218
|
+
Released under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
# drift-or-shift
|
|
2
|
+
|
|
3
|
+
[](https://github.com/DiogoRibeiro7/drift-or-shift/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.python.org/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
[](https://github.com/psf/black)
|
|
7
|
+
[](https://mypy-lang.org/)
|
|
8
|
+
|
|
9
|
+
A reproducible research pipeline for distinguishing **label shift** from **concept
|
|
10
|
+
drift**, and for showing exactly when a training-free logit offset is sufficient —
|
|
11
|
+
and when it is not.
|
|
12
|
+
|
|
13
|
+
The experiments reproduce and extend the analysis in
|
|
14
|
+
[SSRN 6052514](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=6052514): start from synthetic two-class Gaussian
|
|
15
|
+
data, apply the Bayes-optimal offset correction without retraining, then observe
|
|
16
|
+
which metrics are invariant (ROC AUC) and which are not (PR-AUC, effective sample
|
|
17
|
+
size). When the class-conditionals themselves move, the offset provably cannot
|
|
18
|
+
recover the new posterior, and retraining becomes necessary.
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
## Installation
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
# From source, for development
|
|
26
|
+
git clone https://github.com/DiogoRibeiro7/drift-or-shift.git
|
|
27
|
+
cd drift-or-shift
|
|
28
|
+
make install # editable install + dev extras + pre-commit hooks
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Without `make`:
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
python -m pip install -e ".[dev]"
|
|
35
|
+
python -m pre_commit install
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Runtime install only:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
python -m pip install .
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Requires Python 3.10+. This installs the `drift_or_shift` package, the
|
|
45
|
+
`drift_shift_pipeline` compatibility alias, the `caliblab` calibration
|
|
46
|
+
benchmarking helpers, and the eleven `dos-expN` console scripts.
|
|
47
|
+
|
|
48
|
+
## Quickstart
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
# Fast end-to-end sanity check (seconds)
|
|
52
|
+
python scripts/smoke_exp1.py --results-dir results/smoke
|
|
53
|
+
|
|
54
|
+
# A full experiment
|
|
55
|
+
dos-exp1 --pi-train 0.2 --pi-tests 0.05 0.2 0.5 --results-dir results
|
|
56
|
+
|
|
57
|
+
# Aggregate every run into a reviewable dashboard
|
|
58
|
+
python scripts/aggregate_results.py --results-dir results \
|
|
59
|
+
--output reports/dashboard.md --figure reports/best_risk.png
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Each run writes `tables/`, `figures/`, and a `_summary.json` into
|
|
63
|
+
`results/<experiment>/<utc-timestamp>/`.
|
|
64
|
+
|
|
65
|
+
## Core concepts
|
|
66
|
+
|
|
67
|
+
### Label shift vs. concept drift
|
|
68
|
+
|
|
69
|
+
| | Label shift | Concept drift |
|
|
70
|
+
| --- | --- | --- |
|
|
71
|
+
| What moves | Class prior `π` only | The class-conditionals `p(x\|y)` |
|
|
72
|
+
| Posterior recoverable without retraining? | **Yes** — additive logit offset | **No** |
|
|
73
|
+
| Demonstrated by | Exp 1, 2, 5, 8–11 | Exp 4, 7 |
|
|
74
|
+
|
|
75
|
+
### Offset correction
|
|
76
|
+
|
|
77
|
+
Logits are log-odds, so shifting the prior from `π_train` to `π_test` is a
|
|
78
|
+
constant additive term:
|
|
79
|
+
|
|
80
|
+
```
|
|
81
|
+
offset = log( π_test · (1 − π_train) / (π_train · (1 − π_test)) )
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Apply the offset **before** comparing against the cost-derived threshold
|
|
85
|
+
`log(c10 / c01)`; with default costs (`c10 = c01 = 1`) that threshold is zero.
|
|
86
|
+
Because a constant offset preserves the ranking of scores, ROC AUC is invariant
|
|
87
|
+
under label shift — while PR-AUC, which depends on prevalence, is not.
|
|
88
|
+
|
|
89
|
+
## Experiments
|
|
90
|
+
|
|
91
|
+
| Command | What it demonstrates |
|
|
92
|
+
| --- | --- |
|
|
93
|
+
| `dos-exp1` | Synthetic label shift: offset correction vs. oracle risk |
|
|
94
|
+
| `dos-exp2` | ROC AUC invariance and PR-AUC prevalence dependence |
|
|
95
|
+
| `dos-exp3` | ESS fraction shrinking as class weight α grows |
|
|
96
|
+
| `dos-exp4` | Concept drift: standard vs. offset vs. retrained |
|
|
97
|
+
| `dos-exp5` | Breast cancer label shift replication with resampling |
|
|
98
|
+
| `dos-exp6` | Calibration (temperature, isotonic) vs. offset under label shift |
|
|
99
|
+
| `dos-exp7` | Drift-type sweep: covariance, feature, and label shift |
|
|
100
|
+
| `dos-exp8` | Multimodal label shift with mixture components |
|
|
101
|
+
| `dos-exp9` | Covertype: real, high-dimensional label shift |
|
|
102
|
+
| `dos-exp10` | Credit-card fraud benchmark with reweighted prevalences |
|
|
103
|
+
| `dos-exp11` | High-variance medical-style benchmark with nonlinear test shifts |
|
|
104
|
+
|
|
105
|
+
Sources live in [src/drift_or_shift/experiments/](src/drift_or_shift/experiments/).
|
|
106
|
+
Every script accepts `--help`.
|
|
107
|
+
|
|
108
|
+
## Reproducibility
|
|
109
|
+
|
|
110
|
+
- All generators take an explicit `seed`; summaries record seeds, sizes, costs,
|
|
111
|
+
and prevalence grids.
|
|
112
|
+
- Run-directory names and summary timestamps are **UTC** and timezone-aware.
|
|
113
|
+
- `ExperimentConfig` in `drift_or_shift.experiments._common` (or the CLI flags)
|
|
114
|
+
pins a run's configuration.
|
|
115
|
+
- `reproduce/` holds a separate calibration-benchmark replication driven by
|
|
116
|
+
[reproduce/config/default.yaml](reproduce/config/default.yaml).
|
|
117
|
+
|
|
118
|
+
## Tooling utilities
|
|
119
|
+
|
|
120
|
+
- `drift_or_shift.drift_variants` — inject covariance/feature shifts, label
|
|
121
|
+
noise, and density-ratio scoring without rebuilding the generators.
|
|
122
|
+
- `scripts/aggregate_results.py` — collect every `results/*/*_summary.json` into
|
|
123
|
+
a markdown dashboard; `--fail-on-alerts` exits non-zero when drift is detected.
|
|
124
|
+
- `scripts/drift_alerts.py` — flag `feature_max_*` statistics above threshold
|
|
125
|
+
into `reports/drift_alerts.csv`.
|
|
126
|
+
- `scripts/watch_results.py` — poll `results/` and refresh alerts + dashboard as
|
|
127
|
+
new summaries land (`--once` for a single pass).
|
|
128
|
+
|
|
129
|
+
See [docs/CLI_USAGE.md](docs/CLI_USAGE.md) for full flag documentation.
|
|
130
|
+
|
|
131
|
+
## Development
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
make check # ruff + black + mypy + pytest, exactly what CI runs
|
|
135
|
+
make test-cov # tests with an HTML coverage report
|
|
136
|
+
make build # sdist + wheel, validated with twine
|
|
137
|
+
make help # all targets
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
CI runs the suite on Python 3.10–3.13 on Linux, plus Windows and macOS spot
|
|
141
|
+
checks, and verifies that the built wheel installs and runs in a clean
|
|
142
|
+
environment.
|
|
143
|
+
|
|
144
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for branching and PR conventions, and
|
|
145
|
+
[CHANGELOG.md](CHANGELOG.md) for release history.
|
|
146
|
+
|
|
147
|
+
## Project documents
|
|
148
|
+
|
|
149
|
+
| File | Contents |
|
|
150
|
+
| --- | --- |
|
|
151
|
+
| [RESULTS_DIGEST.md](RESULTS_DIGEST.md) | Latest metrics and artifact paths |
|
|
152
|
+
| [REPORT.md](REPORT.md) | Reproduction steps and what each figure claims |
|
|
153
|
+
| [notes/lessons.md](notes/lessons.md) | Counter-intuitive findings worth revisiting |
|
|
154
|
+
| [ROADMAP.md](ROADMAP.md) | Milestones |
|
|
155
|
+
| [FUTURE.md](FUTURE.md) | Candidate extensions |
|
|
156
|
+
| [RELEASE.md](RELEASE.md) | Release checklist |
|
|
157
|
+
|
|
158
|
+
## Citation
|
|
159
|
+
|
|
160
|
+
If this code supports academic work, please cite the underlying paper
|
|
161
|
+
([SSRN 6052514](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=6052514)) and reference this repository:
|
|
162
|
+
|
|
163
|
+
```bibtex
|
|
164
|
+
@software{ribeiro_drift_shift_pipeline,
|
|
165
|
+
author = {Ribeiro, Diogo},
|
|
166
|
+
title = {drift-or-shift: label shift, offset correction and drift diagnostics},
|
|
167
|
+
url = {https://github.com/DiogoRibeiro7/drift-or-shift},
|
|
168
|
+
version = {0.1.0}
|
|
169
|
+
}
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
## License
|
|
173
|
+
|
|
174
|
+
Released under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "drift-or-shift"
|
|
7
|
+
description = "Label shift vs. concept drift: Bayes-optimal offset correction, calibration, and drift diagnostics."
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "Diogo Ribeiro", email = "hansolo.dj@gmail.com" }]
|
|
13
|
+
maintainers = [{ name = "Diogo Ribeiro", email = "hansolo.dj@gmail.com" }]
|
|
14
|
+
keywords = [
|
|
15
|
+
"label-shift",
|
|
16
|
+
"concept-drift",
|
|
17
|
+
"prior-shift",
|
|
18
|
+
"calibration",
|
|
19
|
+
"machine-learning",
|
|
20
|
+
"reproducible-research",
|
|
21
|
+
]
|
|
22
|
+
classifiers = [
|
|
23
|
+
"Development Status :: 3 - Alpha",
|
|
24
|
+
"Intended Audience :: Science/Research",
|
|
25
|
+
"Operating System :: OS Independent",
|
|
26
|
+
"Programming Language :: Python :: 3",
|
|
27
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
28
|
+
"Programming Language :: Python :: 3.10",
|
|
29
|
+
"Programming Language :: Python :: 3.11",
|
|
30
|
+
"Programming Language :: Python :: 3.12",
|
|
31
|
+
"Programming Language :: Python :: 3.13",
|
|
32
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
33
|
+
"Typing :: Typed",
|
|
34
|
+
]
|
|
35
|
+
dynamic = ["version"]
|
|
36
|
+
dependencies = [
|
|
37
|
+
"numpy>=1.23",
|
|
38
|
+
"scikit-learn>=1.2",
|
|
39
|
+
"pandas>=1.5",
|
|
40
|
+
"matplotlib>=3.6",
|
|
41
|
+
"pyyaml>=6.0",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[project.optional-dependencies]
|
|
45
|
+
# Dev tooling is pinned exactly: an unpinned `ruff`/`mypy` means a new release
|
|
46
|
+
# can turn CI red without a single line of project code changing.
|
|
47
|
+
dev = [
|
|
48
|
+
"black==26.5.1",
|
|
49
|
+
"build==1.6.1",
|
|
50
|
+
"mypy==2.3.1",
|
|
51
|
+
# pandas-stubs 3.x requires Python >= 3.11 and this project supports
|
|
52
|
+
# 3.10, so the dev extra cannot install it. See dependabot.yml.
|
|
53
|
+
"pandas-stubs==2.3.3.260113",
|
|
54
|
+
"pre-commit==4.6.2",
|
|
55
|
+
"pytest==9.1.1",
|
|
56
|
+
"pytest-cov==7.1.0",
|
|
57
|
+
"ruff==0.16.8",
|
|
58
|
+
# tomllib is stdlib from 3.11; 3.10 needs the backport.
|
|
59
|
+
"tomli==2.4.1; python_version < '3.11'",
|
|
60
|
+
"types-PyYAML==6.0.12.20260906",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
[project.urls]
|
|
64
|
+
Homepage = "https://github.com/DiogoRibeiro7/drift-or-shift"
|
|
65
|
+
Repository = "https://github.com/DiogoRibeiro7/drift-or-shift"
|
|
66
|
+
Issues = "https://github.com/DiogoRibeiro7/drift-or-shift/issues"
|
|
67
|
+
Changelog = "https://github.com/DiogoRibeiro7/drift-or-shift/blob/main/CHANGELOG.md"
|
|
68
|
+
|
|
69
|
+
[project.scripts]
|
|
70
|
+
dos-exp1 = "drift_or_shift.experiments.exp1_label_shift_synth:main"
|
|
71
|
+
dos-exp2 = "drift_or_shift.experiments.exp2_auc_pr_invariance:main"
|
|
72
|
+
dos-exp3 = "drift_or_shift.experiments.exp3_ess_vs_weight:main"
|
|
73
|
+
dos-exp4 = "drift_or_shift.experiments.exp4_concept_drift:main"
|
|
74
|
+
dos-exp5 = "drift_or_shift.experiments.exp5_realdata_breast_cancer:main"
|
|
75
|
+
dos-exp6 = "drift_or_shift.experiments.exp6_calibration_label_shift:main"
|
|
76
|
+
dos-exp7 = "drift_or_shift.experiments.exp7_drift_types:main"
|
|
77
|
+
dos-exp8 = "drift_or_shift.experiments.exp8_multimodal_label_shift:main"
|
|
78
|
+
dos-exp9 = "drift_or_shift.experiments.exp9_covtype_label_shift:main"
|
|
79
|
+
dos-exp10 = "drift_or_shift.experiments.exp10_credit_card_fraud:main"
|
|
80
|
+
dos-exp11 = "drift_or_shift.experiments.exp11_high_variance_medical:main"
|
|
81
|
+
|
|
82
|
+
# ---------------------------------------------------------------------------
|
|
83
|
+
# Packaging
|
|
84
|
+
# ---------------------------------------------------------------------------
|
|
85
|
+
|
|
86
|
+
[tool.setuptools]
|
|
87
|
+
package-dir = { "" = "src" }
|
|
88
|
+
|
|
89
|
+
[tool.setuptools.packages.find]
|
|
90
|
+
where = ["src"]
|
|
91
|
+
|
|
92
|
+
[tool.setuptools.package-data]
|
|
93
|
+
"*" = ["py.typed"]
|
|
94
|
+
|
|
95
|
+
[tool.setuptools.dynamic]
|
|
96
|
+
version = { attr = "drift_or_shift.__version__" }
|
|
97
|
+
|
|
98
|
+
# ---------------------------------------------------------------------------
|
|
99
|
+
# Lint / format
|
|
100
|
+
# ---------------------------------------------------------------------------
|
|
101
|
+
|
|
102
|
+
[tool.ruff]
|
|
103
|
+
line-length = 88
|
|
104
|
+
target-version = "py310"
|
|
105
|
+
src = ["src", "tests", "scripts", "reproduce"]
|
|
106
|
+
|
|
107
|
+
[tool.ruff.lint]
|
|
108
|
+
select = [
|
|
109
|
+
"E", # pycodestyle errors
|
|
110
|
+
"W", # pycodestyle warnings
|
|
111
|
+
"F", # pyflakes
|
|
112
|
+
"I", # isort
|
|
113
|
+
"B", # flake8-bugbear
|
|
114
|
+
"C4", # flake8-comprehensions
|
|
115
|
+
"UP", # pyupgrade
|
|
116
|
+
"SIM", # flake8-simplify
|
|
117
|
+
"RET", # flake8-return
|
|
118
|
+
"PTH", # flake8-use-pathlib
|
|
119
|
+
"DTZ", # flake8-datetimez
|
|
120
|
+
"RUF", # ruff-specific
|
|
121
|
+
]
|
|
122
|
+
ignore = [
|
|
123
|
+
"E501", # line length is enforced by the formatter
|
|
124
|
+
]
|
|
125
|
+
|
|
126
|
+
[tool.ruff.lint.per-file-ignores]
|
|
127
|
+
"tests/*" = ["B018"]
|
|
128
|
+
|
|
129
|
+
[tool.ruff.lint.isort]
|
|
130
|
+
known-first-party = ["drift_or_shift", "drift_shift_pipeline", "caliblab", "reproduce"]
|
|
131
|
+
|
|
132
|
+
[tool.black]
|
|
133
|
+
line-length = 88
|
|
134
|
+
target-version = ["py310"]
|
|
135
|
+
|
|
136
|
+
# ---------------------------------------------------------------------------
|
|
137
|
+
# Types
|
|
138
|
+
# ---------------------------------------------------------------------------
|
|
139
|
+
|
|
140
|
+
[tool.mypy]
|
|
141
|
+
files = ["src"]
|
|
142
|
+
# NOTE: this is deliberately ahead of `requires-python`. NumPy's bundled stubs
|
|
143
|
+
# use PEP 695 `type` statements, which mypy only parses when python_version is
|
|
144
|
+
# 3.12+, so pinning 3.10 here fails before it checks a single project file.
|
|
145
|
+
# The 3.10 floor is still enforced: ruff runs with target-version = "py310",
|
|
146
|
+
# and CI runs the full test suite on a real 3.10 interpreter.
|
|
147
|
+
python_version = "3.12"
|
|
148
|
+
ignore_missing_imports = true
|
|
149
|
+
warn_unused_configs = true
|
|
150
|
+
warn_redundant_casts = true
|
|
151
|
+
warn_unused_ignores = true
|
|
152
|
+
no_implicit_optional = true
|
|
153
|
+
strict_equality = true
|
|
154
|
+
|
|
155
|
+
# ---------------------------------------------------------------------------
|
|
156
|
+
# Tests / coverage
|
|
157
|
+
# ---------------------------------------------------------------------------
|
|
158
|
+
|
|
159
|
+
[tool.pytest.ini_options]
|
|
160
|
+
minversion = "8.0"
|
|
161
|
+
testpaths = ["tests"]
|
|
162
|
+
# `-m "not network"` keeps the default run offline and fast. Run the
|
|
163
|
+
# dataset-downloading experiments explicitly with `-m network`.
|
|
164
|
+
addopts = "-ra --strict-markers --strict-config -m 'not network'"
|
|
165
|
+
markers = [
|
|
166
|
+
"network: requires downloading an external dataset",
|
|
167
|
+
"slow: takes more than a few seconds",
|
|
168
|
+
]
|
|
169
|
+
filterwarnings = [
|
|
170
|
+
"error::DeprecationWarning:drift_or_shift.*",
|
|
171
|
+
"error::DeprecationWarning:caliblab.*",
|
|
172
|
+
]
|
|
173
|
+
|
|
174
|
+
[tool.coverage.run]
|
|
175
|
+
# The scripts are documented entry points, so they are measured too. Leaving
|
|
176
|
+
# them out reported 89% while a fifth of the shipped code was invisible.
|
|
177
|
+
source = ["drift_or_shift", "caliblab", "scripts", "reproduce"]
|
|
178
|
+
branch = true
|
|
179
|
+
|
|
180
|
+
[tool.coverage.report]
|
|
181
|
+
show_missing = true
|
|
182
|
+
skip_covered = true
|
|
183
|
+
# A floor, not a target. Set below the current figure so ordinary churn does
|
|
184
|
+
# not fail the build, but a real regression does. exp9/exp10 are excluded from
|
|
185
|
+
# the default run (they download datasets), which caps the achievable number.
|
|
186
|
+
fail_under = 85
|
|
187
|
+
exclude_lines = [
|
|
188
|
+
"pragma: no cover",
|
|
189
|
+
"if TYPE_CHECKING:",
|
|
190
|
+
"raise NotImplementedError",
|
|
191
|
+
'if __name__ == "__main__":',
|
|
192
|
+
]
|