drift-or-shift 0.1.0a1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. drift_or_shift-0.1.0a1/LICENSE +21 -0
  2. drift_or_shift-0.1.0a1/PKG-INFO +218 -0
  3. drift_or_shift-0.1.0a1/README.md +174 -0
  4. drift_or_shift-0.1.0a1/pyproject.toml +192 -0
  5. drift_or_shift-0.1.0a1/setup.cfg +4 -0
  6. drift_or_shift-0.1.0a1/src/caliblab/__init__.py +159 -0
  7. drift_or_shift-0.1.0a1/src/caliblab/py.typed +0 -0
  8. drift_or_shift-0.1.0a1/src/drift_or_shift/__init__.py +113 -0
  9. drift_or_shift-0.1.0a1/src/drift_or_shift/calibration.py +92 -0
  10. drift_or_shift-0.1.0a1/src/drift_or_shift/data_synth.py +170 -0
  11. drift_or_shift-0.1.0a1/src/drift_or_shift/drift_monitor.py +124 -0
  12. drift_or_shift-0.1.0a1/src/drift_or_shift/drift_variants.py +66 -0
  13. drift_or_shift-0.1.0a1/src/drift_or_shift/ess.py +26 -0
  14. drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/__init__.py +10 -0
  15. drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/_common.py +106 -0
  16. drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp10_credit_card_fraud.py +179 -0
  17. drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp11_high_variance_medical.py +221 -0
  18. drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp1_label_shift_synth.py +157 -0
  19. drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp2_auc_pr_invariance.py +118 -0
  20. drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp3_ess_vs_weight.py +80 -0
  21. drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp4_concept_drift.py +170 -0
  22. drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp5_realdata_breast_cancer.py +163 -0
  23. drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp6_calibration_label_shift.py +184 -0
  24. drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp7_drift_types.py +199 -0
  25. drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp8_multimodal_label_shift.py +176 -0
  26. drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/exp9_covtype_label_shift.py +171 -0
  27. drift_or_shift-0.1.0a1/src/drift_or_shift/experiments/experiments/__init__.py +31 -0
  28. drift_or_shift-0.1.0a1/src/drift_or_shift/io_utils.py +42 -0
  29. drift_or_shift-0.1.0a1/src/drift_or_shift/metrics.py +56 -0
  30. drift_or_shift-0.1.0a1/src/drift_or_shift/models.py +43 -0
  31. drift_or_shift-0.1.0a1/src/drift_or_shift/plotting.py +75 -0
  32. drift_or_shift-0.1.0a1/src/drift_or_shift/py.typed +0 -0
  33. drift_or_shift-0.1.0a1/src/drift_or_shift/reporting.py +230 -0
  34. drift_or_shift-0.1.0a1/src/drift_or_shift/shift.py +53 -0
  35. drift_or_shift-0.1.0a1/src/drift_or_shift.egg-info/PKG-INFO +218 -0
  36. drift_or_shift-0.1.0a1/src/drift_or_shift.egg-info/SOURCES.txt +68 -0
  37. drift_or_shift-0.1.0a1/src/drift_or_shift.egg-info/dependency_links.txt +1 -0
  38. drift_or_shift-0.1.0a1/src/drift_or_shift.egg-info/entry_points.txt +12 -0
  39. drift_or_shift-0.1.0a1/src/drift_or_shift.egg-info/requires.txt +19 -0
  40. drift_or_shift-0.1.0a1/src/drift_or_shift.egg-info/top_level.txt +3 -0
  41. drift_or_shift-0.1.0a1/src/drift_shift_pipeline/__init__.py +30 -0
  42. drift_or_shift-0.1.0a1/src/drift_shift_pipeline/py.typed +0 -0
  43. drift_or_shift-0.1.0a1/tests/test_aggregate_results_alerts.py +47 -0
  44. drift_or_shift-0.1.0a1/tests/test_array_like_api.py +203 -0
  45. drift_or_shift-0.1.0a1/tests/test_caliblab.py +192 -0
  46. drift_or_shift-0.1.0a1/tests/test_caliblab_leakage.py +112 -0
  47. drift_or_shift-0.1.0a1/tests/test_calibration.py +75 -0
  48. drift_or_shift-0.1.0a1/tests/test_data_synth.py +57 -0
  49. drift_or_shift-0.1.0a1/tests/test_documented_results.py +605 -0
  50. drift_or_shift-0.1.0a1/tests/test_drift_alerts.py +138 -0
  51. drift_or_shift-0.1.0a1/tests/test_drift_monitor.py +30 -0
  52. drift_or_shift-0.1.0a1/tests/test_drift_variants.py +47 -0
  53. drift_or_shift-0.1.0a1/tests/test_ess.py +35 -0
  54. drift_or_shift-0.1.0a1/tests/test_experiments_cli.py +234 -0
  55. drift_or_shift-0.1.0a1/tests/test_experiments_common.py +31 -0
  56. drift_or_shift-0.1.0a1/tests/test_io_utils.py +27 -0
  57. drift_or_shift-0.1.0a1/tests/test_models_metrics.py +47 -0
  58. drift_or_shift-0.1.0a1/tests/test_monitor_drift_script.py +63 -0
  59. drift_or_shift-0.1.0a1/tests/test_package_alias.py +49 -0
  60. drift_or_shift-0.1.0a1/tests/test_packaging.py +79 -0
  61. drift_or_shift-0.1.0a1/tests/test_release_tools.py +311 -0
  62. drift_or_shift-0.1.0a1/tests/test_reporting.py +97 -0
  63. drift_or_shift-0.1.0a1/tests/test_reporting_edge_cases.py +203 -0
  64. drift_or_shift-0.1.0a1/tests/test_reproduce_smoke.py +75 -0
  65. drift_or_shift-0.1.0a1/tests/test_script_entrypoints.py +432 -0
  66. drift_or_shift-0.1.0a1/tests/test_shift.py +55 -0
  67. drift_or_shift-0.1.0a1/tests/test_smoke_aggregate_drift_and_alerts.py +91 -0
  68. drift_or_shift-0.1.0a1/tests/test_smoke_exp1_script.py +24 -0
  69. drift_or_shift-0.1.0a1/tests/test_tooling_pins.py +140 -0
  70. drift_or_shift-0.1.0a1/tests/test_validation_contracts.py +328 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Diogo Ribeiro
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,218 @@
1
+ Metadata-Version: 2.4
2
+ Name: drift-or-shift
3
+ Version: 0.1.0a1
4
+ Summary: Label shift vs. concept drift: Bayes-optimal offset correction, calibration, and drift diagnostics.
5
+ Author-email: Diogo Ribeiro <hansolo.dj@gmail.com>
6
+ Maintainer-email: Diogo Ribeiro <hansolo.dj@gmail.com>
7
+ License-Expression: MIT
8
+ Project-URL: Homepage, https://github.com/DiogoRibeiro7/drift-or-shift
9
+ Project-URL: Repository, https://github.com/DiogoRibeiro7/drift-or-shift
10
+ Project-URL: Issues, https://github.com/DiogoRibeiro7/drift-or-shift/issues
11
+ Project-URL: Changelog, https://github.com/DiogoRibeiro7/drift-or-shift/blob/main/CHANGELOG.md
12
+ Keywords: label-shift,concept-drift,prior-shift,calibration,machine-learning,reproducible-research
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: Intended Audience :: Science/Research
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3 :: Only
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
23
+ Classifier: Typing :: Typed
24
+ Requires-Python: >=3.10
25
+ Description-Content-Type: text/markdown
26
+ License-File: LICENSE
27
+ Requires-Dist: numpy>=1.23
28
+ Requires-Dist: scikit-learn>=1.2
29
+ Requires-Dist: pandas>=1.5
30
+ Requires-Dist: matplotlib>=3.6
31
+ Requires-Dist: pyyaml>=6.0
32
+ Provides-Extra: dev
33
+ Requires-Dist: black==26.5.1; extra == "dev"
34
+ Requires-Dist: build==1.6.1; extra == "dev"
35
+ Requires-Dist: mypy==2.3.1; extra == "dev"
36
+ Requires-Dist: pandas-stubs==2.3.3.260113; extra == "dev"
37
+ Requires-Dist: pre-commit==4.6.2; extra == "dev"
38
+ Requires-Dist: pytest==9.1.1; extra == "dev"
39
+ Requires-Dist: pytest-cov==7.1.0; extra == "dev"
40
+ Requires-Dist: ruff==0.16.8; extra == "dev"
41
+ Requires-Dist: tomli==2.4.1; python_version < "3.11" and extra == "dev"
42
+ Requires-Dist: types-PyYAML==6.0.12.20260906; extra == "dev"
43
+ Dynamic: license-file
44
+
45
+ # drift-or-shift
46
+
47
+ [![CI](https://github.com/DiogoRibeiro7/drift-or-shift/actions/workflows/ci.yml/badge.svg)](https://github.com/DiogoRibeiro7/drift-or-shift/actions/workflows/ci.yml)
48
+ [![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue)](https://www.python.org/)
49
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
50
+ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000)](https://github.com/psf/black)
51
+ [![Checked with mypy](https://img.shields.io/badge/mypy-checked-2a6db2)](https://mypy-lang.org/)
52
+
53
+ A reproducible research pipeline for distinguishing **label shift** from **concept
54
+ drift**, and for showing exactly when a training-free logit offset is sufficient —
55
+ and when it is not.
56
+
57
+ The experiments reproduce and extend the analysis in
58
+ [SSRN 6052514](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=6052514): start from synthetic two-class Gaussian
59
+ data, apply the Bayes-optimal offset correction without retraining, then observe
60
+ which metrics are invariant (ROC AUC) and which are not (PR-AUC, effective sample
61
+ size). When the class-conditionals themselves move, the offset provably cannot
62
+ recover the new posterior, and retraining becomes necessary.
63
+
64
+ ---
65
+
66
+ ## Installation
67
+
68
+ ```bash
69
+ # From source, for development
70
+ git clone https://github.com/DiogoRibeiro7/drift-or-shift.git
71
+ cd drift-or-shift
72
+ make install # editable install + dev extras + pre-commit hooks
73
+ ```
74
+
75
+ Without `make`:
76
+
77
+ ```bash
78
+ python -m pip install -e ".[dev]"
79
+ python -m pre_commit install
80
+ ```
81
+
82
+ Runtime install only:
83
+
84
+ ```bash
85
+ python -m pip install .
86
+ ```
87
+
88
+ Requires Python 3.10+. This installs the `drift_or_shift` package, the
89
+ `drift_shift_pipeline` compatibility alias, the `caliblab` calibration
90
+ benchmarking helpers, and the eleven `dos-expN` console scripts.
91
+
92
+ ## Quickstart
93
+
94
+ ```bash
95
+ # Fast end-to-end sanity check (seconds)
96
+ python scripts/smoke_exp1.py --results-dir results/smoke
97
+
98
+ # A full experiment
99
+ dos-exp1 --pi-train 0.2 --pi-tests 0.05 0.2 0.5 --results-dir results
100
+
101
+ # Aggregate every run into a reviewable dashboard
102
+ python scripts/aggregate_results.py --results-dir results \
103
+ --output reports/dashboard.md --figure reports/best_risk.png
104
+ ```
105
+
106
+ Each run writes `tables/`, `figures/`, and a `_summary.json` into
107
+ `results/<experiment>/<utc-timestamp>/`.
108
+
109
+ ## Core concepts
110
+
111
+ ### Label shift vs. concept drift
112
+
113
+ | | Label shift | Concept drift |
114
+ | --- | --- | --- |
115
+ | What moves | Class prior `π` only | The class-conditionals `p(x\|y)` |
116
+ | Posterior recoverable without retraining? | **Yes** — additive logit offset | **No** |
117
+ | Demonstrated by | Exp 1, 2, 5, 8–11 | Exp 4, 7 |
118
+
119
+ ### Offset correction
120
+
121
+ Logits are log-odds, so shifting the prior from `π_train` to `π_test` is a
122
+ constant additive term:
123
+
124
+ ```
125
+ offset = log( π_test · (1 − π_train) / (π_train · (1 − π_test)) )
126
+ ```
127
+
128
+ Apply the offset **before** comparing against the cost-derived threshold
129
+ `log(c10 / c01)`; with default costs (`c10 = c01 = 1`) that threshold is zero.
130
+ Because a constant offset preserves the ranking of scores, ROC AUC is invariant
131
+ under label shift — while PR-AUC, which depends on prevalence, is not.
132
+
133
+ ## Experiments
134
+
135
+ | Command | What it demonstrates |
136
+ | --- | --- |
137
+ | `dos-exp1` | Synthetic label shift: offset correction vs. oracle risk |
138
+ | `dos-exp2` | ROC AUC invariance and PR-AUC prevalence dependence |
139
+ | `dos-exp3` | ESS fraction shrinking as class weight α grows |
140
+ | `dos-exp4` | Concept drift: standard vs. offset vs. retrained |
141
+ | `dos-exp5` | Breast cancer label shift replication with resampling |
142
+ | `dos-exp6` | Calibration (temperature, isotonic) vs. offset under label shift |
143
+ | `dos-exp7` | Drift-type sweep: covariance, feature, and label shift |
144
+ | `dos-exp8` | Multimodal label shift with mixture components |
145
+ | `dos-exp9` | Covertype: real, high-dimensional label shift |
146
+ | `dos-exp10` | Credit-card fraud benchmark with reweighted prevalences |
147
+ | `dos-exp11` | High-variance medical-style benchmark with nonlinear test shifts |
148
+
149
+ Sources live in [src/drift_or_shift/experiments/](src/drift_or_shift/experiments/).
150
+ Every script accepts `--help`.
151
+
152
+ ## Reproducibility
153
+
154
+ - All generators take an explicit `seed`; summaries record seeds, sizes, costs,
155
+ and prevalence grids.
156
+ - Run-directory names and summary timestamps are **UTC** and timezone-aware.
157
+ - `ExperimentConfig` in `drift_or_shift.experiments._common` (or the CLI flags)
158
+ pins a run's configuration.
159
+ - `reproduce/` holds a separate calibration-benchmark replication driven by
160
+ [reproduce/config/default.yaml](reproduce/config/default.yaml).
161
+
162
+ ## Tooling utilities
163
+
164
+ - `drift_or_shift.drift_variants` — inject covariance/feature shifts, label
165
+ noise, and density-ratio scoring without rebuilding the generators.
166
+ - `scripts/aggregate_results.py` — collect every `results/*/*_summary.json` into
167
+ a markdown dashboard; `--fail-on-alerts` exits non-zero when drift is detected.
168
+ - `scripts/drift_alerts.py` — flag `feature_max_*` statistics above threshold
169
+ into `reports/drift_alerts.csv`.
170
+ - `scripts/watch_results.py` — poll `results/` and refresh alerts + dashboard as
171
+ new summaries land (`--once` for a single pass).
172
+
173
+ See [docs/CLI_USAGE.md](docs/CLI_USAGE.md) for full flag documentation.
174
+
175
+ ## Development
176
+
177
+ ```bash
178
+ make check # ruff + black + mypy + pytest, exactly what CI runs
179
+ make test-cov # tests with an HTML coverage report
180
+ make build # sdist + wheel, validated with twine
181
+ make help # all targets
182
+ ```
183
+
184
+ CI runs the suite on Python 3.10–3.13 on Linux, plus Windows and macOS spot
185
+ checks, and verifies that the built wheel installs and runs in a clean
186
+ environment.
187
+
188
+ See [CONTRIBUTING.md](CONTRIBUTING.md) for branching and PR conventions, and
189
+ [CHANGELOG.md](CHANGELOG.md) for release history.
190
+
191
+ ## Project documents
192
+
193
+ | File | Contents |
194
+ | --- | --- |
195
+ | [RESULTS_DIGEST.md](RESULTS_DIGEST.md) | Latest metrics and artifact paths |
196
+ | [REPORT.md](REPORT.md) | Reproduction steps and what each figure claims |
197
+ | [notes/lessons.md](notes/lessons.md) | Counter-intuitive findings worth revisiting |
198
+ | [ROADMAP.md](ROADMAP.md) | Milestones |
199
+ | [FUTURE.md](FUTURE.md) | Candidate extensions |
200
+ | [RELEASE.md](RELEASE.md) | Release checklist |
201
+
202
+ ## Citation
203
+
204
+ If this code supports academic work, please cite the underlying paper
205
+ ([SSRN 6052514](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=6052514)) and reference this repository:
206
+
207
+ ```bibtex
208
+ @software{ribeiro_drift_shift_pipeline,
209
+ author = {Ribeiro, Diogo},
210
+ title = {drift-or-shift: label shift, offset correction and drift diagnostics},
211
+ url = {https://github.com/DiogoRibeiro7/drift-or-shift},
212
+ version = {0.1.0}
213
+ }
214
+ ```
215
+
216
+ ## License
217
+
218
+ Released under the [MIT License](LICENSE).
@@ -0,0 +1,174 @@
1
+ # drift-or-shift
2
+
3
+ [![CI](https://github.com/DiogoRibeiro7/drift-or-shift/actions/workflows/ci.yml/badge.svg)](https://github.com/DiogoRibeiro7/drift-or-shift/actions/workflows/ci.yml)
4
+ [![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue)](https://www.python.org/)
5
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
6
+ [![Code style: black](https://img.shields.io/badge/code%20style-black-000000)](https://github.com/psf/black)
7
+ [![Checked with mypy](https://img.shields.io/badge/mypy-checked-2a6db2)](https://mypy-lang.org/)
8
+
9
+ A reproducible research pipeline for distinguishing **label shift** from **concept
10
+ drift**, and for showing exactly when a training-free logit offset is sufficient —
11
+ and when it is not.
12
+
13
+ The experiments reproduce and extend the analysis in
14
+ [SSRN 6052514](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=6052514): start from synthetic two-class Gaussian
15
+ data, apply the Bayes-optimal offset correction without retraining, then observe
16
+ which metrics are invariant (ROC AUC) and which are not (PR-AUC, effective sample
17
+ size). When the class-conditionals themselves move, the offset provably cannot
18
+ recover the new posterior, and retraining becomes necessary.
19
+
20
+ ---
21
+
22
+ ## Installation
23
+
24
+ ```bash
25
+ # From source, for development
26
+ git clone https://github.com/DiogoRibeiro7/drift-or-shift.git
27
+ cd drift-or-shift
28
+ make install # editable install + dev extras + pre-commit hooks
29
+ ```
30
+
31
+ Without `make`:
32
+
33
+ ```bash
34
+ python -m pip install -e ".[dev]"
35
+ python -m pre_commit install
36
+ ```
37
+
38
+ Runtime install only:
39
+
40
+ ```bash
41
+ python -m pip install .
42
+ ```
43
+
44
+ Requires Python 3.10+. This installs the `drift_or_shift` package, the
45
+ `drift_shift_pipeline` compatibility alias, the `caliblab` calibration
46
+ benchmarking helpers, and the eleven `dos-expN` console scripts.
47
+
48
+ ## Quickstart
49
+
50
+ ```bash
51
+ # Fast end-to-end sanity check (seconds)
52
+ python scripts/smoke_exp1.py --results-dir results/smoke
53
+
54
+ # A full experiment
55
+ dos-exp1 --pi-train 0.2 --pi-tests 0.05 0.2 0.5 --results-dir results
56
+
57
+ # Aggregate every run into a reviewable dashboard
58
+ python scripts/aggregate_results.py --results-dir results \
59
+ --output reports/dashboard.md --figure reports/best_risk.png
60
+ ```
61
+
62
+ Each run writes `tables/`, `figures/`, and a `_summary.json` into
63
+ `results/<experiment>/<utc-timestamp>/`.
64
+
65
+ ## Core concepts
66
+
67
+ ### Label shift vs. concept drift
68
+
69
+ | | Label shift | Concept drift |
70
+ | --- | --- | --- |
71
+ | What moves | Class prior `π` only | The class-conditionals `p(x\|y)` |
72
+ | Posterior recoverable without retraining? | **Yes** — additive logit offset | **No** |
73
+ | Demonstrated by | Exp 1, 2, 5, 8–11 | Exp 4, 7 |
74
+
75
+ ### Offset correction
76
+
77
+ Logits are log-odds, so shifting the prior from `π_train` to `π_test` is a
78
+ constant additive term:
79
+
80
+ ```
81
+ offset = log( π_test · (1 − π_train) / (π_train · (1 − π_test)) )
82
+ ```
83
+
84
+ Apply the offset **before** comparing against the cost-derived threshold
85
+ `log(c10 / c01)`; with default costs (`c10 = c01 = 1`) that threshold is zero.
86
+ Because a constant offset preserves the ranking of scores, ROC AUC is invariant
87
+ under label shift — while PR-AUC, which depends on prevalence, is not.
88
+
89
+ ## Experiments
90
+
91
+ | Command | What it demonstrates |
92
+ | --- | --- |
93
+ | `dos-exp1` | Synthetic label shift: offset correction vs. oracle risk |
94
+ | `dos-exp2` | ROC AUC invariance and PR-AUC prevalence dependence |
95
+ | `dos-exp3` | ESS fraction shrinking as class weight α grows |
96
+ | `dos-exp4` | Concept drift: standard vs. offset vs. retrained |
97
+ | `dos-exp5` | Breast cancer label shift replication with resampling |
98
+ | `dos-exp6` | Calibration (temperature, isotonic) vs. offset under label shift |
99
+ | `dos-exp7` | Drift-type sweep: covariance, feature, and label shift |
100
+ | `dos-exp8` | Multimodal label shift with mixture components |
101
+ | `dos-exp9` | Covertype: real, high-dimensional label shift |
102
+ | `dos-exp10` | Credit-card fraud benchmark with reweighted prevalences |
103
+ | `dos-exp11` | High-variance medical-style benchmark with nonlinear test shifts |
104
+
105
+ Sources live in [src/drift_or_shift/experiments/](src/drift_or_shift/experiments/).
106
+ Every script accepts `--help`.
107
+
108
+ ## Reproducibility
109
+
110
+ - All generators take an explicit `seed`; summaries record seeds, sizes, costs,
111
+ and prevalence grids.
112
+ - Run-directory names and summary timestamps are **UTC** and timezone-aware.
113
+ - `ExperimentConfig` in `drift_or_shift.experiments._common` (or the CLI flags)
114
+ pins a run's configuration.
115
+ - `reproduce/` holds a separate calibration-benchmark replication driven by
116
+ [reproduce/config/default.yaml](reproduce/config/default.yaml).
117
+
118
+ ## Tooling utilities
119
+
120
+ - `drift_or_shift.drift_variants` — inject covariance/feature shifts, label
121
+ noise, and density-ratio scoring without rebuilding the generators.
122
+ - `scripts/aggregate_results.py` — collect every `results/*/*_summary.json` into
123
+ a markdown dashboard; `--fail-on-alerts` exits non-zero when drift is detected.
124
+ - `scripts/drift_alerts.py` — flag `feature_max_*` statistics above threshold
125
+ into `reports/drift_alerts.csv`.
126
+ - `scripts/watch_results.py` — poll `results/` and refresh alerts + dashboard as
127
+ new summaries land (`--once` for a single pass).
128
+
129
+ See [docs/CLI_USAGE.md](docs/CLI_USAGE.md) for full flag documentation.
130
+
131
+ ## Development
132
+
133
+ ```bash
134
+ make check # ruff + black + mypy + pytest, exactly what CI runs
135
+ make test-cov # tests with an HTML coverage report
136
+ make build # sdist + wheel, validated with twine
137
+ make help # all targets
138
+ ```
139
+
140
+ CI runs the suite on Python 3.10–3.13 on Linux, plus Windows and macOS spot
141
+ checks, and verifies that the built wheel installs and runs in a clean
142
+ environment.
143
+
144
+ See [CONTRIBUTING.md](CONTRIBUTING.md) for branching and PR conventions, and
145
+ [CHANGELOG.md](CHANGELOG.md) for release history.
146
+
147
+ ## Project documents
148
+
149
+ | File | Contents |
150
+ | --- | --- |
151
+ | [RESULTS_DIGEST.md](RESULTS_DIGEST.md) | Latest metrics and artifact paths |
152
+ | [REPORT.md](REPORT.md) | Reproduction steps and what each figure claims |
153
+ | [notes/lessons.md](notes/lessons.md) | Counter-intuitive findings worth revisiting |
154
+ | [ROADMAP.md](ROADMAP.md) | Milestones |
155
+ | [FUTURE.md](FUTURE.md) | Candidate extensions |
156
+ | [RELEASE.md](RELEASE.md) | Release checklist |
157
+
158
+ ## Citation
159
+
160
+ If this code supports academic work, please cite the underlying paper
161
+ ([SSRN 6052514](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=6052514)) and reference this repository:
162
+
163
+ ```bibtex
164
+ @software{ribeiro_drift_shift_pipeline,
165
+ author = {Ribeiro, Diogo},
166
+ title = {drift-or-shift: label shift, offset correction and drift diagnostics},
167
+ url = {https://github.com/DiogoRibeiro7/drift-or-shift},
168
+ version = {0.1.0}
169
+ }
170
+ ```
171
+
172
+ ## License
173
+
174
+ Released under the [MIT License](LICENSE).
@@ -0,0 +1,192 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "drift-or-shift"
7
+ description = "Label shift vs. concept drift: Bayes-optimal offset correction, calibration, and drift diagnostics."
8
+ readme = "README.md"
9
+ requires-python = ">=3.10"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ authors = [{ name = "Diogo Ribeiro", email = "hansolo.dj@gmail.com" }]
13
+ maintainers = [{ name = "Diogo Ribeiro", email = "hansolo.dj@gmail.com" }]
14
+ keywords = [
15
+ "label-shift",
16
+ "concept-drift",
17
+ "prior-shift",
18
+ "calibration",
19
+ "machine-learning",
20
+ "reproducible-research",
21
+ ]
22
+ classifiers = [
23
+ "Development Status :: 3 - Alpha",
24
+ "Intended Audience :: Science/Research",
25
+ "Operating System :: OS Independent",
26
+ "Programming Language :: Python :: 3",
27
+ "Programming Language :: Python :: 3 :: Only",
28
+ "Programming Language :: Python :: 3.10",
29
+ "Programming Language :: Python :: 3.11",
30
+ "Programming Language :: Python :: 3.12",
31
+ "Programming Language :: Python :: 3.13",
32
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
33
+ "Typing :: Typed",
34
+ ]
35
+ dynamic = ["version"]
36
+ dependencies = [
37
+ "numpy>=1.23",
38
+ "scikit-learn>=1.2",
39
+ "pandas>=1.5",
40
+ "matplotlib>=3.6",
41
+ "pyyaml>=6.0",
42
+ ]
43
+
44
+ [project.optional-dependencies]
45
+ # Dev tooling is pinned exactly: an unpinned `ruff`/`mypy` means a new release
46
+ # can turn CI red without a single line of project code changing.
47
+ dev = [
48
+ "black==26.5.1",
49
+ "build==1.6.1",
50
+ "mypy==2.3.1",
51
+ # pandas-stubs 3.x requires Python >= 3.11 and this project supports
52
+ # 3.10, so the dev extra cannot install it. See dependabot.yml.
53
+ "pandas-stubs==2.3.3.260113",
54
+ "pre-commit==4.6.2",
55
+ "pytest==9.1.1",
56
+ "pytest-cov==7.1.0",
57
+ "ruff==0.16.8",
58
+ # tomllib is stdlib from 3.11; 3.10 needs the backport.
59
+ "tomli==2.4.1; python_version < '3.11'",
60
+ "types-PyYAML==6.0.12.20260906",
61
+ ]
62
+
63
+ [project.urls]
64
+ Homepage = "https://github.com/DiogoRibeiro7/drift-or-shift"
65
+ Repository = "https://github.com/DiogoRibeiro7/drift-or-shift"
66
+ Issues = "https://github.com/DiogoRibeiro7/drift-or-shift/issues"
67
+ Changelog = "https://github.com/DiogoRibeiro7/drift-or-shift/blob/main/CHANGELOG.md"
68
+
69
+ [project.scripts]
70
+ dos-exp1 = "drift_or_shift.experiments.exp1_label_shift_synth:main"
71
+ dos-exp2 = "drift_or_shift.experiments.exp2_auc_pr_invariance:main"
72
+ dos-exp3 = "drift_or_shift.experiments.exp3_ess_vs_weight:main"
73
+ dos-exp4 = "drift_or_shift.experiments.exp4_concept_drift:main"
74
+ dos-exp5 = "drift_or_shift.experiments.exp5_realdata_breast_cancer:main"
75
+ dos-exp6 = "drift_or_shift.experiments.exp6_calibration_label_shift:main"
76
+ dos-exp7 = "drift_or_shift.experiments.exp7_drift_types:main"
77
+ dos-exp8 = "drift_or_shift.experiments.exp8_multimodal_label_shift:main"
78
+ dos-exp9 = "drift_or_shift.experiments.exp9_covtype_label_shift:main"
79
+ dos-exp10 = "drift_or_shift.experiments.exp10_credit_card_fraud:main"
80
+ dos-exp11 = "drift_or_shift.experiments.exp11_high_variance_medical:main"
81
+
82
+ # ---------------------------------------------------------------------------
83
+ # Packaging
84
+ # ---------------------------------------------------------------------------
85
+
86
+ [tool.setuptools]
87
+ package-dir = { "" = "src" }
88
+
89
+ [tool.setuptools.packages.find]
90
+ where = ["src"]
91
+
92
+ [tool.setuptools.package-data]
93
+ "*" = ["py.typed"]
94
+
95
+ [tool.setuptools.dynamic]
96
+ version = { attr = "drift_or_shift.__version__" }
97
+
98
+ # ---------------------------------------------------------------------------
99
+ # Lint / format
100
+ # ---------------------------------------------------------------------------
101
+
102
+ [tool.ruff]
103
+ line-length = 88
104
+ target-version = "py310"
105
+ src = ["src", "tests", "scripts", "reproduce"]
106
+
107
+ [tool.ruff.lint]
108
+ select = [
109
+ "E", # pycodestyle errors
110
+ "W", # pycodestyle warnings
111
+ "F", # pyflakes
112
+ "I", # isort
113
+ "B", # flake8-bugbear
114
+ "C4", # flake8-comprehensions
115
+ "UP", # pyupgrade
116
+ "SIM", # flake8-simplify
117
+ "RET", # flake8-return
118
+ "PTH", # flake8-use-pathlib
119
+ "DTZ", # flake8-datetimez
120
+ "RUF", # ruff-specific
121
+ ]
122
+ ignore = [
123
+ "E501", # line length is enforced by the formatter
124
+ ]
125
+
126
+ [tool.ruff.lint.per-file-ignores]
127
+ "tests/*" = ["B018"]
128
+
129
+ [tool.ruff.lint.isort]
130
+ known-first-party = ["drift_or_shift", "drift_shift_pipeline", "caliblab", "reproduce"]
131
+
132
+ [tool.black]
133
+ line-length = 88
134
+ target-version = ["py310"]
135
+
136
+ # ---------------------------------------------------------------------------
137
+ # Types
138
+ # ---------------------------------------------------------------------------
139
+
140
+ [tool.mypy]
141
+ files = ["src"]
142
+ # NOTE: this is deliberately ahead of `requires-python`. NumPy's bundled stubs
143
+ # use PEP 695 `type` statements, which mypy only parses when python_version is
144
+ # 3.12+, so pinning 3.10 here fails before it checks a single project file.
145
+ # The 3.10 floor is still enforced: ruff runs with target-version = "py310",
146
+ # and CI runs the full test suite on a real 3.10 interpreter.
147
+ python_version = "3.12"
148
+ ignore_missing_imports = true
149
+ warn_unused_configs = true
150
+ warn_redundant_casts = true
151
+ warn_unused_ignores = true
152
+ no_implicit_optional = true
153
+ strict_equality = true
154
+
155
+ # ---------------------------------------------------------------------------
156
+ # Tests / coverage
157
+ # ---------------------------------------------------------------------------
158
+
159
+ [tool.pytest.ini_options]
160
+ minversion = "8.0"
161
+ testpaths = ["tests"]
162
+ # `-m "not network"` keeps the default run offline and fast. Run the
163
+ # dataset-downloading experiments explicitly with `-m network`.
164
+ addopts = "-ra --strict-markers --strict-config -m 'not network'"
165
+ markers = [
166
+ "network: requires downloading an external dataset",
167
+ "slow: takes more than a few seconds",
168
+ ]
169
+ filterwarnings = [
170
+ "error::DeprecationWarning:drift_or_shift.*",
171
+ "error::DeprecationWarning:caliblab.*",
172
+ ]
173
+
174
+ [tool.coverage.run]
175
+ # The scripts are documented entry points, so they are measured too. Leaving
176
+ # them out reported 89% while a fifth of the shipped code was invisible.
177
+ source = ["drift_or_shift", "caliblab", "scripts", "reproduce"]
178
+ branch = true
179
+
180
+ [tool.coverage.report]
181
+ show_missing = true
182
+ skip_covered = true
183
+ # A floor, not a target. Set below the current figure so ordinary churn does
184
+ # not fail the build, but a real regression does. exp9/exp10 are excluded from
185
+ # the default run (they download datasets), which caps the achievable number.
186
+ fail_under = 85
187
+ exclude_lines = [
188
+ "pragma: no cover",
189
+ "if TYPE_CHECKING:",
190
+ "raise NotImplementedError",
191
+ 'if __name__ == "__main__":',
192
+ ]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+