robustkit 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- robustkit-0.0.1/LICENSE +21 -0
- robustkit-0.0.1/PKG-INFO +169 -0
- robustkit-0.0.1/README.md +129 -0
- robustkit-0.0.1/pyproject.toml +26 -0
- robustkit-0.0.1/robustkit/__init__.py +54 -0
- robustkit-0.0.1/robustkit/core/__init__.py +0 -0
- robustkit-0.0.1/robustkit/core/consistency.py +33 -0
- robustkit-0.0.1/robustkit/core/diagnostics.py +94 -0
- robustkit-0.0.1/robustkit/core/stability.py +48 -0
- robustkit-0.0.1/robustkit/core/trend.py +86 -0
- robustkit-0.0.1/robustkit/core/uncertainty.py +83 -0
- robustkit-0.0.1/robustkit/information/__init__.py +0 -0
- robustkit-0.0.1/robustkit/information/entropy.py +19 -0
- robustkit-0.0.1/robustkit/information/mutual_info.py +87 -0
- robustkit-0.0.1/robustkit/information/profile.py +32 -0
- robustkit-0.0.1/robustkit/information/quadrants.py +67 -0
- robustkit-0.0.1/robustkit/information/visualization.py +79 -0
- robustkit-0.0.1/robustkit/segmentation/__init__.py +0 -0
- robustkit-0.0.1/robustkit/segmentation/apply.py +49 -0
- robustkit-0.0.1/robustkit/segmentation/hierarchy.py +49 -0
- robustkit-0.0.1/robustkit.egg-info/PKG-INFO +169 -0
- robustkit-0.0.1/robustkit.egg-info/SOURCES.txt +27 -0
- robustkit-0.0.1/robustkit.egg-info/dependency_links.txt +1 -0
- robustkit-0.0.1/robustkit.egg-info/requires.txt +9 -0
- robustkit-0.0.1/robustkit.egg-info/top_level.txt +1 -0
- robustkit-0.0.1/setup.cfg +4 -0
- robustkit-0.0.1/tests/test_core.py +92 -0
- robustkit-0.0.1/tests/test_information.py +120 -0
- robustkit-0.0.1/tests/test_segmentation.py +124 -0
robustkit-0.0.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mikael Lundqvist
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
robustkit-0.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: robustkit
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Practical tools for robust analysis of a single continuous relationship: trend fitting, stability checks, influence diagnostics, and bootstrap uncertainty.
|
|
5
|
+
Author: Mikael Lundqvist
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 Mikael Lundqvist
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Requires-Python: >=3.10
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
License-File: LICENSE
|
|
31
|
+
Requires-Dist: numpy>=1.24
|
|
32
|
+
Requires-Dist: pandas>=2.0
|
|
33
|
+
Requires-Dist: scikit-learn>=1.3
|
|
34
|
+
Requires-Dist: statsmodels>=0.14
|
|
35
|
+
Requires-Dist: scipy>=1.10
|
|
36
|
+
Requires-Dist: matplotlib>=3.7
|
|
37
|
+
Provides-Extra: dev
|
|
38
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
39
|
+
Dynamic: license-file
|
|
40
|
+
|
|
41
|
+
# robustkit
|
|
42
|
+
|
|
43
|
+
> ⚠️ **Under active development.** This is an early placeholder release
|
|
44
|
+
> to claim the package name on PyPI. The API is incomplete and may
|
|
45
|
+
> change without notice. Not yet recommended for production use.
|
|
46
|
+
|
|
47
|
+
Practical tools for robust analysis of a single continuous relationship:
|
|
48
|
+
y as a function of one continuous x.
|
|
49
|
+
|
|
50
|
+
The guiding idea: **a conclusion that survives multiple fitting methods
|
|
51
|
+
is more trustworthy than one that only holds under a single model.**
|
|
52
|
+
`robustkit` makes it easy to compare Huber, Tukey biweight, and OLS
|
|
53
|
+
fits side by side, identify and quantify the influence of individual
|
|
54
|
+
observations, and get honest, bias-corrected uncertainty estimates.
|
|
55
|
+
|
|
56
|
+
## Status
|
|
57
|
+
|
|
58
|
+
`robustkit.core` (trend fitting, stability, diagnostics, uncertainty,
|
|
59
|
+
consistency checks), `robustkit.segmentation` (hierarchical grouping,
|
|
60
|
+
per-segment analysis), and `robustkit.information` (mutual-information
|
|
61
|
+
feature ranking) are stable and tested. A more advanced
|
|
62
|
+
information-theoretic pairing layer (conditional MI for a second
|
|
63
|
+
variable, synergy/redundancy scoring) is planned but not yet included.
|
|
64
|
+
|
|
65
|
+
## Installation
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
git clone https://github.com/<your-username>/robustkit.git
|
|
69
|
+
cd robustkit
|
|
70
|
+
pip install -e ".[dev]"
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## Quickstart
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
import numpy as np
|
|
77
|
+
from robustkit import (
|
|
78
|
+
fit_huber_trend, fit_tukey_trend, predict_trend,
|
|
79
|
+
model_stability_pct, cooks_diagnostic, cook_impact,
|
|
80
|
+
bootstrap_band, bca_bootstrap_ci,
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
# x: a single continuous predictor, y: a single continuous outcome
|
|
84
|
+
x = np.random.default_rng(0).uniform(20, 60, 200)
|
|
85
|
+
y = 1000 + 50 * x - 0.4 * x**2 + np.random.default_rng(1).normal(0, 500, 200)
|
|
86
|
+
|
|
87
|
+
fit = fit_huber_trend(x, y, degree=2)
|
|
88
|
+
y_pred = predict_trend(fit, x_new=[30, 40, 50])
|
|
89
|
+
|
|
90
|
+
stability = model_stability_pct(x, y)
|
|
91
|
+
print("Median % spread between Huber/Tukey/OLS:", stability["median_pct_diff"])
|
|
92
|
+
|
|
93
|
+
diag = cooks_diagnostic(x, y)
|
|
94
|
+
impact = cook_impact(x, y, diag["flagged_indices"])
|
|
95
|
+
print("Median % change in curve if flagged points removed:", impact["median_pct_change"])
|
|
96
|
+
|
|
97
|
+
band = bootstrap_band(x, y)
|
|
98
|
+
ci = bca_bootstrap_ci(x, y, statistic_fn=lambda x_, y_: np.median(y_))
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
See `examples/quickstart_tutorial.py` for a complete, runnable walkthrough.
|
|
102
|
+
|
|
103
|
+
## Segmentation
|
|
104
|
+
|
|
105
|
+
Run any `robustkit.core` analysis independently across subgroups of a
|
|
106
|
+
larger dataset, with automatic fallback to coarser groupings when a
|
|
107
|
+
finer one is too small to analyze reliably:
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
from robustkit import hierarchical_segment, apply_by_segment, model_stability_pct
|
|
111
|
+
|
|
112
|
+
hierarchy = [["department", "level", "status"], ["level", "status"], ["status"]]
|
|
113
|
+
segmented = hierarchical_segment(df, hierarchy, min_size=20)
|
|
114
|
+
|
|
115
|
+
report = apply_by_segment(
|
|
116
|
+
segmented, segment_col="segment_id", x_col="age", y_col="value",
|
|
117
|
+
analysis_fn=model_stability_pct,
|
|
118
|
+
)
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
`apply_by_segment` works with any function shaped like
|
|
122
|
+
`analysis_fn(x, y, **kwargs) -> dict` -- built-in ones
|
|
123
|
+
(`model_stability_pct`, `cook_impact`, `bca_bootstrap_ci`, ...) or your
|
|
124
|
+
own. Only scalar values in the returned dict end up in the report
|
|
125
|
+
table; segments below `min_points` are skipped rather than causing an
|
|
126
|
+
error.
|
|
127
|
+
|
|
128
|
+
## Feature ranking (information)
|
|
129
|
+
|
|
130
|
+
Rank features by mutual information with a target, normalized by each
|
|
131
|
+
feature's own entropy, and classify them into four quadrants:
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
from robustkit import rank_features, quadrant_report, plot_feature_space
|
|
135
|
+
|
|
136
|
+
ranking = rank_features(df, target="value")
|
|
137
|
+
report = quadrant_report(df, target="value") # adds a `quadrant` column
|
|
138
|
+
plot_feature_space(df, target="value") # same quadrants, visualized
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
`quadrant_report` and `plot_feature_space` always agree on quadrant
|
|
142
|
+
assignment -- both route through the same thresholding logic.
|
|
143
|
+
|
|
144
|
+
**Caveat:** default thresholds are the *median* mutual information /
|
|
145
|
+
efficiency across the ranked features. With only a handful of
|
|
146
|
+
features, this can put a genuinely weak feature in the same "high"
|
|
147
|
+
half as a strong one, since roughly half of any list sits above its
|
|
148
|
+
own median regardless of how large the actual gap is. Median
|
|
149
|
+
thresholding becomes meaningful with a reasonably large feature set;
|
|
150
|
+
for a handful of candidates, read the raw `mutual_information` /
|
|
151
|
+
`information_efficiency` values directly rather than relying on the
|
|
152
|
+
quadrant label alone.
|
|
153
|
+
|
|
154
|
+
See `examples/information_tutorial.py` for a complete walkthrough.
|
|
155
|
+
|
|
156
|
+
## Design principles
|
|
157
|
+
|
|
158
|
+
- **One continuous x, one continuous y** at the core. This keeps every
|
|
159
|
+
function's output visually and numerically interpretable (a curve
|
|
160
|
+
you can plot, a band you can read).
|
|
161
|
+
- **Diagnosis and action are separate steps.** `cooks_diagnostic`
|
|
162
|
+
flags candidates; `cook_impact` tells you whether removing them
|
|
163
|
+
actually changes anything.
|
|
164
|
+
- **OLS is a reference point, not the enemy.** Comparing robust fits
|
|
165
|
+
against OLS is how you know whether robustness mattered at all.
|
|
166
|
+
|
|
167
|
+
## License
|
|
168
|
+
|
|
169
|
+
MIT -- see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# robustkit
|
|
2
|
+
|
|
3
|
+
> ⚠️ **Under active development.** This is an early placeholder release
|
|
4
|
+
> to claim the package name on PyPI. The API is incomplete and may
|
|
5
|
+
> change without notice. Not yet recommended for production use.
|
|
6
|
+
|
|
7
|
+
Practical tools for robust analysis of a single continuous relationship:
|
|
8
|
+
y as a function of one continuous x.
|
|
9
|
+
|
|
10
|
+
The guiding idea: **a conclusion that survives multiple fitting methods
|
|
11
|
+
is more trustworthy than one that only holds under a single model.**
|
|
12
|
+
`robustkit` makes it easy to compare Huber, Tukey biweight, and OLS
|
|
13
|
+
fits side by side, identify and quantify the influence of individual
|
|
14
|
+
observations, and get honest, bias-corrected uncertainty estimates.
|
|
15
|
+
|
|
16
|
+
## Status
|
|
17
|
+
|
|
18
|
+
`robustkit.core` (trend fitting, stability, diagnostics, uncertainty,
|
|
19
|
+
consistency checks), `robustkit.segmentation` (hierarchical grouping,
|
|
20
|
+
per-segment analysis), and `robustkit.information` (mutual-information
|
|
21
|
+
feature ranking) are stable and tested. A more advanced
|
|
22
|
+
information-theoretic pairing layer (conditional MI for a second
|
|
23
|
+
variable, synergy/redundancy scoring) is planned but not yet included.
|
|
24
|
+
|
|
25
|
+
## Installation
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
git clone https://github.com/<your-username>/robustkit.git
|
|
29
|
+
cd robustkit
|
|
30
|
+
pip install -e ".[dev]"
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Quickstart
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
import numpy as np
|
|
37
|
+
from robustkit import (
|
|
38
|
+
fit_huber_trend, fit_tukey_trend, predict_trend,
|
|
39
|
+
model_stability_pct, cooks_diagnostic, cook_impact,
|
|
40
|
+
bootstrap_band, bca_bootstrap_ci,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
# x: a single continuous predictor, y: a single continuous outcome
|
|
44
|
+
x = np.random.default_rng(0).uniform(20, 60, 200)
|
|
45
|
+
y = 1000 + 50 * x - 0.4 * x**2 + np.random.default_rng(1).normal(0, 500, 200)
|
|
46
|
+
|
|
47
|
+
fit = fit_huber_trend(x, y, degree=2)
|
|
48
|
+
y_pred = predict_trend(fit, x_new=[30, 40, 50])
|
|
49
|
+
|
|
50
|
+
stability = model_stability_pct(x, y)
|
|
51
|
+
print("Median % spread between Huber/Tukey/OLS:", stability["median_pct_diff"])
|
|
52
|
+
|
|
53
|
+
diag = cooks_diagnostic(x, y)
|
|
54
|
+
impact = cook_impact(x, y, diag["flagged_indices"])
|
|
55
|
+
print("Median % change in curve if flagged points removed:", impact["median_pct_change"])
|
|
56
|
+
|
|
57
|
+
band = bootstrap_band(x, y)
|
|
58
|
+
ci = bca_bootstrap_ci(x, y, statistic_fn=lambda x_, y_: np.median(y_))
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
See `examples/quickstart_tutorial.py` for a complete, runnable walkthrough.
|
|
62
|
+
|
|
63
|
+
## Segmentation
|
|
64
|
+
|
|
65
|
+
Run any `robustkit.core` analysis independently across subgroups of a
|
|
66
|
+
larger dataset, with automatic fallback to coarser groupings when a
|
|
67
|
+
finer one is too small to analyze reliably:
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from robustkit import hierarchical_segment, apply_by_segment, model_stability_pct
|
|
71
|
+
|
|
72
|
+
hierarchy = [["department", "level", "status"], ["level", "status"], ["status"]]
|
|
73
|
+
segmented = hierarchical_segment(df, hierarchy, min_size=20)
|
|
74
|
+
|
|
75
|
+
report = apply_by_segment(
|
|
76
|
+
segmented, segment_col="segment_id", x_col="age", y_col="value",
|
|
77
|
+
analysis_fn=model_stability_pct,
|
|
78
|
+
)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
`apply_by_segment` works with any function shaped like
|
|
82
|
+
`analysis_fn(x, y, **kwargs) -> dict` -- built-in ones
|
|
83
|
+
(`model_stability_pct`, `cook_impact`, `bca_bootstrap_ci`, ...) or your
|
|
84
|
+
own. Only scalar values in the returned dict end up in the report
|
|
85
|
+
table; segments below `min_points` are skipped rather than causing an
|
|
86
|
+
error.
|
|
87
|
+
|
|
88
|
+
## Feature ranking (information)
|
|
89
|
+
|
|
90
|
+
Rank features by mutual information with a target, normalized by each
|
|
91
|
+
feature's own entropy, and classify them into four quadrants:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
from robustkit import rank_features, quadrant_report, plot_feature_space
|
|
95
|
+
|
|
96
|
+
ranking = rank_features(df, target="value")
|
|
97
|
+
report = quadrant_report(df, target="value") # adds a `quadrant` column
|
|
98
|
+
plot_feature_space(df, target="value") # same quadrants, visualized
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
`quadrant_report` and `plot_feature_space` always agree on quadrant
|
|
102
|
+
assignment -- both route through the same thresholding logic.
|
|
103
|
+
|
|
104
|
+
**Caveat:** default thresholds are the *median* mutual information /
|
|
105
|
+
efficiency across the ranked features. With only a handful of
|
|
106
|
+
features, this can put a genuinely weak feature in the same "high"
|
|
107
|
+
half as a strong one, since roughly half of any list sits above its
|
|
108
|
+
own median regardless of how large the actual gap is. Median
|
|
109
|
+
thresholding becomes meaningful with a reasonably large feature set;
|
|
110
|
+
for a handful of candidates, read the raw `mutual_information` /
|
|
111
|
+
`information_efficiency` values directly rather than relying on the
|
|
112
|
+
quadrant label alone.
|
|
113
|
+
|
|
114
|
+
See `examples/information_tutorial.py` for a complete walkthrough.
|
|
115
|
+
|
|
116
|
+
## Design principles
|
|
117
|
+
|
|
118
|
+
- **One continuous x, one continuous y** at the core. This keeps every
|
|
119
|
+
function's output visually and numerically interpretable (a curve
|
|
120
|
+
you can plot, a band you can read).
|
|
121
|
+
- **Diagnosis and action are separate steps.** `cooks_diagnostic`
|
|
122
|
+
flags candidates; `cook_impact` tells you whether removing them
|
|
123
|
+
actually changes anything.
|
|
124
|
+
- **OLS is a reference point, not the enemy.** Comparing robust fits
|
|
125
|
+
against OLS is how you know whether robustness mattered at all.
|
|
126
|
+
|
|
127
|
+
## License
|
|
128
|
+
|
|
129
|
+
MIT -- see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "robustkit"
|
|
7
|
+
version = "0.0.1"
|
|
8
|
+
description = "Practical tools for robust analysis of a single continuous relationship: trend fitting, stability checks, influence diagnostics, and bootstrap uncertainty."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { file = "LICENSE" }
|
|
11
|
+
authors = [{ name = "Mikael Lundqvist" }]
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
dependencies = [
|
|
14
|
+
"numpy>=1.24",
|
|
15
|
+
"pandas>=2.0",
|
|
16
|
+
"scikit-learn>=1.3",
|
|
17
|
+
"statsmodels>=0.14",
|
|
18
|
+
"scipy>=1.10",
|
|
19
|
+
"matplotlib>=3.7",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
dev = ["pytest>=7.0"]
|
|
24
|
+
|
|
25
|
+
[tool.setuptools.packages.find]
|
|
26
|
+
include = ["robustkit*"]
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""
|
|
2
|
+
robustkit
|
|
3
|
+
=========
|
|
4
|
+
|
|
5
|
+
Practical tools for robust analysis of a single continuous relationship
|
|
6
|
+
(y as a function of one continuous x), designed around one core idea:
|
|
7
|
+
a conclusion that survives multiple fitting methods is more trustworthy
|
|
8
|
+
than one that only holds under a single model.
|
|
9
|
+
|
|
10
|
+
Modules:
|
|
11
|
+
robustkit.core -- trend fitting, stability, diagnostics, uncertainty
|
|
12
|
+
robustkit.segmentation -- hierarchical grouping, per-segment analysis
|
|
13
|
+
robustkit.information -- mutual-information-based feature ranking
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from .core.trend import fit_huber_trend, fit_tukey_trend, fit_ols_trend, predict_trend
|
|
17
|
+
from .core.stability import model_stability_pct
|
|
18
|
+
from .core.diagnostics import cooks_diagnostic, cook_impact
|
|
19
|
+
from .core.uncertainty import bootstrap_band, bca_bootstrap_ci
|
|
20
|
+
from .core.consistency import check_row_integrity, compare_row_sets
|
|
21
|
+
from .segmentation.hierarchy import hierarchical_segment, segment_sizes
|
|
22
|
+
from .segmentation.apply import apply_by_segment
|
|
23
|
+
from .information.entropy import entropy
|
|
24
|
+
from .information.mutual_info import rank_features, information_efficiency
|
|
25
|
+
from .information.quadrants import quadrant_report
|
|
26
|
+
from .information.visualization import plot_feature_space, feature_map
|
|
27
|
+
from .information.profile import profile, print_profile
|
|
28
|
+
|
|
29
|
+
__all__ = [
|
|
30
|
+
"fit_huber_trend",
|
|
31
|
+
"fit_tukey_trend",
|
|
32
|
+
"fit_ols_trend",
|
|
33
|
+
"predict_trend",
|
|
34
|
+
"model_stability_pct",
|
|
35
|
+
"cooks_diagnostic",
|
|
36
|
+
"cook_impact",
|
|
37
|
+
"bootstrap_band",
|
|
38
|
+
"bca_bootstrap_ci",
|
|
39
|
+
"check_row_integrity",
|
|
40
|
+
"compare_row_sets",
|
|
41
|
+
"hierarchical_segment",
|
|
42
|
+
"segment_sizes",
|
|
43
|
+
"apply_by_segment",
|
|
44
|
+
"entropy",
|
|
45
|
+
"rank_features",
|
|
46
|
+
"information_efficiency",
|
|
47
|
+
"quadrant_report",
|
|
48
|
+
"plot_feature_space",
|
|
49
|
+
"feature_map",
|
|
50
|
+
"profile",
|
|
51
|
+
"print_profile",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
__version__ = "0.0.1"
|
|
File without changes
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Generic integrity checks, independent of any specific fitting method.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def check_row_integrity(df, group_col):
|
|
7
|
+
total_rows = len(df)
|
|
8
|
+
n_missing_group = int(df[group_col].isna().sum())
|
|
9
|
+
grouped_rows = int(df.groupby(group_col, observed=True).size().sum())
|
|
10
|
+
|
|
11
|
+
return {
|
|
12
|
+
"total_rows": total_rows,
|
|
13
|
+
"rows_accounted_for": grouped_rows,
|
|
14
|
+
"rows_missing_group_label": n_missing_group,
|
|
15
|
+
"consistent": total_rows == grouped_rows + 0,
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def compare_row_sets(before_df, after_df, key_col):
|
|
20
|
+
before_keys = set(before_df[key_col])
|
|
21
|
+
after_keys = set(after_df[key_col])
|
|
22
|
+
|
|
23
|
+
dropped = before_keys - after_keys
|
|
24
|
+
added = after_keys - before_keys
|
|
25
|
+
|
|
26
|
+
return {
|
|
27
|
+
"n_before": len(before_keys),
|
|
28
|
+
"n_after": len(after_keys),
|
|
29
|
+
"n_dropped": len(dropped),
|
|
30
|
+
"n_added": len(added),
|
|
31
|
+
"dropped_keys": dropped,
|
|
32
|
+
"added_keys": added,
|
|
33
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Identify individually influential observations, and quantify how much
|
|
3
|
+
they actually change the fitted trend if removed.
|
|
4
|
+
|
|
5
|
+
cooks_diagnostic() answers "which points stand out?"
|
|
6
|
+
cook_impact() answers "how much does it matter if we remove them?" --
|
|
7
|
+
these are deliberately separate steps, since a flagged point is not
|
|
8
|
+
automatically a point worth acting on.
|
|
9
|
+
|
|
10
|
+
Cook's distance is computed here via plain OLS and the hat matrix
|
|
11
|
+
(pure numpy/scikit-learn), so this module has no statsmodels
|
|
12
|
+
dependency.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
|
|
17
|
+
from .trend import _design_matrix, fit_huber_trend, predict_trend
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def cooks_diagnostic(x, y, degree=2):
|
|
21
|
+
"""
|
|
22
|
+
Compute Cook's distance for each observation, based on an OLS fit
|
|
23
|
+
of y ~ poly(x, degree). Flags points above the conventional 4/n
|
|
24
|
+
threshold.
|
|
25
|
+
"""
|
|
26
|
+
x = np.asarray(x, dtype=float)
|
|
27
|
+
y = np.asarray(y, dtype=float)
|
|
28
|
+
n = len(y)
|
|
29
|
+
|
|
30
|
+
X, _ = _design_matrix(x, degree)
|
|
31
|
+
X_design = np.column_stack([np.ones(n), X])
|
|
32
|
+
p = X_design.shape[1]
|
|
33
|
+
|
|
34
|
+
beta, *_ = np.linalg.lstsq(X_design, y, rcond=None)
|
|
35
|
+
y_hat = X_design @ beta
|
|
36
|
+
residuals = y - y_hat
|
|
37
|
+
|
|
38
|
+
mse = np.sum(residuals ** 2) / (n - p)
|
|
39
|
+
|
|
40
|
+
Q, R = np.linalg.qr(X_design)
|
|
41
|
+
leverage = np.sum(Q ** 2, axis=1)
|
|
42
|
+
leverage = np.clip(leverage, 1e-12, 1 - 1e-12)
|
|
43
|
+
|
|
44
|
+
cooks_d = (residuals ** 2 / (p * mse)) * (leverage / (1 - leverage) ** 2)
|
|
45
|
+
|
|
46
|
+
threshold = 4 / n
|
|
47
|
+
flagged_indices = np.where(cooks_d > threshold)[0]
|
|
48
|
+
|
|
49
|
+
return {
|
|
50
|
+
"cooks_distance": cooks_d,
|
|
51
|
+
"leverage": leverage,
|
|
52
|
+
"threshold": threshold,
|
|
53
|
+
"flagged_indices": flagged_indices,
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def cook_impact(x, y, flagged_indices, degree=2, n_points=50):
|
|
58
|
+
"""
|
|
59
|
+
Compare the Huber-fitted trend curve with and without the given
|
|
60
|
+
flagged observations, to quantify how much they actually pull the
|
|
61
|
+
curve.
|
|
62
|
+
"""
|
|
63
|
+
x = np.asarray(x, dtype=float)
|
|
64
|
+
y = np.asarray(y, dtype=float)
|
|
65
|
+
grid = np.linspace(x.min(), x.max(), n_points)
|
|
66
|
+
|
|
67
|
+
full_fit = fit_huber_trend(x, y, degree=degree)
|
|
68
|
+
full_pred = predict_trend(full_fit, grid)
|
|
69
|
+
|
|
70
|
+
mask = np.ones(len(x), dtype=bool)
|
|
71
|
+
mask[np.asarray(flagged_indices, dtype=int)] = False
|
|
72
|
+
|
|
73
|
+
if mask.sum() < degree + 2:
|
|
74
|
+
raise ValueError(
|
|
75
|
+
"Too few remaining observations to fit a comparison curve "
|
|
76
|
+
"after excluding flagged points."
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
reduced_fit = fit_huber_trend(x[mask], y[mask], degree=degree)
|
|
80
|
+
reduced_pred = predict_trend(reduced_fit, grid)
|
|
81
|
+
|
|
82
|
+
denom = np.abs(full_pred)
|
|
83
|
+
denom[denom == 0] = np.nan
|
|
84
|
+
pct_change = np.abs(full_pred - reduced_pred) / denom * 100
|
|
85
|
+
|
|
86
|
+
return {
|
|
87
|
+
"grid": grid,
|
|
88
|
+
"with_flagged": full_pred,
|
|
89
|
+
"without_flagged": reduced_pred,
|
|
90
|
+
"pct_change": pct_change,
|
|
91
|
+
"median_pct_change": float(np.nanmedian(pct_change)),
|
|
92
|
+
"max_pct_change": float(np.nanmax(pct_change)),
|
|
93
|
+
"n_excluded": int((~mask).sum()),
|
|
94
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Quantify how much a fitted trend curve changes depending on which
|
|
3
|
+
fitting method (Huber, Tukey, or OLS) is used.
|
|
4
|
+
|
|
5
|
+
The central idea: a curve shape that is nearly identical across all
|
|
6
|
+
three methods is a more trustworthy conclusion than one that only
|
|
7
|
+
appears under a single method.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
from .trend import fit_huber_trend, fit_tukey_trend, fit_ols_trend, predict_trend
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def model_stability_pct(x, y, degree=2, n_points=50):
|
|
16
|
+
"""
|
|
17
|
+
Fit Huber, Tukey, and OLS trends to the same (x, y) data, evaluate
|
|
18
|
+
all three on a common grid, and report how far apart they are as a
|
|
19
|
+
percentage of the average predicted value at each grid point.
|
|
20
|
+
"""
|
|
21
|
+
x = np.asarray(x, dtype=float)
|
|
22
|
+
y = np.asarray(y, dtype=float)
|
|
23
|
+
grid = np.linspace(x.min(), x.max(), n_points)
|
|
24
|
+
|
|
25
|
+
huber_fit = fit_huber_trend(x, y, degree=degree)
|
|
26
|
+
tukey_fit = fit_tukey_trend(x, y, degree=degree)
|
|
27
|
+
ols_fit = fit_ols_trend(x, y, degree=degree)
|
|
28
|
+
|
|
29
|
+
huber_pred = predict_trend(huber_fit, grid)
|
|
30
|
+
tukey_pred = predict_trend(tukey_fit, grid)
|
|
31
|
+
ols_pred = predict_trend(ols_fit, grid)
|
|
32
|
+
|
|
33
|
+
curves = np.vstack([huber_pred, tukey_pred, ols_pred])
|
|
34
|
+
spread = curves.max(axis=0) - curves.min(axis=0)
|
|
35
|
+
avg = np.abs(curves.mean(axis=0))
|
|
36
|
+
avg[avg == 0] = np.nan
|
|
37
|
+
pct_spread = spread / avg * 100
|
|
38
|
+
|
|
39
|
+
return {
|
|
40
|
+
"grid": grid,
|
|
41
|
+
"huber": huber_pred,
|
|
42
|
+
"tukey": tukey_pred,
|
|
43
|
+
"ols": ols_pred,
|
|
44
|
+
"pct_spread": pct_spread,
|
|
45
|
+
"median_pct_diff": float(np.nanmedian(pct_spread)),
|
|
46
|
+
"p95_pct_diff": float(np.nanpercentile(pct_spread, 95)),
|
|
47
|
+
"max_pct_diff": float(np.nanmax(pct_spread)),
|
|
48
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Fit a trend curve -- y as a function of a single continuous x -- using
|
|
3
|
+
three different loss functions: Huber, Tukey biweight, and ordinary
|
|
4
|
+
least squares (OLS).
|
|
5
|
+
|
|
6
|
+
All three share the same polynomial design matrix, so their fitted
|
|
7
|
+
curves are directly comparable. OLS is included deliberately: if a
|
|
8
|
+
robust method and OLS agree closely, that agreement is itself useful
|
|
9
|
+
evidence that the conclusion is not driven by a handful of extreme
|
|
10
|
+
points (see robustkit.core.stability).
|
|
11
|
+
|
|
12
|
+
Note: statsmodels is only imported lazily, inside fit_tukey_trend and
|
|
13
|
+
the statsmodels branch of predict_trend. Huber and OLS fitting (via
|
|
14
|
+
scikit-learn) work without statsmodels installed at all.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
from sklearn.linear_model import HuberRegressor, LinearRegression
|
|
19
|
+
from sklearn.preprocessing import PolynomialFeatures
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _design_matrix(x, degree):
|
|
23
|
+
x = np.asarray(x, dtype=float).reshape(-1, 1)
|
|
24
|
+
poly = PolynomialFeatures(degree=degree, include_bias=False)
|
|
25
|
+
return poly.fit_transform(x), poly
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def fit_huber_trend(x, y, degree=2, epsilon=1.35, alpha=0.0001):
|
|
29
|
+
"""
|
|
30
|
+
Fit y ~ poly(x, degree) using Huber loss.
|
|
31
|
+
|
|
32
|
+
epsilon controls the point at which the loss switches from
|
|
33
|
+
quadratic to linear (smaller = more aggressive downweighting of
|
|
34
|
+
outliers; 1.35 is the common default, tuned for ~95% efficiency
|
|
35
|
+
under normal errors).
|
|
36
|
+
"""
|
|
37
|
+
X, poly = _design_matrix(x, degree)
|
|
38
|
+
model = HuberRegressor(epsilon=epsilon, alpha=alpha)
|
|
39
|
+
model.fit(X, y)
|
|
40
|
+
return {"model": model, "poly": poly, "kind": "sklearn"}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def fit_tukey_trend(x, y, degree=2, c=4.685):
|
|
44
|
+
"""
|
|
45
|
+
Fit y ~ poly(x, degree) using Tukey's biweight (redescending) loss
|
|
46
|
+
via statsmodels' robust linear model (RLM).
|
|
47
|
+
|
|
48
|
+
Unlike Huber, Tukey's loss fully suppresses the influence of very
|
|
49
|
+
extreme points rather than merely capping it -- useful when Huber
|
|
50
|
+
still seems pulled by a handful of severe outliers.
|
|
51
|
+
|
|
52
|
+
Requires statsmodels (imported lazily here).
|
|
53
|
+
"""
|
|
54
|
+
import statsmodels.api as sm
|
|
55
|
+
|
|
56
|
+
X, poly = _design_matrix(x, degree)
|
|
57
|
+
X_sm = sm.add_constant(X)
|
|
58
|
+
model = sm.RLM(np.asarray(y, dtype=float), X_sm, M=sm.robust.norms.TukeyBiweight(c=c)).fit()
|
|
59
|
+
return {"model": model, "poly": poly, "kind": "statsmodels"}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def fit_ols_trend(x, y, degree=2):
|
|
63
|
+
"""
|
|
64
|
+
Fit y ~ poly(x, degree) using ordinary least squares. Serves as a
|
|
65
|
+
non-robust reference point, not a method to be discarded.
|
|
66
|
+
"""
|
|
67
|
+
X, poly = _design_matrix(x, degree)
|
|
68
|
+
model = LinearRegression()
|
|
69
|
+
model.fit(X, y)
|
|
70
|
+
return {"model": model, "poly": poly, "kind": "sklearn"}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def predict_trend(fit, x_new):
|
|
74
|
+
"""
|
|
75
|
+
Predict y for new x values from a fit dict returned by
|
|
76
|
+
fit_huber_trend / fit_tukey_trend / fit_ols_trend.
|
|
77
|
+
"""
|
|
78
|
+
x_new = np.asarray(x_new, dtype=float).reshape(-1, 1)
|
|
79
|
+
X_new = fit["poly"].transform(x_new)
|
|
80
|
+
|
|
81
|
+
if fit["kind"] == "statsmodels":
|
|
82
|
+
import statsmodels.api as sm
|
|
83
|
+
X_new = sm.add_constant(X_new, has_constant="add")
|
|
84
|
+
return np.asarray(fit["model"].predict(X_new))
|
|
85
|
+
|
|
86
|
+
return np.asarray(fit["model"].predict(X_new))
|