lensuq 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lensuq-0.2.0/MANIFEST.in +3 -0
- lensuq-0.2.0/PKG-INFO +157 -0
- lensuq-0.2.0/README.md +139 -0
- lensuq-0.2.0/docs/api.md +11 -0
- lensuq-0.2.0/docs/demo-results.json +154 -0
- lensuq-0.2.0/docs/methodology.md +9 -0
- lensuq-0.2.0/examples/breast_cancer_demo.py +9 -0
- lensuq-0.2.0/pyproject.toml +40 -0
- lensuq-0.2.0/setup.cfg +4 -0
- lensuq-0.2.0/src/lensuq/__init__.py +18 -0
- lensuq-0.2.0/src/lensuq/__main__.py +5 -0
- lensuq-0.2.0/src/lensuq/attribution.py +116 -0
- lensuq-0.2.0/src/lensuq/choice.py +129 -0
- lensuq-0.2.0/src/lensuq/cli.py +42 -0
- lensuq-0.2.0/src/lensuq/demo.py +140 -0
- lensuq-0.2.0/src/lensuq/experiment.py +325 -0
- lensuq-0.2.0/src/lensuq/extended_demos.py +77 -0
- lensuq-0.2.0/src/lensuq/metrics.py +147 -0
- lensuq-0.2.0/src/lensuq/report.py +348 -0
- lensuq-0.2.0/src/lensuq/space.py +189 -0
- lensuq-0.2.0/src/lensuq/universe.py +50 -0
- lensuq-0.2.0/src/lensuq.egg-info/PKG-INFO +157 -0
- lensuq-0.2.0/src/lensuq.egg-info/SOURCES.txt +36 -0
- lensuq-0.2.0/src/lensuq.egg-info/dependency_links.txt +1 -0
- lensuq-0.2.0/src/lensuq.egg-info/entry_points.txt +2 -0
- lensuq-0.2.0/src/lensuq.egg-info/requires.txt +10 -0
- lensuq-0.2.0/src/lensuq.egg-info/top_level.txt +1 -0
- lensuq-0.2.0/tests/test_attribution.py +99 -0
- lensuq-0.2.0/tests/test_choice.py +48 -0
- lensuq-0.2.0/tests/test_cli.py +24 -0
- lensuq-0.2.0/tests/test_demo.py +55 -0
- lensuq-0.2.0/tests/test_experiment.py +140 -0
- lensuq-0.2.0/tests/test_final.py +167 -0
- lensuq-0.2.0/tests/test_integration.py +40 -0
- lensuq-0.2.0/tests/test_metrics.py +69 -0
- lensuq-0.2.0/tests/test_report.py +149 -0
- lensuq-0.2.0/tests/test_space.py +102 -0
- lensuq-0.2.0/tests/test_universe.py +21 -0
lensuq-0.2.0/MANIFEST.in
ADDED
lensuq-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: lensuq
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Quantify prediction sensitivity across alternative preprocessing pipelines
|
|
5
|
+
Author: Sarseej Shrestha
|
|
6
|
+
Project-URL: Repository, https://github.com/sarseej-shrestha/LensUQ
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: numpy>=1.24
|
|
10
|
+
Requires-Dist: pandas>=2.0
|
|
11
|
+
Requires-Dist: scikit-learn>=1.3
|
|
12
|
+
Requires-Dist: rich>=13
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
15
|
+
Requires-Dist: ruff>=0.9; extra == "dev"
|
|
16
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
17
|
+
Requires-Dist: twine>=6; extra == "dev"
|
|
18
|
+
|
|
19
|
+
# LensUQ
|
|
20
|
+
|
|
21
|
+
LensUQ measures how predictions vary across a user-defined set of preprocessing pipelines.
|
|
22
|
+
|
|
23
|
+
## The Problem
|
|
24
|
+
|
|
25
|
+
Preprocessing is often treated as fixed, although several representations may be reasonable. A model fitted after mean imputation can disagree with an independently fitted model using median imputation. That variation is easy to miss when only one pipeline is reported.
|
|
26
|
+
|
|
27
|
+
## The LensUQ Question
|
|
28
|
+
|
|
29
|
+
**How dependent is this prediction on preprocessing assumptions?**
|
|
30
|
+
|
|
31
|
+
For a declared pipeline space P, LensUQ fits an estimator separately in every selected universe and examines the predictions for the same held-out observations. The user decides which alternatives are defensible. LensUQ does not decide which choice is scientifically correct.
|
|
32
|
+
|
|
33
|
+
## What LensUQ Is Not
|
|
34
|
+
|
|
35
|
+
- **GridSearchCV:** there is no performance objective or winning configuration.
|
|
36
|
+
- **An ensemble:** universe predictions are not automatically combined for deployment.
|
|
37
|
+
- **An automatic preprocessing selector:** scientific suitability remains the user's judgment.
|
|
38
|
+
|
|
39
|
+
## Installation
|
|
40
|
+
|
|
41
|
+
Python 3.11 or newer:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
python -m pip install lensuq
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
For development from a checkout:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
git clone https://github.com/sarseej-shrestha/LensUQ.git
|
|
51
|
+
cd LensUQ
|
|
52
|
+
python3 -m venv .venv
|
|
53
|
+
source .venv/bin/activate
|
|
54
|
+
python -m pip install -e '.[dev]'
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Quick Start
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from lensuq import Choice, PipelineSpace
|
|
61
|
+
from sklearn.datasets import load_iris
|
|
62
|
+
from sklearn.linear_model import LogisticRegression
|
|
63
|
+
from sklearn.model_selection import train_test_split
|
|
64
|
+
from sklearn.preprocessing import StandardScaler, RobustScaler
|
|
65
|
+
|
|
66
|
+
X, y = load_iris(return_X_y=True)
|
|
67
|
+
X_train, X_test, y_train, _ = train_test_split(
|
|
68
|
+
X, y, test_size=0.2, stratify=y, random_state=42
|
|
69
|
+
)
|
|
70
|
+
space = PipelineSpace(
|
|
71
|
+
scaler=Choice(StandardScaler(), RobustScaler()),
|
|
72
|
+
model=LogisticRegression(max_iter=1000, random_state=42),
|
|
73
|
+
)
|
|
74
|
+
experiment = space.fit(X_train, y_train, cache=True)
|
|
75
|
+
report = experiment.audit(X_test)
|
|
76
|
+
print(report.most_sensitive(5))
|
|
77
|
+
print(report.sample(report.most_sensitive(1).index[0]).contributions)
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## Core Abstractions
|
|
81
|
+
|
|
82
|
+
`Choice` snapshots an ordered set of cloneable sklearn transformers. It supports explicit labels and `"passthrough"`. `ParamChoice` expands finite parameter lists in declaration order. `PipelineSpace` accepts any number of ordered preprocessing stages followed by a fixed `model`. A fixed stage is allowed; a single model-only space remains supported for compatibility. `Universe` holds an ID, factor labels, and its own cloned sklearn pipeline. `Experiment` owns fitted universes and execution metadata. `AuditReport` and `SampleReport` expose data independently of terminal rendering.
|
|
83
|
+
|
|
84
|
+
The first full-product combination is nominal. In a sampled run, the smallest selected Cartesian ID is nominal for that run. Neither is selected by test performance. Estimator family and settings remain fixed across universes.
|
|
85
|
+
|
|
86
|
+
## Full vs Sampled Spaces
|
|
87
|
+
|
|
88
|
+
`space.n_universes` gives the theoretical Cartesian size. Full enumeration uses rightmost-factor-fastest order. For a large space, sample unique IDs without constructing every pipeline:
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
experiment = space.fit(
|
|
92
|
+
X_train,
|
|
93
|
+
y_train,
|
|
94
|
+
strategy="random",
|
|
95
|
+
n_universes=200,
|
|
96
|
+
random_state=42,
|
|
97
|
+
)
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
The same seed and space give the same sorted IDs. Requests above the possible count fail. Sampled audits provide prediction metrics and exports; `report.attribution` is `None`, and `sample()` raises a clear error because classical orthogonal sums of squares are not valid for incomplete designs.
|
|
101
|
+
|
|
102
|
+
## Classification
|
|
103
|
+
|
|
104
|
+
Classifiers must implement `predict_proba`; probabilities are validated. Binary `experiment.predict_proba(X)` preserves the prototype sample-by-universe DataFrame for `classes_[1]`. The binary audit reports nominal and mean probability, population standard deviation, min, max, range, class counts and fractions, nominal flips, decision agreement, and vote entropy. At threshold `t`, equality predicts `classes_[1]`.
|
|
105
|
+
|
|
106
|
+
Multiclass `predict_proba(X)` returns a sample × universe × class NumPy array in `experiment.classes` order. The audit reports nominal and majority class, vote agreement and entropy, per-class frequencies, mean, population standard deviation, min, max, and range. Class decisions use maximum probability; tied votes select the first class in sklearn class order. Binary and multiclass sensitivity ranking sorts by lower decision agreement, then larger probability range, then larger standard deviation, then input order.
|
|
107
|
+
|
|
108
|
+
## Regression
|
|
109
|
+
|
|
110
|
+
Regressors require `predict`. `experiment.predict(X)` returns a sample-by-universe DataFrame. Audits report nominal, mean, population standard deviation, min, max, range, median, and IQR. Regression ranking sorts by larger raw prediction range, then larger standard deviation, then input order. Dispersion retains the target's units; no unstable near-zero normalization is used.
|
|
111
|
+
|
|
112
|
+
## Attribution
|
|
113
|
+
|
|
114
|
+
On a complete equally weighted Cartesian product, LensUQ decomposes the sum of squared prediction deviations from the grand mean. For each factor, the main-effect sum of squares is `sum_l n_l (mean_l - grand_mean)^2`. Each pair uses the two-factor cell mean after subtracting its two main effects. Remaining higher-order variation is reported as `interaction_residual`. Fractions sum to approximately one when total variation is positive; all fractions are zero for exactly constant predictions.
|
|
115
|
+
|
|
116
|
+
Binary attribution analyzes positive-class probability. Multiclass attribution analyzes the majority class probability for each observation by default, or a specified `class_index` passed to `audit`. Regression attribution analyzes the raw prediction. Contributions describe variation in the configured refitted pipeline experiment; they are not causal effects, p-values, or probabilities of error. Incomplete sampled designs have no attribution in this release.
|
|
117
|
+
|
|
118
|
+
## Execution Reuse
|
|
119
|
+
|
|
120
|
+
`cache=True` reuses fitted training transformations for identical preprocessing prefixes on the same training data. Each cached transformed array and fitted transformer is copied before reuse. Every universe still fits an independent model. The cache is local to one `fit` call and never crosses datasets. `cache=False` fits whole sklearn pipelines normally. Metadata records cache hits and training/analysis time.
|
|
121
|
+
|
|
122
|
+
## Exports
|
|
123
|
+
|
|
124
|
+
`report.to_dataframe()` returns sample metrics with the input index. `report.predictions`, `report.universes`, `report.samples`, and `report.attribution` expose other structured data. `to_csv(path)` writes sample metrics. `to_json(path)` writes metadata, universe coordinates, sample metrics, predictions, and attribution without fitted models. File I/O errors propagate to the caller.
|
|
125
|
+
|
|
126
|
+
## Demonstrations
|
|
127
|
+
|
|
128
|
+
All datasets are bundled with sklearn and use fixed seeds:
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
lensuq info
|
|
132
|
+
lensuq demo
|
|
133
|
+
lensuq demo breast-cancer
|
|
134
|
+
lensuq demo wine
|
|
135
|
+
lensuq demo regression
|
|
136
|
+
python examples/breast_cancer_demo.py
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
The breast cancer example uses artificial missingness, a 455/114 split, three imputers, three scalers, and logistic regression. **This is a controlled prototype demonstration of preprocessing sensitivity, not a medical finding.** The wine example uses three scalers and three feature counts with multiclass logistic regression. The diabetes example uses the same factor structure with Ridge regression. Each demo ranks actual observations; no result row is preselected.
|
|
140
|
+
|
|
141
|
+
## Testing
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
python -m pytest
|
|
145
|
+
ruff check .
|
|
146
|
+
ruff format --check .
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
CI runs these commands and the breast cancer demo on Python 3.11 and 3.14.
|
|
150
|
+
|
|
151
|
+
## Limitations
|
|
152
|
+
|
|
153
|
+
Spaces are finite and unconditional Cartesian products. Factor alternatives are equally weighted, so duplicate semantic options can change the implied measure. Attribution requires a complete balanced product. Prefix caching assumes sklearn-compatible transformers without external mutable state. Users must choose scientifically appropriate factors, train/test splits, and seeds for stochastic estimators. LensUQ does not estimate calibration, correctness, confidence intervals, or data-sampling uncertainty. Probability metrics require `predict_proba`.
|
|
154
|
+
|
|
155
|
+
## Future Work
|
|
156
|
+
|
|
157
|
+
Potential extensions include constrained spaces, attribution estimators for incomplete designs, repeated data and model seeds, richer provenance, and non-sklearn backends. These are research directions, not current features. The [research vault](vault/00%20Home.md) records the project rationale and open questions.
|
lensuq-0.2.0/README.md
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# LensUQ
|
|
2
|
+
|
|
3
|
+
LensUQ measures how predictions vary across a user-defined set of preprocessing pipelines.
|
|
4
|
+
|
|
5
|
+
## The Problem
|
|
6
|
+
|
|
7
|
+
Preprocessing is often treated as fixed, although several representations may be reasonable. A model fitted after mean imputation can disagree with an independently fitted model using median imputation. That variation is easy to miss when only one pipeline is reported.
|
|
8
|
+
|
|
9
|
+
## The LensUQ Question
|
|
10
|
+
|
|
11
|
+
**How dependent is this prediction on preprocessing assumptions?**
|
|
12
|
+
|
|
13
|
+
For a declared pipeline space P, LensUQ fits an estimator separately in every selected universe and examines the predictions for the same held-out observations. The user decides which alternatives are defensible. LensUQ does not decide which choice is scientifically correct.
|
|
14
|
+
|
|
15
|
+
## What LensUQ Is Not
|
|
16
|
+
|
|
17
|
+
- **GridSearchCV:** there is no performance objective or winning configuration.
|
|
18
|
+
- **An ensemble:** universe predictions are not automatically combined for deployment.
|
|
19
|
+
- **An automatic preprocessing selector:** scientific suitability remains the user's judgment.
|
|
20
|
+
|
|
21
|
+
## Installation
|
|
22
|
+
|
|
23
|
+
Python 3.11 or newer:
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
python -m pip install lensuq
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
For development from a checkout:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
git clone https://github.com/sarseej-shrestha/LensUQ.git
|
|
33
|
+
cd LensUQ
|
|
34
|
+
python3 -m venv .venv
|
|
35
|
+
source .venv/bin/activate
|
|
36
|
+
python -m pip install -e '.[dev]'
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Quick Start
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
from lensuq import Choice, PipelineSpace
|
|
43
|
+
from sklearn.datasets import load_iris
|
|
44
|
+
from sklearn.linear_model import LogisticRegression
|
|
45
|
+
from sklearn.model_selection import train_test_split
|
|
46
|
+
from sklearn.preprocessing import StandardScaler, RobustScaler
|
|
47
|
+
|
|
48
|
+
X, y = load_iris(return_X_y=True)
|
|
49
|
+
X_train, X_test, y_train, _ = train_test_split(
|
|
50
|
+
X, y, test_size=0.2, stratify=y, random_state=42
|
|
51
|
+
)
|
|
52
|
+
space = PipelineSpace(
|
|
53
|
+
scaler=Choice(StandardScaler(), RobustScaler()),
|
|
54
|
+
model=LogisticRegression(max_iter=1000, random_state=42),
|
|
55
|
+
)
|
|
56
|
+
experiment = space.fit(X_train, y_train, cache=True)
|
|
57
|
+
report = experiment.audit(X_test)
|
|
58
|
+
print(report.most_sensitive(5))
|
|
59
|
+
print(report.sample(report.most_sensitive(1).index[0]).contributions)
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Core Abstractions
|
|
63
|
+
|
|
64
|
+
`Choice` snapshots an ordered set of cloneable sklearn transformers. It supports explicit labels and `"passthrough"`. `ParamChoice` expands finite parameter lists in declaration order. `PipelineSpace` accepts any number of ordered preprocessing stages followed by a fixed `model`. A fixed stage is allowed; a single model-only space remains supported for compatibility. `Universe` holds an ID, factor labels, and its own cloned sklearn pipeline. `Experiment` owns fitted universes and execution metadata. `AuditReport` and `SampleReport` expose data independently of terminal rendering.
|
|
65
|
+
|
|
66
|
+
The first full-product combination is nominal. In a sampled run, the smallest selected Cartesian ID is nominal for that run. Neither is selected by test performance. Estimator family and settings remain fixed across universes.
|
|
67
|
+
|
|
68
|
+
## Full vs Sampled Spaces
|
|
69
|
+
|
|
70
|
+
`space.n_universes` gives the theoretical Cartesian size. Full enumeration uses rightmost-factor-fastest order. For a large space, sample unique IDs without constructing every pipeline:
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
experiment = space.fit(
|
|
74
|
+
X_train,
|
|
75
|
+
y_train,
|
|
76
|
+
strategy="random",
|
|
77
|
+
n_universes=200,
|
|
78
|
+
random_state=42,
|
|
79
|
+
)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
The same seed and space give the same sorted IDs. Requests above the possible count fail. Sampled audits provide prediction metrics and exports; `report.attribution` is `None`, and `sample()` raises a clear error because classical orthogonal sums of squares are not valid for incomplete designs.
|
|
83
|
+
|
|
84
|
+
## Classification
|
|
85
|
+
|
|
86
|
+
Classifiers must implement `predict_proba`; probabilities are validated. Binary `experiment.predict_proba(X)` preserves the prototype sample-by-universe DataFrame for `classes_[1]`. The binary audit reports nominal and mean probability, population standard deviation, min, max, range, class counts and fractions, nominal flips, decision agreement, and vote entropy. At threshold `t`, equality predicts `classes_[1]`.
|
|
87
|
+
|
|
88
|
+
Multiclass `predict_proba(X)` returns a sample × universe × class NumPy array in `experiment.classes` order. The audit reports nominal and majority class, vote agreement and entropy, per-class frequencies, mean, population standard deviation, min, max, and range. Class decisions use maximum probability; tied votes select the first class in sklearn class order. Binary and multiclass sensitivity ranking sorts by lower decision agreement, then larger probability range, then larger standard deviation, then input order.
|
|
89
|
+
|
|
90
|
+
## Regression
|
|
91
|
+
|
|
92
|
+
Regressors require `predict`. `experiment.predict(X)` returns a sample-by-universe DataFrame. Audits report nominal, mean, population standard deviation, min, max, range, median, and IQR. Regression ranking sorts by larger raw prediction range, then larger standard deviation, then input order. Dispersion retains the target's units; no unstable near-zero normalization is used.
|
|
93
|
+
|
|
94
|
+
## Attribution
|
|
95
|
+
|
|
96
|
+
On a complete equally weighted Cartesian product, LensUQ decomposes the sum of squared prediction deviations from the grand mean. For each factor, the main-effect sum of squares is `sum_l n_l (mean_l - grand_mean)^2`. Each pair uses the two-factor cell mean after subtracting its two main effects. Remaining higher-order variation is reported as `interaction_residual`. Fractions sum to approximately one when total variation is positive; all fractions are zero for exactly constant predictions.
|
|
97
|
+
|
|
98
|
+
Binary attribution analyzes positive-class probability. Multiclass attribution analyzes the majority class probability for each observation by default, or a specified `class_index` passed to `audit`. Regression attribution analyzes the raw prediction. Contributions describe variation in the configured refitted pipeline experiment; they are not causal effects, p-values, or probabilities of error. Incomplete sampled designs have no attribution in this release.
|
|
99
|
+
|
|
100
|
+
## Execution Reuse
|
|
101
|
+
|
|
102
|
+
`cache=True` reuses fitted training transformations for identical preprocessing prefixes on the same training data. Each cached transformed array and fitted transformer is copied before reuse. Every universe still fits an independent model. The cache is local to one `fit` call and never crosses datasets. `cache=False` fits whole sklearn pipelines normally. Metadata records cache hits and training/analysis time.
|
|
103
|
+
|
|
104
|
+
## Exports
|
|
105
|
+
|
|
106
|
+
`report.to_dataframe()` returns sample metrics with the input index. `report.predictions`, `report.universes`, `report.samples`, and `report.attribution` expose other structured data. `to_csv(path)` writes sample metrics. `to_json(path)` writes metadata, universe coordinates, sample metrics, predictions, and attribution without fitted models. File I/O errors propagate to the caller.
|
|
107
|
+
|
|
108
|
+
## Demonstrations
|
|
109
|
+
|
|
110
|
+
All datasets are bundled with sklearn and use fixed seeds:
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
lensuq info
|
|
114
|
+
lensuq demo
|
|
115
|
+
lensuq demo breast-cancer
|
|
116
|
+
lensuq demo wine
|
|
117
|
+
lensuq demo regression
|
|
118
|
+
python examples/breast_cancer_demo.py
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
The breast cancer example uses artificial missingness, a 455/114 split, three imputers, three scalers, and logistic regression. **This is a controlled prototype demonstration of preprocessing sensitivity, not a medical finding.** The wine example uses three scalers and three feature counts with multiclass logistic regression. The diabetes example uses the same factor structure with Ridge regression. Each demo ranks actual observations; no result row is preselected.
|
|
122
|
+
|
|
123
|
+
## Testing
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
python -m pytest
|
|
127
|
+
ruff check .
|
|
128
|
+
ruff format --check .
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
CI runs these commands and the breast cancer demo on Python 3.11 and 3.14.
|
|
132
|
+
|
|
133
|
+
## Limitations
|
|
134
|
+
|
|
135
|
+
Spaces are finite and unconditional Cartesian products. Factor alternatives are equally weighted, so duplicate semantic options can change the implied measure. Attribution requires a complete balanced product. Prefix caching assumes sklearn-compatible transformers without external mutable state. Users must choose scientifically appropriate factors, train/test splits, and seeds for stochastic estimators. LensUQ does not estimate calibration, correctness, confidence intervals, or data-sampling uncertainty. Probability metrics require `predict_proba`.
|
|
136
|
+
|
|
137
|
+
## Future Work
|
|
138
|
+
|
|
139
|
+
Potential extensions include constrained spaces, attribution estimators for incomplete designs, repeated data and model seeds, richer provenance, and non-sklearn backends. These are research directions, not current features. The [research vault](vault/00%20Home.md) records the project rationale and open questions.
|
lensuq-0.2.0/docs/api.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# API contract
|
|
2
|
+
|
|
3
|
+
`Choice(*transformers, labels=None, name=None)` stores cloned ordered alternatives. `"passthrough"` is permitted. `ParamChoice(EstimatorClass, parameter=[values], ...)` expands a finite Cartesian product; labels show parameter names and values. Duplicate labels and empty levels are rejected.
|
|
4
|
+
|
|
5
|
+
`PipelineSpace(stage=Choice(...), ..., model=estimator)` uses keyword order for preprocessing order. `model` is fixed and must be a sklearn classifier with `predict_proba` or a regressor with `predict`. `n_universes`, `factors`, `metadata`, and `universes` expose the declared space. `select_universes(strategy="full"|"random", n_universes=None, random_state=None)` returns fresh universes. `fit(X, y, strategy=..., n_universes=..., random_state=..., cache=False)` fits independent models.
|
|
6
|
+
|
|
7
|
+
`Experiment.predict_proba(X)` returns a DataFrame for binary classification and a sample × universe × class array for multiclass. `predict(X)` returns a regression DataFrame. `audit(X, threshold=0.5, class_index=None)` returns an `AuditReport`. `most_sensitive(n)` and `explain(index)` use the latest successful audit.
|
|
8
|
+
|
|
9
|
+
`AuditReport.dataframe` and `to_dataframe()` contain sample metrics. `predictions`, `universes`, `samples`, `metadata`, and `attribution` expose further results. Binary long-form predictions include the positive-class `probability`; multiclass predictions include `winning_probability` and every `class_N_probability`. `contributions` and `level_means` require full factorial coverage. `most_sensitive(n)` ranks sample rows. `sample(index)` returns a `SampleReport` for full designs. `to_csv(path)` and `to_json(path)` write deterministic schema structures; timing fields are measured per run.
|
|
10
|
+
|
|
11
|
+
Sample indices must be unique, flat, and nonmissing. DataFrame X and Series y must have matching index order. The first executed universe is the nominal reference. Fitted universe pipelines are inspectable; mutating them invalidates experiment results.
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
{
|
|
2
|
+
"python": "3.14.3",
|
|
3
|
+
"numpy": "2.5.4",
|
|
4
|
+
"pandas": "3.0.6",
|
|
5
|
+
"scikit_learn": "1.9.1",
|
|
6
|
+
"training_observations": 455,
|
|
7
|
+
"test_observations": 114,
|
|
8
|
+
"universes": 9,
|
|
9
|
+
"missing_cells": 532,
|
|
10
|
+
"eligible_cells": 3414,
|
|
11
|
+
"disagreement_observations": 6,
|
|
12
|
+
"most_sensitive_sample": 385,
|
|
13
|
+
"metrics": {
|
|
14
|
+
"nominal_probability": 0.16790800962770763,
|
|
15
|
+
"mean_probability": 0.3304226179520909,
|
|
16
|
+
"standard_deviation": 0.18121864501908155,
|
|
17
|
+
"minimum_probability": 0.14890945956058965,
|
|
18
|
+
"maximum_probability": 0.6041025790595507,
|
|
19
|
+
"probability_range": 0.455193119498961,
|
|
20
|
+
"positive_prediction_count": 3,
|
|
21
|
+
"negative_prediction_count": 6,
|
|
22
|
+
"positive_prediction_fraction": 0.3333333333333333,
|
|
23
|
+
"negative_prediction_fraction": 0.6666666666666666,
|
|
24
|
+
"decision_agreement": 0.6666666666666666,
|
|
25
|
+
"class_flip_count": 3,
|
|
26
|
+
"decision_entropy": 0.9182958340544896,
|
|
27
|
+
"nominal_decision": 0,
|
|
28
|
+
"majority_decision": 0,
|
|
29
|
+
"perfectly_stable": false,
|
|
30
|
+
"nominal_class": 0,
|
|
31
|
+
"majority_class": 0
|
|
32
|
+
},
|
|
33
|
+
"contributions": {
|
|
34
|
+
"imputer": 0.0009599943034101484,
|
|
35
|
+
"scaler": 0.9962226546914351,
|
|
36
|
+
"interaction_residual": 0.0028173510051548215
|
|
37
|
+
},
|
|
38
|
+
"total_sum_of_squares": 0.29556177572296705,
|
|
39
|
+
"predictions": [
|
|
40
|
+
{
|
|
41
|
+
"universe_id": 0,
|
|
42
|
+
"choice:imputer": "mean",
|
|
43
|
+
"choice:scaler": "standard",
|
|
44
|
+
"probability": 0.16790800962770763,
|
|
45
|
+
"predicted_class": 0
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"universe_id": 1,
|
|
49
|
+
"choice:imputer": "mean",
|
|
50
|
+
"choice:scaler": "robust",
|
|
51
|
+
"probability": 0.25341603601953744,
|
|
52
|
+
"predicted_class": 0
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"universe_id": 2,
|
|
56
|
+
"choice:imputer": "mean",
|
|
57
|
+
"choice:scaler": "minmax",
|
|
58
|
+
"probability": 0.5729237121115737,
|
|
59
|
+
"predicted_class": 1
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"universe_id": 3,
|
|
63
|
+
"choice:imputer": "median",
|
|
64
|
+
"choice:scaler": "standard",
|
|
65
|
+
"probability": 0.15788403044606353,
|
|
66
|
+
"predicted_class": 0
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
"universe_id": 4,
|
|
70
|
+
"choice:imputer": "median",
|
|
71
|
+
"choice:scaler": "robust",
|
|
72
|
+
"probability": 0.24743639488866442,
|
|
73
|
+
"predicted_class": 0
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
"universe_id": 5,
|
|
77
|
+
"choice:imputer": "median",
|
|
78
|
+
"choice:scaler": "minmax",
|
|
79
|
+
"probability": 0.5639892924532462,
|
|
80
|
+
"predicted_class": 1
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
"universe_id": 6,
|
|
84
|
+
"choice:imputer": "knn_5",
|
|
85
|
+
"choice:scaler": "standard",
|
|
86
|
+
"probability": 0.14890945956058965,
|
|
87
|
+
"predicted_class": 0
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"universe_id": 7,
|
|
91
|
+
"choice:imputer": "knn_5",
|
|
92
|
+
"choice:scaler": "robust",
|
|
93
|
+
"probability": 0.2572340474018845,
|
|
94
|
+
"predicted_class": 0
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
"universe_id": 8,
|
|
98
|
+
"choice:imputer": "knn_5",
|
|
99
|
+
"choice:scaler": "minmax",
|
|
100
|
+
"probability": 0.6041025790595507,
|
|
101
|
+
"predicted_class": 1
|
|
102
|
+
}
|
|
103
|
+
],
|
|
104
|
+
"level_means": [
|
|
105
|
+
{
|
|
106
|
+
"factor": "imputer",
|
|
107
|
+
"level": "mean",
|
|
108
|
+
"mean_probability": 0.33141591925293956,
|
|
109
|
+
"universe_count": 3
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
"factor": "imputer",
|
|
113
|
+
"level": "median",
|
|
114
|
+
"mean_probability": 0.32310323926265805,
|
|
115
|
+
"universe_count": 3
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
"factor": "imputer",
|
|
119
|
+
"level": "knn_5",
|
|
120
|
+
"mean_probability": 0.3367486953406749,
|
|
121
|
+
"universe_count": 3
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
"factor": "scaler",
|
|
125
|
+
"level": "standard",
|
|
126
|
+
"mean_probability": 0.1582338332114536,
|
|
127
|
+
"universe_count": 3
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"factor": "scaler",
|
|
131
|
+
"level": "robust",
|
|
132
|
+
"mean_probability": 0.2526954927700288,
|
|
133
|
+
"universe_count": 3
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
"factor": "scaler",
|
|
137
|
+
"level": "minmax",
|
|
138
|
+
"mean_probability": 0.5803385278747902,
|
|
139
|
+
"universe_count": 3
|
|
140
|
+
}
|
|
141
|
+
],
|
|
142
|
+
"top_ten": [
|
|
143
|
+
385,
|
|
144
|
+
41,
|
|
145
|
+
205,
|
|
146
|
+
363,
|
|
147
|
+
261,
|
|
148
|
+
542,
|
|
149
|
+
128,
|
|
150
|
+
375,
|
|
151
|
+
537,
|
|
152
|
+
247
|
|
153
|
+
]
|
|
154
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Measurement methodology
|
|
2
|
+
|
|
3
|
+
LensUQ measures variation over a finite user-defined set of preprocessing alternatives. Each universe receives the same training rows and an independently cloned estimator. Each preprocessing chain is fitted only on training data. Test observations are transformed with that universe's fitted chain. The framework does not rank pipelines by predictive performance.
|
|
4
|
+
|
|
5
|
+
For a sample, classification decisions are votes across universes. Decision agreement is the largest class count divided by the universe count. Decision entropy is Shannon entropy in bits of class frequencies. Probability dispersion uses population standard deviation. Multiclass probability metrics are computed for every class separately. Regression dispersion uses raw target units and percentile IQR.
|
|
6
|
+
|
|
7
|
+
On a complete Cartesian product, all factor contrasts are orthogonal under equal universe weighting. For values v_u, total SS is sum `(v_u - mean(v))²`. A factor main effect is sum over levels of `n_l (mean_l - mean(v))²`. For pair A,B, each cell contributes `n_ab (mean_ab - mean_a - mean_b + mean(v))²`. The residual is the remaining total SS after main and pairwise effects and may contain third and higher order interactions. Numerical roundoff is clipped at zero. Exactly constant values have zero fractions.
|
|
8
|
+
|
|
9
|
+
For multiclass, default attribution selects each observation's majority class probability; `class_index` selects a single probability column across all observations. For binary, the second class probability is used. For regression, the predicted value is used. Sampled designs retain their sample metrics but do not receive this decomposition. No inferential, causal, or calibration claim follows from these fractions.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "lensuq"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Quantify prediction sensitivity across alternative preprocessing pipelines"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
authors = [{name = "Sarseej Shrestha"}]
|
|
12
|
+
dependencies = [
|
|
13
|
+
"numpy>=1.24",
|
|
14
|
+
"pandas>=2.0",
|
|
15
|
+
"scikit-learn>=1.3",
|
|
16
|
+
"rich>=13",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[project.optional-dependencies]
|
|
20
|
+
dev = ["pytest>=8", "ruff>=0.9", "build>=1.2", "twine>=6"]
|
|
21
|
+
|
|
22
|
+
[project.scripts]
|
|
23
|
+
lensuq = "lensuq.cli:main"
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Repository = "https://github.com/sarseej-shrestha/LensUQ"
|
|
27
|
+
|
|
28
|
+
[tool.setuptools.packages.find]
|
|
29
|
+
where = ["src"]
|
|
30
|
+
|
|
31
|
+
[tool.pytest.ini_options]
|
|
32
|
+
testpaths = ["tests"]
|
|
33
|
+
filterwarnings = ["error"]
|
|
34
|
+
|
|
35
|
+
[tool.ruff]
|
|
36
|
+
target-version = "py311"
|
|
37
|
+
line-length = 88
|
|
38
|
+
|
|
39
|
+
[tool.ruff.lint]
|
|
40
|
+
select = ["E", "F", "I", "UP", "B"]
|
lensuq-0.2.0/setup.cfg
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Prediction sensitivity across preprocessing choices."""
|
|
2
|
+
|
|
3
|
+
from lensuq.choice import Choice, ParamChoice
|
|
4
|
+
from lensuq.experiment import Experiment
|
|
5
|
+
from lensuq.report import AuditReport, SampleReport
|
|
6
|
+
from lensuq.space import PipelineSpace
|
|
7
|
+
from lensuq.universe import Universe
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"Choice",
|
|
11
|
+
"ParamChoice",
|
|
12
|
+
"PipelineSpace",
|
|
13
|
+
"Universe",
|
|
14
|
+
"Experiment",
|
|
15
|
+
"AuditReport",
|
|
16
|
+
"SampleReport",
|
|
17
|
+
]
|
|
18
|
+
__version__ = "0.2.0"
|