diag3d 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diag3d/__init__.py +67 -0
- diag3d/__main__.py +3 -0
- diag3d/_version.py +1 -0
- diag3d/baselines.py +246 -0
- diag3d/cli.py +849 -0
- diag3d/config.py +27 -0
- diag3d/dataset.py +160 -0
- diag3d/diagnostics.py +283 -0
- diag3d/lifecycle/__init__.py +39 -0
- diag3d/lifecycle/figures.py +103 -0
- diag3d/lifecycle/findings.py +128 -0
- diag3d/lifecycle/investigate.py +347 -0
- diag3d/lifecycle/monitor.py +720 -0
- diag3d/lifecycle/posthoc.py +984 -0
- diag3d/lifecycle/predictions.py +529 -0
- diag3d/lifecycle/preflight.py +783 -0
- diag3d/lifecycle/report.py +199 -0
- diag3d/lifecycle/run.py +694 -0
- diag3d/lifecycle/splits.py +289 -0
- diag3d/lifecycle/workflow.py +121 -0
- diag3d/model_space/__init__.py +6 -0
- diag3d/model_space/adapters.py +138 -0
- diag3d/model_space/assets/explorer.css +10 -0
- diag3d/model_space/assets/explorer.html +29 -0
- diag3d/model_space/assets/explorer.js +298 -0
- diag3d/model_space/diagnostics.py +143 -0
- diag3d/model_space/glb.py +605 -0
- diag3d/model_space/session.py +498 -0
- diag3d/modeling.py +265 -0
- diag3d/profiling.py +137 -0
- diag3d/relationships.py +166 -0
- diag3d/reporting.py +566 -0
- diag3d/research/__init__.py +19 -0
- diag3d/research/analyzer.py +287 -0
- diag3d/research/benchmarks.py +196 -0
- diag3d/research/calibration.py +298 -0
- diag3d/research/config.py +94 -0
- diag3d/research/diagnostics.py +158 -0
- diag3d/research/discovery.py +155 -0
- diag3d/research/explorer.py +523 -0
- diag3d/research/fields.py +115 -0
- diag3d/research/figures.py +1098 -0
- diag3d/research/nested.py +457 -0
- diag3d/research/nulls.py +73 -0
- diag3d/research/oof.py +197 -0
- diag3d/research/refit.py +220 -0
- diag3d/research/repairs.py +205 -0
- diag3d/research/stability.py +135 -0
- diag3d/research/types.py +360 -0
- diag3d/spatial.py +86 -0
- diag3d/surfaces.py +94 -0
- diag3d/utils.py +71 -0
- diag3d/visualization.py +423 -0
- diag3d-0.2.0.dist-info/METADATA +327 -0
- diag3d-0.2.0.dist-info/RECORD +59 -0
- diag3d-0.2.0.dist-info/WHEEL +4 -0
- diag3d-0.2.0.dist-info/entry_points.txt +2 -0
- diag3d-0.2.0.dist-info/licenses/LICENSE +21 -0
- diag3d-0.2.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +38 -0
diag3d/__init__.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Diag3D: have a CSV? Point Diag3D at it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import TYPE_CHECKING, Any
|
|
7
|
+
|
|
8
|
+
from .config import VERSION, Config
|
|
9
|
+
from .dataset import Dataset
|
|
10
|
+
from .model_space import ModelAdapter, ModelSpace, SklearnAdapter, explore
|
|
11
|
+
from .research import (
|
|
12
|
+
AnalysisConfig,
|
|
13
|
+
AnalysisResult,
|
|
14
|
+
CalibrationConfig,
|
|
15
|
+
CharacterizationConfig,
|
|
16
|
+
FailureAnalyzer,
|
|
17
|
+
NestedRepairConfig,
|
|
18
|
+
RepairGuardrail,
|
|
19
|
+
)
|
|
20
|
+
from .utils import Diag3DError
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
import plotly.graph_objects as go
|
|
24
|
+
|
|
25
|
+
from .reporting import Report
|
|
26
|
+
|
|
27
|
+
__version__ = VERSION
|
|
28
|
+
__all__ = [
|
|
29
|
+
"Config",
|
|
30
|
+
"Dataset",
|
|
31
|
+
"Diag3DError",
|
|
32
|
+
"inspect_csv",
|
|
33
|
+
"report_csv",
|
|
34
|
+
"visualize3d",
|
|
35
|
+
"__version__",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def inspect_csv(path: str | Path, **loading: Any) -> dict[str, Any]:
|
|
40
|
+
"""Profile a CSV. Keyword arguments are forwarded to :class:`Dataset`."""
|
|
41
|
+
return Dataset(path, **loading).inspect()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def report_csv(path: str | Path, *, target: str | None = None, **options: Any) -> Report:
|
|
45
|
+
"""Build an HTML Report with default loading limits."""
|
|
46
|
+
return Dataset(path).report(target=target, **options)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def visualize3d(path: str | Path, *, x: str, y: str, z: str, **options: Any) -> go.Figure:
|
|
50
|
+
"""Create a Plotly relationship cloud directly from a CSV."""
|
|
51
|
+
return Dataset(path).visualize3d(x=x, y=y, z=z, **options)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
__all__ += ["FailureAnalyzer", "AnalysisConfig", "CharacterizationConfig", "AnalysisResult"]
|
|
55
|
+
|
|
56
|
+
__all__ += ["CalibrationConfig", "NestedRepairConfig", "RepairGuardrail"]
|
|
57
|
+
|
|
58
|
+
__all__ += ["ModelSpace", "ModelAdapter", "SklearnAdapter", "explore"]
|
|
59
|
+
|
|
60
|
+
from .baselines import ( # noqa: E402
|
|
61
|
+
BASELINES,
|
|
62
|
+
BaselineBenchmark,
|
|
63
|
+
baseline_models,
|
|
64
|
+
benchmark_baselines,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
__all__ += ["BASELINES", "BaselineBenchmark", "baseline_models", "benchmark_baselines"]
|
diag3d/__main__.py
ADDED
diag3d/_version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.0"
|
diag3d/baselines.py
ADDED
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"""Reference baseline models for diagnosis and benchmarking. Not AutoML.
|
|
2
|
+
|
|
3
|
+
A compact suite of regression baselines that share one split manifest, one
|
|
4
|
+
evaluation population, one preprocessing rule (median imputation and standard
|
|
5
|
+
scaling fitted on the training rows of each partition), documented default
|
|
6
|
+
parameters, deterministic seeds where the estimator supports them, and wall-clock
|
|
7
|
+
timings for fitting and predicting. The result is a validated
|
|
8
|
+
:class:`~diag3d.lifecycle.PredictionTable`, so every post-hoc diagnostic applies
|
|
9
|
+
to the baselines exactly as it applies to a user's model.
|
|
10
|
+
|
|
11
|
+
Optional frameworks stay optional: ``xgb`` needs ``xgboost`` and ``lgbm`` needs
|
|
12
|
+
``lightgbm``; they are imported only when requested and ``import diag3d`` never
|
|
13
|
+
imports them.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import platform
|
|
19
|
+
import sys
|
|
20
|
+
import time
|
|
21
|
+
from dataclasses import dataclass, field
|
|
22
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any, Mapping, Sequence
|
|
25
|
+
|
|
26
|
+
import numpy as np
|
|
27
|
+
import polars as pl
|
|
28
|
+
|
|
29
|
+
from .config import VERSION
|
|
30
|
+
from .lifecycle.predictions import PredictionTable
|
|
31
|
+
from .lifecycle.splits import SplitManifest, partitions
|
|
32
|
+
from .research.analyzer import default_models
|
|
33
|
+
from .research.oof import seeded_clone, train_transform
|
|
34
|
+
from .utils import write_json
|
|
35
|
+
|
|
36
|
+
BASELINES = ("linear", "rf", "hgb", "xgb", "lgbm", "mlp")
|
|
37
|
+
DEFAULT_BASELINES = ("linear", "rf", "hgb")
|
|
38
|
+
PREPROCESSING = ("median imputation (keep_empty_features=True) then standard scaling, "
|
|
39
|
+
"both fitted on the partition's training rows only")
|
|
40
|
+
_OPTIONAL = {"xgb": "xgboost", "lgbm": "lightgbm"}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def baseline_models(names: Sequence[str] = DEFAULT_BASELINES, *, seed: int = 42,
|
|
44
|
+
n_jobs: int = 1) -> dict[str, Any]:
|
|
45
|
+
"""Unfitted estimators by name (``linear``, ``rf``, ``hgb``, ``xgb``, ``lgbm``, ``mlp``).
|
|
46
|
+
|
|
47
|
+
Defaults: Ridge(alpha=1.0); RandomForestRegressor(n_estimators=100,
|
|
48
|
+
min_samples_leaf=3); HistGradientBoostingRegressor(max_iter=100, max_leaf_nodes=15);
|
|
49
|
+
XGBRegressor(n_estimators=100, max_depth=4); LGBMRegressor(n_estimators=100);
|
|
50
|
+
MLPRegressor(hidden_layer_sizes=(48, 24), max_iter=400, early_stopping=True).
|
|
51
|
+
``seed`` is passed as ``random_state`` wherever supported.
|
|
52
|
+
"""
|
|
53
|
+
unknown = [n for n in names if n not in BASELINES]
|
|
54
|
+
if unknown:
|
|
55
|
+
raise ValueError(f"Unknown baselines {unknown}; choose from {BASELINES}")
|
|
56
|
+
return default_models(tuple(names), seed=seed, n_jobs=n_jobs)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def available_baselines() -> dict[str, bool]:
|
|
60
|
+
"""Which baseline names can be built in this environment."""
|
|
61
|
+
out = {}
|
|
62
|
+
for name in BASELINES:
|
|
63
|
+
package = _OPTIONAL.get(name)
|
|
64
|
+
if package is None:
|
|
65
|
+
out[name] = True
|
|
66
|
+
continue
|
|
67
|
+
try:
|
|
68
|
+
version(package)
|
|
69
|
+
out[name] = True
|
|
70
|
+
except PackageNotFoundError:
|
|
71
|
+
out[name] = False
|
|
72
|
+
return out
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _params(estimator) -> dict[str, Any]:
|
|
76
|
+
try:
|
|
77
|
+
raw = estimator.get_params(deep=False)
|
|
78
|
+
except Exception: # pragma: no cover - non-sklearn estimators
|
|
79
|
+
return {"repr": repr(estimator)}
|
|
80
|
+
return {k: (v if isinstance(v, (int, float, str, bool, type(None))) else repr(v))
|
|
81
|
+
for k, v in raw.items()}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass
|
|
85
|
+
class BaselineBenchmark:
|
|
86
|
+
"""Predictions and timings of the reference baselines on one split manifest."""
|
|
87
|
+
|
|
88
|
+
predictions: PredictionTable
|
|
89
|
+
timings: pl.DataFrame
|
|
90
|
+
metrics: pl.DataFrame
|
|
91
|
+
parameters: dict[str, dict[str, Any]]
|
|
92
|
+
split: dict[str, Any]
|
|
93
|
+
environment: dict[str, Any]
|
|
94
|
+
preprocessing: str = PREPROCESSING
|
|
95
|
+
notes: list[str] = field(default_factory=list)
|
|
96
|
+
|
|
97
|
+
def to_dict(self) -> dict[str, Any]:
|
|
98
|
+
return {"metrics": self.metrics.to_dicts(), "timings": self.timings.to_dicts(),
|
|
99
|
+
"parameters": self.parameters, "split": self.split,
|
|
100
|
+
"environment": self.environment, "preprocessing": self.preprocessing,
|
|
101
|
+
"notes": self.notes, "predictions": self.predictions.metadata()}
|
|
102
|
+
|
|
103
|
+
def save(self, directory: str | Path) -> Path:
|
|
104
|
+
"""Write ``baselines.json``, ``metrics.csv``, ``timings.csv`` and the predictions."""
|
|
105
|
+
dest = Path(directory)
|
|
106
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
107
|
+
write_json(dest / "baselines.json", self.to_dict())
|
|
108
|
+
self.metrics.write_csv(dest / "metrics.csv")
|
|
109
|
+
self.timings.write_csv(dest / "timings.csv")
|
|
110
|
+
self.predictions.to_parquet(dest / "predictions.parquet")
|
|
111
|
+
return dest
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def benchmark_baselines(
|
|
115
|
+
samples: pl.DataFrame,
|
|
116
|
+
*,
|
|
117
|
+
sample_id: str,
|
|
118
|
+
target: str,
|
|
119
|
+
features: Sequence[str],
|
|
120
|
+
split: SplitManifest,
|
|
121
|
+
models: Sequence[str] | Mapping[str, Any] = DEFAULT_BASELINES,
|
|
122
|
+
seed: int = 42,
|
|
123
|
+
n_jobs: int = 1,
|
|
124
|
+
) -> BaselineBenchmark:
|
|
125
|
+
"""Fit and evaluate the baselines on every partition of an explicit split manifest.
|
|
126
|
+
|
|
127
|
+
``models`` is a sequence of baseline names or a mapping name → unfitted estimator.
|
|
128
|
+
Fold-level partitions produce ``oof`` predictions; without folds, the role-level
|
|
129
|
+
``train`` → ``validation`` partition produces ``validation`` predictions. Training
|
|
130
|
+
rows are exactly the manifest's ``train`` members of each partition; nothing is
|
|
131
|
+
inferred from fold labels.
|
|
132
|
+
"""
|
|
133
|
+
if not isinstance(split, SplitManifest):
|
|
134
|
+
raise TypeError("split must be a SplitManifest with explicit training membership")
|
|
135
|
+
for c in (sample_id, target, *features):
|
|
136
|
+
if c not in samples.columns:
|
|
137
|
+
raise ValueError(f"Column {c!r} not in samples")
|
|
138
|
+
if not features:
|
|
139
|
+
raise ValueError("At least one feature is required")
|
|
140
|
+
if isinstance(models, Mapping):
|
|
141
|
+
estimators = dict(models)
|
|
142
|
+
else:
|
|
143
|
+
estimators = baseline_models(tuple(models), seed=seed, n_jobs=n_jobs)
|
|
144
|
+
if not estimators:
|
|
145
|
+
raise ValueError("Select at least one model")
|
|
146
|
+
frame = (samples.with_columns(pl.col(sample_id).cast(pl.String).alias("__id"))
|
|
147
|
+
.sort("__id").rechunk())
|
|
148
|
+
ids = frame["__id"].to_numpy().astype(str)
|
|
149
|
+
X = frame.select(list(features)).cast(pl.Float64).to_numpy()
|
|
150
|
+
y = frame[target].cast(pl.Float64).to_numpy()
|
|
151
|
+
if not np.isfinite(y).all():
|
|
152
|
+
raise ValueError("The target must be finite for every row")
|
|
153
|
+
parts = partitions(split, ids)
|
|
154
|
+
folds = [p for p in parts if p.level == "fold"]
|
|
155
|
+
use = folds or [p for p in parts if p.level == "role" and p.evaluation_role == "validation"]
|
|
156
|
+
if not use:
|
|
157
|
+
raise ValueError("The manifest defines no fold-level or train->validation partition "
|
|
158
|
+
"with members in the sample table")
|
|
159
|
+
rows, timings = [], []
|
|
160
|
+
for part in use:
|
|
161
|
+
train, test = train_transform(X[part.train_rows], X[part.evaluation_rows])
|
|
162
|
+
y_train = y[part.train_rows]
|
|
163
|
+
for name, estimator in estimators.items():
|
|
164
|
+
model = seeded_clone(estimator, seed, n_jobs)
|
|
165
|
+
t0 = time.perf_counter()
|
|
166
|
+
model.fit(train, y_train)
|
|
167
|
+
fit_s = time.perf_counter() - t0
|
|
168
|
+
t1 = time.perf_counter()
|
|
169
|
+
pred = np.asarray(model.predict(test), dtype=float)
|
|
170
|
+
predict_s = time.perf_counter() - t1
|
|
171
|
+
rows.append(pl.DataFrame({
|
|
172
|
+
"sample_id": ids[part.evaluation_rows], "model": name, "repeat": part.repeat,
|
|
173
|
+
"fold": part.fold, "role": "oof" if part.level == "fold" else "validation",
|
|
174
|
+
"y_true": y[part.evaluation_rows], "y_pred": pred}))
|
|
175
|
+
timings.append({"model": name, "repeat": part.repeat, "fold": part.fold,
|
|
176
|
+
"train_rows": int(len(part.train_rows)),
|
|
177
|
+
"evaluation_rows": int(len(part.evaluation_rows)),
|
|
178
|
+
"fit_seconds": fit_s, "predict_seconds": predict_s})
|
|
179
|
+
table = PredictionTable.from_frame(
|
|
180
|
+
pl.concat(rows, how="vertical_relaxed").with_columns(pl.col("fold").cast(pl.Int64),
|
|
181
|
+
pl.col("repeat").cast(pl.Int64)),
|
|
182
|
+
source={"origin": "diag3d.baselines.benchmark_baselines", "seed": seed,
|
|
183
|
+
"preprocessing": PREPROCESSING})
|
|
184
|
+
timing_tbl = pl.DataFrame(timings, strict=False)
|
|
185
|
+
metrics = _metrics(table, timing_tbl)
|
|
186
|
+
env = {"diag3d_version": VERSION, "python": platform.python_version(),
|
|
187
|
+
"platform": platform.platform(), "cpu_count": _cpu_count(), "seed": seed,
|
|
188
|
+
"n_jobs": n_jobs, "peak_rss_mb": _peak_rss_mb(),
|
|
189
|
+
"packages": {p: _version_or_none(p) for p in
|
|
190
|
+
("numpy", "polars", "scikit-learn", "scipy", "xgboost", "lightgbm")}}
|
|
191
|
+
return BaselineBenchmark(table, timing_tbl, metrics, {n: _params(e) for n, e in
|
|
192
|
+
estimators.items()},
|
|
193
|
+
split.metadata(), env,
|
|
194
|
+
notes=["Baseline estimators with documented defaults; not tuned "
|
|
195
|
+
"models.",
|
|
196
|
+
"Every model sees the same partitions and the same "
|
|
197
|
+
"preprocessed inputs; timings are wall-clock seconds on "
|
|
198
|
+
"this machine."])
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _metrics(table: PredictionTable, timings: pl.DataFrame) -> pl.DataFrame:
|
|
202
|
+
f = table.frame.with_columns((pl.col("y_true") - pl.col("y_pred")).alias("residual"))
|
|
203
|
+
per = (f.group_by(["model", "sample_id"], maintain_order=True)
|
|
204
|
+
.agg(pl.col("y_true").first(), pl.col("y_pred").mean().alias("pred_mean"),
|
|
205
|
+
pl.col("residual").abs().mean().alias("abs_error"),
|
|
206
|
+
(pl.col("residual") ** 2).mean().alias("squared_error")))
|
|
207
|
+
rows = []
|
|
208
|
+
for name in sorted(per["model"].unique().to_list()):
|
|
209
|
+
s = per.filter(pl.col("model") == name)
|
|
210
|
+
y, p = s["y_true"].to_numpy(), s["pred_mean"].to_numpy()
|
|
211
|
+
r2 = (float(1 - np.sum((y - p) ** 2) / np.sum((y - y.mean()) ** 2))
|
|
212
|
+
if len(y) > 1 and np.ptp(y) > 0 else None)
|
|
213
|
+
t = timings.filter(pl.col("model") == name)
|
|
214
|
+
rows.append({"model": name, "samples": s.height,
|
|
215
|
+
"mae": float(s["abs_error"].mean()),
|
|
216
|
+
"rmse": float(np.sqrt(s["squared_error"].mean())),
|
|
217
|
+
"bias_mean_residual": float((y - p).mean()),
|
|
218
|
+
"r2_of_mean_prediction": r2,
|
|
219
|
+
"fit_seconds_total": float(t["fit_seconds"].sum()),
|
|
220
|
+
"predict_seconds_total": float(t["predict_seconds"].sum()),
|
|
221
|
+
"partitions": t.height})
|
|
222
|
+
return pl.DataFrame(rows, strict=False)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _cpu_count() -> int | None:
|
|
226
|
+
try:
|
|
227
|
+
return len(__import__("os").sched_getaffinity(0))
|
|
228
|
+
except AttributeError:
|
|
229
|
+
return __import__("os").cpu_count()
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _peak_rss_mb() -> float | None:
|
|
233
|
+
try:
|
|
234
|
+
import resource
|
|
235
|
+
|
|
236
|
+
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
|
237
|
+
except (ImportError, AttributeError): # Windows has no resource module
|
|
238
|
+
return None
|
|
239
|
+
return float(peak / 1024) if sys.platform != "darwin" else float(peak / 2**20)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _version_or_none(name: str) -> str | None:
|
|
243
|
+
try:
|
|
244
|
+
return version(name)
|
|
245
|
+
except PackageNotFoundError:
|
|
246
|
+
return None
|