framework-py 0.1.0.dev4__py3-none-any.whl → 0.1.0.dev5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,930 @@
1
+ """Structured exports for objects published to Framework Cloud.
2
+
3
+ The document format is ``framework.artifact/v1``. Tables carry the rows a person
4
+ chose to publish. Models carry coefficients, fit statistics, and other aggregates,
5
+ never the observations they were fit on or a serialized model. Optional libraries
6
+ (numpy aside) are detected by module and class name and imported only when an object
7
+ from them is published.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import math
14
+ from collections.abc import Callable, Iterable, Sequence
15
+ from datetime import date, datetime
16
+ from pathlib import Path
17
+ from typing import Any
18
+
19
+ import numpy as np
20
+ import pandas as pd
21
+ from pandas.api import types as dtypes
22
+
23
+ from framework.errors import FrameworkError
24
+
25
+ SCHEMA = "framework.artifact/v1"
26
+ SUMMARY_COLUMNS = 200
27
+ HISTOGRAM_BINS = 20
28
+ TOP_VALUES = 8
29
+ TOP_VALUE_LENGTH = 80
30
+ SVG_LIMIT = 5 * 1024 * 1024
31
+ CONFIDENCE_LEVEL = 0.95
32
+
33
+
34
+ # --------------------------------------------------------------------------- helpers
35
+
36
+
37
+ def scalar(value: Any) -> Any:
38
+ """Return a JSON scalar while preserving missing values and finite numbers."""
39
+ if value is None or value is pd.NA or value is pd.NaT:
40
+ return None
41
+ if isinstance(value, (datetime, date)):
42
+ return value.isoformat()
43
+ if hasattr(value, "item"):
44
+ value = value.item()
45
+ if isinstance(value, float) and not math.isfinite(value):
46
+ return None
47
+ if isinstance(value, (str, int, float, bool)):
48
+ return value
49
+ if pd.api.types.is_scalar(value) and pd.isna(value):
50
+ return None
51
+ return str(value)
52
+
53
+
54
+ def _classes(source: Any) -> list[tuple[str, str]]:
55
+ return [(cls.__module__, cls.__name__) for cls in type(source).__mro__]
56
+
57
+
58
+ def _from(source: Any, *modules: str) -> bool:
59
+ return any(module.split(".")[0] in modules for module, _ in _classes(source))
60
+
61
+
62
+ def _has_class(source: Any, module: str, *names: str) -> bool:
63
+ return any(m.split(".")[0] == module and n in names for m, n in _classes(source))
64
+
65
+
66
+ def _metric(name: str, value: Any) -> dict[str, Any] | None:
67
+ if value is None or isinstance(value, (list, tuple, dict, set)):
68
+ return None
69
+ if not pd.api.types.is_scalar(value):
70
+ return None
71
+ value = scalar(value)
72
+ return None if value is None else {"name": name, "value": value}
73
+
74
+
75
+ def _metrics(pairs: Iterable[tuple[str, Any]]) -> list[dict[str, Any]]:
76
+ return [m for name, value in pairs if (m := _metric(name, value)) is not None]
77
+
78
+
79
+ def _document(source: Any, kind: str) -> dict[str, Any]:
80
+ return {
81
+ "schema": SCHEMA,
82
+ "kind": kind,
83
+ "source": {
84
+ "language": "python",
85
+ "class": [f"{module}.{name}" for module, name in _classes(source) if name != "object"],
86
+ },
87
+ "tables": [],
88
+ "metrics": [],
89
+ }
90
+
91
+
92
+ # --------------------------------------------------------------------------- tables
93
+
94
+
95
+ def _column_type(series: pd.Series) -> str:
96
+ dtype = series.dtype
97
+ if dtypes.is_bool_dtype(dtype):
98
+ return "boolean"
99
+ if dtypes.is_numeric_dtype(dtype) and not dtypes.is_complex_dtype(dtype):
100
+ # Browser JSON numbers cannot represent large integer identifiers exactly.
101
+ unsafe_integer = dtypes.is_integer_dtype(dtype) and any(
102
+ abs(int(value)) > 2**53 - 1 for value in series.dropna()
103
+ )
104
+ return "string" if unsafe_integer else "number"
105
+ if isinstance(dtype, pd.ArrowDtype) and dtype.kind == "M" and "date" in str(dtype):
106
+ return "date"
107
+ if dtypes.is_datetime64_any_dtype(dtype):
108
+ return "datetime"
109
+ values = series.dropna()
110
+ if len(values) and all(isinstance(v, date) and not isinstance(v, datetime) for v in values):
111
+ return "date"
112
+ if len(values) and all(isinstance(v, datetime) for v in values):
113
+ return "datetime"
114
+ return "string"
115
+
116
+
117
+ def _top(values: pd.Series) -> list[dict[str, Any]]:
118
+ counts = values.value_counts(dropna=True, sort=True).head(TOP_VALUES)
119
+ top = []
120
+ for value, count in counts.items():
121
+ value = scalar(value)
122
+ if isinstance(value, str) and len(value) > TOP_VALUE_LENGTH:
123
+ value = value[:TOP_VALUE_LENGTH]
124
+ top.append({"value": value, "count": int(count)})
125
+ return top
126
+
127
+
128
+ def _histogram(values: np.ndarray[Any, Any]) -> dict[str, list[Any]]:
129
+ low, high = float(values.min()), float(values.max())
130
+ if low == high:
131
+ return {"edges": [low, high], "counts": [len(values)]}
132
+ bins = min(HISTOGRAM_BINS, len(np.unique(values)))
133
+ counts, edges = np.histogram(values, bins=bins, range=(low, high))
134
+ return {"edges": [float(e) for e in edges], "counts": [int(c) for c in counts]}
135
+
136
+
137
+ def _summary(series: pd.Series, kind: str) -> dict[str, Any]:
138
+ present = series.dropna()
139
+ if kind == "string":
140
+ present = present.map(str)
141
+ try:
142
+ distinct = int(present.nunique())
143
+ except TypeError:
144
+ distinct = int(present.map(str).nunique())
145
+ summary: dict[str, Any] = {
146
+ "count": len(series),
147
+ "missing": int(series.isna().sum()),
148
+ "distinct": distinct,
149
+ }
150
+ if kind == "number":
151
+ values = present.to_numpy(dtype="float64")
152
+ values = values[np.isfinite(values)]
153
+ if len(values):
154
+ q1, median, q3 = np.quantile(values, [0.25, 0.5, 0.75])
155
+ summary |= {
156
+ "mean": float(values.mean()),
157
+ **({"sd": float(values.std(ddof=1))} if len(values) > 1 else {}),
158
+ "min": float(values.min()),
159
+ "q1": float(q1),
160
+ "median": float(median),
161
+ "q3": float(q3),
162
+ "max": float(values.max()),
163
+ "histogram": _histogram(values),
164
+ }
165
+ elif kind in ("string", "boolean"):
166
+ summary["top"] = _top(present)
167
+ elif kind in ("date", "datetime") and len(present):
168
+ summary |= {"min": scalar(present.min()), "max": scalar(present.max())}
169
+ return summary
170
+
171
+
172
+ def _unique(names: Sequence[str]) -> list[str]:
173
+ seen: set[str] = set()
174
+ result = []
175
+ for name in names:
176
+ candidate = name
177
+ while candidate in seen:
178
+ candidate += "_"
179
+ seen.add(candidate)
180
+ result.append(candidate)
181
+ return result
182
+
183
+
184
+ def _flat_columns(frame: pd.DataFrame) -> list[str]:
185
+ if isinstance(frame.columns, pd.MultiIndex):
186
+ return [
187
+ "_".join(str(part) for part in key if str(part) != "") or "column"
188
+ for key in frame.columns
189
+ ]
190
+ return [str(name) for name in frame.columns]
191
+
192
+
193
+ def _default_index(index: pd.Index) -> bool:
194
+ return (
195
+ isinstance(index, pd.RangeIndex)
196
+ and index.start == 0
197
+ and index.step == 1
198
+ and index.name is None
199
+ )
200
+
201
+
202
+ def frame_for_export(frame: pd.DataFrame) -> pd.DataFrame:
203
+ """Return the frame with flat column names and any meaningful index as columns."""
204
+ frame = frame.copy(deep=False)
205
+ columns = _flat_columns(frame)
206
+ frame.columns = pd.Index(range(len(columns)))
207
+ if not _default_index(frame.index):
208
+ index = frame.index
209
+ names = [
210
+ str(name) if name is not None else ("index" if index.nlevels == 1 else f"index_{i}")
211
+ for i, name in enumerate(index.names)
212
+ ]
213
+ levels = [index.get_level_values(i) for i in range(index.nlevels)]
214
+ leading = pd.DataFrame({f"i{i}": pd.Series(level) for i, level in enumerate(levels)})
215
+ frame = pd.concat([leading, frame.reset_index(drop=True)], axis=1)
216
+ columns = names + columns
217
+ frame.columns = pd.Index(_unique(columns))
218
+ return frame
219
+
220
+
221
+ def table(
222
+ frame: pd.DataFrame,
223
+ title: str,
224
+ role: str | None = None,
225
+ summarize: bool = False,
226
+ ) -> dict[str, Any]:
227
+ """Return one ``tables[]`` entry. The frame's index is not exported."""
228
+ if len(frame.columns) == 0:
229
+ raise FrameworkError("Artifact tables require at least one column.")
230
+ columns: list[dict[str, Any]] = []
231
+ for position, name in enumerate(frame.columns):
232
+ series = frame.iloc[:, position]
233
+ kind = _column_type(series)
234
+ column: dict[str, Any] = {"name": str(name), "type": kind}
235
+ if summarize and position < SUMMARY_COLUMNS:
236
+ column["summary"] = _summary(series, kind)
237
+ columns.append(column)
238
+ rows = [
239
+ [
240
+ None
241
+ if scalar(value) is None
242
+ else str(value)
243
+ if columns[i]["type"] == "string"
244
+ else scalar(value)
245
+ for i, value in enumerate(row)
246
+ ]
247
+ for row in frame.itertuples(index=False, name=None)
248
+ ]
249
+ result: dict[str, Any] = {"title": title}
250
+ if role is not None:
251
+ result["role"] = role
252
+ return result | {"columns": columns, "rows": rows, "row_count": len(frame)}
253
+
254
+
255
+ def _numpy_frame(array: np.ndarray[Any, Any]) -> pd.DataFrame:
256
+ if array.dtype.names:
257
+ if array.ndim != 1:
258
+ raise FrameworkError("Structured numpy arrays must have one dimension.")
259
+ return pd.DataFrame({name: array[name] for name in array.dtype.names})
260
+ if array.ndim == 1:
261
+ return pd.DataFrame({"value": array})
262
+ if array.ndim == 2:
263
+ return pd.DataFrame(array, columns=[f"V{i + 1}" for i in range(array.shape[1])])
264
+ raise FrameworkError(
265
+ f"publish() accepts numpy arrays with 1 or 2 dimensions, not {array.ndim}."
266
+ )
267
+
268
+
269
+ def as_frame(source: Any) -> pd.DataFrame | None:
270
+ """Convert table-like objects to pandas, or return None."""
271
+ if isinstance(source, pd.DataFrame):
272
+ return source
273
+ if isinstance(source, pd.Series):
274
+ name = "value" if source.name is None or source.name == "" else str(source.name)
275
+ return source.to_frame(name)
276
+ if isinstance(source, np.ndarray):
277
+ return _numpy_frame(source)
278
+ if _has_class(source, "polars", "DataFrame", "Series"):
279
+ if _has_class(source, "polars", "Series"):
280
+ source = source.to_frame(source.name or "value")
281
+ # Arrow-backed columns keep polars dates as dates.
282
+ frame: pd.DataFrame = source.to_pandas(use_pyarrow_extension_array=True)
283
+ return frame
284
+ return None
285
+
286
+
287
+ def _anova(frame: pd.DataFrame) -> bool:
288
+ names = {str(name) for name in frame.columns}
289
+ return (
290
+ "F" in names
291
+ and bool(names & {"PR(>F)", "Pr(>F)"})
292
+ and bool(names & {"df", "df_resid", "df_diff"})
293
+ )
294
+
295
+
296
+ def _table_document(source: Any, frame: pd.DataFrame) -> dict[str, Any]:
297
+ role = "anova" if _anova(frame) else None
298
+ entry = table(frame_for_export(frame), "Table", role=role, summarize=True)
299
+ return _document(source, "table") | {"tables": [entry]}
300
+
301
+
302
+ # --------------------------------------------------------------------------- tests
303
+
304
+ _ALTERNATIVES = {"two-sided": "two.sided", "less": "less", "greater": "greater"}
305
+
306
+ # Class name -> (method, statistic name). scipy result classes don't record the
307
+ # function that made them, so names describe the class.
308
+ _TESTS: dict[str, tuple[str, str]] = {
309
+ "PearsonRResult": ("Pearson correlation", "r"),
310
+ "MannwhitneyuResult": ("Mann-Whitney U test", "U"),
311
+ "WilcoxonResult": ("Wilcoxon signed-rank test", "W"),
312
+ "KruskalResult": ("Kruskal-Wallis rank sum test", "H"),
313
+ "F_onewayResult": ("One-way ANOVA", "F"),
314
+ "Chi2ContingencyResult": ("Chi-squared test of independence", "X-squared"),
315
+ "ShapiroResult": ("Shapiro-Wilk normality test", "W"),
316
+ "KstestResult": ("Kolmogorov-Smirnov test", "D"),
317
+ "BinomTestResult": ("Exact binomial test", "proportion"),
318
+ "Power_divergenceResult": ("Chi-squared goodness-of-fit test", "X-squared"),
319
+ "AndersonResult": ("Anderson-Darling test", "A-squared"),
320
+ "BartlettResult": ("Bartlett test of homogeneity of variances", "K-squared"),
321
+ "LeveneResult": ("Levene test of homogeneity of variances", "W"),
322
+ "FlignerResult": ("Fligner-Killeen test of homogeneity of variances", "X-squared"),
323
+ "NormaltestResult": ("D'Agostino-Pearson normality test", "K-squared"),
324
+ "FriedmanchisquareResult": ("Friedman rank sum test", "X-squared"),
325
+ }
326
+
327
+
328
+ def _readable(name: str) -> str:
329
+ name = name.removesuffix("Result").replace("_", " ")
330
+ words, current = [], ""
331
+ for char in name:
332
+ if char.isupper() and current and not current[-1].isupper():
333
+ words.append(current)
334
+ current = char
335
+ else:
336
+ current += char
337
+ words.append(current)
338
+ text = " ".join(w.strip() for w in words if w.strip())
339
+ return (text[:1].upper() + text[1:].lower()) if text else "Hypothesis test"
340
+
341
+
342
+ def _alternative(result: Any) -> str | None:
343
+ value = getattr(result, "alternative", getattr(result, "_alternative", None))
344
+ if isinstance(value, str):
345
+ return _ALTERNATIVES.get(value)
346
+ if isinstance(value, (int, float, np.number)) and not isinstance(value, bool):
347
+ return {-1: "less", 0: "two.sided", 1: "greater"}.get(int(value))
348
+ return None
349
+
350
+
351
+ def _single(value: Any, what: str) -> Any:
352
+ if np.ndim(value) != 0:
353
+ raise FrameworkError(
354
+ f"publish() accepts tests with a single {what}; this result has several."
355
+ )
356
+ return scalar(value)
357
+
358
+
359
+ def _conf_int(result: Any) -> dict[str, Any] | None:
360
+ method = getattr(result, "confidence_interval", None) or getattr(result, "proportion_ci", None)
361
+ if not callable(method):
362
+ return None
363
+ try:
364
+ interval = method(confidence_level=CONFIDENCE_LEVEL)
365
+ except Exception:
366
+ return None
367
+ low: Any = getattr(interval, "low", None)
368
+ high: Any = getattr(interval, "high", None)
369
+ if np.ndim(low) != 0 or np.ndim(high) != 0:
370
+ return None
371
+ return {"level": CONFIDENCE_LEVEL, "low": scalar(low), "high": scalar(high)}
372
+
373
+
374
+ def _test_document(result: Any) -> dict[str, Any]:
375
+ name = type(result).__name__
376
+ if name == "TtestResult":
377
+ df = scalar(getattr(result, "df", None))
378
+ welch = isinstance(df, float) and not float(df).is_integer()
379
+ method, statistic_name = ("Welch's t-test" if welch else "t-test"), "t"
380
+ elif name == "SignificanceResult" and hasattr(result, "correlation"):
381
+ method, statistic_name = "Rank correlation", "correlation"
382
+ else:
383
+ method, statistic_name = _TESTS.get(name, (_readable(name), "statistic"))
384
+ test: dict[str, Any] = {"method": method}
385
+ alternative = _alternative(result)
386
+ if alternative is not None:
387
+ test["alternative"] = alternative
388
+ test["statistic"] = {"name": statistic_name, "value": _single(result.statistic, "statistic")}
389
+ parameters = _metrics(
390
+ (key, getattr(result, attr, None))
391
+ for key, attr in (("df", "df"), ("df", "dof"))
392
+ if hasattr(result, attr)
393
+ )
394
+ if parameters:
395
+ test["parameters"] = parameters[:1]
396
+ test["p_value"] = _single(result.pvalue, "p-value")
397
+ estimates = []
398
+ if name == "TtestResult" and hasattr(result, "_estimate"):
399
+ label = "difference in means" if test["method"] == "Welch's t-test" else "estimate"
400
+ estimates = _metrics([(label, getattr(result, "_estimate", None))])
401
+ elif name == "PearsonRResult" or hasattr(result, "correlation"):
402
+ estimates = _metrics([("correlation", getattr(result, "correlation", result.statistic))])
403
+ elif name == "BinomTestResult":
404
+ estimates = _metrics([("probability of success", result.proportion_estimate)])
405
+ if estimates:
406
+ test["estimates"] = estimates
407
+ interval = _conf_int(result)
408
+ if interval is not None:
409
+ test["conf_int"] = interval
410
+ document = _document(result, "test") | {"test": test}
411
+ expected = getattr(result, "expected_freq", None)
412
+ if isinstance(expected, np.ndarray) and expected.ndim in (1, 2):
413
+ document["tables"] = [table(_numpy_frame(expected), "Expected frequencies")]
414
+ return document
415
+
416
+
417
+ def _is_test(source: Any) -> bool:
418
+ return hasattr(source, "statistic") and hasattr(source, "pvalue")
419
+
420
+
421
+ # --------------------------------------------------------------------------- statsmodels
422
+
423
+ _FAMILIES = {
424
+ "gaussian": "gaussian",
425
+ "binomial": "binomial",
426
+ "poisson": "poisson",
427
+ "negativebinomial": "negative binomial",
428
+ "gamma": "gamma",
429
+ "inversegaussian": "inverse gaussian",
430
+ "tweedie": "tweedie",
431
+ }
432
+ _LINKS = {
433
+ "identity": "identity",
434
+ "logit": "logit",
435
+ "probit": "probit",
436
+ "log": "log",
437
+ "inversepower": "inverse",
438
+ "inversesquared": "inverse squared",
439
+ "cloglog": "cloglog",
440
+ "loglog": "loglog",
441
+ "sqrt": "sqrt",
442
+ "cauchy": "cauchy",
443
+ "power": "power",
444
+ "nbinom": "nbinom",
445
+ }
446
+ _LINEAR = {"OLS", "WLS", "GLS", "GLSAR", "RecursiveLS"}
447
+ _DISCRETE = {
448
+ "Logit": ("binomial", "logit"),
449
+ "Probit": ("binomial", "probit"),
450
+ "Poisson": ("poisson", "log"),
451
+ "NegativeBinomial": ("negative binomial", "log"),
452
+ "NegativeBinomialP": ("negative binomial", "log"),
453
+ "GeneralizedPoisson": ("generalized poisson", "log"),
454
+ }
455
+
456
+
457
+ def _exponentiate(family: str | None, link: str | None) -> str | None:
458
+ if link == "logit":
459
+ return "Odds ratio"
460
+ if link == "log" and family in ("poisson", "negative binomial", "generalized poisson"):
461
+ return "Rate ratio"
462
+ return None
463
+
464
+
465
+ def _description(model_type: str, **fields: Any) -> dict[str, Any]:
466
+ family, link = fields.get("family"), fields.get("link")
467
+ exponentiate = fields.pop("exponentiate", None)
468
+ if exponentiate is None:
469
+ exponentiate = _exponentiate(family, link)
470
+ return (
471
+ {"type": model_type}
472
+ | {k: v for k, v in fields.items() if v is not None}
473
+ | ({"exponentiate": exponentiate} if exponentiate else {})
474
+ )
475
+
476
+
477
+ def _statsmodels_description(result: Any) -> dict[str, Any]:
478
+ model = result.model
479
+ name = type(model).__name__
480
+ response = getattr(model, "endog_names", None)
481
+ response = response if isinstance(response, str) else None
482
+ if name in _LINEAR:
483
+ return _description("linear", family="gaussian", link="identity", response=response)
484
+ if name == "GLM":
485
+ family = _FAMILIES.get(type(model.family).__name__.lower(), type(model.family).__name__)
486
+ link_name = type(model.family.link).__name__.lower()
487
+ return _description(
488
+ "glm", family=family.lower(), link=_LINKS.get(link_name, link_name), response=response
489
+ )
490
+ if name in _DISCRETE:
491
+ family, link = _DISCRETE[name]
492
+ return _description("glm", family=family, link=link, response=response)
493
+ if name == "MixedLM":
494
+ return _description("mixed", family="gaussian", link="identity", response=response)
495
+ if name == "QuantReg":
496
+ return _description("quantile", response=response)
497
+ if name == "MNLogit":
498
+ return _description("multinomial", family="multinomial", link="logit", response=response)
499
+ if name == "OrderedModel":
500
+ distr = getattr(model, "distr", None)
501
+ links = {"logistic": "logit", "norm": "probit"}
502
+ return _description(
503
+ "ordinal", link=links.get(getattr(distr, "name", "")), response=response
504
+ )
505
+ if name == "PHReg":
506
+ return _description("survival", response=response, exponentiate="Hazard ratio")
507
+ return _description("other", response=response)
508
+
509
+
510
+ _INTERCEPTS = {"Intercept": "(Intercept)", "const": "(Intercept)"}
511
+
512
+
513
+ def _values(value: Any) -> list[Any]:
514
+ return list(np.asarray(value).ravel(order="F"))
515
+
516
+
517
+ def _coefficients(result: Any, description: dict[str, Any]) -> pd.DataFrame:
518
+ params = result.params
519
+ model = result.model
520
+ if getattr(params, "ndim", 1) == 2:
521
+ # Multi-equation models: one row per equation and term, column by column.
522
+ frame = pd.DataFrame(params)
523
+ labels = getattr(model, "_ynames_map", None) or {}
524
+ names = [
525
+ f"{labels.get(j + 1, frame.columns[j])}: {_INTERCEPTS.get(str(term), term)}"
526
+ for j in range(frame.shape[1])
527
+ for term in frame.index
528
+ ]
529
+ else:
530
+ names = [str(n) for n in getattr(params, "index", getattr(model, "exog_names", []))]
531
+ if len(names) != len(params):
532
+ names = [f"term_{i + 1}" for i in range(len(params))]
533
+ confidence = np.asarray(result.conf_int())
534
+ if confidence.ndim == 3:
535
+ confidence = confidence.reshape(-1, 2)
536
+ frame = pd.DataFrame(
537
+ {
538
+ "term": names,
539
+ "estimate": _values(params),
540
+ "std_error": _values(result.bse),
541
+ "statistic": _values(result.tvalues),
542
+ "p_value": _values(result.pvalues),
543
+ "conf_low": confidence[:, 0] if len(confidence) == len(names) else np.nan,
544
+ "conf_high": confidence[:, 1] if len(confidence) == len(names) else np.nan,
545
+ }
546
+ )
547
+ frame["term"] = frame["term"].replace(_INTERCEPTS)
548
+ if type(model).__name__ == "MixedLM":
549
+ frame = frame.iloc[: model.k_fe]
550
+ if description.get("family") == "negative binomial" and type(model).__name__ != "GLM":
551
+ frame = frame[frame["term"] != "alpha"]
552
+ return frame.reset_index(drop=True)
553
+
554
+
555
+ def _random_effects(result: Any) -> tuple[pd.DataFrame, list[dict[str, Any]]]:
556
+ model = result.model
557
+ cov = pd.DataFrame(result.cov_re)
558
+ names = [str(n) for n in cov.columns]
559
+ exog_re = np.asarray(getattr(model, "exog_re", np.empty((0, 0))))
560
+ intercept = bool(exog_re.size) and bool(np.all(exog_re[:, 0] == 1))
561
+ group = names[0] if intercept and names else "Group"
562
+ matrix = cov.to_numpy(dtype="float64")
563
+ variances = np.diag(matrix)
564
+ rows = []
565
+ for i, name in enumerate(names):
566
+ correlation = None
567
+ if i > 0 and variances[0] > 0 and variances[i] > 0:
568
+ correlation = matrix[0, i] / math.sqrt(variances[0] * variances[i])
569
+ rows.append(
570
+ {
571
+ "group": group,
572
+ "term": "(Intercept)" if i == 0 and intercept else name,
573
+ "variance": variances[i],
574
+ "std_dev": math.sqrt(variances[i]) if variances[i] >= 0 else np.nan,
575
+ "correlation": correlation,
576
+ }
577
+ )
578
+ vc_names = list(getattr(getattr(model, "exog_vc", None), "names", []) or [])
579
+ for name, variance in zip(vc_names, np.ravel(getattr(result, "vcomp", [])), strict=False):
580
+ rows.append(
581
+ {
582
+ "group": str(name),
583
+ "term": "(Intercept)",
584
+ "variance": variance,
585
+ "std_dev": math.sqrt(variance) if variance >= 0 else np.nan,
586
+ "correlation": None,
587
+ }
588
+ )
589
+ scale = float(result.scale)
590
+ rows.append(
591
+ {
592
+ "group": "Residual",
593
+ "term": "",
594
+ "variance": scale,
595
+ "std_dev": math.sqrt(scale),
596
+ "correlation": None,
597
+ }
598
+ )
599
+ frame = pd.DataFrame(rows)
600
+ if frame["correlation"].isna().all():
601
+ frame = frame.drop(columns="correlation")
602
+ return frame, _metrics([(f"n_groups.{group}", getattr(model, "n_groups", None))])
603
+
604
+
605
+ _STATSMODELS_METRICS = (
606
+ "nobs",
607
+ "df_model",
608
+ "df_resid",
609
+ "rsquared",
610
+ "rsquared_adj",
611
+ "aic",
612
+ "bic",
613
+ "llf",
614
+ "llnull",
615
+ "deviance",
616
+ "null_deviance",
617
+ "scale",
618
+ "converged",
619
+ )
620
+
621
+
622
+ def _statsmodels_document(result: Any) -> dict[str, Any]:
623
+ if not all(hasattr(result, key) for key in ("params", "bse", "pvalues", "conf_int")):
624
+ raise FrameworkError("This statsmodels result does not expose coefficient statistics.")
625
+ model = result.model
626
+ name = type(model).__name__
627
+ description = _statsmodels_description(result)
628
+ tables = [table(_coefficients(result, description), "Coefficients", role="coefficients")]
629
+ metrics = _metrics(
630
+ (key, getattr(result, key)) for key in _STATSMODELS_METRICS if hasattr(result, key)
631
+ )
632
+ if name != "MixedLM" and hasattr(result, "prsquared"):
633
+ metrics += _metrics([("pseudo_rsquared", result.prsquared)])
634
+ if description.get("family") == "negative binomial":
635
+ alpha = (
636
+ getattr(model.family, "alpha", None)
637
+ if name == "GLM"
638
+ else dict(zip(result.params.index, result.params, strict=False)).get("alpha")
639
+ )
640
+ if alpha:
641
+ metrics += _metrics([("alpha", alpha), ("theta", 1 / float(alpha))])
642
+ if name == "MixedLM":
643
+ random, group_metrics = _random_effects(result)
644
+ tables.append(table(random, "Random effects", role="random_effects"))
645
+ metrics += group_metrics
646
+ if name == "QuantReg":
647
+ metrics += _metrics([("quantile", getattr(result, "q", None))])
648
+ use_t = bool(getattr(result, "use_t", False))
649
+ metrics += [
650
+ {"name": "confidence_level", "value": CONFIDENCE_LEVEL},
651
+ {"name": "interval_method", "value": "wald-t" if use_t else "wald-normal"},
652
+ ]
653
+ document = _document(result, "model") | {
654
+ "model": description,
655
+ "tables": tables,
656
+ "metrics": metrics,
657
+ }
658
+ formula = getattr(model, "formula", None)
659
+ if isinstance(formula, str):
660
+ document["formula"] = formula
661
+ return document
662
+
663
+
664
+ # --------------------------------------------------------------------------- machine learning
665
+
666
+
667
+ def _feature_names(model: Any, count: int) -> list[str]:
668
+ names = getattr(model, "feature_names_in_", None)
669
+ if names is not None and len(names) == count:
670
+ return [str(n) for n in names]
671
+ return [f"x{i}" for i in range(count)]
672
+
673
+
674
+ def _params(model: Any, keys: Sequence[str]) -> list[tuple[str, Any]]:
675
+ getter = getattr(model, "get_params", None)
676
+ params = getter(deep=False) if callable(getter) else {}
677
+ return [(key, _hyperparameter(params.get(key))) for key in keys]
678
+
679
+
680
+ def _hyperparameter(value: Any) -> Any:
681
+ """Drop placeholders such as "deprecated" and -1 for "no limit"; trim float32 noise."""
682
+ if isinstance(value, str) and value in ("deprecated", "warn"):
683
+ return None
684
+ if isinstance(value, (int, float, np.number)) and not isinstance(value, bool):
685
+ if value < 0:
686
+ return None
687
+ if isinstance(value, (float, np.floating)):
688
+ return float(f"{float(value):.7g}")
689
+ return value
690
+
691
+
692
+ def _importance_document(
693
+ source: Any, features: Sequence[str], importance: Sequence[float], importance_type: str
694
+ ) -> dict[str, Any]:
695
+ frame = pd.DataFrame({"feature": list(features), "importance": list(importance)})
696
+ frame = frame.sort_values("importance", ascending=False, kind="stable").reset_index(drop=True)
697
+ document = _document(source, "model") | {
698
+ "model": _description("tree_ensemble"),
699
+ "tables": [table(frame, "Feature importance", role="importance")],
700
+ }
701
+ document["metrics"] = _metrics([("importance_type", importance_type)])
702
+ return document
703
+
704
+
705
+ def _sklearn_document(model: Any) -> dict[str, Any]:
706
+ name = type(model).__name__
707
+ if name.startswith("HistGradientBoosting"):
708
+ raise FrameworkError(
709
+ f"{name} has no feature_importances_. Compute permutation importance with "
710
+ "sklearn.inspection.permutation_importance() and publish the result as a DataFrame."
711
+ )
712
+ if hasattr(model, "coef_"):
713
+ coef = np.atleast_2d(np.asarray(model.coef_, dtype="float64"))
714
+ intercept = np.atleast_1d(np.asarray(getattr(model, "intercept_", 0.0), dtype="float64"))
715
+ features = _feature_names(model, coef.shape[1])
716
+ fit_intercept = bool(getattr(model, "fit_intercept", True))
717
+ classes = list(getattr(model, "classes_", []))
718
+ if len(coef) == 1:
719
+ labels = [""]
720
+ elif len(classes) == len(coef):
721
+ labels = [f"{c}: " for c in classes]
722
+ else:
723
+ labels = [f"y{i}: " for i in range(len(coef))]
724
+ terms, estimates = [], []
725
+ for row, label in enumerate(labels):
726
+ if fit_intercept:
727
+ terms.append(f"{label}(Intercept)")
728
+ estimates.append(float(intercept[row] if len(intercept) > row else intercept[0]))
729
+ terms += [f"{label}{feature}" for feature in features]
730
+ estimates += list(coef[row])
731
+ logistic = name == "LogisticRegression"
732
+ binary = logistic and len(coef) == 1
733
+ description = _description(
734
+ "linear_ml",
735
+ family="binomial" if binary else "multinomial" if logistic else None,
736
+ link="logit" if logistic else None,
737
+ # Softmax coefficients have no reference class, so they aren't odds ratios.
738
+ exponentiate=None if binary else False,
739
+ )
740
+ frame = pd.DataFrame({"term": terms, "estimate": estimates})
741
+ metrics = _metrics(
742
+ [
743
+ *_params(model, ("alpha", "l1_ratio", "C", "penalty", "fit_intercept")),
744
+ ("n_features", getattr(model, "n_features_in_", None)),
745
+ ("n_classes", len(classes) if classes else None),
746
+ ]
747
+ )
748
+ return _document(model, "model") | {
749
+ "model": description,
750
+ "tables": [table(frame, "Coefficients", role="coefficients")],
751
+ "metrics": metrics,
752
+ }
753
+ if hasattr(model, "feature_importances_"):
754
+ importance = np.asarray(model.feature_importances_, dtype="float64")
755
+ document = _importance_document(
756
+ model, _feature_names(model, len(importance)), list(importance), "impurity"
757
+ )
758
+ document["metrics"] += _metrics(
759
+ [
760
+ *_params(model, ("n_estimators", "max_depth", "learning_rate", "min_samples_leaf")),
761
+ ("n_features", getattr(model, "n_features_in_", None)),
762
+ ]
763
+ )
764
+ return document
765
+ raise FrameworkError(
766
+ f"publish() accepts fitted scikit-learn models with coef_ or feature_importances_; "
767
+ f"{name} has neither (is it fitted?)."
768
+ )
769
+
770
+
771
+ def _xgboost_document(source: Any) -> dict[str, Any]:
772
+ booster = source.get_booster() if hasattr(source, "get_booster") else source
773
+ scores = booster.get_score(importance_type="gain")
774
+ features = list(booster.feature_names or []) or [f"f{i}" for i in range(booster.num_features())]
775
+ document = _importance_document(
776
+ source, features, [float(scores.get(f, 0.0)) for f in features], "gain"
777
+ )
778
+ try:
779
+ config = json.loads(booster.save_config())
780
+ learner = config["learner"]
781
+ tree = learner["gradient_booster"].get("tree_train_param", {})
782
+ objective = learner["objective"]["name"]
783
+ except (KeyError, TypeError, ValueError):
784
+ tree, objective = {}, None
785
+ params = dict(_params(source, ("n_estimators", "max_depth", "learning_rate")))
786
+ document["metrics"] += _metrics(
787
+ [
788
+ ("n_estimators", params.get("n_estimators") or booster.num_boosted_rounds()),
789
+ ("max_depth", params.get("max_depth") or _number(tree.get("max_depth"))),
790
+ ("learning_rate", params.get("learning_rate") or _number(tree.get("eta"))),
791
+ ("n_features", booster.num_features()),
792
+ ("objective", objective),
793
+ ]
794
+ )
795
+ return document
796
+
797
+
798
+ def _number(value: Any) -> float | int | None:
799
+ try:
800
+ number = float(value)
801
+ except (TypeError, ValueError):
802
+ return None
803
+ return _hyperparameter(int(number) if number.is_integer() else number) # type: ignore[no-any-return]
804
+
805
+
806
+ def _lightgbm_document(source: Any) -> dict[str, Any]:
807
+ wrapper = hasattr(source, "booster_")
808
+ booster = source.booster_ if wrapper else source
809
+ importance = [float(v) for v in booster.feature_importance(importance_type="gain")]
810
+ features = _feature_names(source, len(importance)) if wrapper else list(booster.feature_name())
811
+ document = _importance_document(source, features, importance, "gain")
812
+ if wrapper:
813
+ params = _params(
814
+ source,
815
+ ("n_estimators", "max_depth", "learning_rate", "num_leaves", "min_child_samples"),
816
+ )
817
+ objective = source.get_params().get("objective") or getattr(source, "objective_", None)
818
+ else:
819
+ params = [
820
+ (key, _hyperparameter(booster.params.get(key)))
821
+ for key in ("max_depth", "learning_rate", "num_leaves", "min_child_samples")
822
+ ]
823
+ params.insert(0, ("n_estimators", booster.current_iteration()))
824
+ objective = booster.params.get("objective")
825
+ document["metrics"] += _metrics(
826
+ [*params, ("n_features", booster.num_feature()), ("objective", objective)]
827
+ )
828
+ return document
829
+
830
+
831
+ # --------------------------------------------------------------------------- entry points
832
+
833
+
834
+ def structured(source: Any) -> dict[str, Any] | None:
835
+ """Return a typed document for a supported object, or None.
836
+
837
+ Model documents carry aggregates only: no observations, residuals, fitted
838
+ values, or serialized model state.
839
+ """
840
+ document = _structured(source)
841
+ if document is None:
842
+ return None
843
+ # Readable key order: what it is, then the description, then tables and metrics.
844
+ order = ("schema", "kind", "source", "formula", "model", "test", "tables", "metrics")
845
+ return {key: document[key] for key in order if key in document} | document
846
+
847
+
848
+ def _structured(source: Any) -> dict[str, Any] | None:
849
+ frame = as_frame(source)
850
+ if frame is not None:
851
+ return _table_document(source, frame)
852
+ if _from(source, "statsmodels") and hasattr(source, "params"):
853
+ return _statsmodels_document(source)
854
+ if _has_class(source, "xgboost", "XGBModel", "Booster"):
855
+ return _xgboost_document(source)
856
+ if _has_class(source, "lightgbm", "LGBMModel", "Booster"):
857
+ return _lightgbm_document(source)
858
+ if _from(source, "sklearn"):
859
+ return _sklearn_document(source)
860
+ if _is_test(source):
861
+ return _test_document(source)
862
+ if _from(source, "statsmodels"):
863
+ raise FrameworkError("This statsmodels result does not expose coefficient statistics.")
864
+ return None
865
+
866
+
867
+ def _figure(source: Any) -> Any:
868
+ """Return the matplotlib Figure behind an Axes, Figure, or seaborn grid."""
869
+ figure = source
870
+ for _ in range(3):
871
+ if callable(getattr(figure, "savefig", None)):
872
+ return figure
873
+ figure = getattr(figure, "figure", None) or getattr(figure, "fig", None)
874
+ return None
875
+
876
+
877
+ def is_figure(source: Any) -> bool:
878
+ return _from(source, "matplotlib", "seaborn") and _figure(source) is not None
879
+
880
+
881
+ def save_figure(source: Any, directory: Path, svg: bool = True) -> Path:
882
+ """Save a matplotlib-backed figure as SVG, or PNG when SVG is too large."""
883
+ figure = _figure(source)
884
+ options: dict[str, Any] = {"bbox_inches": "tight", "facecolor": "white"}
885
+ if svg:
886
+ path = directory / "figure.svg"
887
+ figure.savefig(path, format="svg", **options)
888
+ if path.stat().st_size <= SVG_LIMIT:
889
+ return path
890
+ path.unlink()
891
+ path = directory / "figure.png"
892
+ figure.savefig(path, format="png", dpi=300 if svg else 150, **options)
893
+ return path
894
+
895
+
896
+ def _plotly_html(source: Any, directory: Path) -> Path:
897
+ path = directory / "figure.html"
898
+ source.write_html(path, include_plotlyjs="cdn", full_html=True)
899
+ return path
900
+
901
+
902
+ def _altair_spec(source: Any, directory: Path) -> Path:
903
+ import altair as alt # type: ignore[import-not-found,unused-ignore]
904
+
905
+ # The default transformer inlines data; lift its row limit for this call only.
906
+ with alt.data_transformers.enable("default", max_rows=None):
907
+ spec = source.to_dict()
908
+ path = directory / "figure.vl.json"
909
+ path.write_text(json.dumps(spec, allow_nan=False, default=str), encoding="utf-8")
910
+ return path
911
+
912
+
913
+ def _styler_html(source: Any, directory: Path) -> Path:
914
+ path = directory / "table.html"
915
+ path.write_text(source.to_html(doctype_html=True), encoding="utf-8")
916
+ return path
917
+
918
+
919
+ def rendered(source: Any, directory: Path, cloud: bool) -> Path | None:
920
+ """Write figures, formatted tables, and charts to their file formats, or return None."""
921
+ exporters: list[tuple[Callable[[Any], bool], Callable[[Any, Path], Path]]] = [
922
+ (is_figure, lambda s, d: save_figure(s, d, svg=cloud)),
923
+ (lambda s: _has_class(s, "pandas", "Styler"), _styler_html),
924
+ (lambda s: _has_class(s, "plotly", "BaseFigure"), _plotly_html),
925
+ (lambda s: _has_class(s, "altair", "TopLevelMixin"), _altair_spec),
926
+ ]
927
+ for matches, export in exporters:
928
+ if matches(source):
929
+ return export(source, directory)
930
+ return None