doe-analyzer 0.3.0__py3-none-any.whl

doe_analyzer/__init__.py

@@ -0,0 +1,14 @@
+"""DOE Analyzer — analyse design-of-experiments results from CSV."""
+
+import warnings as _warnings
+
+__version__ = "0.3.0"
+
+# scikit-learn's forests warn about joblib's `delayed` on every fit when run
+# from worker threads; it is noise for this application.
+_warnings.filterwarnings("ignore", message=r".*sklearn\.utils\.parallel\.delayed.*", category=UserWarning)
+# MinCovDet (robust Mahalanobis outliers) warns on every fit when the data are
+# (nearly) singular, as with few discrete levels; the estimate is still used.
+_warnings.filterwarnings("ignore", message=r"Determinant has increased", category=RuntimeWarning)
+_warnings.filterwarnings("ignore", message=r"The covariance matrix associated to your dataset is not full rank",
+                         category=UserWarning)

doe_analyzer/__main__.py

@@ -0,0 +1,3 @@
+from doe_analyzer.main import main
+
+raise SystemExit(main())

doe_analyzer/api.py

@@ -0,0 +1,311 @@
+"""DOE Analyzer in a script: the steps behind a plot, one function each.
+
+Export plot as Python script… writes a script of these calls; they are kept
+working across versions. Each step returns a new dataset (nothing changes in
+place), so a study is built in order:
+
+    import doe_analyzer.api as da
+
+    ds = da.read_csv("study.csv")
+    ds = da.set_column(ds, "y", role="result", goal="min")
+    runs = da.runs(ds, [da.hide_failed()])
+    snap = da.snapshot(ds, runs)
+    opts = da.options(ds, "scatter", x="x1", y="y")
+    result = da.compute(snap, "scatter", opts)
+    fig = da.figure(snap, "scatter", "scatter", opts, result, da.publication_theme(font_pt=9))
+    da.export(fig, "scatter.png", width=160, height=100, unit="mm", dpi=300)
+
+Pages and plots are named by their keys, as in the names Export current plot
+gives its files ("surrogates_parity": page "surrogates", plot "parity").
+"""
+from __future__ import annotations
+
+import os
+
+# before numpy loads, as the app starts (doe_analyzer.main): the same speed, and the same numbers
+for _var in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS"):
+    os.environ.setdefault(_var, "1")
+
+from dataclasses import fields, replace  # noqa: E402
+from pathlib import Path  # noqa: E402
+
+import numpy as np  # noqa: E402
+
+from doe_analyzer.core import columns as C  # noqa: E402
+from doe_analyzer.core import csvio, derived, fmt, runfiles, scales  # noqa: E402
+from doe_analyzer.core.dataset import Dataset, Snapshot  # noqa: E402
+from doe_analyzer.core.filters import Filter, combined_mask, id_mask, row_mask  # noqa: E402
+from doe_analyzer.core.tasks import NULL_TOKEN  # noqa: E402
+
+COLUMN_SETTINGS = ("role", "vtype", "scale", "goal", "target", "constraint", "importance", "bounds")
+
+
+# ------------------------------------------------------------------------------------------------ data
+def demo(name: str) -> Dataset:
+    """One of the demo datasets: branin, ishigami, zdt_constrained, mixed_categorical, large_20k."""
+    from doe_analyzer.core import demo as demos
+    if name not in demos.DEMOS:
+        raise ValueError(f"no demo dataset {name!r}; there are {', '.join(demos.DEMOS)}")
+    return demos.dataset(name)
+
+
+def read_csv(path, encoding: str | None = None, delimiter: str | None = None, decimal: str | None = None) -> Dataset:
+    """The CSV with the column settings DOE Analyzer infers (and a mixture its parameters add up to); what is
+    not given is detected, as on opening it."""
+    from doe_analyzer.core import space
+    path = Path(path)
+    opts = csvio.sniff(path)
+    opts = csvio.CsvOptions(encoding or opts.encoding, delimiter or opts.delimiter, decimal or opts.decimal)
+    df, report, opts = csvio.read_csv(path, opts)
+    return space.with_detected(Dataset(df, C.infer_specs(df), str(path), opts, report))[0]
+
+
+def set_column(ds: Dataset, name: str, **settings) -> Dataset:
+    """A column's settings, as the Data page's column panel sets them:
+
+    * role: "param", "result", "id", "ignore" or "fidelity"
+    * vtype: "continuous", "discrete", "categorical" (or "image", "curve" for file names)
+    * scale: "linear", "log10" or ("symlog", C)
+    * goal: None, "min", "max" or "target" (with target, a number)
+    * constraint: None, ("<=", limit), (">=", limit) or ("between", lower, upper)
+    * importance: 1 to 5
+    * bounds: None or (lower, upper), a parameter's range in original units
+    """
+    unknown = set(settings) - set(COLUMN_SETTINGS)
+    if unknown:
+        raise TypeError(f"no column setting {', '.join(sorted(unknown))}; there are {', '.join(COLUMN_SETTINGS)}")
+    if not ds.has(name):
+        raise ValueError(f"no column {name!r}")
+    spec = ds.spec(name)
+    kw = dict(settings)
+    if "role" in kw and kw["role"] not in C.ROLES:
+        raise ValueError(f"{name}: role is one of {', '.join(C.ROLES)}")
+    if "vtype" in kw:
+        if kw["vtype"] not in C.VTYPES + C.FILE_VTYPES:
+            raise ValueError(f"{name}: vtype is one of {', '.join(C.VTYPES + C.FILE_VTYPES)}")
+        # the levels follow the values, as the panel finds them
+        kw["levels"] = C.levels_of(ds.df[name], spec.numeric) if kw["vtype"] in ("discrete", "categorical") else None
+    if "scale" in kw:
+        kw["scale"] = _scale(kw["scale"])
+    if kw.get("goal") not in C.GOALS:
+        raise ValueError(f"{name}: goal is one of None, min, max, target")
+    if kw.get("target") is not None:
+        kw["target"] = float(kw["target"])
+    if kw.get("constraint"):
+        op, *limits = kw["constraint"]
+        if op not in C.OPS:
+            raise ValueError(f"{name}: a constraint's operator is one of {', '.join(C.OPS)}")
+        kw["constraint"] = (op, *(float(x) for x in limits))
+    if "importance" in kw:
+        kw["importance"] = int(kw["importance"])
+    if kw.get("bounds") is not None:
+        kw["bounds"] = (float(kw["bounds"][0]), float(kw["bounds"][1]))
+    return ds.with_spec(replace(spec, **kw))
+
+
+def _scale(value) -> scales.Scale:
+    if isinstance(value, str):
+        if value == "symlog":
+            raise ValueError('a symlog scale needs its C: scale=("symlog", C)')
+        return scales.Scale(value)
+    kind, c = value
+    return scales.Scale(kind, float(c))
+
+
+def add_formula(ds: Dataset, name: str, expr: str) -> Dataset:
+    """A column computed from the others ("Add derived column"): a result, on the scale its values suggest."""
+    values, _ = derived.evaluate(ds.df, expr)
+    return ds.with_specs(list(ds.specs) + [C.ColumnSpec(name, role="result", expr=expr,
+                                                        scale=scales.suggest(values))])
+
+
+def add_curve_match(ds: Dataset, name: str, column: str, x, y, source: str = "") -> Dataset:
+    """A result to minimise: each run's RMSE from a measured curve (x, y) (Compare runs: Match a measured
+    curve…); ``column`` is the column of the runs' curve files, ``source`` the measured curve's name."""
+    spec = runfiles.match_spec(ds, column, x, y, source)
+    return ds.with_specs(list(ds.specs) + [replace(spec, name=name)])
+
+
+def set_space(ds: Dataset, constraints: list[str]) -> Dataset:
+    """Constraints between parameters, as typed in the Data page's Constraints… ("x1 + x2 <= 1")."""
+    return ds.with_space(list(constraints))
+
+
+def set_network(ds: Dataset, layers=(64, 64), passes: int = 3000, more=()) -> Dataset:
+    """The neural network's hidden layer widths, the passes a training may take, and the passes of each
+    round it was trained further (Surrogate models: Neural network…)."""
+    from doe_analyzer.core.surrogates import Network
+    return ds.with_network(Network(tuple(int(w) for w in layers), int(passes), tuple(int(p) for p in more)))
+
+
+def set_digits(ds: Dataset, digits: int) -> Dataset:
+    """Significant digits of the numbers the plots show (Settings); for every plot this script makes."""
+    fmt.DEFAULT_SIG = int(digits)
+    return ds.with_sig(int(digits))
+
+
+# ------------------------------------------------------------------------------------------------ runs
+def range_filter(column: str, lo: float, hi: float) -> Filter:
+    """The runs with lo ≤ column ≤ hi (original units)."""
+    return Filter("range", column=column, lo=float(lo), hi=float(hi))
+
+
+def levels_filter(column: str, levels) -> Filter:
+    """The runs at these levels of a categorical column."""
+    return Filter("levels", column=column, levels=tuple(str(x) for x in levels))
+
+
+def runs_filter(ids, keep: bool = True, by_row: bool = False) -> Filter:
+    """Only these runs (``keep``), or every run but these; by run ID, or with ``by_row`` by row number in
+    the file (1 the first run)."""
+    return Filter("ids", ids=tuple(str(i) for i in ids), keep=keep, by_row=by_row)
+
+
+def hide_failed() -> Filter:
+    """The runs whose results are all there."""
+    return Filter("hide_failed")
+
+
+def feasible_only() -> Filter:
+    """The runs that meet every constraint."""
+    return Filter("feasible_only")
+
+
+def runs(ds: Dataset, filters) -> np.ndarray:
+    """The runs that pass every filter (a mask of the dataset's rows). A filter on a column the data do not
+    have is an error, not one that keeps every run."""
+    out = []
+    for f in filters:
+        if f.column is not None and not ds.has(f.column):
+            raise ValueError(f"no column {f.column!r}")
+        if f.kind == "levels" and ds.spec(f.column).numeric:
+            # numbers as the column's levels name them: levels_filter("layers", [1, 2]) is at 1.0 and 2.0
+            named = {v: str(v) for v in ds.levels(f.column)}
+            f = replace(f, levels=tuple(named.get(csvio.parse_number(x, "."), x) for x in f.levels))
+        out.append(f)
+    return combined_mask(ds, out)
+
+
+def select(ds: Dataset, ids, by_row: bool = False) -> np.ndarray:
+    """A selection: these runs, by run ID, or with ``by_row`` by row number in the file."""
+    ids = [str(i) for i in ids]
+    return row_mask(ds, ids) if by_row else id_mask(ds, ids)
+
+
+def snapshot(ds: Dataset, runs=None, selected=None, selected_on: str | None = None) -> Snapshot:
+    """What a page sees: the runs in play and, highlighted among them, the selection; ``selected_on``: the
+    page it was made on, which shows it as made there (a brush, a lasso) rather than highlighted."""
+    active = np.ones(ds.n, dtype=bool) if runs is None else np.asarray(runs, dtype=bool)
+    if selected is not None:
+        selected = np.asarray(selected, dtype=bool) & active
+    return Snapshot(ds, active, selected, selected_on if selected is not None else None)
+
+
+# ------------------------------------------------------------------------------------------------ pages
+def _page(page: str):
+    from doe_analyzer.figures import TOPICS
+    for t in TOPICS:
+        if t.KEY == page:
+            return t
+    raise ValueError(f"no page {page!r}; there are {', '.join(t.KEY for t in TOPICS)}")
+
+
+def _plain(v):
+    if isinstance(v, (list, tuple)):
+        return [_plain(x) for x in v]
+    if isinstance(v, dict):
+        return {k: _plain(x) for k, x in v.items()}
+    return v
+
+
+def options(ds: Dataset, page: str, **values):
+    """A page's options: the ones given, the page's defaults for the others. A value the page cannot take
+    here (a column the data do not have) is an error, not quietly replaced."""
+    t = _page(page)
+    names = [f.name for f in fields(t.Options)]
+    unknown = [k for k in values if k not in names]
+    if unknown:
+        raise TypeError(f"{page} has no option {', '.join(unknown)}; its options: {', '.join(names)}")
+    opts = t.options(ds, values)
+    for k, v in values.items():
+        got = getattr(opts, k)
+        if _plain(got) != _plain(v):
+            raise ValueError(f"{page}: {k}={v!r} cannot be used with these data (it would be {got!r})")
+    return opts
+
+
+def compute(snap: Snapshot, page: str, opts):
+    """The page's computation (models fitted, an embedding, …): what its plots draw."""
+    return _page(page).compute(snap, opts, NULL_TOKEN)
+
+
+def figure(snap: Snapshot, page: str, name: str, opts, result, theme) -> dict:
+    """One plot of the page, as a plotly figure (a dict)."""
+    t = _page(page)
+    plots = [k for k, _ in t.FIGURES]
+    if name not in plots:
+        raise ValueError(f"{page} has no plot {name!r}; its plots: {', '.join(plots)}")
+    return t.build(snap, opts, result, theme, only={name})[name]
+
+
+def publication_theme(font_pt: float = 9.0):
+    """White, for print (the export's Publication theme)."""
+    from doe_analyzer.figures.theme import publication_theme as theme
+    return theme(font_pt * 96.0 / 72.0)
+
+
+def dark_theme(font_pt: float = 9.0):
+    """The export's Dark theme."""
+    from doe_analyzer.figures.theme import dark_export_theme
+    return dark_export_theme(font_pt * 96.0 / 72.0)
+
+
+def ui_theme(seed: str, dark: bool = False, contrast: float = 0.0, font_pt: float = 9.0):
+    """The app's look (the export's Current UI theme): its colour, light or dark, and contrast (Settings)."""
+    from doe_analyzer.figures.theme import ui_theme as theme
+    from doe_analyzer.tokens import scheme_tokens
+    return theme(scheme_tokens(seed, dark, contrast), dark, font_pt * 96.0 / 72.0)
+
+
+def zoom(fig: dict, ranges: dict) -> dict:
+    """The figure zoomed to these axis ranges ({"xaxis": [lo, hi], …}, in the plotted units)."""
+    from doe_analyzer.gui.exporter import apply_zoom
+    return apply_zoom(fig, ranges)
+
+
+# ------------------------------------------------------------------------------------------------ image
+def export(fig: dict, path, width: float = 160.0, height: float = 100.0, unit: str = "mm", dpi: int = 300) -> list[str]:
+    """Write the figure as PNG, SVG or PDF (by the file's suffix), as Export current plot writes it, without a
+    window; ``unit`` is "mm", "in" or "px", ``dpi`` the PNG's. Returns warnings (a WebGL plot in an SVG)."""
+    path = Path(path)
+    kind = path.suffix.lower().lstrip(".")
+    if kind not in ("png", "svg", "pdf"):
+        raise ValueError(f"{path.name}: the image is a .png, .svg or .pdf file")
+    _qt()
+    from doe_analyzer.gui.exporter import ExportOptions, export_figure
+    return export_figure(fig, path, ExportOptions(fmt=kind, width=float(width), height=float(height), unit=unit,
+                                                  dpi=int(dpi)))
+
+
+def _qt() -> None:
+    """The Qt application the exporter draws in: offscreen, so a script needs no display."""
+    from PySide6.QtWidgets import QApplication
+    if QApplication.instance() is not None:
+        return
+    import atexit
+    # not the desktop's platform (WSL setups name xcb): that drew through the display and its GPU, and Qt 6.10's
+    # GPU thread crashed there as the script ended
+    os.environ["QT_QPA_PLATFORM"] = "offscreen"
+    # offscreen Qt warned of the GPU contexts it could not make (7 lines, every run) and draws all the same;
+    # a failed drawing raises ExportError. QtWebEngine said "Sandboxing disabled by user." every run (on Linux
+    # main.configure_webengine_env turns the sandbox off)
+    os.environ.setdefault("QT_LOGGING_RULES", "*.warning=false;default.info=false")
+    from doe_analyzer.main import prepare_qt
+    prepare_qt(["doe_analyzer"], gl="auto")
+
+    def close():
+        from doe_analyzer.gui.exporter import shutdown_renderer
+        from doe_analyzer.gui.plotview import shutdown
+        shutdown_renderer()
+        shutdown()
+    atexit.register(close)

doe_analyzer/core/__init__.py

@@ -0,0 +1 @@
+"""Pure computation. Nothing in this package imports Qt."""

doe_analyzer/core/calibration.py

@@ -0,0 +1,191 @@
+"""Bayesian calibration: the parameters that the runs (simulations) vary but an experiment cannot set,
+inferred from what experiments measured.
+
+Each experiment is a row of a measurements CSV: the parameters known in it (a column each, named as in the
+runs), the results measured (a column each) and, optionally, each result's standard deviation (a column
+``<result>_sd``) and a measured curve (a file name in the column of the runs' curve files). The parameters
+without a column are calibrated: one value for all the experiments, with a uniform prior between their
+bounds. Each measured value is the result's surrogate at the experiment's known settings and the calibrated
+values, plus Gaussian errors, the measurement's and the surrogate's own (its σ), independent (no model
+discrepancy, Kennedy and O'Hagan's δ, is fitted); a curve enters by its coefficients on the shapes (POD modes)
+of the runs' curves (Higdon et al., 2008). Everything is in each result's scale space.
+
+The posterior is sampled by the affine-invariant ensemble sampler of Goodman and Weare (2010) (the stretch
+move, as in emcee), its walkers started around the most probable point that differential evolution finds; the
+first half of the steps is dropped as burn-in. The integrated autocorrelation time (Sokal's window on the
+walkers' mean autocorrelation) gives the effective number of independent samples.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+import pandas as pd
+
+from doe_analyzer.core import space
+from doe_analyzer.core.tasks import NULL_TOKEN, Cancelled, Token
+
+SD_SUFFIX = "_sd"
+STRETCH = 2.0       # the stretch move's scale a
+MIN_WALKERS = 32
+KEEP = 4000         # posterior samples kept (thinned)
+
+
+def table_of(df: pd.DataFrame, ds, curve: str | None = None) -> tuple[dict, str]:
+    """(the experiments as stored with the page: {column: values} for the CSV's columns that are parameters,
+    numeric results, their ``_sd`` and the curve column; the message that says what was read). ValueError when
+    none measures a result or every parameter is known."""
+    params = [p for p in ds.params if not ds.spec(p).is_computed]
+    results = [r for r in ds.numeric(ds.results) if r in df.columns]
+    if not results and not (curve and curve in df.columns):
+        raise ValueError("the CSV has no column named as a numeric result of the runs: nothing measured")
+    known = [p for p in params if p in df.columns]
+    if len(known) == len(params):
+        raise ValueError("the CSV gives every parameter: nothing is left to calibrate")
+    mix = space.mixture_in(ds)
+    missing = [c for c in (mix.components if mix is not None else ()) if c not in known]
+    if missing:         # one calibrated alone would take its share from the shares the experiments set
+        raise ValueError(f"{', '.join(missing)} {'is a component' if len(missing) == 1 else 'are components'} of "
+                         f"{mix.describe()}: give each experiment's blend, a column for every component")
+    keep = known + results + [r + SD_SUFFIX for r in results if r + SD_SUFFIX in df.columns] + \
+        ([curve] if curve and curve in df.columns else [])
+    out = {}
+    for c in keep:
+        v = df[c]
+        if c == curve:
+            out[c] = ["" if x is None or (isinstance(x, float) and np.isnan(x)) else str(x) for x in v]
+        elif ds.has(c) and ds.spec(c).is_categorical:
+            out[c] = [None if x is None or (isinstance(x, float) and np.isnan(x)) else str(x) for x in v]
+        else:       # None where not measured: a project is strict JSON, without NaN
+            out[c] = [float(x) if np.isfinite(x) else None for x in pd.to_numeric(v, errors="coerce")]
+    free = [p for p in params if p not in known]
+    measured = results + ([f"the curves of {curve}"] if curve in out else [])
+    return out, (f"{len(df):,} experiment{'s' if len(df) != 1 else ''}: {', '.join(measured)} measured"
+                 + (f" at known {', '.join(known)}" if known else "") + f"; calibrating {', '.join(free)}")
+
+
+@dataclass
+class Problem:
+    """The posterior of the calibrated parameters' unit coordinates ``theta`` (d,): ``logp(Theta)`` for rows
+    of them (walkers × d)."""
+    enc: object                 # the surrogates' encoder (tied: a formula parameter follows the others)
+    free: list                  # indices of the calibrated parameters among enc.codes
+    base: np.ndarray            # (experiments, enc.dim): the known settings' unit coordinates
+    terms: list                 # (surrogate, observed (experiments,), measurement σ (experiments,)) in scale space
+    curves: list = field(default_factory=list)  # (coefficient surrogates, observed (experiments, modes), σ)
+
+    @property
+    def d(self) -> int:
+        return len(self.free)
+
+    def points(self, Theta: np.ndarray) -> np.ndarray:
+        """Every experiment at every row of ``Theta``: (rows × experiments, dim), row-major."""
+        Theta = np.atleast_2d(Theta)
+        U = np.repeat(self.base[None, :, :], len(Theta), axis=0)
+        U[:, :, self.free] = Theta[:, None, :]
+        return U.reshape(-1, self.base.shape[1])
+
+    def logp(self, Theta: np.ndarray) -> np.ndarray:
+        Theta = np.atleast_2d(Theta)
+        inside = ((Theta >= 0) & (Theta <= 1)).all(axis=1)
+        out = np.full(len(Theta), -np.inf)
+        if not inside.any():
+            return out
+        U = self.points(np.clip(Theta[inside], 0.0, 1.0))
+        m = int(inside.sum())
+        total = np.zeros(m)
+        for sur, obs, sd in self.terms:
+            total += _gauss(sur, U, m, obs, sd)
+        for surs, obs, sd in self.curves:
+            for k, sur in enumerate(surs):
+                total += _gauss(sur, U, m, obs[:, k], sd)
+        out[inside] = total
+        return out
+
+
+def _gauss(sur, U: np.ndarray, m: int, obs: np.ndarray, sd: np.ndarray) -> np.ndarray:
+    """Σ over the experiments that measured it of log N(observed; μ, σ_meas² + σ_model²), for m walkers."""
+    mean, std = sur.predict_unit(U)
+    mean, std = mean.reshape(m, -1), std.reshape(m, -1)
+    ok = np.isfinite(obs)
+    v = sd[ok] ** 2 + std[:, ok] ** 2
+    return -0.5 * (((obs[ok] - mean[:, ok]) ** 2) / v + np.log(2 * np.pi * v)).sum(axis=1)
+
+
+def most_probable(prob: Problem, token: Token = NULL_TOKEN, seed: int = 0) -> np.ndarray:
+    """The unit coordinates of the highest posterior density differential evolution finds (no polish)."""
+    from scipy.optimize import differential_evolution
+    gens = []
+
+    def callback(intermediate_result):
+        gens.append(1)
+        token.progress(0.2 * min(len(gens) / 60, 1.0), "finding the most probable values")
+
+    res = differential_evolution(lambda T: -prob.logp(np.atleast_2d(T.T)), [(0.0, 1.0)] * prob.d, vectorized=True,
+                                 seed=seed, maxiter=60, popsize=15, tol=1e-8, polish=False, updating="deferred",
+                                 callback=callback)
+    return np.clip(res.x, 0.0, 1.0)
+
+
+def sample(prob: Problem, start: np.ndarray, steps: int, token: Token = NULL_TOKEN, seed: int = 0) -> dict:
+    """The ensemble sampler's chain from a ball around ``start``: {"chain": (steps, walkers, d), "logp":
+    (steps, walkers), "accepted": the share of moves accepted}. Each step moves one half of the walkers, each
+    along the line to a walker of the other half by a factor z with density ∝ 1/√z on [1/a, a], accepted
+    with probability min(1, z^(d−1) p(new)/p(old))."""
+    rng = np.random.default_rng(seed)
+    d = prob.d
+    W = max(MIN_WALKERS, 4 * d + (4 * d) % 2)
+    X = np.clip(start + 1e-3 * rng.standard_normal((W, d)), 1e-6, 1 - 1e-6)
+    lp = prob.logp(X)
+    chain, logp = np.empty((steps, W, d)), np.empty((steps, W))
+    accepted = 0
+    halves = (np.arange(0, W, 2), np.arange(1, W, 2))
+    for s in range(steps):
+        if s % 20 == 0:
+            if token.cancelled:
+                raise Cancelled()
+            token.progress(0.2 + 0.75 * s / steps, f"sampling: step {s:,} of {steps:,}")
+        for a, b in (halves, halves[::-1]):
+            z = ((STRETCH - 1) * rng.random(len(a)) + 1) ** 2 / STRETCH
+            other = X[rng.choice(b, len(a))]
+            new = other + z[:, None] * (X[a] - other)
+            lp_new = prob.logp(new)
+            with np.errstate(invalid="ignore", over="ignore"):
+                ratio = (d - 1) * np.log(z) + lp_new - lp[a]
+            ok = np.log(rng.random(len(a))) < ratio
+            X[a[ok]], lp[a[ok]] = new[ok], lp_new[ok]
+            accepted += int(ok.sum())
+        chain[s], logp[s] = X, lp
+    return {"chain": chain, "logp": logp, "accepted": accepted / (steps * W)}
+
+
+def autocorr_time(chain: np.ndarray, c: float = 5.0) -> np.ndarray:
+    """Integrated autocorrelation time per parameter of (steps, walkers, d): the walkers' mean
+    autocorrelation summed up to Sokal's window, the first M with M ≥ c τ(M) (as emcee does)."""
+    n = chain.shape[0]
+    out = np.empty(chain.shape[2])
+    size = int(2 ** np.ceil(np.log2(2 * n)))
+    for j in range(chain.shape[2]):
+        x = chain[:, :, j] - chain[:, :, j].mean(axis=0)
+        f = np.fft.rfft(x, n=size, axis=0)
+        acf = np.fft.irfft(f * np.conj(f), n=size, axis=0)[:n].mean(axis=1)
+        if not acf[0] > 0:
+            out[j] = np.inf
+            continue
+        taus = 2.0 * np.cumsum(acf / acf[0]) - 1.0
+        m = np.arange(n) < c * taus
+        out[j] = taus[np.argmin(m)] if not m.all() else taus[-1]
+    return out
+
+
+def summary(chain: np.ndarray, logp: np.ndarray, seed: int = 0) -> dict:
+    """The second half of the chain: thinned samples (at most KEEP), the highest-density sample, τ and the
+    effective sample size per parameter."""
+    half = chain[chain.shape[0] // 2:]
+    lp = logp[logp.shape[0] // 2:]
+    flat = half.reshape(-1, half.shape[2])
+    tau = autocorr_time(half)
+    best = flat[int(np.argmax(lp.reshape(-1)))]
+    idx = np.random.default_rng(seed).choice(len(flat), min(KEEP, len(flat)), replace=False)
+    return {"samples": flat[np.sort(idx)], "best": best, "tau": tau,
+            "ess": half.shape[0] * half.shape[1] / np.maximum(tau, 1.0), "steps_kept": half.shape[0]}

doe_analyzer/core/clustering.py

@@ -0,0 +1,144 @@
+"""Clustering of runs on standardised scale-space features."""
+from __future__ import annotations
+
+import os
+from concurrent.futures import ThreadPoolExecutor
+
+import numpy as np
+
+from doe_analyzer.core.tasks import NULL_TOKEN, Token
+
+METHODS = {"kmeans": "k-means", "gmm": "Gaussian mixture", "agglomerative": "Agglomerative (Ward)",
+           "hdbscan": "HDBSCAN (density)", "dbscan": "DBSCAN (density)"}
+AGGLO_MAX_ROWS = 5000
+SILHOUETTE_SAMPLE = 5000
+DENDRO_ROWS = 1500
+THREADED_ROWS = 5000     # above it the Gaussian mixtures of a sweep are fitted on threads
+
+
+def silhouette(X: np.ndarray, labels: np.ndarray, seed: int = 0) -> float:
+    from sklearn.metrics import silhouette_score
+    ok = labels >= 0
+    if len(set(labels[ok])) < 2:
+        return float("nan")
+    n = min(SILHOUETTE_SAMPLE, int(ok.sum()))
+    # n_jobs: the distances in parallel blocks (threads), the same values; 1.7x faster on 5 000
+    # runs, slower below about 2 000, where starting the threads costs more than they save
+    return float(silhouette_score(X[ok], labels[ok], sample_size=n, random_state=seed,
+                                  n_jobs=-1 if n > 2000 else None))
+
+
+class _OneTree:
+    """``memory`` for AgglomerativeClustering (joblib.Memory's ``cache`` interface):
+    without connectivity it builds the full Ward tree whatever k is, and only the
+    cut depends on k, so a sweep over k on the same runs builds it once."""
+
+    def __init__(self):
+        self.tree = None
+
+    def cache(self, build):
+        def cached(*args, **kwargs):
+            if self.tree is None:
+                self.tree = build(*args, **kwargs)
+            return self.tree
+        return cached
+
+
+def sweep(method: str, X: np.ndarray, kmax: int = 10, token: Token = NULL_TOKEN, seed: int = 0) -> dict:
+    """Quality per k: inertia and silhouette (k-means), BIC (GMM), silhouette (Ward);
+    the GMM and Ward labels per k, which ``cluster`` reuses."""
+    from sklearn.cluster import AgglomerativeClustering, KMeans
+    from sklearn.mixture import GaussianMixture
+    ks = list(range(2, max(3, min(kmax, X.shape[0] - 1)) + 1))
+    out = {"k": ks, "silhouette": [], "inertia": [], "bic": [], "labels": []}
+    Xa = X if X.shape[0] <= AGGLO_MAX_ROWS else X[np.random.default_rng(seed).choice(X.shape[0], AGGLO_MAX_ROWS, replace=False)]
+    tree = _OneTree()
+    if method == "gmm":
+        def fit(k):
+            token.check()
+            g = GaussianMixture(k, random_state=seed).fit(X)
+            return float(g.bic(X)), g.predict(X)
+        # independent fits, mostly BLAS (without the GIL): on threads 20 000 runs took 7 s instead of
+        # 17 s, the same labels; at a few thousand runs starting the threads costs what they save
+        workers = min(len(ks), os.cpu_count() or 1) if X.shape[0] > THREADED_ROWS else 1
+        with ThreadPoolExecutor(workers) as pool:
+            for i, (bic, labels) in enumerate(pool.map(fit, ks)):
+                token.progress(i / len(ks), f"k = {ks[i]}")
+                out["bic"].append(bic)
+                out["labels"].append(labels)
+                out["silhouette"].append(silhouette(X, labels, seed))
+        return out
+    for i, k in enumerate(ks):
+        token.progress(i / len(ks), f"k = {k}")
+        if method == "kmeans":
+            km = KMeans(k, n_init=4, random_state=seed).fit(X)
+            out["inertia"].append(float(km.inertia_))
+            out["silhouette"].append(silhouette(X, km.labels_, seed))
+        elif method == "agglomerative":
+            out["labels"].append(AgglomerativeClustering(k, memory=tree).fit_predict(Xa))
+            out["silhouette"].append(silhouette(Xa, out["labels"][-1], seed))
+    return out
+
+
+def best_k(sw: dict, method: str) -> int:
+    if method == "gmm" and sw["bic"]:
+        return sw["k"][int(np.argmin(sw["bic"]))]
+    sil = np.asarray(sw["silhouette"], dtype=float)
+    if sil.size and np.isfinite(sil).any():
+        return sw["k"][int(np.nanargmax(sil))]
+    return 3
+
+
+def cluster(method: str, X: np.ndarray, k: int, min_size: int = 10, seed: int = 0,
+            sweep: dict | None = None) -> np.ndarray:
+    """Labels (-1 = noise for density methods). A ``sweep`` of the same method, runs and
+    seed already holds the Gaussian mixture and Ward labels of its k."""
+    from sklearn.cluster import DBSCAN, HDBSCAN, AgglomerativeClustering, KMeans
+    from sklearn.mixture import GaussianMixture
+    swept = sweep["labels"][sweep["k"].index(k)] if sweep and sweep["labels"] and k in sweep["k"] else None
+    if method == "kmeans":
+        return KMeans(k, n_init=8, random_state=seed).fit_predict(X)
+    if method == "gmm":
+        return swept if swept is not None else GaussianMixture(k, random_state=seed).fit(X).predict(X)
+    if method == "agglomerative":
+        if X.shape[0] <= AGGLO_MAX_ROWS:
+            return swept if swept is not None else AgglomerativeClustering(k).fit_predict(X)
+        # fit on a subsample (the sweep's), assign the rest to the nearest cluster centroid
+        idx = np.random.default_rng(seed).choice(X.shape[0], AGGLO_MAX_ROWS, replace=False)
+        lab = swept if swept is not None else AgglomerativeClustering(k).fit_predict(X[idx])
+        cent = np.stack([X[idx][lab == c].mean(axis=0) for c in range(k)])
+        out = np.linalg.norm(X[:, None, :] - cent[None], axis=2).argmin(axis=1)
+        out[idx] = lab      # the fitted runs keep Ward's clusters (the sweep's silhouettes are of these)
+        return out
+    if method == "hdbscan":
+        # scikit-learn refuses a size above the run count; then every run is noise anyway
+        return HDBSCAN(min_cluster_size=max(min(min_size, X.shape[0]), 2)).fit_predict(X)
+    if method == "dbscan":
+        from scipy.spatial import cKDTree
+        # the run itself counts toward Minimum cluster size, as in DBSCAN's min_samples: its
+        # (min_size − 1)-th other run (one further merged two groups of exactly that size)
+        d, _ = cKDTree(X).query(X, k=max(min(min_size, X.shape[0]), 2), workers=-1)
+        # 0 when most runs are exact replicates; DBSCAN needs eps > 0, and the
+        # smallest one groups the replicates
+        eps = max(float(np.quantile(d[:, -1], 0.9)), np.finfo(float).tiny)
+        # n_jobs: the neighbourhood queries in parallel (threads), the same result; 2.5x faster
+        # on 2 000 runs and more, while on a few hundred starting the threads costs more
+        return DBSCAN(eps=eps, min_samples=min_size, n_jobs=-1 if X.shape[0] > 1000 else None).fit_predict(X)
+    raise ValueError(method)
+
+
+def profiles(X: np.ndarray, labels: np.ndarray) -> tuple[list[int], np.ndarray, np.ndarray]:
+    """(cluster ids, mean standardised value per cluster × feature, sizes)."""
+    ids = sorted(int(c) for c in set(labels.tolist()) if c >= 0)
+    M = np.stack([X[labels == c].mean(axis=0) for c in ids]) if ids else np.zeros((0, X.shape[1]))
+    n = np.array([(labels == c).sum() for c in ids])
+    return ids, M, n
+
+
+def dendrogram(X: np.ndarray, seed: int = 0) -> dict:
+    """scipy dendrogram coordinates of a Ward linkage (subsampled)."""
+    from scipy.cluster.hierarchy import dendrogram as dg, linkage
+    if X.shape[0] > DENDRO_ROWS:
+        X = X[np.random.default_rng(seed).choice(X.shape[0], DENDRO_ROWS, replace=False)]
+    Z = linkage(X, method="ward")
+    return dg(Z, no_plot=True, truncate_mode="lastp", p=40)