process-improve 2.2.2__tar.gz → 2.3.0__tar.gz

{process_improve-2.2.2 → process_improve-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: process-improve
-Version: 2.2.2
+Version: 2.3.0
 Summary: Designed Experiments; Latent Variables (PCA, PLS, multivariate methods with missing data); Process Monitoring; Batch data analysis.
 Keywords: Designed Experiments,Design of Experiments,DoE,Optimal Design,Response Surface,Latent Variables,PCA,PLS,Multivariate Data Analysis,Chemometrics,Process Monitoring,Batch data analysis
 Author: Kevin Dunn

{process_improve-2.2.2 → process_improve-2.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "process-improve"
-version = "2.2.2"
+version = "2.3.0"
 description = "Designed Experiments; Latent Variables (PCA, PLS, multivariate methods with missing data); Process Monitoring; Batch data analysis."
 readme = "README.md"
 license = "MIT"
@@ -107,7 +107,7 @@ dev = [
     "pytest-cov>=7.0.0",
     "pytest-xdist>=3.8.0",
     "pydata-sphinx-theme>=0.16.1",
-    "ruff>=0.11.0,<0.17",
+    "ruff>=0.11.0,<0.18",
     "sphinx>=8.1.3",
     "nbsphinx>=0.9.5",
     "ipykernel>=6.29",
@@ -309,6 +309,27 @@ only_mutate = [
     "src/process_improve/monitoring/metrics.py",
     "src/process_improve/monitoring/evop.py",
     "src/process_improve/monitoring/control_charts.py",
+    "src/process_improve/multivariate/_nipals.py",
+    "src/process_improve/multivariate/_preprocessing.py",
+    "src/process_improve/multivariate/_projection.py",
+    "src/process_improve/multivariate/_impute.py",
+    "src/process_improve/multivariate/_robust.py",
+    "src/process_improve/multivariate/_diagnostics.py",
+    "src/process_improve/multivariate/_resampling.py",
+    "src/process_improve/multivariate/_null.py",
+    "src/process_improve/multivariate/_procrustes.py",
+    "src/process_improve/multivariate/_prm.py",
+    "src/process_improve/multivariate/_opls.py",
+    "src/process_improve/multivariate/_pca.py",
+    "src/process_improve/multivariate/_pls.py",
+    "src/process_improve/multivariate/_mbpca.py",
+    "src/process_improve/multivariate/_mbpls.py",
+    "src/process_improve/multivariate/_tpls.py",
+    "src/process_improve/multivariate/_cv_criteria.py",
+    "src/process_improve/multivariate/_plsda.py",
+    "src/process_improve/multivariate/_asca.py",
+    "src/process_improve/multivariate/_categorical.py",
+    "src/process_improve/multivariate/_adaptive.py",
 ]
 pytest_add_cli_args = [
     "-o",
@@ -329,4 +350,42 @@ pytest_add_cli_args_test_selection = [
     "tests/test_control_chart_cusum.py",
     "tests/test_control_chart_ewma.py",
     "tests/test_control_chart_hw_golden.py",
+    "tests/test_nipals_quick_regress.py",
+    "tests/test_multivariate_adaptive.py",
+    "tests/test_multivariate_centring_traps.py",
+    "tests/test_multivariate_projection.py",
+    "tests/test_multivariate_pls_missing.py",
+    "tests/test_multivariate_prm.py",
+    "tests/test_contributions_missing_cells.py",
+    "tests/test_multivariate_contribution_plots.py",
+    "tests/test_multivariate_meda.py",
+    "tests/test_multivariate_resampler.py",
+    "tests/test_multivariate_null.py",
+    "tests/test_multivariate_gpa.py",
+    "tests/test_multivariate_opls.py",
+    "tests/test_multivariate_tsr.py",
+    "tests/test_pca_nipals_mixomics_crosscheck.py",
+    "tests/test_multivariate_nipals_seed.py",
+    "tests/test_r2_per_variable_cumulative.py",
+    "tests/test_multivariate_sample_weight.py",
+    "tests/test_multivariate_inversion.py",
+    "tests/test_multivariate_inversion_worked_example.py",
+    "tests/test_multivariate_feature_names.py",
+    "tests/test_sklearn_compat.py",
+    "tests/test_multivariate_robustness.py",
+    "tests/test_tool_spec.py",
+    "tests/test_multiblock_reference.py",
+    "--deselect",
+    "tests/test_multiblock_reference.py::TestFMCReference",
+    "tests/test_multiblock_weighting.py",
+    "tests/test_multivariate_categorical.py",
+    "tests/test_mbpls_cross_validation.py",
+    "tests/test_mbpls_regressor.py",
+    "tests/test_multivariate_tpls_validation.py",
+    "tests/test_multivariate_tpls_display.py",
+    "tests/test_multivariate_tpls_feature_importance.py",
+    "tests/test_multivariate_dataframedict.py",
+    "tests/test_multivariate_cv_criteria.py",
+    "tests/test_multivariate_plsda.py",
+    "tests/test_multivariate_asca.py",
 ]

{process_improve-2.2.2 → process_improve-2.3.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "process-improve"
-version = "2.2.2"
+version = "2.3.0"
 description = 'Designed Experiments; Latent Variables (PCA, PLS, multivariate methods with missing data); Process Monitoring; Batch data analysis.'
 readme = "README.md"
 license = "MIT"
@@ -152,7 +152,7 @@ dev = [
     "pytest-cov>=7.0.0",
     "pytest-xdist>=3.8.0",
     "pydata-sphinx-theme>=0.16.1",
-    "ruff>=0.11.0,<0.17", # upper-bound so a future ruff release cannot silently break the lint gate (see mypy)
+    "ruff>=0.11.0,<0.18", # upper-bound so a future ruff release cannot silently break the lint gate (see mypy)
     "sphinx>=8.1.3",
     "nbsphinx>=0.9.5",
     "ipykernel>=6.29",
@@ -460,6 +460,27 @@ only_mutate = [
     "src/process_improve/monitoring/metrics.py",
     "src/process_improve/monitoring/evop.py",
     "src/process_improve/monitoring/control_charts.py",
+    "src/process_improve/multivariate/_nipals.py",
+    "src/process_improve/multivariate/_preprocessing.py",
+    "src/process_improve/multivariate/_projection.py",
+    "src/process_improve/multivariate/_impute.py",
+    "src/process_improve/multivariate/_robust.py",
+    "src/process_improve/multivariate/_diagnostics.py",
+    "src/process_improve/multivariate/_resampling.py",
+    "src/process_improve/multivariate/_null.py",
+    "src/process_improve/multivariate/_procrustes.py",
+    "src/process_improve/multivariate/_prm.py",
+    "src/process_improve/multivariate/_opls.py",
+    "src/process_improve/multivariate/_pca.py",
+    "src/process_improve/multivariate/_pls.py",
+    "src/process_improve/multivariate/_mbpca.py",
+    "src/process_improve/multivariate/_mbpls.py",
+    "src/process_improve/multivariate/_tpls.py",
+    "src/process_improve/multivariate/_cv_criteria.py",
+    "src/process_improve/multivariate/_plsda.py",
+    "src/process_improve/multivariate/_asca.py",
+    "src/process_improve/multivariate/_categorical.py",
+    "src/process_improve/multivariate/_adaptive.py",
 ]
 # The runner drops pytest.ini's addopts: no xdist workers and no coverage inside each mutant run.
 pytest_add_cli_args = ["-o", "addopts=", "-p", "no:cacheprovider"]
@@ -476,4 +497,43 @@ pytest_add_cli_args_test_selection = [
     "tests/test_control_chart_cusum.py",
     "tests/test_control_chart_ewma.py",
     "tests/test_control_chart_hw_golden.py",
+    "tests/test_nipals_quick_regress.py",
+    "tests/test_multivariate_adaptive.py",
+    "tests/test_multivariate_centring_traps.py",
+    "tests/test_multivariate_projection.py",
+    "tests/test_multivariate_pls_missing.py",
+    "tests/test_multivariate_prm.py",
+    "tests/test_contributions_missing_cells.py",
+    "tests/test_multivariate_contribution_plots.py",
+    "tests/test_multivariate_meda.py",
+    "tests/test_multivariate_resampler.py",
+    "tests/test_multivariate_null.py",
+    "tests/test_multivariate_gpa.py",
+    "tests/test_multivariate_opls.py",
+    "tests/test_multivariate_tsr.py",
+    "tests/test_pca_nipals_mixomics_crosscheck.py",
+    "tests/test_multivariate_nipals_seed.py",
+    "tests/test_r2_per_variable_cumulative.py",
+    "tests/test_multivariate_sample_weight.py",
+    "tests/test_multivariate_inversion.py",
+    "tests/test_multivariate_inversion_worked_example.py",
+    "tests/test_multivariate_feature_names.py",
+    "tests/test_sklearn_compat.py",
+    "tests/test_multivariate_robustness.py",
+    "tests/test_tool_spec.py",
+    "tests/test_multiblock_reference.py",
+    # Its FMC case study reads a script from docs/, which mutmut does not copy, and its data from the web.
+    "--deselect",
+    "tests/test_multiblock_reference.py::TestFMCReference",
+    "tests/test_multiblock_weighting.py",
+    "tests/test_multivariate_categorical.py",
+    "tests/test_mbpls_cross_validation.py",
+    "tests/test_mbpls_regressor.py",
+    "tests/test_multivariate_tpls_validation.py",
+    "tests/test_multivariate_tpls_display.py",
+    "tests/test_multivariate_tpls_feature_importance.py",
+    "tests/test_multivariate_dataframedict.py",
+    "tests/test_multivariate_cv_criteria.py",
+    "tests/test_multivariate_plsda.py",
+    "tests/test_multivariate_asca.py",
 ]

{process_improve-2.2.2 → process_improve-2.3.0}/src/process_improve/batch/control.py

@@ -22,21 +22,29 @@ solves that choice as a quadratic program in the scaled space of a fitted
 With the quadratic caps the problem is a convex quadratically-constrained QP.
 The workhorse is the penalty-form pure QP solved with `osqp
 <https://osqp.org>`_ (the ``control`` extra); the hard-cap mode wraps the
-same QP in an outer scalar iteration on the two penalty multipliers, which is
-exact for this convex problem and converges in a handful of inner solves at
-this size (a few dozen decision variables).
+same QP in an outer iteration on the two penalty multipliers. It raises the
+multiplier of each broken cap until both caps hold, then bisects each
+multiplier down in turn, taking a step only when both caps still hold: a
+schedule returned with status ``"ok"`` meets both caps to within 1%, and a
+cap that binds alone ends with its statistic just inside it. When both caps
+bind, a step down in one multiplier can break the other cap, so a statistic
+can stay well inside its cap and the schedule can fall short of the capped
+optimum.
 
 The formulation follows the latent-variable batch control literature.
 Flores-Cerrillo and MacGregor (2004) give the quality-tracking objective, the
-movement-suppression term, the soft T2 term, and the SPE check on the
-measurements so far that gates whether a correction is computed at all;
-Yabuki and MacGregor (1997) give the no-correction dead band, their
-"no-control region"; Garcia-Munoz, Kourti and MacGregor (2004) give the
-per-decision-point score covariance and limits, and the prediction interval
-at a decision point is built in the same spirit, from the training batches
-re-projected under that point's missingness pattern; Arteaga and Ferrer (2002)
-give the trimmed score regression used to estimate the scores of a partially
-observed row, which Golshan et al. (2010) also apply in the LV-MPC setting.
+movement-suppression term and the soft T2 term, and they check the SPE of the
+measurements so far before a correction is computed, suggesting that the
+schedule from the last decision point be kept when that check fails, which is
+what the gate here does. The dead band plays the role of the "no-control
+region" of Yabuki and MacGregor (1997). Garcia-Munoz, Kourti and MacGregor
+(2004) compute Hotelling's T2 of on-line score estimates against the
+covariance of the reference batches' estimates at each time, as Nomikos and
+MacGregor (1995) suggested; the per-decision-point limits here, and the
+prediction interval at a decision point, are built the same way, from the
+training batches re-projected under that point's missingness pattern.
+Arteaga and Ferrer (2002) give the trimmed score regression used to estimate
+the scores of a partially observed row.
 
 Two departures from Flores-Cerrillo and MacGregor are deliberate. Their
 optimisation is over an adjustment to the scores, with the remaining
@@ -44,18 +52,20 @@ trajectories recovered by inverting the PLS model, which is what keeps those
 trajectories consistent with past operation. Here the decision variables are
 the future MV columns themselves, so actuator bounds and rate limits apply
 exactly in engineering units, and the terms that hold the answer inside the
-model's region are stated explicitly instead. Second, the SPE of the
-*candidate* row is penalised and capped, not only checked on the
-measurements so far, because a candidate written directly in the MV columns
-can leave the model plane in a way a score adjustment cannot.
-
-A practical caveat: models of this kind are identified on *recorded* (noisy,
-realised) trajectories, while the corrector outputs *setpoints*. That is the
-standard identification practice, and it attenuates the apparent gain
-slightly (the regression sees the control error as input noise); the
-executed-policy evaluation in :func:`evaluate_control_policies`
-(:mod:`process_improve.simulation`) measures the realised effect rather than
-trusting the model's own prediction.
+model's region are stated explicitly instead. Golshan et al. (2010) give both
+forms for latent-variable MPC: over a correction to the scores, and directly
+over the future manipulated variables. Second, the SPE of the *candidate* row
+is penalised and capped, not only checked on the measurements so far: a
+trajectory reconstructed from adjusted scores lies on the model plane by
+construction, and one written directly in the MV columns need not.
+
+A practical caveat: models of this kind are identified on *recorded*
+trajectories (the realised values plus measurement noise), while the
+corrector outputs *setpoints*. The substitution is sound where the control
+loops track their setpoints; the executed-policy evaluation in
+:func:`evaluate_control_policies` (:mod:`process_improve.simulation`)
+measures the realised effect rather than trusting the model's own
+prediction.
 
 References
 ----------
@@ -71,6 +81,9 @@ Garcia-Munoz, S., Kourti, T. and MacGregor, J.F., "Model Predictive
 Monitoring for Batch Processes", Industrial & Engineering Chemistry
 Research, 43, 5929-5941, 2004.
 
+Nomikos, P. and MacGregor, J.F., "Multivariate SPC Charts for Monitoring
+Batch Processes", Technometrics, 37, 41-59, 1995.
+
 Yabuki, Y. and MacGregor, J.F., "Product quality control in semibatch
 reactors using midcourse correction policies", Industrial & Engineering
 Chemistry Research, 36, 1268-1275, 1997.
@@ -138,9 +151,10 @@ def _solve_qp(
         eps_abs=1e-10,
         eps_rel=1e-10,
         max_iter=100_000,
-        polish=True,
+        polishing=True,
     )
-    result = problem.solve()
+    # Explicit, as osqp warns that its default will become True; the status check below raises instead.
+    result = problem.solve(raise_error=False)
     status = str(result.info.status)
     if "solved" not in status.lower():
         raise RuntimeError(
@@ -660,10 +674,11 @@ class MidCourseCorrector:
     answers the monitoring question at a decision point (the predicted final
     quality with its interval, and the validity statistics of the batch so
     far). :meth:`correct` adds the decision: it checks the batch-so-far
-    against the model (the SPE validity gate of Flores-Cerrillo and
-    MacGregor, 2004), applies the no-correction dead band (Yabuki and
-    MacGregor, 1997) in target mode, builds the per-decision-point reference
-    limits (Garcia-Munoz et al., 2004), solves the QP, and returns the full
+    against the model (the SPE check of Flores-Cerrillo and MacGregor, 2004,
+    keeping the current schedule when it fails), applies the no-correction
+    dead band (in the role of the no-control region of Yabuki and MacGregor,
+    1997) in target mode, builds the per-decision-point reference limits
+    (Garcia-Munoz et al., 2004), solves the QP, and returns the full
     corrected schedule ready to implement (or to hand to
     :meth:`process_improve.simulation.BioreactorSimulator.simulate_batch`).
 
@@ -795,13 +810,15 @@ class MidCourseCorrector:
         and the *candidate* pattern (monitoring plus the future MV columns,
         which the optimiser treats as observed) for the QP's SPE cap and the
         score covariance behind its T2 term. Limits: the g-chi-squared SPE
-        limit of Nomikos and MacGregor on each pattern's training SPE values,
-        and the F-distribution T2 limit on the candidate-pattern score
-        estimates with their own covariance (Garcia-Munoz et al., 2004).
+        limit (Box's approximation, as Nomikos and MacGregor, 1995, use it)
+        on each pattern's training SPE values, and the F-distribution T2
+        limit on the candidate-pattern score estimates with their own
+        covariance (Garcia-Munoz et al., 2004).
 
         Also returned: ``rmse_k`` (Series over the targets, original units),
         the prediction error of the model at this decision point, from the
-        candidate-pattern estimates of the training batches; and the
+        candidate-pattern estimates of the training batches (an in-sample
+        error, since the model was fitted on those batches); and the
         condition numbers of the two pattern operators. Results are cached
         per ``k``.
         """
@@ -903,7 +920,10 @@ class MidCourseCorrector:
         uses the model's prediction error *at this decision point*
         (``limits_at(k).rmse_k``) with the leverage of the estimated scores,
         so it is wide early in the batch, when the estimate rests on few
-        observed columns, and narrows as the batch runs.
+        observed columns, and narrows as the batch runs. It is the prediction
+        interval of a least-squares regression of the quality on the scores,
+        with the scores treated as known; check its coverage on held-out
+        batches before relying on it.
 
         Parameters
         ----------
@@ -1190,9 +1210,12 @@ def _oracle_remaining(  # noqa: PLR0913 - explicit oracle inputs
     Direct search (Nelder-Mead, then Powell from the same start) over
     ``n_knots`` linearly interpolated knot values per manipulated variable
     from sample ``k`` onward, with the identical seed, so the disturbances
-    match the batch being corrected. This is the ceiling for any mid-course
-    scheme at that decision point, because the objective is the true process,
-    not a model of it.
+    match the batch being corrected. It estimates the ceiling for any
+    mid-course scheme at that decision point: the objective is the true
+    process, not a model of it, and the disturbances still to come are
+    known to it, as they are to no scheme running on a plant. The search is
+    local, within the full operating bounds and without rate limits, so the
+    value is a lower bound on that ceiling.
     """
     from scipy.optimize import minimize  # noqa: PLC0415
 
@@ -1325,13 +1348,18 @@ def evaluate_control_policies(  # noqa: PLR0913, PLR0915, C901 - one executed co
       run the nominal schedule.
     - **oracle-from-k**: for every batch the mid-course policy corrected,
       the remaining schedule is instead optimised against the simulator
-      itself at the same decision point (:func:`_oracle_remaining`). This is
-      the ceiling for any mid-course scheme at that decision point; the gap
-      to the mid-course row is the price of using an empirical model with
-      limited historical excitation.
-    - **adapted**: every batch runs the true optimal schedule for its own
-      initial conditions from time zero
-      (``simulator.optimal_trajectory``), the perfect-feedforward ceiling.
+      itself at the same decision point, with the batch's own seed
+      (:func:`_oracle_remaining`). This estimates the ceiling for any
+      mid-course scheme at that decision point. The gap to the mid-course
+      row mixes the error of an empirical model with limited historical
+      excitation, the oracle's knowledge of the disturbances still to come,
+      and the corrector's own limits (movement penalty, validity caps,
+      tighter bounds and rate limits).
+    - **adapted**: every batch runs the schedule that maximises the
+      disturbance-free titer for its own initial conditions, found before
+      the batch starts from the simulator's own equations
+      (``simulator.optimal_trajectory``): feedforward adaptation with a
+      perfect process model, an estimate of the feedforward ceiling.
 
     Parameters
     ----------
@@ -1357,10 +1385,11 @@ def evaluate_control_policies(  # noqa: PLR0913, PLR0915, C901 - one executed co
         Fit one model per feed class (labels from the training campaign;
         test batches are assigned to the nearest class centroid in
         standardised Z). With ``False``, or when class labels are
-        unavailable, a single global model is used; the executed experiments
-        behind this module found the global linear model averages the
-        class-dependent gain direction away, so per-class models are the
-        default.
+        unavailable, a single global model is used. The effect of a schedule
+        change depends on the feed class and one linear model averages it:
+        on the default configuration the global model corrects five batches
+        for a mean executed gain of about 0.5 g/L, against eight batches and
+        about 1.4 g/L per class, so per-class models are the default.
     target_side, dead_band, weights, bounds, rate_limits, spe_cap, t2_cap, n_knots, method, ridge
         Corrector settings, passed to :class:`MidCourseCorrector`. ``bounds``
         defaults to the simulator's operating bounds tightened inward by

{process_improve-2.2.2 → process_improve-2.3.0}/src/process_improve/batch/datasets.py

@@ -72,9 +72,8 @@ def _per_batch_table(sheet: pd.DataFrame, batch_ids: list[Hashable]) -> pd.DataF
 def load_nylon() -> dict[Hashable, pd.DataFrame]:
     """Return the nylon autoclave reactor batch dataset.
 
-    Trajectory data from an industrial nylon polymerization autoclave,
-    used widely in the batch analysis and monitoring literature. Variables
-    ``Tag01`` to ``Tag10`` are temperatures, pressures, and flows recorded
+    Trajectory data from an industrial nylon polymerization autoclave.
+    Variables ``Tag01`` to ``Tag10`` are temperatures, pressures, and flows recorded
     during each batch. Batch durations vary slightly (113 to 135 samples),
     so resample or align the batches to a common length before unfolding
     (see :func:`process_improve.batch.resample_to_reference`).

process_improve-2.3.0/src/process_improve/datasets/experiments/biscuit/NOTICE

@@ -0,0 +1,38 @@
+Biscuit-dough NIR spectra
+=========================
+
+Source
+------
+The ``cookie`` data set of the R package ppls 2.0.0 (CRAN), file data/cookie.rda.
+The R packages groc 1.0.10 and seedCCA 3.1 (CRAN) carry the same values.
+
+References
+----------
+Osborne, B. G., Fearn, T., Miller, A. R. and Douglas, S. *Application of near infrared
+reflectance spectroscopy to the compositional analysis of biscuits and biscuit doughs.*
+Journal of the Science of Food and Agriculture, 35(1), 99-105 (1984).
+https://doi.org/10.1002/jsfa.2740350116
+
+Brown, P. J., Fearn, T. and Vannucci, M. *Bayesian wavelet regression on curves with
+application to a spectroscopic calibration problem.* Journal of the American Statistical
+Association, 96(454), 398-408 (2001). https://doi.org/10.1198/016214501753168118
+
+Contents
+--------
+biscuit.csv: 72 unbaked biscuit doughs, one row each. ``sample`` numbers them 1 to 72 in
+the upstream order; ``set`` is ``calibration`` for the first 40 and ``validation`` for the
+other 32; ``fat``, ``sucrose``, ``dry_flour`` and ``water`` are the percentages of each
+ingredient; the 700 columns ``1100`` to ``2498`` are the NIR reflectance spectrum, one
+column per wavelength in nm, at 2 nm intervals. The values are the upstream values,
+unchanged; only the column names of the spectrum are the wavelengths instead of 1 to 700.
+
+Terms
+-----
+The values are measurements reported by Osborne et al. (1984): facts, not a creative work.
+The MIT licence of process-improve covers its code, not this file, and claims nothing over
+the measurements. Cite Osborne et al. (1984) when you use them.
+
+The R packages ppls, groc and seedCCA are distributed under the GNU General Public License,
+version 2 or later. That is the licence of those packages, whose authors did not make the
+measurements. This file is kept separate from the code that reads it, so it is an aggregate
+in the sense of the GPL and does not change the licence of process-improve.