optimized-svm 0.1.0__py3-none-any.whl

optimized_svm/training.py

@@ -0,0 +1,258 @@
+"""Two-step training: SMO warm start (``smo_solver``) and Newton refinement (Strasdat, Ch. 8.4)."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+import numpy as np
+import smo_solver
+from numpy.typing import NDArray
+
+from .linalg import solve_bordered
+from .problem import TrainingProblem
+
+Array = NDArray[np.float64]
+
+
+class ConvergenceError(RuntimeError):
+    """The Newton method could not find a descent step."""
+
+
+@dataclass
+class Solution:
+    """Solution ``(a, b, c)`` of a training problem with ``ka = K a / lam``."""
+
+    a: Array
+    b: float
+    c: float
+    ka: Array
+    residual: float = np.nan
+    newton_steps: int = 0
+    smo_steps: int = 0
+    time_smo: float = 0.0
+    time_newton: float = 0.0
+    info: dict[str, Any] = field(default_factory=dict)
+
+
+def residual(prob: TrainingProblem, a: Array, b: float, c: float, ka: Array, use_c: bool) -> float:
+    """Mean squared violation of the optimality conditions ``a + phi'(dec) = 0``, ``sum(a) = 0``
+    and (if the 1-norm constraint is active) ``s^T phi'(dec) + A = 0``."""
+    g = prob.loss.d1(prob.decision(b, c, ka), prob.terms)
+    res = float(np.mean((a + g) ** 2) + g.sum() ** 2)
+    if use_c:
+        res += float(g @ prob.sign + prob.max_asum) ** 2
+    return res
+
+
+def _newton_direction(
+    prob: TrainingProblem, a: Array, g: Array, h: Array, use_c: bool
+) -> tuple[Array, float, float]:
+    """Semismooth Newton direction for ``a + phi'(K a / lam + b + s c) = 0`` and constraints."""
+    lam = prob.lam
+    p = h > 0.0
+    z = ~p
+    da = np.zeros_like(a)
+    da[z] = a[z] + g[z]  # terms with vanishing curvature: a_Z = -phi'_Z after the step
+    k_pz_dz = prob.kernel(p, z) @ da[z] / lam if z.any() and p.any() else np.zeros(p.sum())
+    q = prob.kernel(p, p) / lam + np.diag(1.0 / h[p])
+    rhs = (a[p] + g[p]) / h[p] - k_pz_dz
+    cols = [np.ones(p.sum())]
+    rhs0 = [a[p].sum() - g[z].sum()]
+    if use_c:
+        s = prob.sign
+        cols.append(s[p])
+        rhs0.append(a @ s - prob.max_asum - s[z] @ da[z])
+    d_p, v = solve_bordered(q, np.stack(cols, axis=1), rhs, np.array(rhs0))
+    da[p] = d_p
+    return da, float(v[0]), float(v[1]) if use_c else 0.0
+
+
+def newton(
+    prob: TrainingProblem,
+    a: Array,
+    b: float,
+    c: float,
+    *,
+    tol: float = 1e-12,
+    max_steps: int = 100,
+    max_backtracks: int = 20,
+    sigma: float = 1e-4,
+    beta: float = 0.5,
+    verbose: int = 0,
+) -> Solution:
+    """Damped semismooth Newton method for the (smoothed) primal training problem.
+
+    Requires a loss with (piecewise) second derivative. The 1-norm constraint is treated as
+    active iff ``c > 0`` at the starting point.
+    """
+    use_c = prob.max_asum is not None and c > 0.0
+    if not use_c:
+        c = 0.0
+    a = np.array(a, dtype=np.float64)
+    ka = prob.ka(a)
+    obj = prob.objective(a, b, c, ka)
+    for step in range(max_steps + 1):
+        dec = prob.decision(b, c, ka)
+        g = prob.loss.d1(dec, prob.terms)
+        r = a + g
+        res = float(np.mean(r**2) + g.sum() ** 2)
+        grad_c = float(g @ prob.sign + prob.max_asum) if use_c else 0.0
+        res += grad_c**2
+        if verbose:
+            print(f"[newton {step:3d}] f = {obj:.10g}, residual = {res:.3e}")
+        if res < tol or step == max_steps:
+            break
+        # gradient of the objective w.r.t. (a, b, c): (K r / lam, sum g, s^T g + A)
+        grad_b = float(g.sum())
+        h = prob.loss.d2(dec, prob.terms)
+        directions = []
+        try:
+            directions.append(_newton_direction(prob, a, g, h, use_c))
+        except np.linalg.LinAlgError:
+            pass
+        # preconditioned gradient as fallback
+        directions.append((r, prob.lam * grad_b, prob.lam * grad_c))
+        for da, db, dc in directions:
+            kda = prob.ka(da)
+            slope = float(r @ kda) + db * grad_b + dc * grad_c
+            if slope <= 0.0:
+                continue
+            t = 1.0
+            for _ in range(max_backtracks):
+                a_new, b_new, c_new = a - t * da, b - t * db, c - t * dc
+                ka_new = ka - t * kda
+                obj_new = prob.objective(a_new, b_new, c_new, ka_new)
+                if obj_new <= obj - sigma * t * slope + 1e-15 * abs(obj):
+                    break
+                t *= beta
+            else:
+                continue
+            a, b, c, ka, obj = a_new, b_new, c_new, ka_new, obj_new
+            break
+        else:
+            if res < 1e3 * tol:
+                break
+            raise ConvergenceError(f"no descent step found (residual {res:.3e})")
+    # recompute ka from scratch to remove accumulated rounding errors
+    ka = prob.ka(a)
+    return Solution(
+        a, float(b), float(c), ka, residual(prob, a, b, c, ka, use_c), newton_steps=step
+    )
+
+
+#: SMO tolerance used before a Newton refinement
+ROUGH_SMO_TOL = 0.1
+
+_SMO_OPTIONS = {
+    "tol",
+    "max_steps",
+    "time_limit",
+    "working_set",
+    "two_sided",
+    "shrinking",
+    "shrinking_period",
+    "cache_size_mb",
+}
+
+
+def smo_warm_start(
+    prob: TrainingProblem, alpha0: Array | None = None, **options: Any
+) -> tuple[Array, float, float, int]:
+    """Solves the dual with quadratic smoothing by SMO (``smo_solver``); zeros if not applicable.
+
+    ``alpha0`` (e.g. the solution for neighbouring hyperparameters) is used as starting point
+    if it is feasible.
+    """
+    unknown = set(options) - _SMO_OPTIONS
+    if unknown:
+        raise ValueError(f"unknown SMO options: {sorted(unknown)}")
+    loss = prob.loss
+    if loss.smo_kind is None:
+        return np.zeros(prob.size), 0.0, 0.0, 0
+    common = dict(gamma=1.0, weights=prob.w, smoothing=loss.smoothing, max_asum=prob.max_asum)
+
+    def run(start: Array | None) -> Any:
+        if loss.smo_kind == "classification":
+            return smo_solver.solve_classification(
+                prob.x, prob.y, prob.lam, shift=loss.shift, alpha0=start, **common, **options
+            )
+        return smo_solver.solve_regression(
+            prob.x, prob.y, prob.lam, epsilon=loss.epsilon, alpha0=start, **common, **options
+        )
+
+    res = None
+    if alpha0 is not None:
+        start = _feasible_start(prob, alpha0)
+        if start is not None:
+            try:
+                res = run(start)
+            except ValueError:
+                res = None
+    if res is None:
+        res = run(None)
+    return np.asarray(res.alpha), res.b, res.c, res.steps
+
+
+def _feasible_start(prob: TrainingProblem, alpha0: Array) -> Array | None:
+    """Projects a previous solution onto the box and repairs ``sum(a) = 0``; ``None`` if this
+    fails (e.g. different problem size)."""
+    if alpha0.shape != (prob.size,):
+        return None
+    t = prob.terms
+    lo = np.where(t.sign > 0, 0.0, -t.w) if prob.loss.signed else np.full(prob.size, -np.inf)
+    hi = np.where(t.sign > 0, t.w, 0.0) if prob.loss.signed else np.full(prob.size, np.inf)
+    a = np.clip(alpha0, lo, hi)
+    if prob.max_asum is not None and (t.sign * a).sum() > prob.max_asum:
+        a = a * (prob.max_asum / (t.sign * a).sum()) * (1 - 1e-12)
+    # distribute the violation of sum(a) = 0 over variables with room in that direction
+    excess = a.sum()
+    room = (a - lo) if excess > 0 else (hi - a)
+    total = room.sum()
+    if not total > abs(excess):
+        return None
+    a = a - excess * room / total
+    if prob.max_asum is not None and (t.sign * a).sum() > prob.max_asum:
+        return None
+    return np.ascontiguousarray(a)
+
+
+def train(
+    prob: TrainingProblem,
+    smo_options: dict[str, Any] | None = None,
+    newton_options: dict[str, Any] | None = None,
+    alpha0: Array | None = None,
+) -> Solution:
+    """Two-step training: SMO warm start followed by Newton refinement.
+
+    For non-smooth losses (``smoothing = 0``) the SMO solution is already exact; for losses
+    without SMO formulation (squared, Poisson) Newton starts at zero.
+    """
+    smo_options = dict(smo_options or {})
+    smooth = getattr(prob.loss, "smoothing", 1.0) > 0.0
+    if smooth:
+        # Newton refines the solution, so a rough SMO solution suffices as starting point
+        smo_options.setdefault("tol", ROUGH_SMO_TOL)
+    t0 = time.perf_counter()
+    a, b, c, smo_steps = smo_warm_start(prob, alpha0, **smo_options)
+    t_smo = time.perf_counter() - t0
+    t0 = time.perf_counter()
+    if not smooth:
+        sol = Solution(a, b, c, prob.ka(a))
+    else:
+        try:
+            sol = newton(prob, a, b, c, **(newton_options or {}))
+            if prob.max_asum is not None and sol.c < 0.0:
+                # the constraint is inactive at the solution: solve without it
+                sol = newton(prob, sol.a, sol.b, 0.0, **(newton_options or {}))
+        except ConvergenceError:
+            # fall back to an accurate SMO solution (exact for quadratic smoothing)
+            smo_options["tol"] = 1e-6
+            a, b, c, more = smo_warm_start(prob, a, **smo_options)
+            smo_steps += more
+            sol = Solution(a, b, c, prob.ka(a), info={"newton": "failed"})
+    sol.smo_steps = smo_steps
+    sol.time_smo = t_smo
+    sol.time_newton = time.perf_counter() - t0
+    return sol

optimized_svm/validation.py

@@ -0,0 +1,76 @@
+"""Validation error measures ``E(f) = sum_v L_v(f_v)`` for the upper level problem (Sec. 9.1.4).
+
+All measures are differentiable so that the hyperparameter objective has a gradient.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+from numpy.typing import NDArray
+from scipy.special import gammaln
+
+from .smooth_max import Degree, smooth_max
+
+Array = NDArray[np.float64]
+
+
+class ValidationLoss:
+    """Returns the summed loss and its gradient w.r.t. the predictions."""
+
+    def __call__(self, f: Array, y: Array) -> tuple[float, Array]:
+        raise NotImplementedError
+
+
+@dataclass(frozen=True)
+class SmoothHingeValidation(ValidationLoss):
+    """``sum_v w_v m_delta(shift - y_v f_v)``; ``balanced`` weights both classes equally."""
+
+    shift: float = 1.0
+    smoothing: float = 0.5
+    degree: Degree = "6"
+    balanced: bool = False
+
+    def __call__(self, f: Array, y: Array) -> tuple[float, Array]:
+        m, dm, _ = smooth_max(self.smoothing, self.degree)
+        w = np.ones_like(y)
+        if self.balanced:
+            pos = y > 0
+            frac = pos.mean()
+            if 0 < frac < 1:
+                w = 0.5 / np.where(pos, frac, 1.0 - frac)
+        arg = self.shift - y * f
+        return float(w @ m(arg)), -w * y * dm(arg)
+
+
+@dataclass(frozen=True)
+class SmoothAbsoluteValidation(ValidationLoss):
+    """Smoothed absolute error ``sum_v m_delta(f_v - y_v) + m_delta(y_v - f_v)`` (Huber-like)."""
+
+    smoothing: float = 0.1
+    degree: Degree = "2"
+
+    def __call__(self, f: Array, y: Array) -> tuple[float, Array]:
+        m, dm, _ = smooth_max(self.smoothing, self.degree)
+        e = f - y
+        return float(m(e).sum() + m(-e).sum()), dm(e) - dm(-e)
+
+
+@dataclass(frozen=True)
+class SquaredValidation(ValidationLoss):
+    """``1/2 sum_v (f_v - y_v)^2``."""
+
+    def __call__(self, f: Array, y: Array) -> tuple[float, Array]:
+        e = f - y
+        return 0.5 * float(e @ e), e
+
+
+@dataclass(frozen=True)
+class PoissonValidation(ValidationLoss):
+    """Poisson deviance-type loss ``sum_v exp(f_v) - y_v f_v + log(y_v!)``."""
+
+    def __call__(self, f: Array, y: Array) -> tuple[float, Array]:
+        with np.errstate(over="ignore"):
+            ef = np.exp(f)
+        return float(ef.sum() - f @ y + gammaln(y + 1).sum()), ef - y

optimized_svm-0.1.0.dist-info/METADATA

@@ -0,0 +1,131 @@
+Metadata-Version: 2.5
+Name: optimized-svm
+Version: 0.1.0
+Summary: Support vector machines with hyperparameters optimized by exact gradients (implicit function theorem)
+Project-URL: Homepage, https://wotzlaff.github.io/svm-site/
+Project-URL: Documentation, https://wotzlaff.github.io/optimized-svm/
+Project-URL: Repository, https://github.com/wotzlaff/optimized-svm
+Author-email: Nico Strasdat <nstrasdat@gmail.com>
+License-Expression: BSD-3-Clause
+License-File: LICENSE
+Keywords: bilevel optimization,hyperparameter optimization,kernel methods,svm
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Mathematics
+Requires-Python: >=3.10
+Requires-Dist: numpy>=1.23
+Requires-Dist: scikit-learn>=1.6
+Requires-Dist: scipy>=1.10
+Requires-Dist: smo-solver>=0.1
+Description-Content-Type: text/markdown
+
+# optimized-svm
+
+**Support vector machines whose hyperparameters are tuned by exact gradients instead of grid search.**
+
+`optimized-svm` treats hyperparameter selection as a bilevel optimization problem: the upper level
+minimizes a (smoothed) cross-validation error, the lower level is the SVM training problem. Smoothing
+the loss makes the training solution differentiable in the hyperparameters, so the gradient of the
+CV error follows from the implicit function theorem and a quasi-Newton method (L-BFGS-B) can be used.
+This works for the regularization parameter **and** for per-feature kernel scalings (automatic
+relevance determination), where grid search is hopeless.
+
+The methods are developed in
+
+> N. Strasdat, *Duality, Derivative-Based Training Methods and Hyperparameter Optimization for
+> Support Vector Machines*, Dissertation, Technische Universität Dresden, 2023.
+> [urn:nbn:de:bsz:14-qucosa2-874731](https://nbn-resolving.org/urn:nbn:de:bsz:14-qucosa2-874731)
+
+Training uses the Rust SMO solver [`smo-solver`](https://github.com/wotzlaff/smo-solver) as a warm start
+followed by a semismooth Newton method on the smoothed primal problem.
+
+## Features
+
+- scikit-learn estimators: `OptimizedSVC`, `OptimizedSVR` (tuned) and `SmoothSVC`, `SmoothSVR`
+  (fixed hyperparameters); binary and one-vs-rest classification
+- losses: smoothed hinge, smoothed ε-insensitive, squared (LS-SVM), Poisson; smoothing of degree
+  2, 3 or 6; sample/class weights; optional bound on `Σ|α|` (generalized ν-SVM)
+- exact hypergradients via one adjoint linear system per fold (verified against finite differences)
+- ARD kernels (`blocks="features"`) with two-stage optimization, warm starts, time limits,
+  full optimization history
+- `decision_gradient()` for local feature sensitivities
+
+## Installation
+
+```bash
+pip install optimized-svm
+```
+
+During development (sibling checkout of `smo-solver`, Rust toolchain required):
+
+```bash
+git clone https://github.com/wotzlaff/smo-solver
+git clone https://github.com/wotzlaff/optimized-svm && cd optimized-svm
+uv sync --group dev
+uv run pytest
+```
+
+## Quick start
+
+```python
+from sklearn.datasets import make_friedman1
+from optimized_svm import OptimizedSVR
+
+x, y = make_friedman1(600, noise=0.5, random_state=0)
+model = OptimizedSVR(blocks="features").fit(x, y)   # one kernel scaling per feature
+
+model.lam_            # optimized regularization parameter
+model.scaling_opt_    # optimized scalings: features 5..9 (pure noise) are switched off
+model.history_        # every evaluation of the CV error
+model.predict(x[:5])
+```
+
+On a 1000-sample classification task (`examples/quickstart.py`) the optimizer needs 16 CV
+evaluations (≈10 s) versus 143 grid points (≈20 s) for `GridSearchCV` with `sklearn.svm.SVC`,
+with comparable test accuracy (0.844 vs. 0.856).
+
+## How it works
+
+| step | method | thesis |
+|---|---|---|
+| training, warm start | SMO on the dual with quadratic smoothing (`smo-solver`) | Ch. 7 |
+| training, refinement | semismooth Newton on `a + φ'(Ka/λ + b + sc) = 0` | Ch. 8.4 |
+| CV error gradient | implicit function theorem, adjoint system per fold | Ch. 9.2 |
+| hyperparameter search | L-BFGS-B over `(log λ, log s₁, …, log s_K)` | Ch. 9.1, 10.4 |
+
+## Reproducing the thesis
+
+`benchmarks/thesis_regression.py` repeats the protocol of Table 10.4.1 (LIBSVM regression data,
+30 random 80/20 splits, ε-SVR with cubic smoothing and LS-SVM):
+
+```bash
+python benchmarks/thesis_regression.py --datasets pyrim triazines mpg housing --splits 30
+```
+
+## Documentation
+
+Full documentation including the mathematical background: <https://wotzlaff.github.io/optimized-svm/>.
+Build locally with `uv run --group docs mkdocs serve`.
+
+## Citation
+
+If you use this software, please cite the dissertation:
+
+```bibtex
+@phdthesis{strasdat2023,
+  author = {Nico Strasdat},
+  title  = {Duality, Derivative-Based Training Methods and
+            Hyperparameter Optimization for Support Vector Machines},
+  school = {Technische Universit{\"a}t Dresden},
+  type   = {Dissertation},
+  year   = {2023},
+  url    = {https://nbn-resolving.org/urn:nbn:de:bsz:14-qucosa2-874731},
+  note   = {urn:nbn:de:bsz:14-qucosa2-874731}
+}
+```
+
+## License
+
+BSD-3-Clause. The 2022 version (Nim backend) is preserved in the git tag `legacy-2022`.

optimized_svm-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+optimized_svm/__init__.py,sha256=P7uqJ31jT2DxWYp1vjG7A3ikMIq7Gx5VeYpwi3I-mr4,1052
+optimized_svm/estimators.py,sha256=RrwaFgIV29pcYzDaMLeY3_q3WiIejj7bX619w0N_1V4,27355
+optimized_svm/hypergradient.py,sha256=Le2MNLtJz-q70SHW-KuAHuuKjArCYOkmIP9nQCGNq4U,5187
+optimized_svm/kernels.py,sha256=DmaZUZTwhrTew-OPQVBAPb-FDw6zgsqFOmZSUz4dpE8,2439
+optimized_svm/linalg.py,sha256=F324x9fuJic2EAEWgU3OXreeqMsdUbfX99zzSjN8AlA,734
+optimized_svm/losses.py,sha256=5MzjNkyq4X2zeI-iE1UFFd3tuKwiyPlgNiC1wi8XEfQ,5651
+optimized_svm/problem.py,sha256=plrjiZ4n4UfWUOGZ_7IPhIfJ8po8nVVXBhrShAeTbFM,3699
+optimized_svm/smooth_max.py,sha256=l3Kwt9Gj68l-WAYVECW7yFX4PE_ODyPuOZUR79e5hDg,2966
+optimized_svm/training.py,sha256=UJDhaldrKmVe8QjBsFs5wbV48YNBGgHermKfvoFqkoc,9141
+optimized_svm/validation.py,sha256=SILMorbg1s3cDEMm8Jkuw8bGXJeahs2w2BMm2ryTkD0,2306
+optimized_svm-0.1.0.dist-info/METADATA,sha256=wvknVL2hnhYEjPQXE_lz00nlhhYwu5Ybmwos7YM7YYw,5280
+optimized_svm-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+optimized_svm-0.1.0.dist-info/licenses/LICENSE,sha256=xg1gX4NDaA0vqOtx1QTqjFg-r-edA4M823f1r876NK0,1505
+optimized_svm-0.1.0.dist-info/RECORD,,

optimized_svm-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

optimized_svm-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2022-2026, Nico Strasdat
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.