regspline 25.5.2__tar.gz

regspline-25.5.2/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2023, mvds314
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

regspline-25.5.2/PKG-INFO

@@ -0,0 +1,107 @@
+Metadata-Version: 2.4
+Name: regspline
+Version: 25.5.2
+Summary: Regression spline
+Keywords: statistics,regression,splines
+Author: Martin van der Schans
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: statsmodels
+Requires-Dist: pyqreg ; extra == "fastqr"
+Requires-Dist: cvxopt!=1.3.0 ; extra == "lasso"
+Requires-Dist: scikit-learn ; extra == "svr"
+Project-URL: repository, https://github.com/mvds314/regspline
+Provides-Extra: fastqr
+Provides-Extra: lasso
+Provides-Extra: svr
+
+# Regression splines
+
+This module includes two spline implementations splines suitable for regression: linear splines using a hinge function basis, and natural cubic splines.
+
+```python
+import numpy as np
+import matplotlib.pyplot as plt
+from regspline import LinearSpline
+plt.close('all')
+
+knots = [0,1,2]
+coeffs = [1,2,3]
+s = LinearSpline(knots, coeffs)
+y=s(np.linspace(0,1))
+
+x=np.linspace(0,np.pi)
+y=np.sin(x)
+xobs = np.repeat(x,50)
+yobs = np.repeat(y,50) + 0.01*np.random.randn(*xobs.shape)
+
+s, res = LinearSpline.from_data(xobs, yobs,
+                                knots=np.linspace(0,np.pi,30),
+                                method='OLS',
+                                return_estim_result=True,
+                                prune=True)
+
+plt.plot(x,y)
+plt.plot(x,s(x))
+```
+
+Several regression types are supported to extract the splines from data, including OLS, LASSO, and quantile regression. See the example files.
+
+## Installation
+
+You can install this library directly from github:
+
+```bash
+pip install git+https://github.com/mvds314/regspline.git
+```
+
+There are two optional dependencies: `scikit-learn`, and `cvxopt`. They are only required to estimate splines on data with, respectively, support vector regressions, and LASSO.
+
+## Background
+
+The module contains two splines:
+
+- A linear spline represented by Hinge functions: $h_i(x) = \max(x-k_i,0)$, where $k_i$ are the knots.
+- A natural cubic spline.
+
+The splines chosen:
+
+- have coefficents that have a one-to-one correspondence with the knots.
+- have the ability that knots can be removed, e.g., when the corresponding coefficient is small or insignificant, without changing the basis functions corresponding to other knots.
+- have the ability to represent functions with sparse basis.
+
+One way to interpret, e.g., the linear spline in the hings basis is as follows. $h_1(x)$ sets an initial slope from the first knot onwards. Then next basis function $h_2(x)$ can adjust the slope at the knot $k_2$, if no adjustment is required, its coefficient is insignificant and the knot can be removed from the spline without any impact on the other basis functions.
+
+## Related projects
+
+Some projects with related methods:
+
+- [basis-expansions](https://github.com/madrury/basis-expansions)
+- [py-earth](https://github.com/scikit-learn-contrib/py-earth)
+- Quantile regression using decision trees [scikit-garden](https://scikit-garden.github.io/)
+
+The module differs from these implementations as it implements the splines as functions, and they are not integrated within an estimation framework.
+
+## Development
+
+For development purposes, clone the repo:
+
+```bash
+git clone https://github.com/mvds314/regspline.git
+```
+
+Then navigate to the folder containing `setup.py` and run
+
+```bash
+pip install -e .
+```
+
+to install the package in edit mode.
+
+Run unittests with `pytest`.
+
+Install the optional dependencies to test all functionality.
+

regspline-25.5.2/README.md

@@ -0,0 +1,86 @@
+# Regression splines
+
+This module includes two spline implementations splines suitable for regression: linear splines using a hinge function basis, and natural cubic splines.
+
+```python
+import numpy as np
+import matplotlib.pyplot as plt
+from regspline import LinearSpline
+plt.close('all')
+
+knots = [0,1,2]
+coeffs = [1,2,3]
+s = LinearSpline(knots, coeffs)
+y=s(np.linspace(0,1))
+
+x=np.linspace(0,np.pi)
+y=np.sin(x)
+xobs = np.repeat(x,50)
+yobs = np.repeat(y,50) + 0.01*np.random.randn(*xobs.shape)
+
+s, res = LinearSpline.from_data(xobs, yobs,
+                                knots=np.linspace(0,np.pi,30),
+                                method='OLS',
+                                return_estim_result=True,
+                                prune=True)
+
+plt.plot(x,y)
+plt.plot(x,s(x))
+```
+
+Several regression types are supported to extract the splines from data, including OLS, LASSO, and quantile regression. See the example files.
+
+## Installation
+
+You can install this library directly from github:
+
+```bash
+pip install git+https://github.com/mvds314/regspline.git
+```
+
+There are two optional dependencies: `scikit-learn`, and `cvxopt`. They are only required to estimate splines on data with, respectively, support vector regressions, and LASSO.
+
+## Background
+
+The module contains two splines:
+
+- A linear spline represented by Hinge functions: $h_i(x) = \max(x-k_i,0)$, where $k_i$ are the knots.
+- A natural cubic spline.
+
+The splines chosen:
+
+- have coefficents that have a one-to-one correspondence with the knots.
+- have the ability that knots can be removed, e.g., when the corresponding coefficient is small or insignificant, without changing the basis functions corresponding to other knots.
+- have the ability to represent functions with sparse basis.
+
+One way to interpret, e.g., the linear spline in the hings basis is as follows. $h_1(x)$ sets an initial slope from the first knot onwards. Then next basis function $h_2(x)$ can adjust the slope at the knot $k_2$, if no adjustment is required, its coefficient is insignificant and the knot can be removed from the spline without any impact on the other basis functions.
+
+## Related projects
+
+Some projects with related methods:
+
+- [basis-expansions](https://github.com/madrury/basis-expansions)
+- [py-earth](https://github.com/scikit-learn-contrib/py-earth)
+- Quantile regression using decision trees [scikit-garden](https://scikit-garden.github.io/)
+
+The module differs from these implementations as it implements the splines as functions, and they are not integrated within an estimation framework.
+
+## Development
+
+For development purposes, clone the repo:
+
+```bash
+git clone https://github.com/mvds314/regspline.git
+```
+
+Then navigate to the folder containing `setup.py` and run
+
+```bash
+pip install -e .
+```
+
+to install the package in edit mode.
+
+Run unittests with `pytest`.
+
+Install the optional dependencies to test all functionality.

regspline-25.5.2/pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+name = "regspline"
+description = "Regression spline"
+version = "25.5.2"
+authors = [{ name = "Martin van der Schans" }]
+readme = "README.md"
+keywords = ["statistics", "regression", "splines"]
+requires-python = ">=3.9"
+dependencies = ["numpy", "pandas", "statsmodels"]
+license = { text = "BSD-3-Clause" }
+
+[project.urls]
+repository = "https://github.com/mvds314/regspline"
+
+[project.optional-dependencies]
+SVR = ["scikit-learn"]
+LASSO = ["cvxopt!=1.3.0"] # Note: 1.3.0 has a domain error bug:
+FASTQR = ["pyqreg"]
+
+[build-system]
+requires = ["flit_core >=3.2,<4"]
+build-backend = "flit_core.buildapi"
+
+[tool.ruff]
+line-length = 99
+
+[tool.ruff.lint]
+select = ["E", "F", "W"] # Pycodestyle, pyflakes, bugbear, and isort
+ignore = [
+  # Ignore common conflicts with Blackjj
+  "E203",
+  "E501",
+  "E731", # Ignore lamda expression warning
+  "E402", # Ignore module level import not at top of file
+]
+
+[tool.pytest.ini_options]
+markers = [
+  "tofix: marks tests as to be fixed (deselect with '-m \"not tofix\"')",
+]

regspline-25.5.2/regspline/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from .linear_spline import LinearSpline, HingeBasisFunction
+from .natural_cubic_spline import NaturalCubicSpline, NaturalCubicSplineBasisFunction
+from .util import Timer

regspline-25.5.2/regspline/base.py

@@ -0,0 +1,451 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import pandas as pd
+import statsmodels.api as sm
+from abc import ABC, abstractmethod
+
+try:
+    from sklearn.svm import LinearSVR, NuSVR
+    from sklearn.linear_model import QuantileRegressor
+except ImportError:
+    _has_sklearn = False
+else:
+    _has_sklearn = True
+
+try:
+    import cvxopt
+
+    if cvxopt.__version__ == "1.3.0":
+        raise ImportError("SVR doesn not work well with cvxopt 1.3.0")
+        # Note there is a domain error bug in 1.3.0: https://github.com/cvxopt/cvxopt/issues/202
+except ImportError:
+    _has_cvxopt = False
+else:
+    _has_cvxopt = True
+
+try:
+    from pyqreg import QuantReg as qrQuantReg
+except ImportError:
+    _has_pyqreg = False
+else:
+    _has_pyqreg = True
+
+
+from .util import type_wrapper, PandasWrapper
+
+
+class BasisFuncInterface(ABC):
+    r"""
+    Abstraction of basis functions
+    """
+
+    def __init__(self, xmin=-np.inf, xmax=np.inf, val=0):
+        assert np.isscalar(xmin) and np.isreal(xmin)
+        assert np.isscalar(xmax) and np.isreal(xmax)
+        assert np.isscalar(val) and np.isreal(val)
+        assert xmin < xmax
+        self.xmax = xmax
+        self.xmin = xmin
+        self.val = val
+
+    @abstractmethod
+    def _apply(self, x):
+        pass
+
+    @type_wrapper(xloc=1)
+    def __call__(self, x):
+        y = self._apply(x)
+        if np.isfinite(self.xmax):
+            y = np.where(x <= self.xmax, y, self.val)
+        if np.isfinite(self.xmin):
+            y = np.where(x >= self.xmin, y, self.val)
+        return y
+
+
+class KnotsInterface(ABC):
+    """
+    Abstraction for a class containing knots
+    """
+
+    def __init__(self, knots):
+        self._knots = None
+        self.knots = knots
+
+    @property
+    def knots(self):
+        return self._knots
+
+    @knots.setter
+    def knots(self, value):
+        if value is not None:
+            value = np.asanyarray(value)
+            assert len(value) >= 2, "Must specify at least 2 knots"
+            assert np.all(value[:-1] < value[1:]), "Knots are assumed to be sorted and unique"
+        self._knots = value
+
+    @property
+    def n_knots(self):
+        return None if self.knots is None else len(self.knots)
+
+    def __len__(self):
+        return self.n_knots
+
+
+class RegressionSplineBase(KnotsInterface, ABC):
+    r"""
+    Regression linear spline represented by basis functions
+    :math:`b_1\ldots b_M`:, and with knots :math:`k_1<k_2\ldots <k{N}`.
+    The result is a function of the form
+    .. math::
+        s(x)=c_0+\sum\limits_{i=1}^{M} c_i b_i(x).
+
+    The relation between the basis functions and the knots differs per
+    spline. Nevertheless, in all cases, :math:`c_0` is a constant
+    and the other :math:`c_i` are coefficients.
+
+    The spline intended to be defined for :math:`k_1\leq x \leq k_N`. Outside
+    of this range, the function can be extrapolated by either:
+
+    * simply evaluating the basis functions;
+    * with the value :math:`s(k_0)=c_0` left of :math:`k_0`, and with the value :math:`s(k_N)` right of :math:`k_N`;
+    * with NaN.
+    """
+
+    def __init__(self, knots, coeffs, extrapolation_method="nan"):
+        self._knots = None
+        self._coeffs = None
+        super().__init__(knots)
+        self.coeffs = coeffs
+        self.extrapolation_method = extrapolation_method
+
+    def __hash__(self):
+        return hash(
+            (
+                self.__class__.__name__,
+                tuple(self.knots),
+                tuple(self.coeffs),
+            )
+        )
+
+    @property
+    def extrapolation_method(self):
+        return self._extrapolation_method
+
+    @extrapolation_method.setter
+    def extrapolation_method(self, value):
+        assert value in ["nan", "const", "basis", "linear"]
+        self._extrapolation_method = value
+        if hasattr(self, "_bi_cache"):
+            del self._bi_cache
+
+    @KnotsInterface.knots.setter
+    def knots(self, value):
+        if value is not None and self.coeffs is not None:
+            value = np.asanyarray(value)
+            assert self._validate_knots_coeffs(value, self.coeffs)
+        super(RegressionSplineBase, RegressionSplineBase).knots.__set__(self, value)
+        if hasattr(self, "_bi_cache"):
+            del self._bi_cache
+
+    @property
+    def coeffs(self):
+        return self._coeffs
+
+    @coeffs.setter
+    def coeffs(self, value):
+        if value is not None:
+            value = np.asanyarray(value)
+            assert self._validate_knots_coeffs(self.knots, value)
+        self._coeffs = value
+
+    @abstractmethod
+    def _validate_knots_coeffs(self, knots, coeffs):
+        return False
+
+    @property
+    @abstractmethod
+    def _bi(self):
+        pass
+
+    @property
+    def _ci(self):
+        assert self.knots is not None and self.coeffs is not None
+        return self.coeffs[1:] if self.has_const else self.coeffs
+
+    @property
+    def const(self):
+        if self.coeffs is None:
+            return None
+        elif self.knots is None:
+            return 0
+        else:
+            return self.coeffs[0] if self.has_const else 0
+
+    @property
+    @abstractmethod
+    def has_const(self):
+        return True
+
+    @const.setter
+    def const(self, value):
+        assert self.knots is not None, "Cannot set constant if knots are not specified"
+        assert self.coeffs is not None, "Cannot set constant if coeffs are not specified"
+        assert np.isscalar(value) and np.isreal(value)
+        if self.has_const:
+            self.coeffs[0] = value
+        else:
+            self.coeffs = np.insert(self.coeffs, 0, value, axis=0)
+
+    @property
+    def n_coeffs(self):
+        return None if self.coeffs is None else len(self.coeffs)
+
+    @type_wrapper(xloc=1)
+    def __call__(self, x):
+        if self.knots is None or self.coeffs is None:
+            return np.nan * x
+        if self.extrapolation_method == "const":
+            x = np.clip(x, self.knots[0], self.knots[-1])
+        # Note: bi takes care of other extrapolation methods
+        return self.const + self._ci.dot([bi(x) for bi in self._bi])
+
+    def eval_basis(self, x, include_constant=False):
+        """
+        Evaluates the basis functions of the spline on x, usefull for
+        linear regression pruposes.
+        """
+        wrapper = PandasWrapper(x)
+        x = np.asanyarray(x, dtype=np.float64)
+        y = [np.ones(x.shape)] if include_constant else []
+        y += [bi(x) for bi in self._bi]
+        y = np.asanyarray(y).T
+        y = y.tolist() if len(y.shape) == 0 else wrapper.wrap(y)
+        if isinstance(y, pd.DataFrame):
+            columns = ["const"] if include_constant else []
+            columns += [f"b{i}" for i, _ in enumerate(self._bi)]
+            y.columns = columns
+        return y
+
+    @abstractmethod
+    def prune_knots(self, method="isclose", tol=1e-6, **kwargs):
+        """
+        Prunes knots based on criterion
+
+        Parameters
+        ----------
+        method : string, optional
+            Method used to determine which knots to prune. The default is 'isclose'.
+        tol : float, optional
+            Tolerance, passed to method. The default is 1e-6.
+        kwargs : dictionary
+            Other keyword arguments passed to method.
+        """
+        raise NotImplementedError(
+            "Which knots can be removed depends on coeffs and knots are related"
+        )
+
+    @classmethod
+    def from_data(
+        cls,
+        x,
+        y,
+        knots=None,
+        method="OLS",
+        add_constant=True,
+        prune=False,
+        return_estim_result=False,
+        backend=None,
+        **kwargs,
+    ):
+        """
+        Estimates a spline from data
+
+        Parameters
+        ----------
+        x : numpy array
+            observations of the independent variable
+        y : numpy array
+            observations of the dependent variable
+        knots : numpy array, optional
+            knots, defaults to 10 knots between min and max x-value
+        method : string, optional
+            Estimation  method. The default is "OLS".
+        add_constant : boolean, optional
+            Add a constant to the spline. The default is True.
+        prune : boolean, optional
+            Prunes insignificant knots after estimation, and estimates again. The default is False.
+        return_estim_result : boolean, optional
+            If True, estimation results are returned. The default is False.
+        backend : None or string
+            Force a certain backend, can be statsmodels, sklearn, or pyqreg. Defaults to using statsmodels where possible.
+
+        Notable kwargs
+        ------
+        Dictionary with additional kwargs passed to estimation (statsmodels fit methods).
+        q : float
+            quantile used for quantile regression, defaults to the median (0.5)
+        C : float
+            Regularization parameter for SVR. The strength of the regularization is inversely proportional to C. Must be strictly positive.
+        epsilon : float
+            Epsilon parameter in the epsilon-insensitive loss function when using SVR. Note that the value of this parameter depends on the scale of the target variable y.
+        nu : float
+            Used in NuSVR. An upper bound on the fraction of training errors and a lower bound of the fraction of support vectors. Should be in the interval (0, 1]. By default 0.5 will be taken.
+
+
+        Returns
+        -------
+        Spline object
+        """
+        # Initialize
+        x = np.asanyarray(x)
+        y = np.asanyarray(y)
+        if knots is None:
+            knots = np.linspace(np.min(x), np.max(x), num=10)
+        elif isinstance(knots, int):
+            knots = np.linspace(np.min(x), np.max(x), num=knots)
+        else:
+            knots = np.asanyarray(knots)
+        spline = cls(knots, None, extrapolation_method=kwargs.pop("extrapolation_method", "nan"))
+        # Estimate
+        if method == "OLS":
+            assert backend is None or backend == "statsmodels", "sklearn backend not implemented"
+            smkwargs = dict(
+                exog=spline.eval_basis(x, include_constant=add_constant),
+                hasconst=True,
+                missing=kwargs.pop("missing", "none"),
+            )
+            model = sm.OLS(y, **smkwargs)
+            result = model.fit(**kwargs)
+            spline.coeffs = result.params
+            insignificant = np.abs(result.tvalues) < 1.96
+            if prune and np.any(insignificant):
+                add_constant = add_constant and not insignificant[0]
+                spline.prune_knots(method="coeffs", coeffs_to_prune=insignificant)
+                return cls.from_data(
+                    x,
+                    y,
+                    knots=spline.knots,
+                    method=method,
+                    add_constant=add_constant,
+                    prune=False,
+                    return_estim_result=return_estim_result,
+                    **kwargs,
+                )
+        elif method == "LASSO":
+            assert backend is None or backend == "statsmodels", "sklearn backend not implemented"
+            assert _has_cvxopt, "Mising optional dependency cvxopt"
+            smkwargs = dict(
+                exog=spline.eval_basis(x, include_constant=add_constant),
+                hasconst=True,
+                missing=kwargs.pop("missing", "none"),
+            )
+            model = sm.OLS(y, **smkwargs)
+            result = model.fit_regularized(method="sqrt_lasso", **kwargs)
+            spline.coeffs = result.params
+            if prune:
+                spline.prune_knots()
+        elif method == "QuantileRegression":
+            if backend is None or backend == "statsmodels":
+                smkwargs = dict(
+                    exog=spline.eval_basis(x, include_constant=add_constant),
+                    hasconst=True,
+                    missing=kwargs.pop("missing", "none"),
+                )
+                model = sm.QuantReg(y, **smkwargs)
+                kwargs.setdefault("q", 0.5)
+                result = model.fit(**kwargs)
+                spline.coeffs = result.params
+                insignificant = np.abs(result.tvalues) < 1.96
+                if prune and np.any(insignificant):
+                    add_constant = add_constant and not insignificant[0]
+                    spline.prune_knots(method="coeffs", coeffs_to_prune=insignificant)
+                    return cls.from_data(
+                        x,
+                        y,
+                        knots=spline.knots,
+                        method=method,
+                        add_constant=add_constant,
+                        prune=False,
+                        return_estim_result=return_estim_result,
+                        **kwargs,
+                    )
+            elif backend == "sklearn":
+                assert _has_sklearn, "Mising optional dependency scikit learn"
+                kwargs.setdefault("fit_intercept", add_constant)
+                kwargs.setdefault("solver", "highs")
+                kwargs.setdefault("quantile", kwargs.pop("q", 0.5))
+                kwargs.setdefault("alpha", 0)
+                model = QuantileRegressor(**kwargs)
+                result = model.fit(spline.eval_basis(x, include_constant=False), y)
+                spline.coeffs = np.append(result.intercept_, result.coef_)
+                assert np.allclose(spline(x), result.predict(spline.eval_basis(x))), (
+                    "Something is wrong, this should give the same result"
+                )
+                if prune:
+                    spline.prune_knots()
+            elif backend == "pyqreg":
+                assert _has_pyqreg, "Mising optional dependency pyqreg"
+                exog = spline.eval_basis(x, include_constant=add_constant)
+                model = qrQuantReg(y, exog)
+                q = kwargs.pop("q", 0.5)
+                result = model.fit(q, **kwargs)
+                spline.coeffs = result.params
+                insignificant = np.abs(result.tvalues) < 1.96
+                if prune and np.any(insignificant):
+                    add_constant = add_constant and not insignificant[0]
+                    spline.prune_knots(method="coeffs", coeffs_to_prune=insignificant)
+                    return cls.from_data(
+                        x,
+                        y,
+                        knots=spline.knots,
+                        method=method,
+                        add_constant=add_constant,
+                        prune=False,
+                        return_estim_result=return_estim_result,
+                        **kwargs,
+                    )
+            else:
+                raise ValueError("Invalid backend")
+        elif method == "SVR":
+            assert backend is None or backend == "sklearn", "statsmodels backend not implemented"
+            assert _has_sklearn, "Mising optional dependency scikit learn"
+            kwargs.setdefault("random_state", 0)
+            kwargs.setdefault("tol", 1e-5)
+            kwargs.setdefault("loss", "epsilon_insensitive")
+            kwargs.setdefault("C", 1)
+            kwargs.setdefault("max_iter", int(1e7))
+            kwargs.setdefault("fit_intercept", add_constant)
+            model = LinearSVR(**kwargs)
+            result = model.fit(spline.eval_basis(x, include_constant=False), y)
+            spline.coeffs = np.append(result.intercept_, result.coef_)
+            assert np.allclose(spline(x), result.predict(spline.eval_basis(x))), (
+                "Something is wrong, this should give the same result"
+            )
+            if prune:
+                spline.prune_knots()
+        elif method == "NuSVR":
+            assert backend is None or backend == "sklearn", "statsmodels backend not implemented"
+            assert _has_sklearn, "Mising optional dependency scikit learn"
+            assert add_constant, "A constant is always fitted for NuSVR"
+            kwargs.setdefault("tol", 1e-5)
+            kwargs.setdefault("nu", 0.5)
+            kwargs.setdefault("C", 1)
+            kwargs.setdefault("kernel", "linear")
+            kwargs.setdefault("max_iter", int(1e7))
+            model = NuSVR(**kwargs)
+            result = model.fit(spline.eval_basis(x, include_constant=False), y)
+            spline.coeffs = np.append(result.intercept_, result.coef_)
+            assert np.allclose(spline(x), result.predict(spline.eval_basis(x))), (
+                "Something is wrong, this should give the same result"
+            )
+            if prune:
+                spline.prune_knots()
+        else:
+            raise ValueError(f"Unknown method: {method}")
+        # Return
+        if return_estim_result:
+            return spline, result
+        else:
+            return spline

regspline-25.5.2/regspline/linear_spline.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import numpy as np
+from .base import BasisFuncInterface, RegressionSplineBase
+
+
+class HingeBasisFunction(BasisFuncInterface):
+    r"""
+    Defines the Hinge basis function
+
+    .. math::
+        \text{max}(x-\text{ref}, 0)
+
+    Furthermore, input values outside the range xmin to xmax are mapped to val.
+
+    """
+
+    def __init__(self, ref, xmin=-np.inf, xmax=np.inf, val=0):
+        super().__init__(xmin=xmin, xmax=xmax, val=val)
+        assert np.isscalar(ref) and np.isreal(ref) and np.isfinite(ref)
+        if np.isfinite(xmin):
+            assert xmin <= ref
+        if np.isfinite(xmax):
+            assert xmax > ref
+        self.ref = ref
+
+    def _apply(self, x):
+        return np.fmax(x - self.ref, 0)
+
+
+class LinearSpline(RegressionSplineBase):
+    r"""
+    Linear regression spline represented by Hinge basis functions :math:`h_i`:
+
+    .. math:: h_i(x) = \max(x-k_i,0),
+
+    for :math:`i=1\ldots N-1`, and where :math:`k_1<k_2<\ldots <k_{N}`
+    are the knots. The result is a piecewise linear function :math:`s(x)`
+    between the knots of the form:
+
+    .. math::
+        s(x)=c_0+\sum\limits_{i=1}^{N-1} c_i h_i(x),
+
+    where :math:`c_0` is a constant and the other :math:`c_i` are coefficients.
+    Note that, by construction, :math:`s(k_1)=c_0`.
+
+    The spline intended to be defined for :math:`k_1\leq x \leq k_N`. Outside
+    of this range, the function can be extrapolated by either:
+
+    * simply evaluating the basis functions;
+    * with the value :math:`s(k_0)=c_0` left of :math:`k_0`, and with the value :math:`s(k_N)` right of :math:`k_N`;
+    * with NaN.
+
+    One to interpret the spline in this basis is:
+
+    * :math:`h_1(x)` equals zero at :math:`k_0`,sets slope in the interval :math:`k_1\leq x\leq k_1`.
+    * :math:`h_2(x)` compensates and sets the slope in the interval :math:`k_2 \leq x\leq k_3`.
+    * etc.
+
+    Contrary to other basis functions,, such as spikes, one can represent a linear
+    function with only one basis function. Each kink adds requires another basis
+    function. Consequently, in regression, coefficients of basis functions become
+    insignificant, unless there is a kink at the knot. This keeps the regression
+    sparse.
+
+    Note that this approach is similar to the MARS splines, see the py-earth package.
+    This implementation is more lean, and merely implements the spline (in 1 dimension)
+    without algorithms, or a modelling framework around it.
+
+    Params
+    --
+        x           - float, 1d numpy array or pd.Series object
+        coeffs      - pd.Series with coeffs and knots as index
+                        - coeffs created by linear regression on :math:`f_i(x)` and the observed y's
+    """
+
+    def _validate_knots_coeffs(self, knots, coeffs):
+        if knots is not None:
+            knots = np.asanyarray(knots)
+            assert len(knots) >= 2, "Must specify at least 2 knots"
+        if coeffs is not None:
+            coeffs = np.asanyarray(coeffs)
+        if coeffs is not None and knots is not None:
+            assert len(knots) == len(coeffs) or len(knots) - 1 == len(coeffs)
+        return True
+
+    @property
+    def _bi(self):
+        assert self.knots is not None
+        if not hasattr(self, "_bi_cache"):
+            if self.extrapolation_method == "nan":
+                kwargs = dict(xmin=self.knots[0], xmax=self.knots[-1], val=np.nan)
+            else:
+                kwargs = {}
+            if self.extrapolation_method == "linear":
+                self._bi_cache = [lambda x, ref=self.knots[0]: x - ref] + [
+                    HingeBasisFunction(k, **kwargs) for k in self.knots[1:-1]
+                ]
+            else:
+                self._bi_cache = [HingeBasisFunction(k, **kwargs) for k in self.knots[:-1]]
+        return self._bi_cache
+
+    @property
+    def has_const(self):
+        return self.n_knots == self.n_coeffs
+
+    def prune_knots(self, method="isclose", tol=1e-6, coeffs_to_prune=None, **kwargs):
+        """
+        Prunes knots based on criterion
+
+        Parameters
+        ----------
+        method : string, optional
+            Method used to determine which knots to prune. The default is 'isclose'.
+        tol : float, optional
+            Tolerance, passed to method. The default is 1e-6.
+        kwargs : dictionary
+            Other keyword arguments passed to method.
+        """
+        # Initialize
+        coeffs = np.copy(self.coeffs)
+        knots = np.copy(self.knots)
+        # Determine to prune
+        if method == "isclose":
+            kwargs.setdefault("atol", tol)
+            to_prune = np.isclose(coeffs, 0, **kwargs)
+        elif method == "coeffs":
+            assert len(coeffs_to_prune) == self.n_coeffs
+            to_prune = np.asanyarray(coeffs_to_prune)
+        else:
+            raise ValueError(f"Method {method} invalid")
+        # Prune
+        if len(to_prune) > 0:
+            try:
+                # Note first can correspond to const, last knot has no coeff
+                self.coeffs = None
+                self.knots = None
+                self.coeffs = coeffs[~to_prune]
+                to_prune = np.append(
+                    to_prune[1:] if len(knots) == len(coeffs) else to_prune, False
+                )
+                self.knots = knots[~to_prune]
+            except:
+                # Cleanup, don't leave the spline in an invalid state
+                self.knots = None
+                self.coeffs = None
+                self.knots = knots
+                self.coeffs = coeffs
+                raise

regspline-25.5.2/regspline/natural_cubic_spline.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import numpy as np
+
+from abc import ABC
+from .base import BasisFuncInterface, KnotsInterface, RegressionSplineBase
+
+
+class _NaturalCubicSplineBasisFuncInterface(KnotsInterface, BasisFuncInterface, ABC):
+    r"""
+    Abstraction for basis functions relative to knot :math:`k_i`, where
+    :math:`k=1\ldots N`
+    """
+
+    def __init__(self, i, knots, xmin=-np.inf, xmax=np.inf, val=0):
+        KnotsInterface.__init__(self, knots)
+        BasisFuncInterface.__init__(self, xmin=xmin, xmax=xmax, val=val)
+        assert 1 <= i <= self.n_knots - 1, (
+            f"Cannot initialize for {i}th knot, only possible up to {self.n_knots - 1}th knot"
+        )
+        self._i = i
+        if np.isfinite(xmin):
+            assert xmin <= self.knots[0]
+        if np.isfinite(xmax):
+            assert xmax >= self.knots[-1]
+
+    @property
+    def ref(self):
+        return self.knots[self._i - 1]
+
+
+class di(_NaturalCubicSplineBasisFuncInterface):
+    r"""
+    Defines helper functions for the Natural spline basis
+
+    .. math::
+        d_i(x) = \frac{\max(x-k_i,0)^3-\max(x-k_N,0)^3}{k_N-k_i}
+               = \frac{ h(x,k_i)^3-h(x,k_n)^3}{k_N-k_i}\\,
+
+    for :math:`i=1\ldots N-1`. Here :math:`k_1<k_2<\ldots <k_{N}`
+    are the knots, and the :math:`h(x,k_i)=\max(x-k_i,0)` are Hinge functions.
+
+    See [1] Chapter 5 for further details.
+
+    References
+    ----------
+    [1] Hastie, Tibshirani, Friedman (2009) - The elements of statistical learning
+
+    """
+
+    def _apply(self, x):
+        return (np.fmax(x - self.ref, 0) ** 3 - np.fmax(x - self.knots[-1], 0) ** 3) / (
+            self.knots[-1] - self.ref
+        )
+
+
+class NaturalCubicSplineBasisFunction(_NaturalCubicSplineBasisFuncInterface):
+    r"""
+    Defines the natural cubic spline basis function
+
+    .. math::
+        N_1(x) = x-k_1,
+
+    .. math::
+        N_i(x) = d_{i-1}(x)-d_{N-1}(x)
+
+    for :math:`i=2\ldots N-1`. Here :math:`k_1<k_2<\ldots <k_{N}`
+    are the knots. In total, we have :math:`N-1` basis functions, plus
+    a constant.
+
+    See [1] Chapter 5.3 for further details. Note we have changed notation
+    compared to the referenc. First, the index has shifted. We left out
+    the constant and handle it manually. Sometimes we can to leave it out
+    of regression to prevent misspecification. Second, The first basis
+    is relative to the first knot, this keeps the splines more properly
+    scaled.
+
+    References
+    ----------
+    [1] Hastie, Tibshirani, Friedman (2009) - The elements of statistical learning
+
+    """
+
+    @property
+    def ref(self):
+        # Note: first and second basis function are both relative to first knot
+        return self.knots[self._i - 1] if self._i == 1 else self.knots[self._i - 2]
+
+    @property
+    def _dim1(self):
+        if not hasattr(self, "_dim1_cache"):
+            assert self._i > 1 and self._i <= self.n_knots - 1
+            self._dim1_cache = di(
+                self._i - 1, self.knots, xmin=self.xmin, xmax=self.xmax, val=self.val
+            )
+        return self._dim1_cache
+
+    @property
+    def _dNm1(self):
+        if not hasattr(self, "_dNm1_cache"):
+            self._dNm1_cache = di(
+                self.n_knots - 1,
+                self.knots,
+                xmin=self.xmin,
+                xmax=self.xmax,
+                val=self.val,
+            )
+        return self._dNm1_cache
+
+    def _apply(self, x):
+        if self._i == 1:
+            return x - self.ref
+        else:
+            return self._dim1(x) - self._dNm1(x)
+
+
+class NaturalCubicSpline(RegressionSplineBase):
+    r"""
+    Natural cubic spline represented by basis functions :math:`N_i`:
+
+    .. math::
+        N_1(x) = x-k_1,
+
+    .. math::
+        N_i(x) = d_{i-1}(x)-d_{N-1}(x)
+
+    for :math:`i=2\ldots N-1`. Here :math:`k_1<k_2<\ldots <k_{N}`
+    are the knots. In total, we have :math:`N-1` basis functions, plus
+    a constant.
+
+    The result is a cubic spline linear function :math:`s(x)`
+    between the knots of the form:
+
+    .. math::
+        s(x)=c_0+\sum\limits_{i=1}^{N-1} c_i h_i(x),
+
+    where :math:`c_0` is a constant and the other :math:`c_i` are coefficients.
+    Note that, by construction, :math:`s(k_1)=c_0`. Also, by construction,
+    the spline is linear in the range outside of the knots.
+
+    The spline intended to be defined for :math:`k_1\leq x \leq k_N`. Outside
+    of this range, the function can be extrapolated by either:
+
+    * simply evaluating the basis functions, in which case it becomes linear;
+    * with the value :math:`s(k_0)=c_0` left of :math:`k_0`, and with the value :math:`s(k_N)` right of :math:`k_N`;
+    * with NaN.
+
+    Some notes on the interpretation of the coefficients:
+
+    * First coefficient represent the constant, if there is one.
+    * Next one is the coefficient of a linear function which is zero at the first knot
+    * Next ones are the spline basis functions for knots :math:`k_1` to :math:`k_{N-2}`
+    * Interval in between the last two knots is 'cleanup', there is no coefficient.
+    """
+
+    def _validate_knots_coeffs(self, knots, coeffs):
+        if knots is not None:
+            knots = np.asanyarray(knots)
+            assert len(knots) >= 3, "Must specify at least 3 knots"
+        if coeffs is not None:
+            coeffs = np.asanyarray(coeffs)
+        if coeffs is not None and knots is not None:
+            assert len(knots) == len(coeffs) or len(knots) - 1 == len(coeffs)
+        return True
+
+    @property
+    def _bi(self):
+        assert self.knots is not None
+        if not hasattr(self, "_bi_cache"):
+            if self.extrapolation_method == "nan":
+                kwargs = dict(xmin=self.knots[0], xmax=self.knots[-1], val=np.nan)
+            else:
+                kwargs = {}
+            self._bi_cache = [
+                NaturalCubicSplineBasisFunction(k, self.knots, **kwargs)
+                for k in range(1, self.n_knots)
+            ]
+        return self._bi_cache
+
+    @property
+    def has_const(self):
+        return self.n_knots == self.n_coeffs
+
+    def prune_knots(self, method="isclose", tol=1e-6, coeffs_to_prune=None, **kwargs):
+        """
+        Prunes knots based on criterion
+
+        Parameters
+        ----------
+        method : string, optional
+            Method used to determine which knots to prune. The default is 'isclose'.
+        tol : float, optional
+            Tolerance, passed to method. The default is 1e-6.
+        kwargs : dictionary
+            Other keyword arguments passed to method.
+        """
+        # Initialize
+        coeffs = np.copy(self.coeffs)
+        knots = np.copy(self.knots)
+        # Determine to prune
+        if method == "isclose":
+            kwargs.setdefault("atol", tol)
+            to_prune = np.isclose(coeffs, 0, **kwargs)
+        elif method == "coeffs":
+            assert len(coeffs_to_prune) == self.n_coeffs
+            to_prune = np.asanyarray(coeffs_to_prune)
+        else:
+            raise ValueError(f"Method {method} invalid")
+        # Prune
+        if len(to_prune) > 0:
+            try:
+                # Note first coeff can correspond to const, last knot has no coeff
+                # Never prune the first knot, and never prune the last knot
+                if self.has_const and to_prune[0]:
+                    self.coeffs = self.coeffs[1:]
+                    to_prune = to_prune[1:]
+                # Never prune the first knot
+                # And never prune the last two knots
+                if self.n_knots <= 3:
+                    return
+                if self.has_const:
+                    knots_to_prune = np.append(np.append(False, to_prune[3:]), [False, False])
+                    coeffs_to_prune = np.append([False, False, False], to_prune[3:])
+                else:
+                    knots_to_prune = np.append(np.append(False, to_prune[2:]), [False, False])
+                    coeffs_to_prune = np.append([False, False], to_prune[2:])
+                self.coeffs = None
+                self.knots = None
+                self.coeffs = coeffs[~coeffs_to_prune]
+                self.knots = knots[~knots_to_prune]
+            except:
+                # Cleanup, don't leave the spline in an invalid state
+                self.knots = None
+                self.coeffs = None
+                self.knots = knots
+                self.coeffs = coeffs
+                raise

regspline-25.5.2/regspline/util.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import time
+import functools as ft
+import numpy as np
+from statsmodels.tools.validation import PandasWrapper
+
+
+class Timer:
+    def __init__(self, msg="Running"):
+        self.msg = f"{msg}..."
+        self.t0 = 0
+
+    def __enter__(self):
+        print(self.msg, end="")
+        self.t0 = time.time()
+        return self.t0
+
+    def __exit__(self, ex_type, ex_value, ex_traceback):
+        print(f" finished in {time.time() - self.t0:.2f} seconds")
+        return False
+
+
+def type_wrapper(xloc=0):
+    def decorator(f):
+        @ft.wraps(f)
+        def wrapped(*args, **kwargs):
+            x = args[xloc]
+            wrapper = PandasWrapper(x)
+            args = tuple(np.asanyarray(x) if i == xloc else a for i, a in enumerate(args))
+            y = np.asanyarray(f(*args, **kwargs))
+            return y.tolist() if len(y.shape) == 0 else wrapper.wrap(y)
+
+        return wrapped
+
+    return decorator