torch-linear-regression 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- torch_linear_regression-0.1.0/LICENSE +21 -0
- torch_linear_regression-0.1.0/MANIFEST.in +1 -0
- torch_linear_regression-0.1.0/PKG-INFO +82 -0
- torch_linear_regression-0.1.0/README.md +67 -0
- torch_linear_regression-0.1.0/requirements.txt +5 -0
- torch_linear_regression-0.1.0/setup.cfg +7 -0
- torch_linear_regression-0.1.0/setup.py +55 -0
- torch_linear_regression-0.1.0/tests/test_regression.py +84 -0
- torch_linear_regression-0.1.0/torch_linear_regression/__init__.py +4 -0
- torch_linear_regression-0.1.0/torch_linear_regression/linear_regression.py +599 -0
- torch_linear_regression-0.1.0/torch_linear_regression/utils.py +33 -0
- torch_linear_regression-0.1.0/torch_linear_regression.egg-info/PKG-INFO +82 -0
- torch_linear_regression-0.1.0/torch_linear_regression.egg-info/SOURCES.txt +15 -0
- torch_linear_regression-0.1.0/torch_linear_regression.egg-info/dependency_links.txt +1 -0
- torch_linear_regression-0.1.0/torch_linear_regression.egg-info/requires.txt +4 -0
- torch_linear_regression-0.1.0/torch_linear_regression.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 RichieHakim
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
include requirements.txt
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: torch_linear_regression
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A library of useful modules for data analysis.
|
|
5
|
+
Home-page: https://github.com/RichieHakim/torch_linear_regression
|
|
6
|
+
Author: Richard Hakim
|
|
7
|
+
License: LICENSE
|
|
8
|
+
Keywords: data analysis,machine learning,neuroscience
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: numpy
|
|
12
|
+
Requires-Dist: scikit-learn
|
|
13
|
+
Requires-Dist: torch
|
|
14
|
+
Requires-Dist: hypothesis
|
|
15
|
+
|
|
16
|
+
[](https://badge.fury.io/py/torch_linear_regression)
|
|
17
|
+
[](https://pepy.tech/project/torch_linear_regression)
|
|
18
|
+
[](https://github.com/RichieHakim/torch_linear_regression/)
|
|
19
|
+
|
|
20
|
+
# torch_linear_regression
|
|
21
|
+
A very simple library containing closed-form linear regression models using PyTorch.
|
|
22
|
+
Includes:
|
|
23
|
+
- Ordinary Least Squares (OLS) Linear Regression: `(X'X)^-1 X'Y`
|
|
24
|
+
- Ridge Regression: `(X'X + λI)^-1 X'Y`
|
|
25
|
+
- Reduced Rank Regression (RRR) with Ridge penalty: Ridge regression followed by SVD on the weights matrix
|
|
26
|
+
|
|
27
|
+
The closed-form approach results in fast and accurate results under most
|
|
28
|
+
conditions. However, when ``n_features`` is large and/or underdetermined
|
|
29
|
+
(``n_samples`` <= ``n_features``), the solution will start to diverge from
|
|
30
|
+
gradient-based / sklearn solutions.
|
|
31
|
+
|
|
32
|
+
Each model also includes a `model.prefit()` method that can be used to precompute
|
|
33
|
+
the inverse matrix and the ridge penalty matrix. This can be useful when the model
|
|
34
|
+
is used multiple times with the same `X` input data.
|
|
35
|
+
|
|
36
|
+
Because the models are based on PyTorch, they are significantly faster than sklearn's
|
|
37
|
+
models, and can be further accelerated by using GPU. Also the models can be used
|
|
38
|
+
in conjunction with PyTorch's autograd for gradient-based optimization.
|
|
39
|
+
|
|
40
|
+
## Installation
|
|
41
|
+
Install stable version:
|
|
42
|
+
```
|
|
43
|
+
pip install torch_linear_regression
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Install development version:
|
|
47
|
+
```
|
|
48
|
+
pip install git+https://github.com/RichieHakim/torch_linear_regression.git
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Usage
|
|
52
|
+
See the notebook for more examples: [demo notebook](https://github.com/RichieHakim/torch_linear_regression/blob/master/demo_notebook.ipynb)
|
|
53
|
+
```
|
|
54
|
+
import torch_linear_regression as tlr
|
|
55
|
+
|
|
56
|
+
import torch
|
|
57
|
+
import numpy as np
|
|
58
|
+
import sklearn
|
|
59
|
+
import sklearn.datasets
|
|
60
|
+
import matplotlib.pyplot as plt
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
## Generate data for regression
|
|
64
|
+
X, Y = sklearn.datasets.make_regression(
|
|
65
|
+
n_samples=100,
|
|
66
|
+
n_features=2,
|
|
67
|
+
n_informative=10,
|
|
68
|
+
bias=2,
|
|
69
|
+
noise=50,
|
|
70
|
+
random_state=42,
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
## Create model
|
|
74
|
+
model_ols = tlr.OLS()
|
|
75
|
+
## Fit model
|
|
76
|
+
model_ols.fit(X=X, y=Y)
|
|
77
|
+
## Predict
|
|
78
|
+
Y_pred = model_ols.predict(X)
|
|
79
|
+
## Score model
|
|
80
|
+
score = model_ols.score(X=X, y=Y)
|
|
81
|
+
print(f"R^2: {score}")
|
|
82
|
+
```
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
[](https://badge.fury.io/py/torch_linear_regression)
|
|
2
|
+
[](https://pepy.tech/project/torch_linear_regression)
|
|
3
|
+
[](https://github.com/RichieHakim/torch_linear_regression/)
|
|
4
|
+
|
|
5
|
+
# torch_linear_regression
|
|
6
|
+
A very simple library containing closed-form linear regression models using PyTorch.
|
|
7
|
+
Includes:
|
|
8
|
+
- Ordinary Least Squares (OLS) Linear Regression: `(X'X)^-1 X'Y`
|
|
9
|
+
- Ridge Regression: `(X'X + λI)^-1 X'Y`
|
|
10
|
+
- Reduced Rank Regression (RRR) with Ridge penalty: Ridge regression followed by SVD on the weights matrix
|
|
11
|
+
|
|
12
|
+
The closed-form approach results in fast and accurate results under most
|
|
13
|
+
conditions. However, when ``n_features`` is large and/or underdetermined
|
|
14
|
+
(``n_samples`` <= ``n_features``), the solution will start to diverge from
|
|
15
|
+
gradient-based / sklearn solutions.
|
|
16
|
+
|
|
17
|
+
Each model also includes a `model.prefit()` method that can be used to precompute
|
|
18
|
+
the inverse matrix and the ridge penalty matrix. This can be useful when the model
|
|
19
|
+
is used multiple times with the same `X` input data.
|
|
20
|
+
|
|
21
|
+
Because the models are based on PyTorch, they are significantly faster than sklearn's
|
|
22
|
+
models, and can be further accelerated by using GPU. Also the models can be used
|
|
23
|
+
in conjunction with PyTorch's autograd for gradient-based optimization.
|
|
24
|
+
|
|
25
|
+
## Installation
|
|
26
|
+
Install stable version:
|
|
27
|
+
```
|
|
28
|
+
pip install torch_linear_regression
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Install development version:
|
|
32
|
+
```
|
|
33
|
+
pip install git+https://github.com/RichieHakim/torch_linear_regression.git
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Usage
|
|
37
|
+
See the notebook for more examples: [demo notebook](https://github.com/RichieHakim/torch_linear_regression/blob/master/demo_notebook.ipynb)
|
|
38
|
+
```
|
|
39
|
+
import torch_linear_regression as tlr
|
|
40
|
+
|
|
41
|
+
import torch
|
|
42
|
+
import numpy as np
|
|
43
|
+
import sklearn
|
|
44
|
+
import sklearn.datasets
|
|
45
|
+
import matplotlib.pyplot as plt
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
## Generate data for regression
|
|
49
|
+
X, Y = sklearn.datasets.make_regression(
|
|
50
|
+
n_samples=100,
|
|
51
|
+
n_features=2,
|
|
52
|
+
n_informative=10,
|
|
53
|
+
bias=2,
|
|
54
|
+
noise=50,
|
|
55
|
+
random_state=42,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
## Create model
|
|
59
|
+
model_ols = tlr.OLS()
|
|
60
|
+
## Fit model
|
|
61
|
+
model_ols.fit(X=X, y=Y)
|
|
62
|
+
## Predict
|
|
63
|
+
Y_pred = model_ols.predict(X)
|
|
64
|
+
## Score model
|
|
65
|
+
score = model_ols.score(X=X, y=Y)
|
|
66
|
+
print(f"R^2: {score}")
|
|
67
|
+
```
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
## setup.py file for roicat
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
import copy
|
|
4
|
+
import platform
|
|
5
|
+
|
|
6
|
+
from distutils.core import setup
|
|
7
|
+
|
|
8
|
+
## Get the parent directory of this file
|
|
9
|
+
dir_parent = Path(__file__).parent
|
|
10
|
+
|
|
11
|
+
## Get requirements from requirements.txt
|
|
12
|
+
def read_requirements():
|
|
13
|
+
with open(str(dir_parent / "requirements.txt"), "r") as req:
|
|
14
|
+
content = req.read() ## read the file
|
|
15
|
+
requirements = content.split("\n") ## make a list of requirements split by (\n) which is the new line character
|
|
16
|
+
|
|
17
|
+
## Filter out any empty strings from the list
|
|
18
|
+
requirements = [req for req in requirements if req]
|
|
19
|
+
## Filter out any lines starting with #
|
|
20
|
+
requirements = [req for req in requirements if not req.startswith("#")]
|
|
21
|
+
## Remove any commas, quotation marks, and spaces from each requirement
|
|
22
|
+
requirements = [req.replace(",", "").replace("\"", "").replace("\'", "").strip() for req in requirements]
|
|
23
|
+
|
|
24
|
+
return requirements
|
|
25
|
+
deps_all = read_requirements()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
## Get README.md
|
|
29
|
+
with open(str(dir_parent / "README.md"), "r") as f:
|
|
30
|
+
readme = f.read()
|
|
31
|
+
|
|
32
|
+
## Get version number
|
|
33
|
+
with open(str(dir_parent / "torch_linear_regression" / "__init__.py"), "r") as f:
|
|
34
|
+
for line in f:
|
|
35
|
+
if line.startswith("__version__"):
|
|
36
|
+
version = line.split("=")[1].strip().replace("\"", "").replace("\'", "")
|
|
37
|
+
break
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
setup(
|
|
41
|
+
name='torch_linear_regression',
|
|
42
|
+
version=version,
|
|
43
|
+
author='Richard Hakim',
|
|
44
|
+
keywords=['data analysis', 'machine learning', 'neuroscience'],
|
|
45
|
+
license='LICENSE',
|
|
46
|
+
description='A library of useful modules for data analysis.',
|
|
47
|
+
long_description=open('README.md').read(),
|
|
48
|
+
long_description_content_type='text/markdown',
|
|
49
|
+
url='https://github.com/RichieHakim/torch_linear_regression',
|
|
50
|
+
|
|
51
|
+
packages=['torch_linear_regression'],
|
|
52
|
+
|
|
53
|
+
install_requires=deps_all,
|
|
54
|
+
# extras_require=extras_require,
|
|
55
|
+
)
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from sklearn.linear_model import LinearRegression
|
|
3
|
+
from sklearn.metrics import mean_squared_error
|
|
4
|
+
from sklearn.datasets import make_regression
|
|
5
|
+
import numpy as np
|
|
6
|
+
from hypothesis import given, strategies as st
|
|
7
|
+
import hypothesis
|
|
8
|
+
|
|
9
|
+
from torch_linear_regression import OLS
|
|
10
|
+
|
|
11
|
+
# Basic functionality and accuracy
|
|
12
|
+
def test_basic_functionality():
|
|
13
|
+
X, y = make_regression(n_samples=100, n_features=90, noise=0.1)
|
|
14
|
+
model_sklearn = LinearRegression().fit(X, y)
|
|
15
|
+
model_ols = OLS().fit(X, y)
|
|
16
|
+
|
|
17
|
+
np.testing.assert_allclose(model_sklearn.coef_, model_ols.coef_, rtol=1e-5)
|
|
18
|
+
np.testing.assert_allclose(model_sklearn.intercept_, model_ols.intercept_, rtol=1e-5)
|
|
19
|
+
predictions_sklearn = model_sklearn.predict(X)
|
|
20
|
+
predictions_ols = model_ols.predict(X)
|
|
21
|
+
assert mean_squared_error(predictions_sklearn, predictions_ols) < 1e-5
|
|
22
|
+
|
|
23
|
+
# High dimensionality
|
|
24
|
+
def test_high_dimensionality():
|
|
25
|
+
"""
|
|
26
|
+
OLS solution is expected to diverge when n_features >= n_samples. Expect a
|
|
27
|
+
warning.
|
|
28
|
+
"""
|
|
29
|
+
X, y = make_regression(n_samples=100, n_features=100, noise=0.1)
|
|
30
|
+
with pytest.warns(UserWarning):
|
|
31
|
+
model_ols = OLS().fit(X, y)
|
|
32
|
+
|
|
33
|
+
# Single feature test
|
|
34
|
+
def test_single_feature():
|
|
35
|
+
X, y = make_regression(n_samples=100, n_features=1, noise=0.1)
|
|
36
|
+
model_sklearn = LinearRegression().fit(X, y)
|
|
37
|
+
model_ols = OLS().fit(X, y)
|
|
38
|
+
|
|
39
|
+
np.testing.assert_allclose(model_sklearn.coef_, model_ols.coef_, rtol=1e-5)
|
|
40
|
+
np.testing.assert_allclose(model_sklearn.intercept_, model_ols.intercept_, rtol=1e-5)
|
|
41
|
+
|
|
42
|
+
# Zero variation test
|
|
43
|
+
def test_zero_variation():
|
|
44
|
+
X = np.ones((100, 1)) # no variation in X
|
|
45
|
+
y = np.random.randn(100)
|
|
46
|
+
with pytest.raises(Exception):
|
|
47
|
+
model_ols = OLS().fit(X, y)
|
|
48
|
+
|
|
49
|
+
# Extreme values test
|
|
50
|
+
def test_extreme_values():
|
|
51
|
+
X = np.array([
|
|
52
|
+
[1e10, 1e11, 1e12],
|
|
53
|
+
[1e-10, 1e-11, 1e-12],
|
|
54
|
+
[1e-10, 1e11, 1e12]
|
|
55
|
+
])
|
|
56
|
+
y = np.array([1, 2, 3])
|
|
57
|
+
|
|
58
|
+
model_sklearn = LinearRegression().fit(X, y)
|
|
59
|
+
model_ols = OLS().fit(X, y)
|
|
60
|
+
|
|
61
|
+
np.testing.assert_allclose(model_sklearn.coef_, model_ols.coef_, atol=1e-5)
|
|
62
|
+
np.testing.assert_allclose(model_sklearn.intercept_, model_ols.intercept_, atol=1e-5)
|
|
63
|
+
|
|
64
|
+
# Hypothesis test
|
|
65
|
+
@given(
|
|
66
|
+
n=st.integers(min_value=3, max_value=100),
|
|
67
|
+
m=st.integers(min_value=3, max_value=99),
|
|
68
|
+
noise=st.floats(min_value=0.01, max_value=1.0),
|
|
69
|
+
)
|
|
70
|
+
@hypothesis.settings(max_examples=10, deadline=2000)
|
|
71
|
+
def test_hypothesis(n, m, noise):
|
|
72
|
+
X, y = make_regression(n_samples=n, n_features=m, noise=noise)
|
|
73
|
+
model_sklearn = LinearRegression().fit(X, y)
|
|
74
|
+
if m >= n:
|
|
75
|
+
with pytest.warns(UserWarning):
|
|
76
|
+
model_ols = OLS().fit(X, y)
|
|
77
|
+
else:
|
|
78
|
+
model_ols = OLS().fit(X, y)
|
|
79
|
+
|
|
80
|
+
np.testing.assert_allclose(model_sklearn.coef_, model_ols.coef_, rtol=1e-5)
|
|
81
|
+
np.testing.assert_allclose(model_sklearn.intercept_, model_ols.intercept_, rtol=1e-5)
|
|
82
|
+
predictions_sklearn = model_sklearn.predict(X)
|
|
83
|
+
predictions_ols = model_ols.predict(X)
|
|
84
|
+
assert mean_squared_error(predictions_sklearn, predictions_ols) < 1e-5
|
|
@@ -0,0 +1,599 @@
|
|
|
1
|
+
from typing import Dict, Type, Any, Union, Optional, Callable, Tuple, List
|
|
2
|
+
import functools
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import sklearn
|
|
6
|
+
import sklearn.linear_model
|
|
7
|
+
import torch
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class LinearRegression_sk(sklearn.base.BaseEstimator, sklearn.base.RegressorMixin, torch.nn.Module):
|
|
11
|
+
"""
|
|
12
|
+
Implements a basic linear regression estimator following the scikit-learn
|
|
13
|
+
estimator interface.
|
|
14
|
+
RH 2023
|
|
15
|
+
|
|
16
|
+
Attributes:
|
|
17
|
+
coef_ (Union[np.ndarray, torch.Tensor]):
|
|
18
|
+
Coefficients of the linear model.
|
|
19
|
+
intercept_ (Union[float, np.ndarray, torch.Tensor]):
|
|
20
|
+
Intercept of the linear model.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(self, **kwargs):
|
|
24
|
+
"""
|
|
25
|
+
Initializes the LinearRegression_sk model.
|
|
26
|
+
"""
|
|
27
|
+
super(LinearRegression_sk, self).__init__()
|
|
28
|
+
|
|
29
|
+
self.coef_ = None
|
|
30
|
+
self.intercept_ = None
|
|
31
|
+
|
|
32
|
+
def get_backend_namespace(
|
|
33
|
+
self,
|
|
34
|
+
X: Union[np.ndarray, torch.Tensor],
|
|
35
|
+
y: Optional[Union[np.ndarray, torch.Tensor]] = None
|
|
36
|
+
) -> Dict[str, Callable]:
|
|
37
|
+
"""
|
|
38
|
+
Determines the appropriate numerical backend (NumPy or PyTorch) based on
|
|
39
|
+
the type of input data.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
X (Union[np.ndarray, torch.Tensor]):
|
|
43
|
+
Input features array or tensor.
|
|
44
|
+
y (Optional[Union[np.ndarray, torch.Tensor]]):
|
|
45
|
+
Optional target array or tensor. (Default is ``None``)
|
|
46
|
+
|
|
47
|
+
Raises:
|
|
48
|
+
NotImplementedError:
|
|
49
|
+
If the input type is neither NumPy array nor PyTorch tensor.
|
|
50
|
+
|
|
51
|
+
Returns:
|
|
52
|
+
Dict[str, Callable]:
|
|
53
|
+
Dictionary containing backend-specific functions and
|
|
54
|
+
constructors.
|
|
55
|
+
"""
|
|
56
|
+
if isinstance(X, torch.Tensor):
|
|
57
|
+
backend = 'torch'
|
|
58
|
+
if y is not None:
|
|
59
|
+
assert X.device == y.device, 'X and y must be on the same device'
|
|
60
|
+
device = X.device
|
|
61
|
+
elif isinstance(X, np.ndarray):
|
|
62
|
+
backend = 'numpy'
|
|
63
|
+
else:
|
|
64
|
+
raise NotImplementedError
|
|
65
|
+
|
|
66
|
+
if y is not None:
|
|
67
|
+
assert isinstance(y, type(X)), 'X and y must be the same type'
|
|
68
|
+
|
|
69
|
+
dtype = X.dtype
|
|
70
|
+
|
|
71
|
+
ns = {}
|
|
72
|
+
|
|
73
|
+
if backend == 'numpy':
|
|
74
|
+
ns['inv'] = np.linalg.inv
|
|
75
|
+
ns['eye'] = functools.partial(np.eye, dtype=dtype)
|
|
76
|
+
ns['ones'] = functools.partial(np.ones, dtype=dtype)
|
|
77
|
+
ns['zeros'] = functools.partial(np.zeros, dtype=dtype)
|
|
78
|
+
ns['cat'] = np.concatenate
|
|
79
|
+
ns['svd'] = np.linalg.svd
|
|
80
|
+
ns['eigh'] = np.linalg.eigh
|
|
81
|
+
elif backend == 'torch':
|
|
82
|
+
ns['inv'] = torch.linalg.inv
|
|
83
|
+
ns['eye'] = functools.partial(torch.eye, device=device, dtype=dtype)
|
|
84
|
+
ns['ones'] = functools.partial(torch.ones, device=device, dtype=dtype)
|
|
85
|
+
ns['zeros'] = functools.partial(torch.zeros, device=device, dtype=dtype)
|
|
86
|
+
ns['cat'] = torch.cat
|
|
87
|
+
ns['svd'] = torch.linalg.svd
|
|
88
|
+
ns['eigh'] = torch.linalg.eigh
|
|
89
|
+
else:
|
|
90
|
+
raise NotImplementedError
|
|
91
|
+
|
|
92
|
+
return ns
|
|
93
|
+
|
|
94
|
+
def _check_X(self, X: Union[np.ndarray, torch.Tensor]):
|
|
95
|
+
"""
|
|
96
|
+
Passes X through the _check_array method, and gives a UserWarning if
|
|
97
|
+
n_features >= n_samples.
|
|
98
|
+
"""
|
|
99
|
+
X = self._check_array(X, atleast_2d=True)
|
|
100
|
+
|
|
101
|
+
## Give a UserWarning if n_features >= n_samples
|
|
102
|
+
if X.shape[1] >= X.shape[0]:
|
|
103
|
+
import warnings
|
|
104
|
+
warnings.warn('Regression solution is expected to diverge from sklearn solution when n_features >= n_samples')
|
|
105
|
+
return X
|
|
106
|
+
|
|
107
|
+
def _check_y(self, y: Union[np.ndarray, torch.Tensor]):
|
|
108
|
+
"""
|
|
109
|
+
Passes y through the _check_array method.
|
|
110
|
+
"""
|
|
111
|
+
y = self._check_array(y, atleast_2d=False)
|
|
112
|
+
return y
|
|
113
|
+
|
|
114
|
+
def _check_array(self, arr: Union[np.ndarray, torch.Tensor], atleast_2d: bool = False):
|
|
115
|
+
"""
|
|
116
|
+
Checks input array for type and dimensions. Must be numpy or torch
|
|
117
|
+
array, and must be 1 or 2D, and if 1D it can be converted to 2D if
|
|
118
|
+
atleast_2d is True.
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
arr (Union[np.ndarray, torch.Tensor]):
|
|
122
|
+
Input array or tensor.
|
|
123
|
+
atleast_2d (bool):
|
|
124
|
+
If True, converts 1D array to 2D.
|
|
125
|
+
|
|
126
|
+
Returns:
|
|
127
|
+
Union[np.ndarray, torch.Tensor]:
|
|
128
|
+
Input array or tensor, converted to 2D if 1D.
|
|
129
|
+
|
|
130
|
+
Raises:
|
|
131
|
+
AssertionError:
|
|
132
|
+
If the input is not a NumPy array or PyTorch tensor.
|
|
133
|
+
AssertionError:
|
|
134
|
+
If the input is not 1D or 2D.
|
|
135
|
+
"""
|
|
136
|
+
assert isinstance(arr, (np.ndarray, torch.Tensor)), 'array must be a NumPy array or PyTorch tensor'
|
|
137
|
+
assert arr.ndim in [1, 2], 'array must be 1D or 2D'
|
|
138
|
+
|
|
139
|
+
if (arr.ndim == 1) and atleast_2d:
|
|
140
|
+
arr = arr[:, None]
|
|
141
|
+
return arr
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def predict(self, X: Union[np.ndarray, torch.Tensor]) -> Union[np.ndarray, torch.Tensor]:
|
|
145
|
+
"""
|
|
146
|
+
Predicts the target values using the linear model.
|
|
147
|
+
|
|
148
|
+
Args:
|
|
149
|
+
X (Union[np.ndarray, torch.Tensor]):
|
|
150
|
+
Input features array or tensor.
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
Union[np.ndarray, torch.Tensor]:
|
|
154
|
+
Predicted target values.
|
|
155
|
+
"""
|
|
156
|
+
if X.ndim==1:
|
|
157
|
+
X = X[:, None]
|
|
158
|
+
|
|
159
|
+
return X @ self.coef_ + self.intercept_
|
|
160
|
+
|
|
161
|
+
def fit_predict(
|
|
162
|
+
self,
|
|
163
|
+
X: Union[np.ndarray, torch.Tensor],
|
|
164
|
+
y: Union[np.ndarray, torch.Tensor],
|
|
165
|
+
) -> Union[np.ndarray, torch.Tensor]:
|
|
166
|
+
"""
|
|
167
|
+
Fits the model to the data and returns the predicted target values.
|
|
168
|
+
|
|
169
|
+
Args:
|
|
170
|
+
X (Union[np.ndarray, torch.Tensor]):
|
|
171
|
+
Input features array or tensor.
|
|
172
|
+
y (Union[np.ndarray, torch.Tensor]):
|
|
173
|
+
Target values.
|
|
174
|
+
|
|
175
|
+
Returns:
|
|
176
|
+
Union[np.ndarray, torch.Tensor]:
|
|
177
|
+
Predicted target values.
|
|
178
|
+
"""
|
|
179
|
+
self.fit(X, y)
|
|
180
|
+
return self.predict(X)
|
|
181
|
+
|
|
182
|
+
def score(
|
|
183
|
+
self,
|
|
184
|
+
X: Union[np.ndarray, torch.Tensor],
|
|
185
|
+
y: Union[np.ndarray, torch.Tensor],
|
|
186
|
+
sample_weight: Optional[np.ndarray] = None
|
|
187
|
+
) -> float:
|
|
188
|
+
"""
|
|
189
|
+
Calculates the coefficient of determination R^2 of the prediction.
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
X (Union[np.ndarray, torch.Tensor]):
|
|
193
|
+
Input features for which to predict the targets.
|
|
194
|
+
y (Union[np.ndarray, torch.Tensor]):
|
|
195
|
+
True target values.
|
|
196
|
+
sample_weight (Optional[np.ndarray]):
|
|
197
|
+
Sample weights. (Default is ``None``)
|
|
198
|
+
|
|
199
|
+
Raises:
|
|
200
|
+
NotImplementedError:
|
|
201
|
+
If the input type is neither NumPy array nor PyTorch tensor.
|
|
202
|
+
|
|
203
|
+
Returns:
|
|
204
|
+
float:
|
|
205
|
+
R^2 score indicating the prediction accuracy. Uses
|
|
206
|
+
sklearn.metrics.r2_score if input is NumPy array, and implements
|
|
207
|
+
similar calculation if input is PyTorch tensor.
|
|
208
|
+
"""
|
|
209
|
+
y_pred = self.predict(X)
|
|
210
|
+
if isinstance(X, torch.Tensor):
|
|
211
|
+
if sample_weight is not None:
|
|
212
|
+
assert sample_weight.ndim == 1
|
|
213
|
+
assert isinstance(sample_weight, torch.Tensor)
|
|
214
|
+
weight = sample_weight[:, None]
|
|
215
|
+
else:
|
|
216
|
+
weight, sample_weight = 1.0, 1.0
|
|
217
|
+
|
|
218
|
+
numerator = torch.sum(weight * (y - y_pred) ** 2, dim=0)
|
|
219
|
+
denominator = (weight * (y - torch.mean(y * sample_weight, dim=0)) ** 2).sum(dim=0)
|
|
220
|
+
|
|
221
|
+
output_scores = 1 - (numerator / denominator)
|
|
222
|
+
output_scores[denominator == 0.0] = 0.0
|
|
223
|
+
return torch.mean(output_scores) ## 'uniform_average' in sklearn
|
|
224
|
+
|
|
225
|
+
elif isinstance(X, np.ndarray):
|
|
226
|
+
return sklearn.metrics.r2_score(
|
|
227
|
+
y_true=y,
|
|
228
|
+
y_pred=y_pred,
|
|
229
|
+
sample_weight=sample_weight,
|
|
230
|
+
multioutput='uniform_average',
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
else:
|
|
234
|
+
raise NotImplementedError
|
|
235
|
+
|
|
236
|
+
def to(self, device):
|
|
237
|
+
self.coef_ = torch.as_tensor(self.coef_, device=device)
|
|
238
|
+
self.intercept_ = torch.as_tensor(self.intercept_, device=device)
|
|
239
|
+
return self
|
|
240
|
+
def cpu(self):
|
|
241
|
+
return self.to('cpu')
|
|
242
|
+
def numpy(self):
|
|
243
|
+
def check_and_convert(x):
|
|
244
|
+
if isinstance(x, torch.Tensor):
|
|
245
|
+
return x.detach().cpu().numpy()
|
|
246
|
+
else:
|
|
247
|
+
return x
|
|
248
|
+
self.coef_, self.intercept_ = (check_and_convert(x) for x in (self.coef_, self.intercept_))
|
|
249
|
+
return self
|
|
250
|
+
|
|
251
|
+
def forward(self, X: Union[np.ndarray, torch.Tensor]) -> Union[np.ndarray, torch.Tensor]:
|
|
252
|
+
return self.predict(X)
|
|
253
|
+
|
|
254
|
+
# def __repr__(self):
|
|
255
|
+
# return f'LinearRegression_sk(coef={self.coef_}, intercept={self.intercept_})'
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
class OLS(LinearRegression_sk):
|
|
259
|
+
"""
|
|
260
|
+
Implements Ordinary Least Squares (OLS) regression. \n
|
|
261
|
+
|
|
262
|
+
NOTE: This class does not check for NaNs, Infs, singular matrices, etc. \n
|
|
263
|
+
This method is fast and accurate for small to medium-sized datasets. When
|
|
264
|
+
``n_features`` is large and/or underdetermined (``n_samples`` <= ``n_features``),
|
|
265
|
+
the solution will likely diverge from the sklearn solution. \n
|
|
266
|
+
|
|
267
|
+
RH 2023
|
|
268
|
+
|
|
269
|
+
Args:
|
|
270
|
+
fit_intercept (bool):
|
|
271
|
+
Specifies whether to calculate the intercept for this model. If set
|
|
272
|
+
to ``True``, a column of ones is added to the feature matrix \(X\).
|
|
273
|
+
(Default is ``True``)
|
|
274
|
+
prefit_X (Optional[Union[np.ndarray, torch.Tensor]]):
|
|
275
|
+
Optional. If the ``X`` array is provided here, then ``inv(X^T X) @
|
|
276
|
+
X^T`` will be precomputed and stored in the model.
|
|
277
|
+
|
|
278
|
+
Attributes:
|
|
279
|
+
fit_intercept (bool):
|
|
280
|
+
Specifies if the intercept should be calculated.
|
|
281
|
+
X_precompute (bool):
|
|
282
|
+
Indicates if ``inv(X.T @ X) @ X.T`` is precomputed.
|
|
283
|
+
iXTXXT (Optional[np.ndarray]):
|
|
284
|
+
Stores the precomputed result of ``inv(X.T @ X) @ X.T`` if provided,
|
|
285
|
+
otherwise ``None``.
|
|
286
|
+
"""
|
|
287
|
+
|
|
288
|
+
def __init__(
|
|
289
|
+
self,
|
|
290
|
+
fit_intercept=True,
|
|
291
|
+
prefit_X=None,
|
|
292
|
+
**kwargs,
|
|
293
|
+
):
|
|
294
|
+
"""
|
|
295
|
+
Initializes the OLS regression model with the specified `fit_intercept`
|
|
296
|
+
and an optional `prefit_X`.
|
|
297
|
+
"""
|
|
298
|
+
super(OLS, self).__init__()
|
|
299
|
+
self.fit_intercept = fit_intercept
|
|
300
|
+
|
|
301
|
+
self.iXTXXT = self.prefit(prefit_X) if prefit_X is not None else None
|
|
302
|
+
self.prefit_X = True if prefit_X is not None else False
|
|
303
|
+
|
|
304
|
+
def prefit(self, X):
|
|
305
|
+
"""
|
|
306
|
+
Precomputes the ``inv(X.T @ X) @ X.T`` to be used in the ``fit`` method.
|
|
307
|
+
|
|
308
|
+
Args:
|
|
309
|
+
X (np.ndarray):
|
|
310
|
+
The input features matrix.
|
|
311
|
+
|
|
312
|
+
Returns:
|
|
313
|
+
np.ndarray:
|
|
314
|
+
Precomputed result of ``inv(X.T @ X) @ X.T``.
|
|
315
|
+
"""
|
|
316
|
+
ns = self.get_backend_namespace(X=X)
|
|
317
|
+
cat, inv, ones = (ns[key] for key in ['cat', 'inv', 'ones'])
|
|
318
|
+
if self.fit_intercept:
|
|
319
|
+
X = cat((X, ones((X.shape[0], 1))), axis=1)
|
|
320
|
+
|
|
321
|
+
inv_XT_X_XT = inv(X.T @ X) @ X.T
|
|
322
|
+
return inv_XT_X_XT
|
|
323
|
+
|
|
324
|
+
def fit(self, X, y):
|
|
325
|
+
"""
|
|
326
|
+
Fits the OLS regression model to the data.
|
|
327
|
+
|
|
328
|
+
Args:
|
|
329
|
+
X (np.ndarray):
|
|
330
|
+
Training data features.
|
|
331
|
+
y (np.ndarray):
|
|
332
|
+
Target values.
|
|
333
|
+
|
|
334
|
+
Returns:
|
|
335
|
+
OLS:
|
|
336
|
+
The instance of this OLS model.
|
|
337
|
+
"""
|
|
338
|
+
self._check_X(X)
|
|
339
|
+
|
|
340
|
+
self.n_features_in_ = X.shape[1]
|
|
341
|
+
|
|
342
|
+
ns = self.get_backend_namespace(X=X, y=y)
|
|
343
|
+
zeros = ns['zeros']
|
|
344
|
+
|
|
345
|
+
## Regression algorithm
|
|
346
|
+
inv_XT_X_XT = self.prefit(X) if self.iXTXXT is None else self.iXTXXT
|
|
347
|
+
theta = inv_XT_X_XT @ y
|
|
348
|
+
|
|
349
|
+
### Extract bias terms
|
|
350
|
+
if self.fit_intercept:
|
|
351
|
+
self.intercept_ = theta[-1]
|
|
352
|
+
self.coef_ = theta[:-1]
|
|
353
|
+
else:
|
|
354
|
+
self.intercept_ = zeros((y.shape[1],)) if y.ndim == 2 else 0
|
|
355
|
+
self.coef_ = theta
|
|
356
|
+
|
|
357
|
+
return self
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
class Ridge(LinearRegression_sk):
|
|
361
|
+
"""
|
|
362
|
+
Implements Ridge regression based on the closed-form / 'analytic solution'. \n
|
|
363
|
+
NOTE: This class does not check for NaNs, Infs, singular matrices, etc. \n
|
|
364
|
+
This method is fast and accurate for small to medium-sized datasets. When
|
|
365
|
+
``n_features`` is large and/or underdetermined (``n_samples`` <
|
|
366
|
+
``n_features``), the solution will likely diverge from the sklearn solution. \n
|
|
367
|
+
|
|
368
|
+
RH 2023
|
|
369
|
+
|
|
370
|
+
Args:
|
|
371
|
+
alpha (float):
|
|
372
|
+
The regularization strength which must be a positive float.
|
|
373
|
+
Regularizes the estimate to prevent overfitting by constraining the
|
|
374
|
+
size of the coefficients. Usually ~1e5. (Default is 1)
|
|
375
|
+
fit_intercept (bool):
|
|
376
|
+
Whether to calculate the intercept for this model. If set to
|
|
377
|
+
``False``, no intercept will be used in calculations (i.e., data is
|
|
378
|
+
expected to be centered). (Default is ``False``)
|
|
379
|
+
prefit_X (Optional[np.ndarray, torch.Tensor]):
|
|
380
|
+
Optional. If the ``X`` array is provided here, then ``inv(X.T @ X +
|
|
381
|
+
alpha*eye(X.shape[1])) @ X.T`` will be precomputed and stored in the
|
|
382
|
+
model.
|
|
383
|
+
|
|
384
|
+
Attributes:
|
|
385
|
+
alpha (float):
|
|
386
|
+
Regularization parameter.
|
|
387
|
+
fit_intercept (bool):
|
|
388
|
+
Specifies if the intercept should be calculated.
|
|
389
|
+
iXTXaeXT (Optional[np.ndarray]):
|
|
390
|
+
Stores the precomputed ``inv(X.T @ X + alpha*eye(X.shape[1])) @
|
|
391
|
+
X.T`` if precomputed, otherwise ``None``.
|
|
392
|
+
"""
|
|
393
|
+
def __init__(
|
|
394
|
+
self,
|
|
395
|
+
alpha=1,
|
|
396
|
+
fit_intercept=True,
|
|
397
|
+
prefit_X=None,
|
|
398
|
+
**kwargs,
|
|
399
|
+
):
|
|
400
|
+
"""
|
|
401
|
+
Initializes the Ridge regression model with the specified alpha,
|
|
402
|
+
fit_intercept, and optional X_precompute.
|
|
403
|
+
"""
|
|
404
|
+
super(Ridge, self).__init__()
|
|
405
|
+
self.alpha = alpha
|
|
406
|
+
self.fit_intercept = fit_intercept
|
|
407
|
+
|
|
408
|
+
self.iXTXaeXT = self.prefit(prefit_X) if prefit_X is not None else None
|
|
409
|
+
self.prefit_X = True if prefit_X is not None else False
|
|
410
|
+
|
|
411
|
+
def prefit(self, X):
|
|
412
|
+
"""
|
|
413
|
+
Precomputes ``inv(X.T @ X + alpha*eye(X.shape[1])) @ X.T`` to be used in
|
|
414
|
+
the `fit` method.
|
|
415
|
+
|
|
416
|
+
Args:
|
|
417
|
+
X (np.ndarray):
|
|
418
|
+
The input features matrix.
|
|
419
|
+
|
|
420
|
+
Returns:
|
|
421
|
+
np.ndarray:
|
|
422
|
+
Precomputed inverse of ``inv(X.T @ X + alpha*eye(X.shape[1])) @
|
|
423
|
+
X.T``.
|
|
424
|
+
"""
|
|
425
|
+
ns = self.get_backend_namespace(X=X)
|
|
426
|
+
cat, eye, inv, ones = (ns[key] for key in ['cat', 'eye', 'inv', 'ones'])
|
|
427
|
+
if self.fit_intercept:
|
|
428
|
+
X = cat((X, ones((X.shape[0], 1))), axis=1)
|
|
429
|
+
|
|
430
|
+
inv_XT_X_plus_alpha_eye_XT = inv(X.T @ X + self.alpha*eye(X.shape[1])) @ X.T
|
|
431
|
+
return inv_XT_X_plus_alpha_eye_XT
|
|
432
|
+
|
|
433
|
+
def fit(self, X, y):
|
|
434
|
+
"""
|
|
435
|
+
Fits the Ridge regression model to the data.
|
|
436
|
+
|
|
437
|
+
Args:
|
|
438
|
+
X (np.ndarray):
|
|
439
|
+
Training data features.
|
|
440
|
+
y (np.ndarray):
|
|
441
|
+
Target values.
|
|
442
|
+
|
|
443
|
+
Returns:
|
|
444
|
+
Ridge:
|
|
445
|
+
The instance of this Ridge model.
|
|
446
|
+
"""
|
|
447
|
+
self.n_features_in_ = X.shape[1]
|
|
448
|
+
|
|
449
|
+
## Give a UserWarning if n_features >= n_samples
|
|
450
|
+
if X.shape[1] >= X.shape[0]:
|
|
451
|
+
import warnings
|
|
452
|
+
warnings.warn('OLS solution is expected to diverge from sklearn solution when n_features >= n_samples')
|
|
453
|
+
|
|
454
|
+
ns = self.get_backend_namespace(X=X, y=y)
|
|
455
|
+
zeros = ns['zeros']
|
|
456
|
+
|
|
457
|
+
## Regression algorithm
|
|
458
|
+
inv_XT_X_plus_alpha_eye_XT = self.prefit(X) if self.iXTXaeXT is None else self.iXTXaeXT
|
|
459
|
+
theta = inv_XT_X_plus_alpha_eye_XT @ y
|
|
460
|
+
|
|
461
|
+
### Extract bias terms
|
|
462
|
+
if self.fit_intercept:
|
|
463
|
+
self.intercept_ = theta[-1]
|
|
464
|
+
self.coef_ = theta[:-1]
|
|
465
|
+
else:
|
|
466
|
+
self.intercept_ = zeros((y.shape[1],)) if y.ndim == 2 else 0
|
|
467
|
+
self.coef_ = theta
|
|
468
|
+
|
|
469
|
+
return self
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
class ReducedRankRidgeRegression(LinearRegression_sk):
|
|
473
|
+
"""
|
|
474
|
+
Implements a Reduced Rank Ridge Regression model using a closed-form
|
|
475
|
+
solution based on doing SVD on the coefficients of the Ridge regression Beta
|
|
476
|
+
matrix. This model reduces the rank of the coefficient matrix to address
|
|
477
|
+
multicollinearity and reduce model complexity. \n
|
|
478
|
+
NOTE: self.coef_ is stored as the dense beta coefficients, not as a Kruskal
|
|
479
|
+
tensor. \n
|
|
480
|
+
|
|
481
|
+
RH 2024
|
|
482
|
+
|
|
483
|
+
Args:
|
|
484
|
+
rank (int):
|
|
485
|
+
The target rank for the reduced rank regression model. It determines
|
|
486
|
+
the number of singular values to retain in the model. (Default is 3)
|
|
487
|
+
alpha (float):
|
|
488
|
+
The regularization strength which must be a positive float.
|
|
489
|
+
Regularizes the estimate to prevent overfitting by constraining the
|
|
490
|
+
size of the coefficients. Usually ~1e5. (Default is 1)
|
|
491
|
+
fit_intercept (bool):
|
|
492
|
+
Specifies whether to calculate the intercept for this model. If set
|
|
493
|
+
to ``True``, a column of ones is added to the feature matrix \(X\).
|
|
494
|
+
(Default is ``True``)
|
|
495
|
+
prefit_X (Optional[np.ndarray, torch.Tensor]):
|
|
496
|
+
Optional. If the ``X`` array is provided here, then ``inv(X^T X +
|
|
497
|
+
alpha*eye(X.shape[1])) @ X^T`` will be precomputed and stored
|
|
498
|
+
in the model.
|
|
499
|
+
|
|
500
|
+
Attributes:
|
|
501
|
+
rank (int):
|
|
502
|
+
The rank specified for reducing the regression model.
|
|
503
|
+
alpha (float):
|
|
504
|
+
Regularization parameter.
|
|
505
|
+
fit_intercept (bool):
|
|
506
|
+
Specifies if the intercept should be calculated.
|
|
507
|
+
iXTXaeXT (Optional[np.ndarray]):
|
|
508
|
+
Stores the precomputed ``inv(X.T @ X + alpha*eye(X.shape[1])) @
|
|
509
|
+
X.T`` if precomputed, otherwise ``None``.
|
|
510
|
+
"""
|
|
511
|
+
|
|
512
|
+
def __init__(
|
|
513
|
+
self,
|
|
514
|
+
rank=3,
|
|
515
|
+
alpha=1,
|
|
516
|
+
fit_intercept=True,
|
|
517
|
+
prefit_X=None,
|
|
518
|
+
**kwargs,
|
|
519
|
+
):
|
|
520
|
+
"""
|
|
521
|
+
Initializes the Reduced Rank Regression model with the specified rank
|
|
522
|
+
and an optional
|
|
523
|
+
"""
|
|
524
|
+
super(ReducedRankRidgeRegression, self).__init__()
|
|
525
|
+
self.fit_intercept = fit_intercept
|
|
526
|
+
self.rank = rank
|
|
527
|
+
self.alpha = alpha
|
|
528
|
+
|
|
529
|
+
self.iXTXXT = self.prefit(prefit_X) if prefit_X is not None else None
|
|
530
|
+
self.prefit_X = True if prefit_X is not None else False
|
|
531
|
+
|
|
532
|
+
def prefit(self, X):
|
|
533
|
+
"""
|
|
534
|
+
Precomputes the \(X^T X + \alpha I\) inverse times \(X^T\) to be used in
|
|
535
|
+
the `fit` method.
|
|
536
|
+
|
|
537
|
+
Args:
|
|
538
|
+
X (np.ndarray):
|
|
539
|
+
The input features matrix.
|
|
540
|
+
|
|
541
|
+
Returns:
|
|
542
|
+
np.ndarray:
|
|
543
|
+
Precomputed inverse of \(X^T X + \alpha I\) times \(X^T\).
|
|
544
|
+
"""
|
|
545
|
+
ns = self.get_backend_namespace(X=X)
|
|
546
|
+
cat, eye, inv, ones = (ns[key] for key in ['cat', 'eye', 'inv', 'ones'])
|
|
547
|
+
if self.fit_intercept:
|
|
548
|
+
X = cat((X, ones((X.shape[0], 1))), axis=1)
|
|
549
|
+
|
|
550
|
+
inv_XT_X_plus_alpha_eye_XT = inv(X.T @ X + self.alpha*eye(X.shape[1])) @ X.T
|
|
551
|
+
return inv_XT_X_plus_alpha_eye_XT
|
|
552
|
+
|
|
553
|
+
def fit(self, X, y, rank=None):
|
|
554
|
+
"""
|
|
555
|
+
Fits the Reduced Rank Regression model to the data, applying a rank
|
|
556
|
+
reduction on the OLS regression coefficients.
|
|
557
|
+
|
|
558
|
+
Args:
|
|
559
|
+
X (np.ndarray):
|
|
560
|
+
Training data features.
|
|
561
|
+
y (np.ndarray):
|
|
562
|
+
Target values.
|
|
563
|
+
rank (Optional[int]):
|
|
564
|
+
The target rank for the reduced rank regression model. It
|
|
565
|
+
determines the number of singular values to retain in the model.
|
|
566
|
+
(Default is ``None``)
|
|
567
|
+
|
|
568
|
+
Returns:
|
|
569
|
+
ReducedRankRegression:
|
|
570
|
+
The instance of this Reduced Rank Regression model.
|
|
571
|
+
"""
|
|
572
|
+
self.rank = self.rank if rank is None else rank
|
|
573
|
+
self.n_features_in_ = X.shape[1]
|
|
574
|
+
assert self.rank <= X.shape[1], 'Rank must be less than or equal to the number of features'
|
|
575
|
+
|
|
576
|
+
ns = self.get_backend_namespace(X=X, y=y)
|
|
577
|
+
svd, zeros = (ns[key] for key in ['svd', 'zeros'])
|
|
578
|
+
|
|
579
|
+
## Calculate OLS solution
|
|
580
|
+
inv_XT_X_plus_alpha_eye_XT = self.prefit(X) if self.iXTXXT is None else self.iXTXXT
|
|
581
|
+
B_r = inv_XT_X_plus_alpha_eye_XT @ y
|
|
582
|
+
|
|
583
|
+
## Calculate SVD of B_r
|
|
584
|
+
U, S, Vt = svd(B_r, full_matrices=False)
|
|
585
|
+
|
|
586
|
+
## Truncate to rank
|
|
587
|
+
Vt = Vt[:self.rank]
|
|
588
|
+
B_rrr = B_r @ Vt.T @ Vt
|
|
589
|
+
|
|
590
|
+
### Extract bias terms
|
|
591
|
+
if self.fit_intercept:
|
|
592
|
+
self.intercept_ = B_rrr[-1]
|
|
593
|
+
self.coef_ = B_rrr[:-1]
|
|
594
|
+
else:
|
|
595
|
+
self.intercept_ = zeros((y.shape[1],)) if y.ndim == 2 else 0
|
|
596
|
+
self.coef_ = B_rrr
|
|
597
|
+
|
|
598
|
+
return self
|
|
599
|
+
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import torch
|
|
3
|
+
|
|
4
|
+
def check_full_rank(X, a_thresh=None, r_thresh=1e-10):
|
|
5
|
+
"""
|
|
6
|
+
Check if a matrix is full rank.
|
|
7
|
+
|
|
8
|
+
Args:
|
|
9
|
+
X (torch.Tensor):
|
|
10
|
+
The matrix to check.
|
|
11
|
+
Should be a 2D numpy array or torch.Tensor.
|
|
12
|
+
a_thresh (float, optional):
|
|
13
|
+
The absolute threshold for the singular values; singular values
|
|
14
|
+
below this threshold are considered zero.
|
|
15
|
+
If None, then not used.
|
|
16
|
+
r_thresh (float, optional):
|
|
17
|
+
The relative threshold for the singular values; singular values
|
|
18
|
+
below this threshold are considered zero.
|
|
19
|
+
Here, 'relative' means relative to the largest singular value.
|
|
20
|
+
|
|
21
|
+
Returns:
|
|
22
|
+
bool:
|
|
23
|
+
True if the matrix is full rank, False otherwise.
|
|
24
|
+
"""
|
|
25
|
+
U, S, V = torch.linalg.svd(torch.as_tensor(X))
|
|
26
|
+
|
|
27
|
+
S_bool = torch.ones_like(S, dtype=bool)
|
|
28
|
+
if a_thresh is not None:
|
|
29
|
+
S_bool = S_bool & (S > a_thresh)
|
|
30
|
+
if r_thresh is not None:
|
|
31
|
+
S_bool = S_bool & (S > r_thresh * S[0])
|
|
32
|
+
|
|
33
|
+
return S_bool.all()
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: torch_linear_regression
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A library of useful modules for data analysis.
|
|
5
|
+
Home-page: https://github.com/RichieHakim/torch_linear_regression
|
|
6
|
+
Author: Richard Hakim
|
|
7
|
+
License: LICENSE
|
|
8
|
+
Keywords: data analysis,machine learning,neuroscience
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: numpy
|
|
12
|
+
Requires-Dist: scikit-learn
|
|
13
|
+
Requires-Dist: torch
|
|
14
|
+
Requires-Dist: hypothesis
|
|
15
|
+
|
|
16
|
+
[](https://badge.fury.io/py/torch_linear_regression)
|
|
17
|
+
[](https://pepy.tech/project/torch_linear_regression)
|
|
18
|
+
[](https://github.com/RichieHakim/torch_linear_regression/)
|
|
19
|
+
|
|
20
|
+
# torch_linear_regression
|
|
21
|
+
A very simple library containing closed-form linear regression models using PyTorch.
|
|
22
|
+
Includes:
|
|
23
|
+
- Ordinary Least Squares (OLS) Linear Regression: `(X'X)^-1 X'Y`
|
|
24
|
+
- Ridge Regression: `(X'X + λI)^-1 X'Y`
|
|
25
|
+
- Reduced Rank Regression (RRR) with Ridge penalty: Ridge regression followed by SVD on the weights matrix
|
|
26
|
+
|
|
27
|
+
The closed-form approach results in fast and accurate results under most
|
|
28
|
+
conditions. However, when ``n_features`` is large and/or underdetermined
|
|
29
|
+
(``n_samples`` <= ``n_features``), the solution will start to diverge from
|
|
30
|
+
gradient-based / sklearn solutions.
|
|
31
|
+
|
|
32
|
+
Each model also includes a `model.prefit()` method that can be used to precompute
|
|
33
|
+
the inverse matrix and the ridge penalty matrix. This can be useful when the model
|
|
34
|
+
is used multiple times with the same `X` input data.
|
|
35
|
+
|
|
36
|
+
Because the models are based on PyTorch, they are significantly faster than sklearn's
|
|
37
|
+
models, and can be further accelerated by using GPU. Also the models can be used
|
|
38
|
+
in conjunction with PyTorch's autograd for gradient-based optimization.
|
|
39
|
+
|
|
40
|
+
## Installation
|
|
41
|
+
Install stable version:
|
|
42
|
+
```
|
|
43
|
+
pip install torch_linear_regression
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Install development version:
|
|
47
|
+
```
|
|
48
|
+
pip install git+https://github.com/RichieHakim/torch_linear_regression.git
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Usage
|
|
52
|
+
See the notebook for more examples: [demo notebook](https://github.com/RichieHakim/torch_linear_regression/blob/master/demo_notebook.ipynb)
|
|
53
|
+
```
|
|
54
|
+
import torch_linear_regression as tlr
|
|
55
|
+
|
|
56
|
+
import torch
|
|
57
|
+
import numpy as np
|
|
58
|
+
import sklearn
|
|
59
|
+
import sklearn.datasets
|
|
60
|
+
import matplotlib.pyplot as plt
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
## Generate data for regression
|
|
64
|
+
X, Y = sklearn.datasets.make_regression(
|
|
65
|
+
n_samples=100,
|
|
66
|
+
n_features=2,
|
|
67
|
+
n_informative=10,
|
|
68
|
+
bias=2,
|
|
69
|
+
noise=50,
|
|
70
|
+
random_state=42,
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
## Create model
|
|
74
|
+
model_ols = tlr.OLS()
|
|
75
|
+
## Fit model
|
|
76
|
+
model_ols.fit(X=X, y=Y)
|
|
77
|
+
## Predict
|
|
78
|
+
Y_pred = model_ols.predict(X)
|
|
79
|
+
## Score model
|
|
80
|
+
score = model_ols.score(X=X, y=Y)
|
|
81
|
+
print(f"R^2: {score}")
|
|
82
|
+
```
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
MANIFEST.in
|
|
3
|
+
README.md
|
|
4
|
+
requirements.txt
|
|
5
|
+
setup.cfg
|
|
6
|
+
setup.py
|
|
7
|
+
tests/test_regression.py
|
|
8
|
+
torch_linear_regression/__init__.py
|
|
9
|
+
torch_linear_regression/linear_regression.py
|
|
10
|
+
torch_linear_regression/utils.py
|
|
11
|
+
torch_linear_regression.egg-info/PKG-INFO
|
|
12
|
+
torch_linear_regression.egg-info/SOURCES.txt
|
|
13
|
+
torch_linear_regression.egg-info/dependency_links.txt
|
|
14
|
+
torch_linear_regression.egg-info/requires.txt
|
|
15
|
+
torch_linear_regression.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
torch_linear_regression
|