torch-linear-regression 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,4 @@
1
+ from .linear_regression import LinearRegression_sk, OLS, Ridge, ReducedRankRidgeRegression
2
+ from .utils import check_full_rank
3
+
4
+ __version__ = '0.1.0'
@@ -0,0 +1,599 @@
1
+ from typing import Dict, Type, Any, Union, Optional, Callable, Tuple, List
2
+ import functools
3
+
4
+ import numpy as np
5
+ import sklearn
6
+ import sklearn.linear_model
7
+ import torch
8
+
9
+
10
+ class LinearRegression_sk(sklearn.base.BaseEstimator, sklearn.base.RegressorMixin, torch.nn.Module):
11
+ """
12
+ Implements a basic linear regression estimator following the scikit-learn
13
+ estimator interface.
14
+ RH 2023
15
+
16
+ Attributes:
17
+ coef_ (Union[np.ndarray, torch.Tensor]):
18
+ Coefficients of the linear model.
19
+ intercept_ (Union[float, np.ndarray, torch.Tensor]):
20
+ Intercept of the linear model.
21
+ """
22
+
23
+ def __init__(self, **kwargs):
24
+ """
25
+ Initializes the LinearRegression_sk model.
26
+ """
27
+ super(LinearRegression_sk, self).__init__()
28
+
29
+ self.coef_ = None
30
+ self.intercept_ = None
31
+
32
+ def get_backend_namespace(
33
+ self,
34
+ X: Union[np.ndarray, torch.Tensor],
35
+ y: Optional[Union[np.ndarray, torch.Tensor]] = None
36
+ ) -> Dict[str, Callable]:
37
+ """
38
+ Determines the appropriate numerical backend (NumPy or PyTorch) based on
39
+ the type of input data.
40
+
41
+ Args:
42
+ X (Union[np.ndarray, torch.Tensor]):
43
+ Input features array or tensor.
44
+ y (Optional[Union[np.ndarray, torch.Tensor]]):
45
+ Optional target array or tensor. (Default is ``None``)
46
+
47
+ Raises:
48
+ NotImplementedError:
49
+ If the input type is neither NumPy array nor PyTorch tensor.
50
+
51
+ Returns:
52
+ Dict[str, Callable]:
53
+ Dictionary containing backend-specific functions and
54
+ constructors.
55
+ """
56
+ if isinstance(X, torch.Tensor):
57
+ backend = 'torch'
58
+ if y is not None:
59
+ assert X.device == y.device, 'X and y must be on the same device'
60
+ device = X.device
61
+ elif isinstance(X, np.ndarray):
62
+ backend = 'numpy'
63
+ else:
64
+ raise NotImplementedError
65
+
66
+ if y is not None:
67
+ assert isinstance(y, type(X)), 'X and y must be the same type'
68
+
69
+ dtype = X.dtype
70
+
71
+ ns = {}
72
+
73
+ if backend == 'numpy':
74
+ ns['inv'] = np.linalg.inv
75
+ ns['eye'] = functools.partial(np.eye, dtype=dtype)
76
+ ns['ones'] = functools.partial(np.ones, dtype=dtype)
77
+ ns['zeros'] = functools.partial(np.zeros, dtype=dtype)
78
+ ns['cat'] = np.concatenate
79
+ ns['svd'] = np.linalg.svd
80
+ ns['eigh'] = np.linalg.eigh
81
+ elif backend == 'torch':
82
+ ns['inv'] = torch.linalg.inv
83
+ ns['eye'] = functools.partial(torch.eye, device=device, dtype=dtype)
84
+ ns['ones'] = functools.partial(torch.ones, device=device, dtype=dtype)
85
+ ns['zeros'] = functools.partial(torch.zeros, device=device, dtype=dtype)
86
+ ns['cat'] = torch.cat
87
+ ns['svd'] = torch.linalg.svd
88
+ ns['eigh'] = torch.linalg.eigh
89
+ else:
90
+ raise NotImplementedError
91
+
92
+ return ns
93
+
94
+ def _check_X(self, X: Union[np.ndarray, torch.Tensor]):
95
+ """
96
+ Passes X through the _check_array method, and gives a UserWarning if
97
+ n_features >= n_samples.
98
+ """
99
+ X = self._check_array(X, atleast_2d=True)
100
+
101
+ ## Give a UserWarning if n_features >= n_samples
102
+ if X.shape[1] >= X.shape[0]:
103
+ import warnings
104
+ warnings.warn('Regression solution is expected to diverge from sklearn solution when n_features >= n_samples')
105
+ return X
106
+
107
+ def _check_y(self, y: Union[np.ndarray, torch.Tensor]):
108
+ """
109
+ Passes y through the _check_array method.
110
+ """
111
+ y = self._check_array(y, atleast_2d=False)
112
+ return y
113
+
114
+ def _check_array(self, arr: Union[np.ndarray, torch.Tensor], atleast_2d: bool = False):
115
+ """
116
+ Checks input array for type and dimensions. Must be numpy or torch
117
+ array, and must be 1 or 2D, and if 1D it can be converted to 2D if
118
+ atleast_2d is True.
119
+
120
+ Args:
121
+ arr (Union[np.ndarray, torch.Tensor]):
122
+ Input array or tensor.
123
+ atleast_2d (bool):
124
+ If True, converts 1D array to 2D.
125
+
126
+ Returns:
127
+ Union[np.ndarray, torch.Tensor]:
128
+ Input array or tensor, converted to 2D if 1D.
129
+
130
+ Raises:
131
+ AssertionError:
132
+ If the input is not a NumPy array or PyTorch tensor.
133
+ AssertionError:
134
+ If the input is not 1D or 2D.
135
+ """
136
+ assert isinstance(arr, (np.ndarray, torch.Tensor)), 'array must be a NumPy array or PyTorch tensor'
137
+ assert arr.ndim in [1, 2], 'array must be 1D or 2D'
138
+
139
+ if (arr.ndim == 1) and atleast_2d:
140
+ arr = arr[:, None]
141
+ return arr
142
+
143
+
144
+ def predict(self, X: Union[np.ndarray, torch.Tensor]) -> Union[np.ndarray, torch.Tensor]:
145
+ """
146
+ Predicts the target values using the linear model.
147
+
148
+ Args:
149
+ X (Union[np.ndarray, torch.Tensor]):
150
+ Input features array or tensor.
151
+
152
+ Returns:
153
+ Union[np.ndarray, torch.Tensor]:
154
+ Predicted target values.
155
+ """
156
+ if X.ndim==1:
157
+ X = X[:, None]
158
+
159
+ return X @ self.coef_ + self.intercept_
160
+
161
+ def fit_predict(
162
+ self,
163
+ X: Union[np.ndarray, torch.Tensor],
164
+ y: Union[np.ndarray, torch.Tensor],
165
+ ) -> Union[np.ndarray, torch.Tensor]:
166
+ """
167
+ Fits the model to the data and returns the predicted target values.
168
+
169
+ Args:
170
+ X (Union[np.ndarray, torch.Tensor]):
171
+ Input features array or tensor.
172
+ y (Union[np.ndarray, torch.Tensor]):
173
+ Target values.
174
+
175
+ Returns:
176
+ Union[np.ndarray, torch.Tensor]:
177
+ Predicted target values.
178
+ """
179
+ self.fit(X, y)
180
+ return self.predict(X)
181
+
182
+ def score(
183
+ self,
184
+ X: Union[np.ndarray, torch.Tensor],
185
+ y: Union[np.ndarray, torch.Tensor],
186
+ sample_weight: Optional[np.ndarray] = None
187
+ ) -> float:
188
+ """
189
+ Calculates the coefficient of determination R^2 of the prediction.
190
+
191
+ Args:
192
+ X (Union[np.ndarray, torch.Tensor]):
193
+ Input features for which to predict the targets.
194
+ y (Union[np.ndarray, torch.Tensor]):
195
+ True target values.
196
+ sample_weight (Optional[np.ndarray]):
197
+ Sample weights. (Default is ``None``)
198
+
199
+ Raises:
200
+ NotImplementedError:
201
+ If the input type is neither NumPy array nor PyTorch tensor.
202
+
203
+ Returns:
204
+ float:
205
+ R^2 score indicating the prediction accuracy. Uses
206
+ sklearn.metrics.r2_score if input is NumPy array, and implements
207
+ similar calculation if input is PyTorch tensor.
208
+ """
209
+ y_pred = self.predict(X)
210
+ if isinstance(X, torch.Tensor):
211
+ if sample_weight is not None:
212
+ assert sample_weight.ndim == 1
213
+ assert isinstance(sample_weight, torch.Tensor)
214
+ weight = sample_weight[:, None]
215
+ else:
216
+ weight, sample_weight = 1.0, 1.0
217
+
218
+ numerator = torch.sum(weight * (y - y_pred) ** 2, dim=0)
219
+ denominator = (weight * (y - torch.mean(y * sample_weight, dim=0)) ** 2).sum(dim=0)
220
+
221
+ output_scores = 1 - (numerator / denominator)
222
+ output_scores[denominator == 0.0] = 0.0
223
+ return torch.mean(output_scores) ## 'uniform_average' in sklearn
224
+
225
+ elif isinstance(X, np.ndarray):
226
+ return sklearn.metrics.r2_score(
227
+ y_true=y,
228
+ y_pred=y_pred,
229
+ sample_weight=sample_weight,
230
+ multioutput='uniform_average',
231
+ )
232
+
233
+ else:
234
+ raise NotImplementedError
235
+
236
+ def to(self, device):
237
+ self.coef_ = torch.as_tensor(self.coef_, device=device)
238
+ self.intercept_ = torch.as_tensor(self.intercept_, device=device)
239
+ return self
240
+ def cpu(self):
241
+ return self.to('cpu')
242
+ def numpy(self):
243
+ def check_and_convert(x):
244
+ if isinstance(x, torch.Tensor):
245
+ return x.detach().cpu().numpy()
246
+ else:
247
+ return x
248
+ self.coef_, self.intercept_ = (check_and_convert(x) for x in (self.coef_, self.intercept_))
249
+ return self
250
+
251
+ def forward(self, X: Union[np.ndarray, torch.Tensor]) -> Union[np.ndarray, torch.Tensor]:
252
+ return self.predict(X)
253
+
254
+ # def __repr__(self):
255
+ # return f'LinearRegression_sk(coef={self.coef_}, intercept={self.intercept_})'
256
+
257
+
258
+ class OLS(LinearRegression_sk):
259
+ """
260
+ Implements Ordinary Least Squares (OLS) regression. \n
261
+
262
+ NOTE: This class does not check for NaNs, Infs, singular matrices, etc. \n
263
+ This method is fast and accurate for small to medium-sized datasets. When
264
+ ``n_features`` is large and/or underdetermined (``n_samples`` <= ``n_features``),
265
+ the solution will likely diverge from the sklearn solution. \n
266
+
267
+ RH 2023
268
+
269
+ Args:
270
+ fit_intercept (bool):
271
+ Specifies whether to calculate the intercept for this model. If set
272
+ to ``True``, a column of ones is added to the feature matrix \(X\).
273
+ (Default is ``True``)
274
+ prefit_X (Optional[Union[np.ndarray, torch.Tensor]]):
275
+ Optional. If the ``X`` array is provided here, then ``inv(X^T X) @
276
+ X^T`` will be precomputed and stored in the model.
277
+
278
+ Attributes:
279
+ fit_intercept (bool):
280
+ Specifies if the intercept should be calculated.
281
+ X_precompute (bool):
282
+ Indicates if ``inv(X.T @ X) @ X.T`` is precomputed.
283
+ iXTXXT (Optional[np.ndarray]):
284
+ Stores the precomputed result of ``inv(X.T @ X) @ X.T`` if provided,
285
+ otherwise ``None``.
286
+ """
287
+
288
+ def __init__(
289
+ self,
290
+ fit_intercept=True,
291
+ prefit_X=None,
292
+ **kwargs,
293
+ ):
294
+ """
295
+ Initializes the OLS regression model with the specified `fit_intercept`
296
+ and an optional `prefit_X`.
297
+ """
298
+ super(OLS, self).__init__()
299
+ self.fit_intercept = fit_intercept
300
+
301
+ self.iXTXXT = self.prefit(prefit_X) if prefit_X is not None else None
302
+ self.prefit_X = True if prefit_X is not None else False
303
+
304
+ def prefit(self, X):
305
+ """
306
+ Precomputes the ``inv(X.T @ X) @ X.T`` to be used in the ``fit`` method.
307
+
308
+ Args:
309
+ X (np.ndarray):
310
+ The input features matrix.
311
+
312
+ Returns:
313
+ np.ndarray:
314
+ Precomputed result of ``inv(X.T @ X) @ X.T``.
315
+ """
316
+ ns = self.get_backend_namespace(X=X)
317
+ cat, inv, ones = (ns[key] for key in ['cat', 'inv', 'ones'])
318
+ if self.fit_intercept:
319
+ X = cat((X, ones((X.shape[0], 1))), axis=1)
320
+
321
+ inv_XT_X_XT = inv(X.T @ X) @ X.T
322
+ return inv_XT_X_XT
323
+
324
+ def fit(self, X, y):
325
+ """
326
+ Fits the OLS regression model to the data.
327
+
328
+ Args:
329
+ X (np.ndarray):
330
+ Training data features.
331
+ y (np.ndarray):
332
+ Target values.
333
+
334
+ Returns:
335
+ OLS:
336
+ The instance of this OLS model.
337
+ """
338
+ self._check_X(X)
339
+
340
+ self.n_features_in_ = X.shape[1]
341
+
342
+ ns = self.get_backend_namespace(X=X, y=y)
343
+ zeros = ns['zeros']
344
+
345
+ ## Regression algorithm
346
+ inv_XT_X_XT = self.prefit(X) if self.iXTXXT is None else self.iXTXXT
347
+ theta = inv_XT_X_XT @ y
348
+
349
+ ### Extract bias terms
350
+ if self.fit_intercept:
351
+ self.intercept_ = theta[-1]
352
+ self.coef_ = theta[:-1]
353
+ else:
354
+ self.intercept_ = zeros((y.shape[1],)) if y.ndim == 2 else 0
355
+ self.coef_ = theta
356
+
357
+ return self
358
+
359
+
360
+ class Ridge(LinearRegression_sk):
361
+ """
362
+ Implements Ridge regression based on the closed-form / 'analytic solution'. \n
363
+ NOTE: This class does not check for NaNs, Infs, singular matrices, etc. \n
364
+ This method is fast and accurate for small to medium-sized datasets. When
365
+ ``n_features`` is large and/or underdetermined (``n_samples`` <
366
+ ``n_features``), the solution will likely diverge from the sklearn solution. \n
367
+
368
+ RH 2023
369
+
370
+ Args:
371
+ alpha (float):
372
+ The regularization strength which must be a positive float.
373
+ Regularizes the estimate to prevent overfitting by constraining the
374
+ size of the coefficients. Usually ~1e5. (Default is 1)
375
+ fit_intercept (bool):
376
+ Whether to calculate the intercept for this model. If set to
377
+ ``False``, no intercept will be used in calculations (i.e., data is
378
+ expected to be centered). (Default is ``False``)
379
+ prefit_X (Optional[np.ndarray, torch.Tensor]):
380
+ Optional. If the ``X`` array is provided here, then ``inv(X.T @ X +
381
+ alpha*eye(X.shape[1])) @ X.T`` will be precomputed and stored in the
382
+ model.
383
+
384
+ Attributes:
385
+ alpha (float):
386
+ Regularization parameter.
387
+ fit_intercept (bool):
388
+ Specifies if the intercept should be calculated.
389
+ iXTXaeXT (Optional[np.ndarray]):
390
+ Stores the precomputed ``inv(X.T @ X + alpha*eye(X.shape[1])) @
391
+ X.T`` if precomputed, otherwise ``None``.
392
+ """
393
+ def __init__(
394
+ self,
395
+ alpha=1,
396
+ fit_intercept=True,
397
+ prefit_X=None,
398
+ **kwargs,
399
+ ):
400
+ """
401
+ Initializes the Ridge regression model with the specified alpha,
402
+ fit_intercept, and optional X_precompute.
403
+ """
404
+ super(Ridge, self).__init__()
405
+ self.alpha = alpha
406
+ self.fit_intercept = fit_intercept
407
+
408
+ self.iXTXaeXT = self.prefit(prefit_X) if prefit_X is not None else None
409
+ self.prefit_X = True if prefit_X is not None else False
410
+
411
+ def prefit(self, X):
412
+ """
413
+ Precomputes ``inv(X.T @ X + alpha*eye(X.shape[1])) @ X.T`` to be used in
414
+ the `fit` method.
415
+
416
+ Args:
417
+ X (np.ndarray):
418
+ The input features matrix.
419
+
420
+ Returns:
421
+ np.ndarray:
422
+ Precomputed inverse of ``inv(X.T @ X + alpha*eye(X.shape[1])) @
423
+ X.T``.
424
+ """
425
+ ns = self.get_backend_namespace(X=X)
426
+ cat, eye, inv, ones = (ns[key] for key in ['cat', 'eye', 'inv', 'ones'])
427
+ if self.fit_intercept:
428
+ X = cat((X, ones((X.shape[0], 1))), axis=1)
429
+
430
+ inv_XT_X_plus_alpha_eye_XT = inv(X.T @ X + self.alpha*eye(X.shape[1])) @ X.T
431
+ return inv_XT_X_plus_alpha_eye_XT
432
+
433
+ def fit(self, X, y):
434
+ """
435
+ Fits the Ridge regression model to the data.
436
+
437
+ Args:
438
+ X (np.ndarray):
439
+ Training data features.
440
+ y (np.ndarray):
441
+ Target values.
442
+
443
+ Returns:
444
+ Ridge:
445
+ The instance of this Ridge model.
446
+ """
447
+ self.n_features_in_ = X.shape[1]
448
+
449
+ ## Give a UserWarning if n_features >= n_samples
450
+ if X.shape[1] >= X.shape[0]:
451
+ import warnings
452
+ warnings.warn('OLS solution is expected to diverge from sklearn solution when n_features >= n_samples')
453
+
454
+ ns = self.get_backend_namespace(X=X, y=y)
455
+ zeros = ns['zeros']
456
+
457
+ ## Regression algorithm
458
+ inv_XT_X_plus_alpha_eye_XT = self.prefit(X) if self.iXTXaeXT is None else self.iXTXaeXT
459
+ theta = inv_XT_X_plus_alpha_eye_XT @ y
460
+
461
+ ### Extract bias terms
462
+ if self.fit_intercept:
463
+ self.intercept_ = theta[-1]
464
+ self.coef_ = theta[:-1]
465
+ else:
466
+ self.intercept_ = zeros((y.shape[1],)) if y.ndim == 2 else 0
467
+ self.coef_ = theta
468
+
469
+ return self
470
+
471
+
472
+ class ReducedRankRidgeRegression(LinearRegression_sk):
473
+ """
474
+ Implements a Reduced Rank Ridge Regression model using a closed-form
475
+ solution based on doing SVD on the coefficients of the Ridge regression Beta
476
+ matrix. This model reduces the rank of the coefficient matrix to address
477
+ multicollinearity and reduce model complexity. \n
478
+ NOTE: self.coef_ is stored as the dense beta coefficients, not as a Kruskal
479
+ tensor. \n
480
+
481
+ RH 2024
482
+
483
+ Args:
484
+ rank (int):
485
+ The target rank for the reduced rank regression model. It determines
486
+ the number of singular values to retain in the model. (Default is 3)
487
+ alpha (float):
488
+ The regularization strength which must be a positive float.
489
+ Regularizes the estimate to prevent overfitting by constraining the
490
+ size of the coefficients. Usually ~1e5. (Default is 1)
491
+ fit_intercept (bool):
492
+ Specifies whether to calculate the intercept for this model. If set
493
+ to ``True``, a column of ones is added to the feature matrix \(X\).
494
+ (Default is ``True``)
495
+ prefit_X (Optional[np.ndarray, torch.Tensor]):
496
+ Optional. If the ``X`` array is provided here, then ``inv(X^T X +
497
+ alpha*eye(X.shape[1])) @ X^T`` will be precomputed and stored
498
+ in the model.
499
+
500
+ Attributes:
501
+ rank (int):
502
+ The rank specified for reducing the regression model.
503
+ alpha (float):
504
+ Regularization parameter.
505
+ fit_intercept (bool):
506
+ Specifies if the intercept should be calculated.
507
+ iXTXaeXT (Optional[np.ndarray]):
508
+ Stores the precomputed ``inv(X.T @ X + alpha*eye(X.shape[1])) @
509
+ X.T`` if precomputed, otherwise ``None``.
510
+ """
511
+
512
+ def __init__(
513
+ self,
514
+ rank=3,
515
+ alpha=1,
516
+ fit_intercept=True,
517
+ prefit_X=None,
518
+ **kwargs,
519
+ ):
520
+ """
521
+ Initializes the Reduced Rank Regression model with the specified rank
522
+ and an optional
523
+ """
524
+ super(ReducedRankRidgeRegression, self).__init__()
525
+ self.fit_intercept = fit_intercept
526
+ self.rank = rank
527
+ self.alpha = alpha
528
+
529
+ self.iXTXXT = self.prefit(prefit_X) if prefit_X is not None else None
530
+ self.prefit_X = True if prefit_X is not None else False
531
+
532
+ def prefit(self, X):
533
+ """
534
+ Precomputes the \(X^T X + \alpha I\) inverse times \(X^T\) to be used in
535
+ the `fit` method.
536
+
537
+ Args:
538
+ X (np.ndarray):
539
+ The input features matrix.
540
+
541
+ Returns:
542
+ np.ndarray:
543
+ Precomputed inverse of \(X^T X + \alpha I\) times \(X^T\).
544
+ """
545
+ ns = self.get_backend_namespace(X=X)
546
+ cat, eye, inv, ones = (ns[key] for key in ['cat', 'eye', 'inv', 'ones'])
547
+ if self.fit_intercept:
548
+ X = cat((X, ones((X.shape[0], 1))), axis=1)
549
+
550
+ inv_XT_X_plus_alpha_eye_XT = inv(X.T @ X + self.alpha*eye(X.shape[1])) @ X.T
551
+ return inv_XT_X_plus_alpha_eye_XT
552
+
553
+ def fit(self, X, y, rank=None):
554
+ """
555
+ Fits the Reduced Rank Regression model to the data, applying a rank
556
+ reduction on the OLS regression coefficients.
557
+
558
+ Args:
559
+ X (np.ndarray):
560
+ Training data features.
561
+ y (np.ndarray):
562
+ Target values.
563
+ rank (Optional[int]):
564
+ The target rank for the reduced rank regression model. It
565
+ determines the number of singular values to retain in the model.
566
+ (Default is ``None``)
567
+
568
+ Returns:
569
+ ReducedRankRegression:
570
+ The instance of this Reduced Rank Regression model.
571
+ """
572
+ self.rank = self.rank if rank is None else rank
573
+ self.n_features_in_ = X.shape[1]
574
+ assert self.rank <= X.shape[1], 'Rank must be less than or equal to the number of features'
575
+
576
+ ns = self.get_backend_namespace(X=X, y=y)
577
+ svd, zeros = (ns[key] for key in ['svd', 'zeros'])
578
+
579
+ ## Calculate OLS solution
580
+ inv_XT_X_plus_alpha_eye_XT = self.prefit(X) if self.iXTXXT is None else self.iXTXXT
581
+ B_r = inv_XT_X_plus_alpha_eye_XT @ y
582
+
583
+ ## Calculate SVD of B_r
584
+ U, S, Vt = svd(B_r, full_matrices=False)
585
+
586
+ ## Truncate to rank
587
+ Vt = Vt[:self.rank]
588
+ B_rrr = B_r @ Vt.T @ Vt
589
+
590
+ ### Extract bias terms
591
+ if self.fit_intercept:
592
+ self.intercept_ = B_rrr[-1]
593
+ self.coef_ = B_rrr[:-1]
594
+ else:
595
+ self.intercept_ = zeros((y.shape[1],)) if y.ndim == 2 else 0
596
+ self.coef_ = B_rrr
597
+
598
+ return self
599
+
@@ -0,0 +1,33 @@
1
+ import numpy as np
2
+ import torch
3
+
4
+ def check_full_rank(X, a_thresh=None, r_thresh=1e-10):
5
+ """
6
+ Check if a matrix is full rank.
7
+
8
+ Args:
9
+ X (torch.Tensor):
10
+ The matrix to check.
11
+ Should be a 2D numpy array or torch.Tensor.
12
+ a_thresh (float, optional):
13
+ The absolute threshold for the singular values; singular values
14
+ below this threshold are considered zero.
15
+ If None, then not used.
16
+ r_thresh (float, optional):
17
+ The relative threshold for the singular values; singular values
18
+ below this threshold are considered zero.
19
+ Here, 'relative' means relative to the largest singular value.
20
+
21
+ Returns:
22
+ bool:
23
+ True if the matrix is full rank, False otherwise.
24
+ """
25
+ U, S, V = torch.linalg.svd(torch.as_tensor(X))
26
+
27
+ S_bool = torch.ones_like(S, dtype=bool)
28
+ if a_thresh is not None:
29
+ S_bool = S_bool & (S > a_thresh)
30
+ if r_thresh is not None:
31
+ S_bool = S_bool & (S > r_thresh * S[0])
32
+
33
+ return S_bool.all()
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2024 RichieHakim
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,82 @@
1
+ Metadata-Version: 2.1
2
+ Name: torch_linear_regression
3
+ Version: 0.1.0
4
+ Summary: A library of useful modules for data analysis.
5
+ Home-page: https://github.com/RichieHakim/torch_linear_regression
6
+ Author: Richard Hakim
7
+ License: LICENSE
8
+ Keywords: data analysis,machine learning,neuroscience
9
+ Description-Content-Type: text/markdown
10
+ License-File: LICENSE
11
+ Requires-Dist: numpy
12
+ Requires-Dist: scikit-learn
13
+ Requires-Dist: torch
14
+ Requires-Dist: hypothesis
15
+
16
+ [![PyPI version](https://badge.fury.io/py/torch_linear_regression.svg)](https://badge.fury.io/py/torch_linear_regression)
17
+ [![Downloads](https://pepy.tech/badge/torch_linear_regression)](https://pepy.tech/project/torch_linear_regression)
18
+ [![repo size](https://img.shields.io/github/repo-size/RichieHakim/torch_linear_regression)](https://github.com/RichieHakim/torch_linear_regression/)
19
+
20
+ # torch_linear_regression
21
+ A very simple library containing closed-form linear regression models using PyTorch.
22
+ Includes:
23
+ - Ordinary Least Squares (OLS) Linear Regression: `(X'X)^-1 X'Y`
24
+ - Ridge Regression: `(X'X + λI)^-1 X'Y`
25
+ - Reduced Rank Regression (RRR) with Ridge penalty: Ridge regression followed by SVD on the weights matrix
26
+
27
+ The closed-form approach results in fast and accurate results under most
28
+ conditions. However, when ``n_features`` is large and/or underdetermined
29
+ (``n_samples`` <= ``n_features``), the solution will start to diverge from
30
+ gradient-based / sklearn solutions.
31
+
32
+ Each model also includes a `model.prefit()` method that can be used to precompute
33
+ the inverse matrix and the ridge penalty matrix. This can be useful when the model
34
+ is used multiple times with the same `X` input data.
35
+
36
+ Because the models are based on PyTorch, they are significantly faster than sklearn's
37
+ models, and can be further accelerated by using GPU. Also the models can be used
38
+ in conjunction with PyTorch's autograd for gradient-based optimization.
39
+
40
+ ## Installation
41
+ Install stable version:
42
+ ```
43
+ pip install torch_linear_regression
44
+ ```
45
+
46
+ Install development version:
47
+ ```
48
+ pip install git+https://github.com/RichieHakim/torch_linear_regression.git
49
+ ```
50
+
51
+ ## Usage
52
+ See the notebook for more examples: [demo notebook](https://github.com/RichieHakim/torch_linear_regression/blob/master/demo_notebook.ipynb)
53
+ ```
54
+ import torch_linear_regression as tlr
55
+
56
+ import torch
57
+ import numpy as np
58
+ import sklearn
59
+ import sklearn.datasets
60
+ import matplotlib.pyplot as plt
61
+
62
+
63
+ ## Generate data for regression
64
+ X, Y = sklearn.datasets.make_regression(
65
+ n_samples=100,
66
+ n_features=2,
67
+ n_informative=10,
68
+ bias=2,
69
+ noise=50,
70
+ random_state=42,
71
+ )
72
+
73
+ ## Create model
74
+ model_ols = tlr.OLS()
75
+ ## Fit model
76
+ model_ols.fit(X=X, y=Y)
77
+ ## Predict
78
+ Y_pred = model_ols.predict(X)
79
+ ## Score model
80
+ score = model_ols.score(X=X, y=Y)
81
+ print(f"R^2: {score}")
82
+ ```
@@ -0,0 +1,8 @@
1
+ torch_linear_regression/__init__.py,sha256=Y81FNWQ8A4_d-4rhbYrkndgI0EhPTYatQC7nHDXbdx0,148
2
+ torch_linear_regression/linear_regression.py,sha256=tqfj57OYtOpnfT-1vlCiVCkOjhE_mhdzlAsgSf_XYyM,20684
3
+ torch_linear_regression/utils.py,sha256=hqe1RwKr1pmK4BmihrPijxxNBWjPzDnMtBBIwGzgXsQ,1069
4
+ torch_linear_regression-0.1.0.dist-info/LICENSE,sha256=W62j-ciEr8-wG9kAD7NXeiucEWfe3uck_85dn_5c1OY,1068
5
+ torch_linear_regression-0.1.0.dist-info/METADATA,sha256=0_W9uSaYgWlMYtwTL8DrYm3-X5YxJ3yKtgLzlAm-KZ8,2752
6
+ torch_linear_regression-0.1.0.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
7
+ torch_linear_regression-0.1.0.dist-info/top_level.txt,sha256=dMGLxDaNOOCUfmABgI9jDi4392U2TojAe_196mHhk-I,24
8
+ torch_linear_regression-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (73.0.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1 @@
1
+ torch_linear_regression