commsubspace 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- commsubspace/__init__.py +10 -0
- commsubspace/_linalg.py +64 -0
- commsubspace/_noise_floor.py +107 -0
- commsubspace/communication_subspace.py +523 -0
- commsubspace/contrastive_communication_subspace.py +556 -0
- commsubspace-0.1.0.dist-info/METADATA +68 -0
- commsubspace-0.1.0.dist-info/RECORD +10 -0
- commsubspace-0.1.0.dist-info/WHEEL +5 -0
- commsubspace-0.1.0.dist-info/licenses/LICENSE +21 -0
- commsubspace-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,523 @@
|
|
|
1
|
+
# file for common methods used in communication subspace analysis
|
|
2
|
+
#
|
|
3
|
+
# Conventions: data arrays are (n_samples, n_features); X is the source/predictor
|
|
4
|
+
# population and Y the target population. Units are those of the input data unless
|
|
5
|
+
# a method states that it standardizes (z-scores) first.
|
|
6
|
+
|
|
7
|
+
import warnings
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
from scipy.stats import zscore
|
|
11
|
+
from sklearn.linear_model import LinearRegression, Ridge, RidgeCV
|
|
12
|
+
from sklearn.model_selection import KFold
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _column_statistics(data, ddof):
|
|
16
|
+
"""Column means and standard deviations of ``data``, (n, f) -> two (f,) arrays.
|
|
17
|
+
|
|
18
|
+
Columns with zero spread get std = 1 so standardizing them yields zeros and
|
|
19
|
+
never NaN.
|
|
20
|
+
"""
|
|
21
|
+
mean = np.mean(data, axis=0)
|
|
22
|
+
std = np.std(data, axis=0, ddof=ddof)
|
|
23
|
+
return mean, np.where(std == 0, 1.0, std)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _as_2d_float(data):
|
|
27
|
+
"""Convert to a float array and turn a 1-D vector into a single column, (n,) -> (n, 1)."""
|
|
28
|
+
data = np.asarray(data, dtype=float)
|
|
29
|
+
return data.reshape(-1, 1) if data.ndim == 1 else data
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def select_rank_one_standard_error(fold_scores, ranks_vec):
|
|
33
|
+
"""One-standard-error rule for choosing a rank from cross-validated scores.
|
|
34
|
+
|
|
35
|
+
Parameters
|
|
36
|
+
----------
|
|
37
|
+
fold_scores : ndarray, (n_folds, n_ranks)
|
|
38
|
+
Cross-validated score (e.g. R^2) of every candidate rank in every fold.
|
|
39
|
+
ranks_vec : sequence of int, length n_ranks
|
|
40
|
+
Rank evaluated in each column of ``fold_scores``.
|
|
41
|
+
|
|
42
|
+
Returns
|
|
43
|
+
-------
|
|
44
|
+
int
|
|
45
|
+
Smallest rank whose mean score is within one standard error (std over
|
|
46
|
+
folds / sqrt(n_folds), evaluated at the best-scoring rank) of the best
|
|
47
|
+
mean score.
|
|
48
|
+
"""
|
|
49
|
+
fold_scores = np.asarray(fold_scores, dtype=float)
|
|
50
|
+
mean_scores = fold_scores.mean(axis=0)
|
|
51
|
+
standard_errors = fold_scores.std(axis=0) / np.sqrt(fold_scores.shape[0])
|
|
52
|
+
best_index = int(np.argmax(mean_scores))
|
|
53
|
+
threshold = mean_scores[best_index] - standard_errors[best_index]
|
|
54
|
+
qualifying = [rank for rank, mean in zip(ranks_vec, mean_scores) if mean >= threshold]
|
|
55
|
+
return min(qualifying)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class CrossDecomposition:
|
|
59
|
+
"""
|
|
60
|
+
A simplified cross-decomposition class supporting two methods:
|
|
61
|
+
- 'covariance' (equivalent to PLS-SVD)
|
|
62
|
+
- 'correlation' (equivalent to CCA)
|
|
63
|
+
|
|
64
|
+
Parameters
|
|
65
|
+
----------
|
|
66
|
+
method : {'covariance', 'correlation'}
|
|
67
|
+
scale : bool
|
|
68
|
+
If True, X and Y are standardized (mean 0, std 1, ddof=1) with statistics
|
|
69
|
+
estimated in ``fit``; the same training statistics are reused by every
|
|
70
|
+
later ``transform``/``score`` call.
|
|
71
|
+
n_components : int or np.inf
|
|
72
|
+
Requested number of components. ``np.inf`` (default) means "as many as the
|
|
73
|
+
data supports". This hyperparameter is never modified by ``fit``; the number
|
|
74
|
+
actually used is ``n_components_``.
|
|
75
|
+
|
|
76
|
+
Attributes
|
|
77
|
+
----------
|
|
78
|
+
x_weights_ : ndarray, (n_features_x, n_components_)
|
|
79
|
+
Weights (loadings) of X, in the (standardized, if scale) units of X.
|
|
80
|
+
y_weights_ : ndarray, (n_features_y, n_components_)
|
|
81
|
+
singular_values_ : ndarray, (n_components_,)
|
|
82
|
+
Singular values of the cross-covariance (covariance method) or canonical
|
|
83
|
+
correlations (correlation method), computed on the training data.
|
|
84
|
+
n_components_ : int
|
|
85
|
+
Number of components kept, min(requested, rank of X, rank of Y).
|
|
86
|
+
x_mean_, x_std_, y_mean_, y_std_ : ndarray, (n_features,)
|
|
87
|
+
Training mean / std (ddof=1) used for standardization.
|
|
88
|
+
|
|
89
|
+
Notes
|
|
90
|
+
-----
|
|
91
|
+
- The number of dimensions returned is limited by the rank of X and Y.
|
|
92
|
+
- The correlation method is unregularized: with fewer samples than features
|
|
93
|
+
the canonical correlations are inflated (a warning is issued).
|
|
94
|
+
|
|
95
|
+
Example Usage
|
|
96
|
+
-------------
|
|
97
|
+
>>> X = np.random.randn(100, 5) # 5 features
|
|
98
|
+
>>> Y = np.random.randn(100, 4) # 4 features
|
|
99
|
+
|
|
100
|
+
>>> model = CrossDecomposition(method='covariance', scale=True)
|
|
101
|
+
>>> model.fit(X, Y)
|
|
102
|
+
>>> X_scores, Y_scores = model.transform(X, Y)
|
|
103
|
+
>>> print(X_scores.shape, Y_scores.shape) # (100, 4), (100, 4)
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
# ------------------------------------------------------------------
|
|
107
|
+
# USER-FACING METHODS
|
|
108
|
+
# ------------------------------------------------------------------
|
|
109
|
+
def __init__(self, method='covariance', scale=True, n_components=np.inf):
|
|
110
|
+
self.method = method
|
|
111
|
+
self.scale = scale
|
|
112
|
+
self.n_components = n_components
|
|
113
|
+
self.is_fitted_ = False
|
|
114
|
+
|
|
115
|
+
def fit(self, X, Y):
|
|
116
|
+
"""
|
|
117
|
+
Fit the cross-decomposition model on X and Y.
|
|
118
|
+
|
|
119
|
+
Parameters
|
|
120
|
+
----------
|
|
121
|
+
X : ndarray of shape (n_samples, n_features_x)
|
|
122
|
+
Y : ndarray of shape (n_samples, n_features_y)
|
|
123
|
+
|
|
124
|
+
Returns
|
|
125
|
+
-------
|
|
126
|
+
self : object
|
|
127
|
+
Fitted model instance.
|
|
128
|
+
"""
|
|
129
|
+
X = np.asarray(X, dtype=float)
|
|
130
|
+
Y = np.asarray(Y, dtype=float)
|
|
131
|
+
self.x_mean_, self.x_std_ = _column_statistics(X, ddof=1)
|
|
132
|
+
self.y_mean_, self.y_std_ = _column_statistics(Y, ddof=1)
|
|
133
|
+
|
|
134
|
+
Xs = self._standardize(X, self.x_mean_, self.x_std_) if self.scale else X
|
|
135
|
+
Ys = self._standardize(Y, self.y_mean_, self.y_std_) if self.scale else Y
|
|
136
|
+
|
|
137
|
+
max_dimension = int(min(np.linalg.matrix_rank(Xs), np.linalg.matrix_rank(Ys)))
|
|
138
|
+
self.n_components_ = self._resolve_n_components(max_dimension)
|
|
139
|
+
|
|
140
|
+
method = self.method.lower()
|
|
141
|
+
if method == 'correlation':
|
|
142
|
+
x_weights, y_weights, singular_values = self._fit_correlation(Xs, Ys)
|
|
143
|
+
elif method == 'covariance':
|
|
144
|
+
x_weights, y_weights, singular_values = self._fit_covariance(Xs, Ys)
|
|
145
|
+
else:
|
|
146
|
+
raise ValueError("Unsupported method '{}'. Use 'correlation' or 'covariance'.".format(self.method))
|
|
147
|
+
|
|
148
|
+
self.x_weights_ = x_weights[:, :self.n_components_]
|
|
149
|
+
self.y_weights_ = y_weights[:, :self.n_components_]
|
|
150
|
+
self.singular_values_ = singular_values[:self.n_components_]
|
|
151
|
+
self.is_fitted_ = True
|
|
152
|
+
return self
|
|
153
|
+
|
|
154
|
+
def transform(self, X, Y=None, scale=None):
|
|
155
|
+
"""
|
|
156
|
+
Project data X (and optionally Y) onto the latent components.
|
|
157
|
+
|
|
158
|
+
Parameters
|
|
159
|
+
----------
|
|
160
|
+
X : ndarray of shape (n_samples, n_features_x)
|
|
161
|
+
Y : ndarray of shape (n_samples, n_features_y), optional
|
|
162
|
+
scale : bool or None
|
|
163
|
+
Standardize with the *training* statistics before projecting.
|
|
164
|
+
``None`` (default) follows the ``scale`` setting used in ``fit``.
|
|
165
|
+
|
|
166
|
+
Returns
|
|
167
|
+
-------
|
|
168
|
+
X_scores : ndarray of shape (n_samples, n_components_)
|
|
169
|
+
The scores for X in the new latent space.
|
|
170
|
+
Y_scores : ndarray of shape (n_samples, n_components_), optional
|
|
171
|
+
The scores for Y in the new latent space (returned only if Y is provided).
|
|
172
|
+
"""
|
|
173
|
+
self._check_fitted()
|
|
174
|
+
use_scale = self.scale if scale is None else scale
|
|
175
|
+
X = np.asarray(X, dtype=float)
|
|
176
|
+
if use_scale:
|
|
177
|
+
X = self._standardize(X, self.x_mean_, self.x_std_)
|
|
178
|
+
X_scores = X @ self.x_weights_
|
|
179
|
+
if Y is None:
|
|
180
|
+
return X_scores
|
|
181
|
+
|
|
182
|
+
Y = np.asarray(Y, dtype=float)
|
|
183
|
+
if use_scale:
|
|
184
|
+
Y = self._standardize(Y, self.y_mean_, self.y_std_)
|
|
185
|
+
return X_scores, Y @ self.y_weights_
|
|
186
|
+
|
|
187
|
+
def score(self, X, Y):
|
|
188
|
+
"""
|
|
189
|
+
Pearson correlation between the X and Y scores of each component.
|
|
190
|
+
|
|
191
|
+
Parameters
|
|
192
|
+
----------
|
|
193
|
+
X : ndarray of shape (n_samples, n_features_x)
|
|
194
|
+
Y : ndarray of shape (n_samples, n_features_y)
|
|
195
|
+
|
|
196
|
+
Returns
|
|
197
|
+
-------
|
|
198
|
+
ndarray of shape (n_components_,)
|
|
199
|
+
Correlation of the projected scores, per component, for both methods
|
|
200
|
+
(dimensionless). Scores are projected following the ``scale`` setting.
|
|
201
|
+
"""
|
|
202
|
+
X_scores, Y_scores = self.transform(X, Y)
|
|
203
|
+
X_centered = X_scores - X_scores.mean(axis=0)
|
|
204
|
+
Y_centered = Y_scores - Y_scores.mean(axis=0)
|
|
205
|
+
covariance = np.sum(X_centered * Y_centered, axis=0)
|
|
206
|
+
spread = np.sqrt(np.sum(X_centered ** 2, axis=0) * np.sum(Y_centered ** 2, axis=0))
|
|
207
|
+
return covariance / spread
|
|
208
|
+
|
|
209
|
+
# ------------------------------------------------------------------
|
|
210
|
+
# HELPER METHODS
|
|
211
|
+
# ------------------------------------------------------------------
|
|
212
|
+
def _check_fitted(self):
|
|
213
|
+
# method to check if the model has been fitted.
|
|
214
|
+
if not self.is_fitted_:
|
|
215
|
+
raise RuntimeError("The model must be fitted before calling transform().")
|
|
216
|
+
|
|
217
|
+
@staticmethod
|
|
218
|
+
def _standardize(data, mean, std):
|
|
219
|
+
"""(data - mean) / std with stored training statistics; (n, f) -> (n, f)."""
|
|
220
|
+
return (data - mean) / std
|
|
221
|
+
|
|
222
|
+
def _resolve_n_components(self, max_dimension):
|
|
223
|
+
"""Number of components to keep; warns only if the user asked for too many."""
|
|
224
|
+
if np.isinf(self.n_components):
|
|
225
|
+
return max_dimension
|
|
226
|
+
if self.n_components > max_dimension:
|
|
227
|
+
warnings.warn("The number of components is higher than the rank of the data, "
|
|
228
|
+
"reducing the components to: {}".format(max_dimension))
|
|
229
|
+
return max_dimension
|
|
230
|
+
return int(self.n_components)
|
|
231
|
+
|
|
232
|
+
def _fit_covariance(self, Xs, Ys):
|
|
233
|
+
"""PLS-SVD: SVD of the cross-covariance of (scaled) data.
|
|
234
|
+
|
|
235
|
+
Returns weights (n_features_x, k), (n_features_y, k) and singular values (k,),
|
|
236
|
+
with k = min(n_features_x, n_features_y); singular values are those of
|
|
237
|
+
Xs^T Ys / (n_samples - 1).
|
|
238
|
+
"""
|
|
239
|
+
U, S, Vt = np.linalg.svd(Xs.T @ Ys, full_matrices=False)
|
|
240
|
+
return U, Vt.T, S / (Xs.shape[0] - 1)
|
|
241
|
+
|
|
242
|
+
def _fit_correlation(self, Xs, Ys):
|
|
243
|
+
"""CCA: SVD of the whitened cross-covariance.
|
|
244
|
+
|
|
245
|
+
Returns weights (n_features_x, k), (n_features_y, k) and canonical
|
|
246
|
+
correlations (k,), with k = min(n_features_x, n_features_y).
|
|
247
|
+
"""
|
|
248
|
+
n_samples = Xs.shape[0]
|
|
249
|
+
if n_samples - 1 < max(Xs.shape[1], Ys.shape[1]):
|
|
250
|
+
warnings.warn("Fewer samples than features: the canonical correlations are "
|
|
251
|
+
"inflated (rank-deficient covariance matrices); interpret with care.")
|
|
252
|
+
Sigma_XY = (Xs.T @ Ys) / (n_samples - 1)
|
|
253
|
+
inv_sqrt_x = self._inverse_sqrt_covariance(Xs)
|
|
254
|
+
inv_sqrt_y = self._inverse_sqrt_covariance(Ys)
|
|
255
|
+
|
|
256
|
+
U, s, Vt = np.linalg.svd(inv_sqrt_x @ Sigma_XY @ inv_sqrt_y, full_matrices=False)
|
|
257
|
+
return inv_sqrt_x @ U, inv_sqrt_y @ Vt.T, s
|
|
258
|
+
|
|
259
|
+
@staticmethod
|
|
260
|
+
def _inverse_sqrt_covariance(data):
|
|
261
|
+
"""C^(-1/2) of the sample covariance of ``data``; (n, f) -> (f, f).
|
|
262
|
+
|
|
263
|
+
Eigenvalues are floored at 1e-12 to avoid division by zero.
|
|
264
|
+
"""
|
|
265
|
+
# TODO: add a tol parameter.
|
|
266
|
+
covariance = (data.T @ data) / (data.shape[0] - 1)
|
|
267
|
+
eigvals, eigvecs = np.linalg.eigh(covariance)
|
|
268
|
+
eigvals = np.where(eigvals < 1e-12, 1e-12, eigvals)
|
|
269
|
+
return eigvecs @ np.diag(1.0 / np.sqrt(eigvals)) @ eigvecs.T
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
class ReducedRankRegress:
|
|
273
|
+
"""
|
|
274
|
+
Reduced Rank Regression
|
|
275
|
+
- X is an n-by-d matrix of features.
|
|
276
|
+
- Y is an n-by-p matrix of targets.
|
|
277
|
+
- rank is a rank constraint (never modified by ``fit``; see ``rank_``).
|
|
278
|
+
- ridge_init: boolean parameter for intiate the weights with ridge regression
|
|
279
|
+
- scale: if True, X and Y are z-scored (ddof=0) with training statistics
|
|
280
|
+
estimated in ``fit`` and reused by ``predict`` and ``score``.
|
|
281
|
+
|
|
282
|
+
Attributes
|
|
283
|
+
----------
|
|
284
|
+
B : ndarray, (d, p)
|
|
285
|
+
Rank-constrained coefficients in standardized units (if scale) mapping
|
|
286
|
+
standardized X to standardized Y. Alias: ``coef_``.
|
|
287
|
+
B_ : ndarray, (d, rank_)
|
|
288
|
+
Predictive dimensions: standardized X @ B_ gives the latent scores.
|
|
289
|
+
Alias: ``predictive_dims_``.
|
|
290
|
+
rank_ : int
|
|
291
|
+
Rank actually used, min(rank, p).
|
|
292
|
+
x_mean_, x_std_, y_mean_, y_std_ : ndarray
|
|
293
|
+
Training statistics used for standardization.
|
|
294
|
+
"""
|
|
295
|
+
|
|
296
|
+
@staticmethod
|
|
297
|
+
def data_for_test(nsamples=10000, nfeatures=100, ntargets=40, rank=5, noise_sigma=1,
|
|
298
|
+
random_state=None):
|
|
299
|
+
"""Create synthetic data with a rank-``rank`` linear relation.
|
|
300
|
+
|
|
301
|
+
Returns
|
|
302
|
+
-------
|
|
303
|
+
X : ndarray, (nsamples, nfeatures)
|
|
304
|
+
Y : ndarray, (nsamples, ntargets)
|
|
305
|
+
"""
|
|
306
|
+
rng = np.random.default_rng(random_state)
|
|
307
|
+
X = rng.standard_normal((nsamples, nfeatures))
|
|
308
|
+
B = np.dot(rng.standard_normal((nfeatures, rank)), rng.standard_normal((rank, ntargets)))
|
|
309
|
+
Y = np.dot(X, B) + (rng.standard_normal((nsamples, ntargets)) * noise_sigma)
|
|
310
|
+
return X, Y
|
|
311
|
+
|
|
312
|
+
# ------------------------------------------------------------------
|
|
313
|
+
# USER-FACING METHODS
|
|
314
|
+
# ------------------------------------------------------------------
|
|
315
|
+
def __init__(self, rank, ridge_init=True, scale=True):
|
|
316
|
+
self.rank = rank
|
|
317
|
+
self.ridge_init = ridge_init
|
|
318
|
+
self.scale = scale
|
|
319
|
+
|
|
320
|
+
def __str__(self):
|
|
321
|
+
return 'Reduced Rank Regressor (rank = {})'.format(self.rank)
|
|
322
|
+
|
|
323
|
+
def fit(self, X, Y):
|
|
324
|
+
"""Fit the reduced-rank model.
|
|
325
|
+
|
|
326
|
+
Parameters
|
|
327
|
+
----------
|
|
328
|
+
X : ndarray, (n_samples, n_features) (a 1-D vector is treated as one feature)
|
|
329
|
+
Y : ndarray, (n_samples, n_targets)
|
|
330
|
+
|
|
331
|
+
Returns
|
|
332
|
+
-------
|
|
333
|
+
self
|
|
334
|
+
"""
|
|
335
|
+
X, Y = _as_2d_float(X), _as_2d_float(Y)
|
|
336
|
+
self.x_mean_, self.x_std_ = _column_statistics(X, ddof=0)
|
|
337
|
+
self.y_mean_, self.y_std_ = _column_statistics(Y, ddof=0)
|
|
338
|
+
if self.scale:
|
|
339
|
+
X = (X - self.x_mean_) / self.x_std_
|
|
340
|
+
Y = (Y - self.y_mean_) / self.y_std_
|
|
341
|
+
|
|
342
|
+
full_coefficients, Yhat = self._fit_full_regression(X, Y)
|
|
343
|
+
directions = self._principal_directions(Yhat) # (n_targets, n_targets), rows
|
|
344
|
+
|
|
345
|
+
self.rank_ = self._resolve_rank(directions.shape[0])
|
|
346
|
+
top_directions = directions[:self.rank_, :]
|
|
347
|
+
self.B = np.linalg.multi_dot((full_coefficients.T, top_directions.T, top_directions))
|
|
348
|
+
self.B_ = np.dot(full_coefficients.T, top_directions.T)
|
|
349
|
+
return self
|
|
350
|
+
|
|
351
|
+
def predict(self, X):
|
|
352
|
+
"""Predict Y from X.
|
|
353
|
+
|
|
354
|
+
Parameters
|
|
355
|
+
----------
|
|
356
|
+
X : ndarray, (n_samples, n_features)
|
|
357
|
+
|
|
358
|
+
Returns
|
|
359
|
+
-------
|
|
360
|
+
ndarray, (n_samples, n_targets)
|
|
361
|
+
Predictions in the original units of Y (training statistics are used
|
|
362
|
+
to standardize X and to undo the standardization of Y).
|
|
363
|
+
"""
|
|
364
|
+
X = _as_2d_float(X)
|
|
365
|
+
if self.scale:
|
|
366
|
+
X = (X - self.x_mean_) / self.x_std_
|
|
367
|
+
prediction = np.dot(X, self.B)
|
|
368
|
+
if self.scale:
|
|
369
|
+
prediction = prediction * self.y_std_ + self.y_mean_
|
|
370
|
+
return prediction
|
|
371
|
+
|
|
372
|
+
def score(self, X, Y):
|
|
373
|
+
"""R^2 of the model on (X, Y), pooled over targets.
|
|
374
|
+
|
|
375
|
+
With ``scale=True`` both X and Y are standardized with the training
|
|
376
|
+
statistics first, so R^2 is measured in standardized units (every target
|
|
377
|
+
counts equally), consistent with ``get_best_rank``.
|
|
378
|
+
|
|
379
|
+
Parameters
|
|
380
|
+
----------
|
|
381
|
+
X : ndarray, (n_samples, n_features)
|
|
382
|
+
Y : ndarray, (n_samples, n_targets)
|
|
383
|
+
|
|
384
|
+
Returns
|
|
385
|
+
-------
|
|
386
|
+
float
|
|
387
|
+
"""
|
|
388
|
+
X, Y = _as_2d_float(X), _as_2d_float(Y)
|
|
389
|
+
if self.scale:
|
|
390
|
+
X = (X - self.x_mean_) / self.x_std_
|
|
391
|
+
Y = (Y - self.y_mean_) / self.y_std_
|
|
392
|
+
Yhat = np.dot(X, self.B)
|
|
393
|
+
rss = np.sum(np.square(Yhat - Y))
|
|
394
|
+
tss = np.sum(np.square(Y - np.mean(Y, axis=0)))
|
|
395
|
+
return 1 - (rss / tss)
|
|
396
|
+
|
|
397
|
+
def get_ridge_lambda(self, X, dMaxShrink=np.arange(0.5, 1, 0.01), scale=True):
|
|
398
|
+
"""Ridge penalty grid from the largest eigenvalue of the (z-scored) X Gram matrix."""
|
|
399
|
+
m = np.mean(X, axis=0)
|
|
400
|
+
s = np.std(X, axis=0)
|
|
401
|
+
|
|
402
|
+
#exclude neurons that are zeroed variance
|
|
403
|
+
idxs = s != 0
|
|
404
|
+
if np.sum(idxs) > 0:
|
|
405
|
+
X = X[:, idxs]
|
|
406
|
+
m = m[idxs]
|
|
407
|
+
|
|
408
|
+
if scale:
|
|
409
|
+
Z = zscore(X)
|
|
410
|
+
else:
|
|
411
|
+
Z = X - m
|
|
412
|
+
|
|
413
|
+
#determining lambda
|
|
414
|
+
d, _ = np.linalg.eigh(np.dot(Z.T, Z))
|
|
415
|
+
dMax = max(d)
|
|
416
|
+
return np.divide(dMax * (1 - dMaxShrink), dMaxShrink)
|
|
417
|
+
|
|
418
|
+
def get_best_rank(self, X, Y, ranks_vec):
|
|
419
|
+
"""Cross-validated choice of the rank (5 contiguous folds, one-SE rule).
|
|
420
|
+
|
|
421
|
+
Each test fold is standardized with the statistics of its training folds.
|
|
422
|
+
|
|
423
|
+
Parameters
|
|
424
|
+
----------
|
|
425
|
+
X : ndarray, (n_samples, n_features)
|
|
426
|
+
Y : ndarray, (n_samples, n_targets)
|
|
427
|
+
ranks_vec : sequence of int
|
|
428
|
+
Candidate ranks.
|
|
429
|
+
|
|
430
|
+
Returns
|
|
431
|
+
-------
|
|
432
|
+
best_rank : int
|
|
433
|
+
best_score : float
|
|
434
|
+
Mean cross-validated R^2 of ``best_rank``.
|
|
435
|
+
rsquare : ndarray, (n_folds, n_ranks)
|
|
436
|
+
R^2 of each candidate rank in each fold.
|
|
437
|
+
"""
|
|
438
|
+
X, Y = _as_2d_float(X), _as_2d_float(Y)
|
|
439
|
+
fold_scores = [self._score_ranks_on_fold(X[train], Y[train], X[test], Y[test], ranks_vec)
|
|
440
|
+
for train, test in KFold().split(X)]
|
|
441
|
+
rsquare = np.vstack(fold_scores)
|
|
442
|
+
|
|
443
|
+
best_rank = select_rank_one_standard_error(rsquare, ranks_vec)
|
|
444
|
+
best_index = list(ranks_vec).index(best_rank)
|
|
445
|
+
|
|
446
|
+
self.best_rank_ = best_rank
|
|
447
|
+
self.best_rank_score_ = rsquare.mean(axis=0)[best_index]
|
|
448
|
+
self.best_rank_scores_ = rsquare
|
|
449
|
+
return self.best_rank_, self.best_rank_score_, rsquare
|
|
450
|
+
|
|
451
|
+
# aliases with scikit-learn style names
|
|
452
|
+
@property
|
|
453
|
+
def coef_(self):
|
|
454
|
+
return self.B
|
|
455
|
+
|
|
456
|
+
@property
|
|
457
|
+
def predictive_dims_(self):
|
|
458
|
+
return self.B_
|
|
459
|
+
|
|
460
|
+
# ------------------------------------------------------------------
|
|
461
|
+
# HELPER METHODS
|
|
462
|
+
# ------------------------------------------------------------------
|
|
463
|
+
def _fit_full_regression(self, X, Y):
|
|
464
|
+
"""Full-rank ridge (or OLS) fit.
|
|
465
|
+
|
|
466
|
+
Returns
|
|
467
|
+
-------
|
|
468
|
+
coefficients : ndarray, (n_targets, n_features) (scikit-learn ``coef_`` layout)
|
|
469
|
+
Yhat : ndarray, (n_samples, n_targets) fitted values on the training data
|
|
470
|
+
"""
|
|
471
|
+
if self.ridge_init:
|
|
472
|
+
alphas = self.get_ridge_lambda(X)
|
|
473
|
+
best_alpha = RidgeCV(alphas=alphas, cv=KFold(), fit_intercept=False).fit(X, Y).alpha_
|
|
474
|
+
model = Ridge(alpha=best_alpha, fit_intercept=False).fit(X, Y)
|
|
475
|
+
else:
|
|
476
|
+
model = LinearRegression(fit_intercept=False).fit(X, Y)
|
|
477
|
+
return model.coef_, model.predict(X)
|
|
478
|
+
|
|
479
|
+
@staticmethod
|
|
480
|
+
def _principal_directions(Yhat):
|
|
481
|
+
"""Principal directions of the fitted values, as rows sorted by decreasing variance.
|
|
482
|
+
|
|
483
|
+
Yhat : (n_samples, n_targets) -> (n_targets, n_targets)
|
|
484
|
+
Uses an eigen-decomposition of the covariance instead of an SVD for speed.
|
|
485
|
+
"""
|
|
486
|
+
covariance = np.dot(Yhat.T, Yhat) / (Yhat.shape[0] - 1)
|
|
487
|
+
eigenvalues, eigenvectors = np.linalg.eigh(covariance)
|
|
488
|
+
order = np.argsort(eigenvalues)[::-1]
|
|
489
|
+
return eigenvectors[:, order].T
|
|
490
|
+
|
|
491
|
+
def _resolve_rank(self, n_targets):
|
|
492
|
+
"""Rank actually used; warns if the requested rank exceeds the number of targets."""
|
|
493
|
+
if self.rank > n_targets:
|
|
494
|
+
warnings.warn("The number of components is higher than the rank of the data, "
|
|
495
|
+
"reducing the components to: {}".format(n_targets))
|
|
496
|
+
return n_targets
|
|
497
|
+
return self.rank
|
|
498
|
+
|
|
499
|
+
def _score_ranks_on_fold(self, X_train, Y_train, X_test, Y_test, ranks_vec):
|
|
500
|
+
"""R^2 on the test fold of the model at every candidate rank.
|
|
501
|
+
|
|
502
|
+
Standardization (if ``scale``) uses the training fold only.
|
|
503
|
+
Returns ndarray (n_ranks,).
|
|
504
|
+
"""
|
|
505
|
+
if self.scale:
|
|
506
|
+
x_mean, x_std = _column_statistics(X_train, ddof=0)
|
|
507
|
+
y_mean, y_std = _column_statistics(Y_train, ddof=0)
|
|
508
|
+
X_train, X_test = (X_train - x_mean) / x_std, (X_test - x_mean) / x_std
|
|
509
|
+
Y_train, Y_test = (Y_train - y_mean) / y_std, (Y_test - y_mean) / y_std
|
|
510
|
+
# missing values are treated as zero signal (as in the original implementation)
|
|
511
|
+
X_train, X_test = np.nan_to_num(X_train), np.nan_to_num(X_test)
|
|
512
|
+
Y_train, Y_test = np.nan_to_num(Y_train), np.nan_to_num(Y_test)
|
|
513
|
+
|
|
514
|
+
full_coefficients, Yhat = self._fit_full_regression(X_train, Y_train)
|
|
515
|
+
directions = self._principal_directions(Yhat)
|
|
516
|
+
|
|
517
|
+
total_ss = np.sum(np.square(Y_test - np.mean(Y_test, axis=0)))
|
|
518
|
+
scores = []
|
|
519
|
+
for rank in ranks_vec:
|
|
520
|
+
B = np.linalg.multi_dot((full_coefficients.T, directions[:rank].T, directions[:rank]))
|
|
521
|
+
residual_ss = np.sum(np.square(np.dot(X_test, B) - Y_test))
|
|
522
|
+
scores.append(1 - residual_ss / total_ss)
|
|
523
|
+
return np.array(scores)
|