TensorFox 1.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- TensorFox/Alternating_Least_Squares.py +264 -0
- TensorFox/Auxiliar.py +686 -0
- TensorFox/Compression.py +624 -0
- TensorFox/Conversion.py +494 -0
- TensorFox/Critical.py +3320 -0
- TensorFox/Display.py +466 -0
- TensorFox/GaussNewton.py +811 -0
- TensorFox/Initialization.py +386 -0
- TensorFox/MultilinearAlgebra.py +649 -0
- TensorFox/TensorFox.py +1194 -0
- TensorFox/TensorLearning.py +1103 -0
- TensorFox/__init__.py +7 -0
- tensorfox-1.2.2.dist-info/METADATA +257 -0
- tensorfox-1.2.2.dist-info/RECORD +17 -0
- tensorfox-1.2.2.dist-info/WHEEL +5 -0
- tensorfox-1.2.2.dist-info/licenses/LICENSE +674 -0
- tensorfox-1.2.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1103 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
from numpy import prod, uint64
|
|
3
|
+
import sys
|
|
4
|
+
import matplotlib.pyplot as plt
|
|
5
|
+
from numba import njit, prange
|
|
6
|
+
import TensorFox.TensorFox as tfx
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
# PREPROCESSING
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def normalization(X, method='mean_normalization'):
|
|
13
|
+
"""
|
|
14
|
+
Normalize data by shifting all data by a certain constant, and dividing all data by other certain constant. The
|
|
15
|
+
program returns the normalized data X = (X - mu)/sigma. In the case the user want to return to the original format,
|
|
16
|
+
set X = sigma*X + mu.
|
|
17
|
+
|
|
18
|
+
Inputs
|
|
19
|
+
------
|
|
20
|
+
X: 2D array
|
|
21
|
+
Train data in 2D array format. Each row correspond to a single sample of the data.
|
|
22
|
+
method: string
|
|
23
|
+
There are 4 available methods of feature scaling.
|
|
24
|
+
min_max_normalization: X = (X - min(X)) / (max(X) - min(X))
|
|
25
|
+
mean_normalization: X = (X - mean(X)) / (max(X) - min(X))
|
|
26
|
+
standardization: X = (X - mean(X)) / sqrt(var(X))
|
|
27
|
+
unit_length: X = X / norm(X), where norm() is the Frobenius norm
|
|
28
|
+
|
|
29
|
+
Outputs
|
|
30
|
+
-------
|
|
31
|
+
X_new: 2D array
|
|
32
|
+
Transformed data after normalization, that is, X_new = (X - mu)/sigma.
|
|
33
|
+
mu: 2D array
|
|
34
|
+
The value which X is subtracted.
|
|
35
|
+
sigma: float
|
|
36
|
+
The value which X will be divided. In the case max_unit and var_unit are False, then sigma = 1.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
if method == 'min_max_normalization':
|
|
40
|
+
mu = np.min(X)
|
|
41
|
+
sigma = np.max(X) - np.min(X)
|
|
42
|
+
elif method == 'mean_normalization':
|
|
43
|
+
mu = np.mean(X)
|
|
44
|
+
sigma = np.max(X) - np.min(X)
|
|
45
|
+
elif method == 'standardization':
|
|
46
|
+
mu = np.mean(X)
|
|
47
|
+
sigma = np.sqrt(np.var(X))
|
|
48
|
+
elif method == 'unit_length':
|
|
49
|
+
mu = 0
|
|
50
|
+
sigma = np.linalg.norm(X)
|
|
51
|
+
|
|
52
|
+
X_new = (X - mu)/sigma
|
|
53
|
+
|
|
54
|
+
return X_new, mu, sigma
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def pca(X, p):
|
|
58
|
+
"""
|
|
59
|
+
Make principal component analysis of the data. In the case the user want to return to the original format, set
|
|
60
|
+
X = dot(X, U.T) .
|
|
61
|
+
|
|
62
|
+
Inputs
|
|
63
|
+
------
|
|
64
|
+
X: 2D array
|
|
65
|
+
Train data in 2D array format. Each row correspond to a single sample of the data.
|
|
66
|
+
p: Float
|
|
67
|
+
Float between 0 and 1. The program compress the data so that 100*p % is retained. For example, if p = 0.90, then
|
|
68
|
+
90 % of the data is retained after compression.
|
|
69
|
+
|
|
70
|
+
Outputs
|
|
71
|
+
-------
|
|
72
|
+
X_new: 2D array
|
|
73
|
+
Transformed data after PCA.
|
|
74
|
+
U: 2D array
|
|
75
|
+
Left singular vectors of the SVD of the covariance matrix of X.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
# Number of samples and size of inputs
|
|
79
|
+
num_samples, n = X.shape
|
|
80
|
+
# Covariance matrix of X
|
|
81
|
+
cov = (1 / num_samples) * np.dot(X.T, X)
|
|
82
|
+
# SVD of covariance matrix
|
|
83
|
+
U, S, Vt = np.linalg.svd(cov)
|
|
84
|
+
|
|
85
|
+
# Compute size k <= n of compression
|
|
86
|
+
total_energy = np.sum(S)
|
|
87
|
+
for k in range(n):
|
|
88
|
+
energy = np.sum(S[0:k])
|
|
89
|
+
ratio = energy / total_energy
|
|
90
|
+
if ratio > p:
|
|
91
|
+
break
|
|
92
|
+
|
|
93
|
+
# Compute compressed inputs
|
|
94
|
+
U = U[:, :k]
|
|
95
|
+
X_new = np.dot(X, U)
|
|
96
|
+
|
|
97
|
+
return X_new, U
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def prepare_data(X, p):
|
|
101
|
+
"""
|
|
102
|
+
Before the starting the training stage, you will need to prepare your data, that is, make the PCA, construct the one
|
|
103
|
+
hot encoded classes, adding bias, and so on. This function and the function normalize_data are close since both
|
|
104
|
+
prepare the data for training.
|
|
105
|
+
|
|
106
|
+
Inputs
|
|
107
|
+
------
|
|
108
|
+
X: 2D array
|
|
109
|
+
Train data in 2D array format. Each row correspond to a single sample of the data.
|
|
110
|
+
p: Float
|
|
111
|
+
Float between 0 and 1. The program compress the data so that 100*p % is retained. For example, if p = 0.90, then
|
|
112
|
+
90 % of the data is retained after compression.
|
|
113
|
+
|
|
114
|
+
Outputs
|
|
115
|
+
-------
|
|
116
|
+
X_bias: 2D array
|
|
117
|
+
Train data prepared to be used for training.
|
|
118
|
+
U: 2D array
|
|
119
|
+
Left singular vectors of the SVD of the covariance matrix of X.
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
# PCA of inputs
|
|
123
|
+
X_new, U = pca(X, p)
|
|
124
|
+
num_samples, n = X_new.shape
|
|
125
|
+
|
|
126
|
+
# Add bias
|
|
127
|
+
aux = np.ones((num_samples, n + 1))
|
|
128
|
+
for i in range(num_samples):
|
|
129
|
+
aux[i, 1:] = X_new[i, :]
|
|
130
|
+
X_bias = aux
|
|
131
|
+
|
|
132
|
+
return X_bias, U
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def hot_encoded_target(Y):
|
|
136
|
+
"""
|
|
137
|
+
If the labels are not hot encoded, then this function is mandatory, otherwise we get errors. We are using one hot
|
|
138
|
+
encoded outputs of the form [1,0,0,...,0], [0,1,0,...,0], ..., [0,0,...,0,1]. To make the conversion the program
|
|
139
|
+
assumes the classes (the entries of Y) are numbers, from 0 to m-1 (so we have m classes).
|
|
140
|
+
|
|
141
|
+
Inputs
|
|
142
|
+
------
|
|
143
|
+
Y: 1D array
|
|
144
|
+
Each entry i of Y correspond to the class of the ith input (ith row of X).
|
|
145
|
+
|
|
146
|
+
Outputs
|
|
147
|
+
-------
|
|
148
|
+
Y_hot_encoded: 2D array
|
|
149
|
+
Train data classes with hot encoding.
|
|
150
|
+
"""
|
|
151
|
+
|
|
152
|
+
num_samples = Y.size
|
|
153
|
+
m = int(max(Y)) + 1
|
|
154
|
+
Y_hot_encoded = np.zeros((num_samples, m))
|
|
155
|
+
I = np.identity(m)
|
|
156
|
+
|
|
157
|
+
for i in range(num_samples):
|
|
158
|
+
for j in range(m):
|
|
159
|
+
if Y[i] == j:
|
|
160
|
+
Y_hot_encoded[i, :] = I[j, :]
|
|
161
|
+
|
|
162
|
+
return Y_hot_encoded
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def norm_pca(X_new, U, mu, sigma):
|
|
166
|
+
"""
|
|
167
|
+
After normalizing and performing pca over the train dataset, we may be interested in making predictions about some
|
|
168
|
+
new inputs. In any case, normalizing and compressing in the same way it is necessary in order to get meaningful
|
|
169
|
+
results.
|
|
170
|
+
|
|
171
|
+
Inputs
|
|
172
|
+
------
|
|
173
|
+
X_new: 2D array
|
|
174
|
+
Input data in 2D array format. Each row correspond to a single sample of the data.
|
|
175
|
+
U: 2D array
|
|
176
|
+
Left singular vectors of the SVD of the covariance matrix of the train dataset.
|
|
177
|
+
mu: 2D array
|
|
178
|
+
Mean of the train dataset.
|
|
179
|
+
sigma: float
|
|
180
|
+
The value in which the dataset was divided.
|
|
181
|
+
|
|
182
|
+
Outputs
|
|
183
|
+
-------
|
|
184
|
+
X_new: 2D array
|
|
185
|
+
Input data after transformations.
|
|
186
|
+
"""
|
|
187
|
+
|
|
188
|
+
X_new = (X_new - mu) / sigma
|
|
189
|
+
X_new = np.dot(X_new, U)
|
|
190
|
+
|
|
191
|
+
return X_new
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
# CPD LEARNING
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def init_W(m, L, n, R):
|
|
198
|
+
"""
|
|
199
|
+
Generate initial weights to start iterating. We have that W[k] = [W1,...,WL] is a multidimensional array of shape
|
|
200
|
+
L x n x R. For each tensor Tk of the model we have that
|
|
201
|
+
|
|
202
|
+
Tk = sum_{r=1}^R W[k, 0, :, r] ⊗ W[k, 1, :, r] ⊗ ... ⊗ W[k, L-1, :, r].
|
|
203
|
+
|
|
204
|
+
Note that each W[k, l, :, :] is a n x R factor matrix of the CPD of Tk, in other words, W[k] is the set of the
|
|
205
|
+
factor matrices of Tk. Each tensor Tk has shape n x n x ... x n (L times) and has rank <= R.
|
|
206
|
+
|
|
207
|
+
Inputs
|
|
208
|
+
------
|
|
209
|
+
m: int
|
|
210
|
+
Number of tensors of the model, which must coincide with the dimension of the output space.
|
|
211
|
+
L: int
|
|
212
|
+
Order of the tensors.
|
|
213
|
+
n: int
|
|
214
|
+
Dimension of the input space.
|
|
215
|
+
R: int
|
|
216
|
+
Rank of the tensors.
|
|
217
|
+
|
|
218
|
+
Outputs
|
|
219
|
+
-------
|
|
220
|
+
W: 4D array
|
|
221
|
+
"""
|
|
222
|
+
|
|
223
|
+
W = np.zeros((m, L, n, R))
|
|
224
|
+
|
|
225
|
+
for k in range(m):
|
|
226
|
+
for l in range(L):
|
|
227
|
+
W[k, l, :, :] = np.random.uniform(-1, 1, size=(n, R))
|
|
228
|
+
|
|
229
|
+
return W
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def dot_products(x, W, dropout):
|
|
233
|
+
"""
|
|
234
|
+
Function to compute all the dot products between an input x and the weights.
|
|
235
|
+
|
|
236
|
+
Inputs
|
|
237
|
+
------
|
|
238
|
+
x: 1D array
|
|
239
|
+
A single input sample.
|
|
240
|
+
W: L-D array
|
|
241
|
+
Coordinates of all tensors Tk of the model (see function init_W for more details).
|
|
242
|
+
dropout:
|
|
243
|
+
Probability to use or not dropout (see function cpd_train for more details).
|
|
244
|
+
|
|
245
|
+
Outputs
|
|
246
|
+
-------
|
|
247
|
+
dot_prods: 3D array
|
|
248
|
+
Each value dot_prods[k, l, r] is the dot product between x and the weight vector corresponding to the r-th
|
|
249
|
+
column of the l-th factor matrix of the k-th tensor.
|
|
250
|
+
"""
|
|
251
|
+
|
|
252
|
+
# Initialize array to receive all dot products
|
|
253
|
+
m, L, n, R = W.shape
|
|
254
|
+
dot_prods = np.ones((m, L, R))
|
|
255
|
+
|
|
256
|
+
# Compute all the dot products.
|
|
257
|
+
lst = [W[k, :, :, r] for k in range(m) for r in range(R)]
|
|
258
|
+
W_tmp = np.concatenate(lst)
|
|
259
|
+
Wx = np.dot(W_tmp, x)
|
|
260
|
+
Wx = Wx.reshape(m, R, L)
|
|
261
|
+
for k in range(m):
|
|
262
|
+
dot_prods[k, :, :] = Wx[k, :, :].T
|
|
263
|
+
|
|
264
|
+
# The above is an optimized form of the calculations commented below.
|
|
265
|
+
#for k in range(m):
|
|
266
|
+
# for r in range(R):
|
|
267
|
+
# dot_prods[k, :, r] = np.dot(W[k, :, :, r], x)
|
|
268
|
+
|
|
269
|
+
# Apply dropout.
|
|
270
|
+
if dropout < 1:
|
|
271
|
+
dropout_decision = np.random.uniform(0, 1, size=(m, R))
|
|
272
|
+
dropout_decision = dropout_decision <= dropout
|
|
273
|
+
for k in range(m):
|
|
274
|
+
for r in np.arange(R)[~dropout_decision[k, :]]:
|
|
275
|
+
dot_prods[k, :, r] *= 0
|
|
276
|
+
|
|
277
|
+
return dot_prods
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
@njit(nogil=True)
|
|
281
|
+
def h(W, dot_prods):
|
|
282
|
+
"""
|
|
283
|
+
Function to compute the hypothesis function h(x). This is defined by
|
|
284
|
+
|
|
285
|
+
h(x) = f(T1(x,...,x),...,Tm(x,...,x)) = ( f(T1(x,...,x)) ,..., f(Tm(x,...,x)) ),
|
|
286
|
+
|
|
287
|
+
where f is the activation function (in this case we are using the sigmoid function). Note that h(x) is a
|
|
288
|
+
m-dimensional vector.
|
|
289
|
+
|
|
290
|
+
Inputs
|
|
291
|
+
------
|
|
292
|
+
W: 4D array
|
|
293
|
+
Coordinates of all tensors Tk of the model (see function init_W for more details).
|
|
294
|
+
dot_prods: 3D array.
|
|
295
|
+
Dot products between the weights and x (see function dot_products for more details).
|
|
296
|
+
|
|
297
|
+
Outputs
|
|
298
|
+
-------
|
|
299
|
+
hx: 1D array
|
|
300
|
+
Evaluation of h(x).
|
|
301
|
+
dhx: 1D array
|
|
302
|
+
Evaluation of the derivative of h at x, that is, h'(x).
|
|
303
|
+
"""
|
|
304
|
+
|
|
305
|
+
m, L, n, R = W.shape
|
|
306
|
+
# Initialize vector to receive T(x,...,x)
|
|
307
|
+
hx = np.zeros(m, dtype=np.float64)
|
|
308
|
+
|
|
309
|
+
# Compute Tk(x,...,x) for all k=1...m
|
|
310
|
+
for k in range(m):
|
|
311
|
+
for r in range(R):
|
|
312
|
+
temp = 1
|
|
313
|
+
for l in range(L):
|
|
314
|
+
temp *= dot_prods[k, l, r]
|
|
315
|
+
hx[k] += temp
|
|
316
|
+
|
|
317
|
+
# Compute (f'(T1(x,...,x),...,f'(Tm(x,...,x))
|
|
318
|
+
dhx = df(hx)
|
|
319
|
+
# Compute f(T1(x,...,x),...,Tm(x,...,x))
|
|
320
|
+
hx = f(hx)
|
|
321
|
+
|
|
322
|
+
return hx, dhx
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
@njit(nogil=True)
|
|
326
|
+
def f(z):
|
|
327
|
+
"""
|
|
328
|
+
Non-linear (activation) function f at a point z(we are using the sigmoid function here).
|
|
329
|
+
|
|
330
|
+
Inputs
|
|
331
|
+
------
|
|
332
|
+
z: 1D array
|
|
333
|
+
|
|
334
|
+
Outputs
|
|
335
|
+
-------
|
|
336
|
+
fz: 1D array
|
|
337
|
+
"""
|
|
338
|
+
|
|
339
|
+
fz = 1/(1 + np.exp(-z))
|
|
340
|
+
return fz
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
@njit(nogil=True)
|
|
344
|
+
def df(z):
|
|
345
|
+
"""
|
|
346
|
+
Derivative of the non-linear (activation) function f at a point z.
|
|
347
|
+
|
|
348
|
+
Inputs
|
|
349
|
+
------
|
|
350
|
+
z: 1D array
|
|
351
|
+
|
|
352
|
+
Outputs
|
|
353
|
+
-------
|
|
354
|
+
dfz: 1D array
|
|
355
|
+
"""
|
|
356
|
+
|
|
357
|
+
dfz = f(z)*(1 - f(z))
|
|
358
|
+
return dfz
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
@njit(nogil=True, parallel=True)
|
|
362
|
+
def grad(x, y, W, dot_prods, Lambda):
|
|
363
|
+
"""
|
|
364
|
+
Let J(W) be the cost function of the problem, and grad(J) be the gradient of J. Then this function constructs
|
|
365
|
+
grad(J)_{k, l, i, r} with respect to a pair input-output (x, y). In the end, the partial derivative of J with
|
|
366
|
+
respect to W[k, l, i, r] is the sum of grad(J)_{k, l, i, r}(x, y) for all pairs (x, y).
|
|
367
|
+
|
|
368
|
+
Inputs
|
|
369
|
+
------
|
|
370
|
+
x: 1D array
|
|
371
|
+
y: 1D array
|
|
372
|
+
W: 4D array
|
|
373
|
+
dot_prods: 3D array
|
|
374
|
+
Lambda: float
|
|
375
|
+
Regularization parameter for the cost function. We must have Lambda >= 0.
|
|
376
|
+
|
|
377
|
+
Outputs
|
|
378
|
+
-------
|
|
379
|
+
grad_J: 4D array
|
|
380
|
+
"""
|
|
381
|
+
|
|
382
|
+
m, L, n, R = W.shape
|
|
383
|
+
# Initialize gradient vector associated to J at x
|
|
384
|
+
grad_J = np.zeros((m, L, n, R))
|
|
385
|
+
# Compute hypothesis and its derivative at x
|
|
386
|
+
hx, dhx = h(W, dot_prods)
|
|
387
|
+
|
|
388
|
+
for k in prange(m):
|
|
389
|
+
for l in range(L):
|
|
390
|
+
for i in range(n):
|
|
391
|
+
for r in range(R):
|
|
392
|
+
grad_J[k, l, i, r] = compute_grad(x, y, W, dot_prods, hx, dhx, Lambda, L, k, l, i, r)
|
|
393
|
+
|
|
394
|
+
return grad_J
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
@njit(nogil=True)
|
|
398
|
+
def compute_grad(x, y, W, dot_prods, hx, dhx, Lambda, L, k, l, i, r):
|
|
399
|
+
"""
|
|
400
|
+
Auxiliar function to make inner computations of function grad.
|
|
401
|
+
"""
|
|
402
|
+
|
|
403
|
+
term1 = Lambda * W[k, l, i, r]
|
|
404
|
+
term2 = hx[k] - y[k]
|
|
405
|
+
term3 = dhx[k]
|
|
406
|
+
|
|
407
|
+
temp = 1
|
|
408
|
+
for lt in range(L):
|
|
409
|
+
if lt != l:
|
|
410
|
+
temp *= dot_prods[k, lt, r]
|
|
411
|
+
term4 = x[i] * temp
|
|
412
|
+
|
|
413
|
+
return term1 + term2*term3*term4
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def update(alpha, W, grad_J, constraint):
|
|
417
|
+
"""
|
|
418
|
+
Compute the update stage W = W - alpha*grad_J for the weights in the gradient descent algorithm.
|
|
419
|
+
|
|
420
|
+
Inputs
|
|
421
|
+
------
|
|
422
|
+
alpha: float
|
|
423
|
+
Step parameter of the gradient descent algorithm. We must have alpha > 0.
|
|
424
|
+
W: 4D array
|
|
425
|
+
grad_J: 4D array
|
|
426
|
+
See the description of grad_J in the function grad.
|
|
427
|
+
constraint: float
|
|
428
|
+
This paramater limits the size of the weights with the update formula W = constraint * W/max(abs(W)). This way
|
|
429
|
+
the largest magnitude of W is always equal to this parameter.
|
|
430
|
+
|
|
431
|
+
Outputs
|
|
432
|
+
-------
|
|
433
|
+
W: 4D array
|
|
434
|
+
Update of W.
|
|
435
|
+
"""
|
|
436
|
+
|
|
437
|
+
W = W - alpha * grad_J
|
|
438
|
+
|
|
439
|
+
if constraint != np.inf:
|
|
440
|
+
W = constraint * W / np.max(np.abs(W))
|
|
441
|
+
|
|
442
|
+
return W
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def find_class(x, W):
|
|
446
|
+
"""
|
|
447
|
+
Given a set of weights W and a new input x, this function computes the predicted class of x.
|
|
448
|
+
|
|
449
|
+
Inputs
|
|
450
|
+
------
|
|
451
|
+
x: 1D array
|
|
452
|
+
New input for which the program will try to predict its class.
|
|
453
|
+
W:
|
|
454
|
+
Set of weights after the learning stage.
|
|
455
|
+
|
|
456
|
+
Outputs
|
|
457
|
+
-------
|
|
458
|
+
x_class: int
|
|
459
|
+
At the moment the program is limited to classification problems. So each prediction is a number (of a class).
|
|
460
|
+
"""
|
|
461
|
+
|
|
462
|
+
dot_prods = dot_products(x, W, 1)
|
|
463
|
+
hx, dhx = h(W, dot_prods)
|
|
464
|
+
x_class = np.argmax(hx)
|
|
465
|
+
|
|
466
|
+
return x_class
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def cpd_train(X, Y, X_val, Y_val, W, alpha=0.01, alpha_decay=0.5, Lambda=0.1, epochs=10, batch=1, constraint=np.inf, dropout=1, display=True):
|
|
470
|
+
"""
|
|
471
|
+
Function to start the training stage.
|
|
472
|
+
|
|
473
|
+
Inputs
|
|
474
|
+
------
|
|
475
|
+
X: 2D array
|
|
476
|
+
Train data in 2D array format. Each row correspond to a single sample of the data.
|
|
477
|
+
Y: 1D array
|
|
478
|
+
Each entry i of Y correspond to the original class of the ith input (ith row of X).
|
|
479
|
+
X_val: 2D array
|
|
480
|
+
Piece of train dataset used as validation input set. If no validation is intended, set X_val to nan.
|
|
481
|
+
Y_val: 1D array
|
|
482
|
+
Piece of train dataset used as validation target set.
|
|
483
|
+
W: 4D array
|
|
484
|
+
Initial set of weights (see function init_W for more details).
|
|
485
|
+
alpha: float
|
|
486
|
+
Step parameter of the gradient descent algorithm. We must have alpha > 0.
|
|
487
|
+
alpha_decay: float
|
|
488
|
+
The more the program is closer to the optimum, the more it is necessary to take smaller steps. In this case it
|
|
489
|
+
is interesting to decrease alpha gradually. At each epoch we update alpha with alpha = alpha_decay * alpha.
|
|
490
|
+
Default is alpha_decay = 0.5.
|
|
491
|
+
Lambda: float
|
|
492
|
+
Regularization parameter for the cost function. We must have Lambda >= 0. Default is Lambda = 0.1.
|
|
493
|
+
epochs: int
|
|
494
|
+
Desired number of epochs to use in the training stage. Default is 10 epochs.
|
|
495
|
+
batch: int
|
|
496
|
+
Size of the batch in the learning algorithm. After passing through batch inputs and accumulating their costs,
|
|
497
|
+
the corresponding gradient is used to make the next step. Default is 1 batch.
|
|
498
|
+
constraint: float
|
|
499
|
+
Large weights size can be a sign of an unstable network. Ona may force the magnitude of all weights to be below
|
|
500
|
+
a specified value, given by this parameter. At each iteration the program applies the update formula
|
|
501
|
+
W = constraint * W/max(abs(W)). This way the largest magnitude of W is always equal to this parameter. Default
|
|
502
|
+
is constraint = np.inf, which means to not use constraints.
|
|
503
|
+
dropout: float
|
|
504
|
+
This value should be between 1 (no dropout is used) and 0 (drop is used for all nodes). At each iteration the
|
|
505
|
+
program decides, for each node, if it will be used or not, based on this probability. Default is dropout = 1.
|
|
506
|
+
display: bool
|
|
507
|
+
If set to True (default), the program displays some relevant results and plots about the training stage.
|
|
508
|
+
|
|
509
|
+
Outputs
|
|
510
|
+
-------
|
|
511
|
+
W: 4D array
|
|
512
|
+
Optimized set of weights.
|
|
513
|
+
accuracy: list
|
|
514
|
+
Evolution of the accuracy (0 % to 100 %) at each iteration.
|
|
515
|
+
cost_function: list
|
|
516
|
+
Evolution of the cost function at each iteration.
|
|
517
|
+
success: int
|
|
518
|
+
Total number of successes after the training.
|
|
519
|
+
"""
|
|
520
|
+
|
|
521
|
+
# Verify if the parameter dropout is consistent.
|
|
522
|
+
if dropout < 0 or dropout > 1:
|
|
523
|
+
sys.exit('The parameter dropout has to be between 0 ans 1.')
|
|
524
|
+
elif dropout == 0:
|
|
525
|
+
sys.exit('The parameter dropout has to be bigger than 0.')
|
|
526
|
+
|
|
527
|
+
# Number of samples.
|
|
528
|
+
num_samples, n = X.shape
|
|
529
|
+
|
|
530
|
+
# Transform target into hot encoded form.
|
|
531
|
+
Y_hot_encoded = hot_encoded_target(Y)
|
|
532
|
+
|
|
533
|
+
# Identity matrix with size of inputs minus bias.
|
|
534
|
+
I = np.identity(n-1)
|
|
535
|
+
|
|
536
|
+
# Create vectors to visualize some information later.
|
|
537
|
+
accuracy = []
|
|
538
|
+
accuracy_val = []
|
|
539
|
+
cost_function = []
|
|
540
|
+
|
|
541
|
+
for ep in range(epochs):
|
|
542
|
+
count = 0
|
|
543
|
+
success = 0
|
|
544
|
+
cum_cost = 0
|
|
545
|
+
cum_grad = np.zeros(W.shape)
|
|
546
|
+
|
|
547
|
+
for j in range(num_samples):
|
|
548
|
+
x = X[j, :]
|
|
549
|
+
y = Y_hot_encoded[j, :]
|
|
550
|
+
dot_prods = dot_products(x, W, dropout)
|
|
551
|
+
grad_J = grad(x, y, W, dot_prods, Lambda)
|
|
552
|
+
cum_grad += grad_J
|
|
553
|
+
|
|
554
|
+
# Update weights after seeing batch inputs.
|
|
555
|
+
if (j % batch == 0) and (j >= batch):
|
|
556
|
+
W = update(alpha, W, cum_grad/batch, constraint)
|
|
557
|
+
cum_grad = 0*cum_grad
|
|
558
|
+
elif j == num_samples-1:
|
|
559
|
+
W = update(alpha, W, cum_grad/batch, constraint)
|
|
560
|
+
|
|
561
|
+
# Predict class of current input.
|
|
562
|
+
x_class = find_class(x, W)
|
|
563
|
+
if x_class == Y[j]:
|
|
564
|
+
success += 1
|
|
565
|
+
|
|
566
|
+
# Save the value of the cost function with respect to this individual input.
|
|
567
|
+
hx, dhx = h(W, dot_prods)
|
|
568
|
+
cost = (1 / 2) * np.linalg.norm(hx - y) ** 2
|
|
569
|
+
cum_cost += cost
|
|
570
|
+
|
|
571
|
+
# Display progress bar.
|
|
572
|
+
if j % (num_samples//80) == 0 and j >= (num_samples//80):
|
|
573
|
+
count += 1
|
|
574
|
+
s = "Epoch " + str(ep+1) + ": [" + min(80, count) * "=" + min((80-count), 80) * " " + "]" + \
|
|
575
|
+
" " + str(np.round(100*j/num_samples, 1)) + "%"
|
|
576
|
+
sys.stdout.write('\r' + s)
|
|
577
|
+
|
|
578
|
+
# Display progress bar at 100%.
|
|
579
|
+
s = "Epoch " + str(ep+1) + ": [" + min(80, count) * "=" + min((80-count), 80) * " " + "]" + " " + "100.0% / acc=" + str(round(100 * success/num_samples, 2)) + '%'
|
|
580
|
+
sys.stdout.write('\r' + s)
|
|
581
|
+
|
|
582
|
+
# Update relevant information.
|
|
583
|
+
acc = 100 * success/num_samples
|
|
584
|
+
accuracy.append(100 * success/num_samples)
|
|
585
|
+
cost_function.append(cum_cost/num_samples)
|
|
586
|
+
|
|
587
|
+
# Update alpha.
|
|
588
|
+
alpha = alpha_decay * alpha
|
|
589
|
+
|
|
590
|
+
# The X_val dataset is supposed to be already normalized.
|
|
591
|
+
if np.sum(np.isnan(X_val)) == 0:
|
|
592
|
+
accuracy_val.append(cpd_test(X_val, Y_val, W, I))
|
|
593
|
+
print()
|
|
594
|
+
|
|
595
|
+
if display:
|
|
596
|
+
disp_results(accuracy, accuracy_val, cost_function, W, epochs)
|
|
597
|
+
|
|
598
|
+
return W, accuracy, accuracy_val, cost_function, success
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def disp_results(accuracy, accuracy_val, cost_function, W, epochs):
|
|
602
|
+
"""
|
|
603
|
+
Display results of the training stage.
|
|
604
|
+
|
|
605
|
+
Inputs
|
|
606
|
+
------
|
|
607
|
+
accuracy: list
|
|
608
|
+
Evolution of the accuracy (0 % to 100 %) at each iteration.
|
|
609
|
+
accuracy_val list
|
|
610
|
+
Evolution of the validation accuracy (0 % to 100 %) at each iteration.
|
|
611
|
+
cost_function: list
|
|
612
|
+
Evolution of the cost function at each iteration.
|
|
613
|
+
W: 4D array
|
|
614
|
+
Initial set of weights (see function init_W for more details).
|
|
615
|
+
epochs: int
|
|
616
|
+
Number desired of epochs to use in the training stage.
|
|
617
|
+
"""
|
|
618
|
+
|
|
619
|
+
m, L, n, R = W.shape
|
|
620
|
+
print()
|
|
621
|
+
print('Dimensions of input:', n)
|
|
622
|
+
print('Dimensions of target:', m)
|
|
623
|
+
print('Rank:', R)
|
|
624
|
+
print('Number of factor matrices:', L)
|
|
625
|
+
print('Number of epochs:', epochs)
|
|
626
|
+
print('Number of weights:', ((L - 1) * n + m) * R)
|
|
627
|
+
print('Accuracy over training dataset:', np.round(accuracy[-1], 3), '%')
|
|
628
|
+
if len(accuracy_val) > 0:
|
|
629
|
+
print('Accuracy over validation dataset:', np.round(accuracy_val[-1], 3), '%')
|
|
630
|
+
print()
|
|
631
|
+
|
|
632
|
+
plt.figure(figsize=[10, 4])
|
|
633
|
+
plt.plot(np.arange(1, epochs+1), accuracy, markersize=2, label='train')
|
|
634
|
+
if len(accuracy_val) > 0:
|
|
635
|
+
plt.plot(np.arange(1, epochs + 1), accuracy_val, markersize=2, label='validation')
|
|
636
|
+
plt.title('Learning Curve')
|
|
637
|
+
plt.grid()
|
|
638
|
+
plt.legend()
|
|
639
|
+
plt.xlabel('Epoch')
|
|
640
|
+
plt.show()
|
|
641
|
+
|
|
642
|
+
plt.figure(figsize=[10, 4])
|
|
643
|
+
plt.plot(np.arange(1, epochs+1), cost_function, 'r', markersize=2)
|
|
644
|
+
plt.title('Cost function')
|
|
645
|
+
plt.grid()
|
|
646
|
+
plt.xlabel('Epoch')
|
|
647
|
+
plt.show()
|
|
648
|
+
|
|
649
|
+
return
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
def cpd_test(X_test, Y_test, W, U, mu=0, sigma=1):
|
|
653
|
+
"""
|
|
654
|
+
After the training stage it is time to validate the model with the test dataset. This function is responsible for
|
|
655
|
+
that part. The default values for mu and sigma assume that no normalization was performed. Additionally, if no PCA
|
|
656
|
+
was also performed, pass the matrix U as np.identity(X.shape[1]).
|
|
657
|
+
|
|
658
|
+
Inputs
|
|
659
|
+
------
|
|
660
|
+
X_test: 2D array
|
|
661
|
+
Test data in 2D array format. Each row correspond to a single sample of the test data.
|
|
662
|
+
Y_test:
|
|
663
|
+
Each entry i of Y_test correspond to the class of the ith input of the test dataset (ith row of X_test).
|
|
664
|
+
W: 4D array
|
|
665
|
+
Weights obtained after the training stage.
|
|
666
|
+
U: 2D array
|
|
667
|
+
Left singular vectors of the SVD of the covariance matrix of the normalized X_test. The compressed data is given
|
|
668
|
+
by dot(X_test_new, U), where X_test_new is the normalized version of X_test.
|
|
669
|
+
mu, sigma: float
|
|
670
|
+
mu and sigma are such the normalized data is given by (X_test - mu)/sigma.
|
|
671
|
+
|
|
672
|
+
Outputs
|
|
673
|
+
-------
|
|
674
|
+
accuracy: float
|
|
675
|
+
Accuracy of the model over the test dataset.
|
|
676
|
+
"""
|
|
677
|
+
|
|
678
|
+
predictions = cpd_predictions(X_test, W, U, mu=mu, sigma=sigma)
|
|
679
|
+
|
|
680
|
+
num_samples = X_test.shape[0]
|
|
681
|
+
success = 0
|
|
682
|
+
for j in range(num_samples):
|
|
683
|
+
x_class = predictions[j]
|
|
684
|
+
y = Y_test[j]
|
|
685
|
+
if x_class == y:
|
|
686
|
+
success += 1
|
|
687
|
+
|
|
688
|
+
accuracy = 100 * success/num_samples
|
|
689
|
+
|
|
690
|
+
return accuracy
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
def cpd_predictions(X_test, W, U, mu=0, sigma=1):
|
|
694
|
+
"""
|
|
695
|
+
Given set of new inputs X_test, this function makes the corresponding predictions. The default values for mu and
|
|
696
|
+
sigma assume that no normalization was performed. Additionally, if no PCA was also performed, pass the matrix U as
|
|
697
|
+
np.identity(X.shape[1]).
|
|
698
|
+
|
|
699
|
+
Inputs
|
|
700
|
+
------
|
|
701
|
+
X_test: 2D array
|
|
702
|
+
Test data in 2D array format. Each row correspond to a single sample of the test data.
|
|
703
|
+
W: 4D array
|
|
704
|
+
Weights obtained after the training stage.
|
|
705
|
+
U: 2D array
|
|
706
|
+
Left singular vectors of the SVD of the covariance matrix of the normalized X_test. The compressed data is given
|
|
707
|
+
by dot(X_test_new, U), where X_test_new is the normalized version of X_test.
|
|
708
|
+
mu, sigma: float
|
|
709
|
+
mu and sigma are such the normalized data is given by (X_test - mu)/sigma.
|
|
710
|
+
|
|
711
|
+
Outputs
|
|
712
|
+
-------
|
|
713
|
+
predictions: 1D array
|
|
714
|
+
Predictions of the classes of each sample in X_test.
|
|
715
|
+
"""
|
|
716
|
+
|
|
717
|
+
# Transform data to compressed form. This is necessary since the weights W were obtained over compressed data.
|
|
718
|
+
X_test_new = norm_pca(X_test, U, mu, sigma)
|
|
719
|
+
num_samples, n = X_test_new.shape
|
|
720
|
+
|
|
721
|
+
# Add bias
|
|
722
|
+
aux = np.ones((num_samples, n + 1))
|
|
723
|
+
aux[:, 1:] = X_test_new
|
|
724
|
+
X_bias = aux
|
|
725
|
+
|
|
726
|
+
# Create array of predictions, which are integers from 0 to m-1.
|
|
727
|
+
predictions = np.zeros(num_samples, dtype=np.int64)
|
|
728
|
+
|
|
729
|
+
for j in range(num_samples):
|
|
730
|
+
x = X_bias[j, :]
|
|
731
|
+
x_class = find_class(x, W)
|
|
732
|
+
predictions[j] = x_class
|
|
733
|
+
|
|
734
|
+
return predictions
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
def simplify_model(W, r, options=False):
|
|
738
|
+
"""
|
|
739
|
+
With the set of weights W, obtained after training and testing, we can construct the corresponding coordinate
|
|
740
|
+
tensors and compute a rank-r CPD for them. The idea is to simplify the model, using less rank-1 terms.
|
|
741
|
+
|
|
742
|
+
Inputs
|
|
743
|
+
------
|
|
744
|
+
W: 4D array
|
|
745
|
+
Weights obtained after the training stage.
|
|
746
|
+
r: int
|
|
747
|
+
Desired rank.
|
|
748
|
+
options: class
|
|
749
|
+
Set of options to be used in the CPD computation (see the Tensor Fox documentation for more details).
|
|
750
|
+
|
|
751
|
+
Outputs
|
|
752
|
+
-------
|
|
753
|
+
new_W: float 4D-ndarray
|
|
754
|
+
New model based in rank-r approximations. An array with the same shape as W, except for the last dimension.
|
|
755
|
+
errors: list
|
|
756
|
+
Each errors[k] is the relative error between the rank-r approximation and Tk.
|
|
757
|
+
"""
|
|
758
|
+
|
|
759
|
+
# Initialize variables.
|
|
760
|
+
m, L, n, R = W.shape
|
|
761
|
+
new_W = np.zeros((m, L, n, r))
|
|
762
|
+
errors = []
|
|
763
|
+
|
|
764
|
+
for k in range(m):
|
|
765
|
+
# Construct list of factor matrices for Tk.
|
|
766
|
+
factors = []
|
|
767
|
+
for l in range(L):
|
|
768
|
+
factors.append(W[k, l, :, :])
|
|
769
|
+
# Convert factor matrices to coordinates format.
|
|
770
|
+
Tk = tfx.cnv.cpd2tens(factors)
|
|
771
|
+
# Compute rank-r approximation for Tk.
|
|
772
|
+
factors, output = tfx.cpd(Tk, r, options)
|
|
773
|
+
errors.append(output.rel_error)
|
|
774
|
+
for l in range(L):
|
|
775
|
+
new_W[k, l, :, :] = factors[l]
|
|
776
|
+
|
|
777
|
+
return new_W, errors
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
# MLSVD LEARNING
|
|
781
|
+
|
|
782
|
+
|
|
783
|
+
def create_sets(X, Y, num_samples, p, var_type, display):
|
|
784
|
+
"""
|
|
785
|
+
This function separates the inputs by class in a list called samples_per_class. If the number of inputs per
|
|
786
|
+
class is no equal, the program randomly select some inputs to be repeated in a certain class. This is done
|
|
787
|
+
so that all classes have the same number of elements.
|
|
788
|
+
There is the option to add noise to these repeated inputs, so we can have a simple method of data augmentation
|
|
789
|
+
to apply.
|
|
790
|
+
|
|
791
|
+
Inputs
|
|
792
|
+
------
|
|
793
|
+
X: 2D array
|
|
794
|
+
Train data in 2D array format. Each row correspond to a single sample of the data.
|
|
795
|
+
Y: 1D array
|
|
796
|
+
Each entry i of Y correspond to the class of the ith input (ith row of X).
|
|
797
|
+
num_samples: int
|
|
798
|
+
The desired number of samples per class. This number should be equal or greater than the current maximum number
|
|
799
|
+
of samples in a class.
|
|
800
|
+
p: float
|
|
801
|
+
Proportion of the noise with respect to the original data. For example, if p=0.1, then the noise added has
|
|
802
|
+
size less or equal than 10% of the size of the original data.
|
|
803
|
+
var_type: str
|
|
804
|
+
This variable indicates the type or perturbation. In both cases we use uniform distribution to pick the
|
|
805
|
+
perturbations. The only possibilities considered are 'int' and 'float'. We remark that you should use float
|
|
806
|
+
whenever the data is normalized.
|
|
807
|
+
display: bool
|
|
808
|
+
If True (default), the function shows the number of inputs for each class.
|
|
809
|
+
|
|
810
|
+
Outputs
|
|
811
|
+
-------
|
|
812
|
+
X_new, Y_new: arrays
|
|
813
|
+
New versions of X and Y, maybe with more data.
|
|
814
|
+
inputs: list
|
|
815
|
+
Each element of this list is a list with all inputs with same class.
|
|
816
|
+
"""
|
|
817
|
+
|
|
818
|
+
total_num_samples = X.shape[0]
|
|
819
|
+
num_classes = int(max(Y)) + 1
|
|
820
|
+
X_new = []
|
|
821
|
+
Y_new = []
|
|
822
|
+
inputs = [[] for i in range(num_classes)]
|
|
823
|
+
|
|
824
|
+
# Create list with all inputs separated by class.
|
|
825
|
+
for i in range(total_num_samples):
|
|
826
|
+
x = X[i, :]
|
|
827
|
+
y = int(Y[i])
|
|
828
|
+
inputs[y].append(x)
|
|
829
|
+
|
|
830
|
+
# Counting of inputs for each class.
|
|
831
|
+
samples_per_class = []
|
|
832
|
+
for i in range(num_classes):
|
|
833
|
+
samples_per_class.append(len(inputs[i]))
|
|
834
|
+
if display:
|
|
835
|
+
print('Inputs of class', i, '=', len(inputs[i]))
|
|
836
|
+
|
|
837
|
+
# Make each class to have the same number of inputs.
|
|
838
|
+
if num_samples is not None:
|
|
839
|
+
max_class = max(max(samples_per_class), num_samples)
|
|
840
|
+
else:
|
|
841
|
+
max_class = max(samples_per_class)
|
|
842
|
+
|
|
843
|
+
for i in range(num_classes):
|
|
844
|
+
c = samples_per_class[i]
|
|
845
|
+
diff = max_class - c
|
|
846
|
+
if diff > 1:
|
|
847
|
+
for j in range(diff):
|
|
848
|
+
idx = np.random.randint(c)
|
|
849
|
+
x = inputs[i][idx]
|
|
850
|
+
|
|
851
|
+
# Apply data augmentation.
|
|
852
|
+
xp = np.random.uniform(low=min(x), high=1+max(x), size=x.shape[0])
|
|
853
|
+
if var_type == 'float':
|
|
854
|
+
x_new = x + p*xp
|
|
855
|
+
elif var_type == 'int':
|
|
856
|
+
x_new = x + (p*xp).astype(int)
|
|
857
|
+
inputs[i].append(x_new)
|
|
858
|
+
|
|
859
|
+
# Update X_new and Y_new.
|
|
860
|
+
X_new.append(x_new)
|
|
861
|
+
Y_new.append(i)
|
|
862
|
+
|
|
863
|
+
# Convert X_new and Y_new to arrays.
|
|
864
|
+
X_new = np.array(X_new)
|
|
865
|
+
Y_new = np.array(Y_new)
|
|
866
|
+
X_new = np.concatenate([X, X_new])
|
|
867
|
+
Y_new = np.concatenate([Y, Y_new])
|
|
868
|
+
|
|
869
|
+
if display:
|
|
870
|
+
print()
|
|
871
|
+
print('After fixing number of inputs per class:')
|
|
872
|
+
for i in range(num_classes):
|
|
873
|
+
print('Inputs of class', i, '=', len(inputs[i]))
|
|
874
|
+
|
|
875
|
+
return X_new, Y_new, inputs
|
|
876
|
+
|
|
877
|
+
|
|
878
|
+
def data2tens(X, Y, num_samples=None, p=0, var_type='float', display=True):
|
|
879
|
+
"""
|
|
880
|
+
Given the input dataset X and the target dataset Y, this function separates the inputs by class so that each
|
|
881
|
+
class has the same number of inputs m. If n is the dimension of the inputs and each class has p inputs, the
|
|
882
|
+
tensor created has shape m x n x p. Each frontal slice contains all inputs corresponding to a certain class.
|
|
883
|
+
|
|
884
|
+
Inputs
|
|
885
|
+
------
|
|
886
|
+
X: 2D array
|
|
887
|
+
Train data in 2D array format. Each row correspond to a single sample of the data.
|
|
888
|
+
Y: 1D array
|
|
889
|
+
Each entry i of Y correspond to the class of the ith input (ith row of X).
|
|
890
|
+
num_samples: int
|
|
891
|
+
The desired number of samples per class. This number should be equal or greater than the current maximum number
|
|
892
|
+
of samples in a class.
|
|
893
|
+
p: float
|
|
894
|
+
Proportion of the noise with respect to the original data. For example, if p=0.1, then the noise added has
|
|
895
|
+
size less or equal than 10% of the size of the original data. Default is p=0, which means to not perturb.
|
|
896
|
+
var_type: str
|
|
897
|
+
This variable indicates the type or perturbation. In both cases we use uniform distribution to pick the
|
|
898
|
+
perturbations. The only possibilities considered are 'int' and 'float'. Default is 'float'. We remark that you
|
|
899
|
+
should use float whenever the data is normalized.
|
|
900
|
+
display: bool
|
|
901
|
+
If True (default), the function shows the number of inputs for each class.
|
|
902
|
+
|
|
903
|
+
Outputs
|
|
904
|
+
-------
|
|
905
|
+
T: 3D array
|
|
906
|
+
Each vector T[i, :, k] is the ith input of the kth class.
|
|
907
|
+
X_new, Y_new: arrays
|
|
908
|
+
New versions of X and Y, maybe with more data.
|
|
909
|
+
"""
|
|
910
|
+
|
|
911
|
+
# Create list with inputs organized by class.
|
|
912
|
+
X, Y, inputs = create_sets(X, Y, num_samples, p, var_type, display)
|
|
913
|
+
num_inputs = len(inputs[0])
|
|
914
|
+
input_size = inputs[0][0].size
|
|
915
|
+
num_classes = len(inputs)
|
|
916
|
+
|
|
917
|
+
# Create tensor.
|
|
918
|
+
T = np.zeros((num_inputs, input_size, num_classes))
|
|
919
|
+
for i in range(num_inputs):
|
|
920
|
+
for k in range(num_classes):
|
|
921
|
+
T[i, :, k] = np.array(inputs[k][i])
|
|
922
|
+
|
|
923
|
+
return T, X, Y
|
|
924
|
+
|
|
925
|
+
|
|
926
|
+
def mlsvd_train(T, r, options=False):
|
|
927
|
+
"""
|
|
928
|
+
Compute MLSVD (Multilinear Singular Value Decomposition) of T, a tensor of shape m x n x p. More precisely, we have
|
|
929
|
+
that T = (U1, U2, U3)*S (in fact, S is truncated so this is an approximation), where S is a R1 x R2 x R3 tensor (we
|
|
930
|
+
have that R3 = p, that is, the number of classes can't change), S_energy is the total energy retained after
|
|
931
|
+
truncating the original MLSVD, U1 is a m x R1 matrix, U2 is a n x R2 matrix, U3 is a p x R3 matrix, sigma1 is a
|
|
932
|
+
vector with the first R1 singular values of T1 (first unfolding), sigma2 is a vector with the first R2 singular
|
|
933
|
+
values of T2, sigma3 is a vector with the first R3 singular values of T3, rel_error = | T - (U1, U2, U3)*S | / |T|
|
|
934
|
+
is the relative error of the truncated MLSVD.
|
|
935
|
+
|
|
936
|
+
Inputs
|
|
937
|
+
------
|
|
938
|
+
T: 3D array
|
|
939
|
+
Each vector T[i, :, k] is the ith input of the kth class.
|
|
940
|
+
r: int
|
|
941
|
+
Rank used to compute the MLSVD. See the function mlsvd in Tensor Fox for more details about this parameter. If a
|
|
942
|
+
specific truncation trunc_dims is passed, the parameter r is irrelevant.
|
|
943
|
+
options: class
|
|
944
|
+
Options to the computation of the MLSVD. See more in the Tensor Fox documentation. If set to False, the default
|
|
945
|
+
options are used.
|
|
946
|
+
|
|
947
|
+
Outputs
|
|
948
|
+
-------
|
|
949
|
+
F: 3D array
|
|
950
|
+
Each vector F[i, :, k] is the compressed ith input of the kth class. We have F = (I, I, U3)*S =
|
|
951
|
+
= (U1.T, U2.T, I)*T, where is the identity matrix.
|
|
952
|
+
U2: 2D array
|
|
953
|
+
Second orthogonal matrix of the MLSVD of T.
|
|
954
|
+
success: bool
|
|
955
|
+
success is False only if the last dimension of T was trancated. In this case the proccess must be repeated.
|
|
956
|
+
"""
|
|
957
|
+
|
|
958
|
+
print('Training model...')
|
|
959
|
+
m, n, p = T.shape
|
|
960
|
+
num_samples, num_classes = m, p
|
|
961
|
+
print('Original shape:', T.shape)
|
|
962
|
+
|
|
963
|
+
# Set options
|
|
964
|
+
options = tfx.aux.make_options(options)
|
|
965
|
+
|
|
966
|
+
Tsize = np.linalg.norm(T)
|
|
967
|
+
if options.display == 3:
|
|
968
|
+
S, U, UT, sigmas, rel_error = tfx.cmpr.mlsvd(T, Tsize, r, options)
|
|
969
|
+
else:
|
|
970
|
+
S, U, UT, sigmas = tfx.cmpr.mlsvd(T, Tsize, r, options)
|
|
971
|
+
|
|
972
|
+
R1, R2, R3 = S.shape
|
|
973
|
+
U1, U2, U3 = U
|
|
974
|
+
|
|
975
|
+
# Sometimes the MLSVD may truncate the last dimension p. We have to repeat the training in this case.
|
|
976
|
+
if R3 != num_classes:
|
|
977
|
+
print('The number of classes was truncated. This cannot happen. Try again, but if the problem persists, we '
|
|
978
|
+
'recommend to manually pass the truncation with the parameter `trunc_dims`.')
|
|
979
|
+
success = False
|
|
980
|
+
F, U2 = 0, 0
|
|
981
|
+
else:
|
|
982
|
+
success = True
|
|
983
|
+
S1 = tfx.cnv.unfold(S, 1)
|
|
984
|
+
F = tfx.mlinalg.multilin_mult([np.identity(R1), np.identity(R2), U3], S1, S.shape)
|
|
985
|
+
print('MLSVD shape:', F.shape)
|
|
986
|
+
if options.display == 3:
|
|
987
|
+
print('Error:', rel_error)
|
|
988
|
+
print('Working with', np.round(100*int(prod(F.shape, dtype=uint64))/int(prod(T.shape, dtype=uint64)), 4), '% of the original size')
|
|
989
|
+
print()
|
|
990
|
+
|
|
991
|
+
return F, U2, success
|
|
992
|
+
|
|
993
|
+
|
|
994
|
+
def mlsvd_test(X_test, Y_test, F, U2, p, mu=0, sigma=1):
|
|
995
|
+
"""
|
|
996
|
+
After the training stage it is time to validate the model with the test dataset. This function is responsible for
|
|
997
|
+
that part.
|
|
998
|
+
|
|
999
|
+
Inputs
|
|
1000
|
+
------
|
|
1001
|
+
X_test: 2D array
|
|
1002
|
+
Test data in 2D array format. Each row correspond to a single sample of the test data.
|
|
1003
|
+
Y_test:
|
|
1004
|
+
Each entry i of Y_test correspond to the class of the ith input of the test dataset (ith row of X_test).
|
|
1005
|
+
F: 3D array
|
|
1006
|
+
Same tensor of mlsvd_train function.
|
|
1007
|
+
U2: 2D array
|
|
1008
|
+
Same matrix of mlsvd_train function.
|
|
1009
|
+
p: Float
|
|
1010
|
+
Float between 0 and 1. The program compress the data so that 100*p % is retained. For example, if
|
|
1011
|
+
p = 0.90, then 90 % of the data is retained after compression. This compression is used in each slice
|
|
1012
|
+
of F to make the predictions.
|
|
1013
|
+
mu, sigma: float
|
|
1014
|
+
mu and sigma are such the normalized data is given by (X_test - mu)/sigma.
|
|
1015
|
+
|
|
1016
|
+
Outputs
|
|
1017
|
+
-------
|
|
1018
|
+
accuracy: float
|
|
1019
|
+
Accuracy of the model over the test dataset.
|
|
1020
|
+
"""
|
|
1021
|
+
|
|
1022
|
+
predictions = mlsvd_predictions(X_test, F, U2, p, mu=mu, sigma=sigma)
|
|
1023
|
+
|
|
1024
|
+
num_samples = X_test.shape[0]
|
|
1025
|
+
success = 0
|
|
1026
|
+
for j in range(num_samples):
|
|
1027
|
+
x_class = predictions[j]
|
|
1028
|
+
y = Y_test[j]
|
|
1029
|
+
if x_class == y:
|
|
1030
|
+
success += 1
|
|
1031
|
+
|
|
1032
|
+
accuracy = 100 * success/num_samples
|
|
1033
|
+
|
|
1034
|
+
return accuracy
|
|
1035
|
+
|
|
1036
|
+
|
|
1037
|
+
def mlsvd_predictions(X_test, F, U2, p, mu=0, sigma=1):
|
|
1038
|
+
"""
|
|
1039
|
+
In a more geometric interpretation, first we consider the space generated by the columns associated to the digit d,
|
|
1040
|
+
then we check the distance between z_new and this space. This distance is the norm associated with the solution of
|
|
1041
|
+
the least squares problem. After choosing d, this choice means z_new is more close to the space associated to the
|
|
1042
|
+
digit d.
|
|
1043
|
+
|
|
1044
|
+
Inputs
|
|
1045
|
+
------
|
|
1046
|
+
X_test: 2D array
|
|
1047
|
+
Test data in 2D array format. Each row correspond to a single sample of the test data.
|
|
1048
|
+
F: 3D array
|
|
1049
|
+
Same tensor of mlsvd_train function.
|
|
1050
|
+
U2: 2D array
|
|
1051
|
+
Same matrix of mlsvd_train function.
|
|
1052
|
+
p: Float
|
|
1053
|
+
Float between 0 and 1. The program compress the data so that 100*p % is retained. For example, if
|
|
1054
|
+
p = 0.90, then 90 % of the data is retained after compression. This compression is used in each slice
|
|
1055
|
+
of F to make the predictions.
|
|
1056
|
+
mu, sigma: float
|
|
1057
|
+
mu and sigma are such the normalized data is given by (X_test - mu)/sigma.
|
|
1058
|
+
|
|
1059
|
+
Outputs
|
|
1060
|
+
-------
|
|
1061
|
+
predictions: 1D array
|
|
1062
|
+
Predictions of the classes of each sample in X_test.
|
|
1063
|
+
"""
|
|
1064
|
+
|
|
1065
|
+
print('Computing predictions...')
|
|
1066
|
+
X_test_new = (X_test - mu)/sigma
|
|
1067
|
+
num_samples, num_classes = X_test_new.shape[0], F.shape[2]
|
|
1068
|
+
predictions = np.zeros(num_samples, dtype=np.int64)
|
|
1069
|
+
|
|
1070
|
+
# Compute truncated SVD of slices of F.
|
|
1071
|
+
F_slices_svd = []
|
|
1072
|
+
for d in range(num_classes):
|
|
1073
|
+
U, S, Vt = np.linalg.svd(F[:, :, d])
|
|
1074
|
+
# Truncate U
|
|
1075
|
+
n = S.size
|
|
1076
|
+
total_energy = np.sum(S)
|
|
1077
|
+
for k in range(n):
|
|
1078
|
+
energy = np.sum(S[0:k])
|
|
1079
|
+
ratio = energy / total_energy
|
|
1080
|
+
if ratio > p:
|
|
1081
|
+
break
|
|
1082
|
+
F_slices_svd.append(Vt[:k, :].T)
|
|
1083
|
+
|
|
1084
|
+
# Make predictions.
|
|
1085
|
+
for i in range(num_samples):
|
|
1086
|
+
z = X_test_new[i, :]
|
|
1087
|
+
z_new = np.dot(U2.T, z)
|
|
1088
|
+
best_dist = np.inf
|
|
1089
|
+
for d in range(num_classes):
|
|
1090
|
+
x = np.dot(F_slices_svd[d].T, z_new)
|
|
1091
|
+
# Distance between z' and the space associated to the digit d.
|
|
1092
|
+
dist = np.linalg.norm(np.dot(F_slices_svd[d], x) - z_new)
|
|
1093
|
+
if dist < best_dist:
|
|
1094
|
+
best_class = d
|
|
1095
|
+
best_dist = dist
|
|
1096
|
+
|
|
1097
|
+
# Save results.
|
|
1098
|
+
predictions[i] = best_class
|
|
1099
|
+
|
|
1100
|
+
print('Finished')
|
|
1101
|
+
|
|
1102
|
+
return predictions
|
|
1103
|
+
|