TensorFox 1.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1103 @@
1
+ import numpy as np
2
+ from numpy import prod, uint64
3
+ import sys
4
+ import matplotlib.pyplot as plt
5
+ from numba import njit, prange
6
+ import TensorFox.TensorFox as tfx
7
+
8
+
9
+ # PREPROCESSING
10
+
11
+
12
+ def normalization(X, method='mean_normalization'):
13
+ """
14
+ Normalize data by shifting all data by a certain constant, and dividing all data by other certain constant. The
15
+ program returns the normalized data X = (X - mu)/sigma. In the case the user want to return to the original format,
16
+ set X = sigma*X + mu.
17
+
18
+ Inputs
19
+ ------
20
+ X: 2D array
21
+ Train data in 2D array format. Each row correspond to a single sample of the data.
22
+ method: string
23
+ There are 4 available methods of feature scaling.
24
+ min_max_normalization: X = (X - min(X)) / (max(X) - min(X))
25
+ mean_normalization: X = (X - mean(X)) / (max(X) - min(X))
26
+ standardization: X = (X - mean(X)) / sqrt(var(X))
27
+ unit_length: X = X / norm(X), where norm() is the Frobenius norm
28
+
29
+ Outputs
30
+ -------
31
+ X_new: 2D array
32
+ Transformed data after normalization, that is, X_new = (X - mu)/sigma.
33
+ mu: 2D array
34
+ The value which X is subtracted.
35
+ sigma: float
36
+ The value which X will be divided. In the case max_unit and var_unit are False, then sigma = 1.
37
+ """
38
+
39
+ if method == 'min_max_normalization':
40
+ mu = np.min(X)
41
+ sigma = np.max(X) - np.min(X)
42
+ elif method == 'mean_normalization':
43
+ mu = np.mean(X)
44
+ sigma = np.max(X) - np.min(X)
45
+ elif method == 'standardization':
46
+ mu = np.mean(X)
47
+ sigma = np.sqrt(np.var(X))
48
+ elif method == 'unit_length':
49
+ mu = 0
50
+ sigma = np.linalg.norm(X)
51
+
52
+ X_new = (X - mu)/sigma
53
+
54
+ return X_new, mu, sigma
55
+
56
+
57
+ def pca(X, p):
58
+ """
59
+ Make principal component analysis of the data. In the case the user want to return to the original format, set
60
+ X = dot(X, U.T) .
61
+
62
+ Inputs
63
+ ------
64
+ X: 2D array
65
+ Train data in 2D array format. Each row correspond to a single sample of the data.
66
+ p: Float
67
+ Float between 0 and 1. The program compress the data so that 100*p % is retained. For example, if p = 0.90, then
68
+ 90 % of the data is retained after compression.
69
+
70
+ Outputs
71
+ -------
72
+ X_new: 2D array
73
+ Transformed data after PCA.
74
+ U: 2D array
75
+ Left singular vectors of the SVD of the covariance matrix of X.
76
+ """
77
+
78
+ # Number of samples and size of inputs
79
+ num_samples, n = X.shape
80
+ # Covariance matrix of X
81
+ cov = (1 / num_samples) * np.dot(X.T, X)
82
+ # SVD of covariance matrix
83
+ U, S, Vt = np.linalg.svd(cov)
84
+
85
+ # Compute size k <= n of compression
86
+ total_energy = np.sum(S)
87
+ for k in range(n):
88
+ energy = np.sum(S[0:k])
89
+ ratio = energy / total_energy
90
+ if ratio > p:
91
+ break
92
+
93
+ # Compute compressed inputs
94
+ U = U[:, :k]
95
+ X_new = np.dot(X, U)
96
+
97
+ return X_new, U
98
+
99
+
100
+ def prepare_data(X, p):
101
+ """
102
+ Before the starting the training stage, you will need to prepare your data, that is, make the PCA, construct the one
103
+ hot encoded classes, adding bias, and so on. This function and the function normalize_data are close since both
104
+ prepare the data for training.
105
+
106
+ Inputs
107
+ ------
108
+ X: 2D array
109
+ Train data in 2D array format. Each row correspond to a single sample of the data.
110
+ p: Float
111
+ Float between 0 and 1. The program compress the data so that 100*p % is retained. For example, if p = 0.90, then
112
+ 90 % of the data is retained after compression.
113
+
114
+ Outputs
115
+ -------
116
+ X_bias: 2D array
117
+ Train data prepared to be used for training.
118
+ U: 2D array
119
+ Left singular vectors of the SVD of the covariance matrix of X.
120
+ """
121
+
122
+ # PCA of inputs
123
+ X_new, U = pca(X, p)
124
+ num_samples, n = X_new.shape
125
+
126
+ # Add bias
127
+ aux = np.ones((num_samples, n + 1))
128
+ for i in range(num_samples):
129
+ aux[i, 1:] = X_new[i, :]
130
+ X_bias = aux
131
+
132
+ return X_bias, U
133
+
134
+
135
+ def hot_encoded_target(Y):
136
+ """
137
+ If the labels are not hot encoded, then this function is mandatory, otherwise we get errors. We are using one hot
138
+ encoded outputs of the form [1,0,0,...,0], [0,1,0,...,0], ..., [0,0,...,0,1]. To make the conversion the program
139
+ assumes the classes (the entries of Y) are numbers, from 0 to m-1 (so we have m classes).
140
+
141
+ Inputs
142
+ ------
143
+ Y: 1D array
144
+ Each entry i of Y correspond to the class of the ith input (ith row of X).
145
+
146
+ Outputs
147
+ -------
148
+ Y_hot_encoded: 2D array
149
+ Train data classes with hot encoding.
150
+ """
151
+
152
+ num_samples = Y.size
153
+ m = int(max(Y)) + 1
154
+ Y_hot_encoded = np.zeros((num_samples, m))
155
+ I = np.identity(m)
156
+
157
+ for i in range(num_samples):
158
+ for j in range(m):
159
+ if Y[i] == j:
160
+ Y_hot_encoded[i, :] = I[j, :]
161
+
162
+ return Y_hot_encoded
163
+
164
+
165
+ def norm_pca(X_new, U, mu, sigma):
166
+ """
167
+ After normalizing and performing pca over the train dataset, we may be interested in making predictions about some
168
+ new inputs. In any case, normalizing and compressing in the same way it is necessary in order to get meaningful
169
+ results.
170
+
171
+ Inputs
172
+ ------
173
+ X_new: 2D array
174
+ Input data in 2D array format. Each row correspond to a single sample of the data.
175
+ U: 2D array
176
+ Left singular vectors of the SVD of the covariance matrix of the train dataset.
177
+ mu: 2D array
178
+ Mean of the train dataset.
179
+ sigma: float
180
+ The value in which the dataset was divided.
181
+
182
+ Outputs
183
+ -------
184
+ X_new: 2D array
185
+ Input data after transformations.
186
+ """
187
+
188
+ X_new = (X_new - mu) / sigma
189
+ X_new = np.dot(X_new, U)
190
+
191
+ return X_new
192
+
193
+
194
+ # CPD LEARNING
195
+
196
+
197
+ def init_W(m, L, n, R):
198
+ """
199
+ Generate initial weights to start iterating. We have that W[k] = [W1,...,WL] is a multidimensional array of shape
200
+ L x n x R. For each tensor Tk of the model we have that
201
+
202
+ Tk = sum_{r=1}^R W[k, 0, :, r] ⊗ W[k, 1, :, r] ⊗ ... ⊗ W[k, L-1, :, r].
203
+
204
+ Note that each W[k, l, :, :] is a n x R factor matrix of the CPD of Tk, in other words, W[k] is the set of the
205
+ factor matrices of Tk. Each tensor Tk has shape n x n x ... x n (L times) and has rank <= R.
206
+
207
+ Inputs
208
+ ------
209
+ m: int
210
+ Number of tensors of the model, which must coincide with the dimension of the output space.
211
+ L: int
212
+ Order of the tensors.
213
+ n: int
214
+ Dimension of the input space.
215
+ R: int
216
+ Rank of the tensors.
217
+
218
+ Outputs
219
+ -------
220
+ W: 4D array
221
+ """
222
+
223
+ W = np.zeros((m, L, n, R))
224
+
225
+ for k in range(m):
226
+ for l in range(L):
227
+ W[k, l, :, :] = np.random.uniform(-1, 1, size=(n, R))
228
+
229
+ return W
230
+
231
+
232
+ def dot_products(x, W, dropout):
233
+ """
234
+ Function to compute all the dot products between an input x and the weights.
235
+
236
+ Inputs
237
+ ------
238
+ x: 1D array
239
+ A single input sample.
240
+ W: L-D array
241
+ Coordinates of all tensors Tk of the model (see function init_W for more details).
242
+ dropout:
243
+ Probability to use or not dropout (see function cpd_train for more details).
244
+
245
+ Outputs
246
+ -------
247
+ dot_prods: 3D array
248
+ Each value dot_prods[k, l, r] is the dot product between x and the weight vector corresponding to the r-th
249
+ column of the l-th factor matrix of the k-th tensor.
250
+ """
251
+
252
+ # Initialize array to receive all dot products
253
+ m, L, n, R = W.shape
254
+ dot_prods = np.ones((m, L, R))
255
+
256
+ # Compute all the dot products.
257
+ lst = [W[k, :, :, r] for k in range(m) for r in range(R)]
258
+ W_tmp = np.concatenate(lst)
259
+ Wx = np.dot(W_tmp, x)
260
+ Wx = Wx.reshape(m, R, L)
261
+ for k in range(m):
262
+ dot_prods[k, :, :] = Wx[k, :, :].T
263
+
264
+ # The above is an optimized form of the calculations commented below.
265
+ #for k in range(m):
266
+ # for r in range(R):
267
+ # dot_prods[k, :, r] = np.dot(W[k, :, :, r], x)
268
+
269
+ # Apply dropout.
270
+ if dropout < 1:
271
+ dropout_decision = np.random.uniform(0, 1, size=(m, R))
272
+ dropout_decision = dropout_decision <= dropout
273
+ for k in range(m):
274
+ for r in np.arange(R)[~dropout_decision[k, :]]:
275
+ dot_prods[k, :, r] *= 0
276
+
277
+ return dot_prods
278
+
279
+
280
+ @njit(nogil=True)
281
+ def h(W, dot_prods):
282
+ """
283
+ Function to compute the hypothesis function h(x). This is defined by
284
+
285
+ h(x) = f(T1(x,...,x),...,Tm(x,...,x)) = ( f(T1(x,...,x)) ,..., f(Tm(x,...,x)) ),
286
+
287
+ where f is the activation function (in this case we are using the sigmoid function). Note that h(x) is a
288
+ m-dimensional vector.
289
+
290
+ Inputs
291
+ ------
292
+ W: 4D array
293
+ Coordinates of all tensors Tk of the model (see function init_W for more details).
294
+ dot_prods: 3D array.
295
+ Dot products between the weights and x (see function dot_products for more details).
296
+
297
+ Outputs
298
+ -------
299
+ hx: 1D array
300
+ Evaluation of h(x).
301
+ dhx: 1D array
302
+ Evaluation of the derivative of h at x, that is, h'(x).
303
+ """
304
+
305
+ m, L, n, R = W.shape
306
+ # Initialize vector to receive T(x,...,x)
307
+ hx = np.zeros(m, dtype=np.float64)
308
+
309
+ # Compute Tk(x,...,x) for all k=1...m
310
+ for k in range(m):
311
+ for r in range(R):
312
+ temp = 1
313
+ for l in range(L):
314
+ temp *= dot_prods[k, l, r]
315
+ hx[k] += temp
316
+
317
+ # Compute (f'(T1(x,...,x),...,f'(Tm(x,...,x))
318
+ dhx = df(hx)
319
+ # Compute f(T1(x,...,x),...,Tm(x,...,x))
320
+ hx = f(hx)
321
+
322
+ return hx, dhx
323
+
324
+
325
+ @njit(nogil=True)
326
+ def f(z):
327
+ """
328
+ Non-linear (activation) function f at a point z(we are using the sigmoid function here).
329
+
330
+ Inputs
331
+ ------
332
+ z: 1D array
333
+
334
+ Outputs
335
+ -------
336
+ fz: 1D array
337
+ """
338
+
339
+ fz = 1/(1 + np.exp(-z))
340
+ return fz
341
+
342
+
343
+ @njit(nogil=True)
344
+ def df(z):
345
+ """
346
+ Derivative of the non-linear (activation) function f at a point z.
347
+
348
+ Inputs
349
+ ------
350
+ z: 1D array
351
+
352
+ Outputs
353
+ -------
354
+ dfz: 1D array
355
+ """
356
+
357
+ dfz = f(z)*(1 - f(z))
358
+ return dfz
359
+
360
+
361
+ @njit(nogil=True, parallel=True)
362
+ def grad(x, y, W, dot_prods, Lambda):
363
+ """
364
+ Let J(W) be the cost function of the problem, and grad(J) be the gradient of J. Then this function constructs
365
+ grad(J)_{k, l, i, r} with respect to a pair input-output (x, y). In the end, the partial derivative of J with
366
+ respect to W[k, l, i, r] is the sum of grad(J)_{k, l, i, r}(x, y) for all pairs (x, y).
367
+
368
+ Inputs
369
+ ------
370
+ x: 1D array
371
+ y: 1D array
372
+ W: 4D array
373
+ dot_prods: 3D array
374
+ Lambda: float
375
+ Regularization parameter for the cost function. We must have Lambda >= 0.
376
+
377
+ Outputs
378
+ -------
379
+ grad_J: 4D array
380
+ """
381
+
382
+ m, L, n, R = W.shape
383
+ # Initialize gradient vector associated to J at x
384
+ grad_J = np.zeros((m, L, n, R))
385
+ # Compute hypothesis and its derivative at x
386
+ hx, dhx = h(W, dot_prods)
387
+
388
+ for k in prange(m):
389
+ for l in range(L):
390
+ for i in range(n):
391
+ for r in range(R):
392
+ grad_J[k, l, i, r] = compute_grad(x, y, W, dot_prods, hx, dhx, Lambda, L, k, l, i, r)
393
+
394
+ return grad_J
395
+
396
+
397
+ @njit(nogil=True)
398
+ def compute_grad(x, y, W, dot_prods, hx, dhx, Lambda, L, k, l, i, r):
399
+ """
400
+ Auxiliar function to make inner computations of function grad.
401
+ """
402
+
403
+ term1 = Lambda * W[k, l, i, r]
404
+ term2 = hx[k] - y[k]
405
+ term3 = dhx[k]
406
+
407
+ temp = 1
408
+ for lt in range(L):
409
+ if lt != l:
410
+ temp *= dot_prods[k, lt, r]
411
+ term4 = x[i] * temp
412
+
413
+ return term1 + term2*term3*term4
414
+
415
+
416
+ def update(alpha, W, grad_J, constraint):
417
+ """
418
+ Compute the update stage W = W - alpha*grad_J for the weights in the gradient descent algorithm.
419
+
420
+ Inputs
421
+ ------
422
+ alpha: float
423
+ Step parameter of the gradient descent algorithm. We must have alpha > 0.
424
+ W: 4D array
425
+ grad_J: 4D array
426
+ See the description of grad_J in the function grad.
427
+ constraint: float
428
+ This paramater limits the size of the weights with the update formula W = constraint * W/max(abs(W)). This way
429
+ the largest magnitude of W is always equal to this parameter.
430
+
431
+ Outputs
432
+ -------
433
+ W: 4D array
434
+ Update of W.
435
+ """
436
+
437
+ W = W - alpha * grad_J
438
+
439
+ if constraint != np.inf:
440
+ W = constraint * W / np.max(np.abs(W))
441
+
442
+ return W
443
+
444
+
445
+ def find_class(x, W):
446
+ """
447
+ Given a set of weights W and a new input x, this function computes the predicted class of x.
448
+
449
+ Inputs
450
+ ------
451
+ x: 1D array
452
+ New input for which the program will try to predict its class.
453
+ W:
454
+ Set of weights after the learning stage.
455
+
456
+ Outputs
457
+ -------
458
+ x_class: int
459
+ At the moment the program is limited to classification problems. So each prediction is a number (of a class).
460
+ """
461
+
462
+ dot_prods = dot_products(x, W, 1)
463
+ hx, dhx = h(W, dot_prods)
464
+ x_class = np.argmax(hx)
465
+
466
+ return x_class
467
+
468
+
469
+ def cpd_train(X, Y, X_val, Y_val, W, alpha=0.01, alpha_decay=0.5, Lambda=0.1, epochs=10, batch=1, constraint=np.inf, dropout=1, display=True):
470
+ """
471
+ Function to start the training stage.
472
+
473
+ Inputs
474
+ ------
475
+ X: 2D array
476
+ Train data in 2D array format. Each row correspond to a single sample of the data.
477
+ Y: 1D array
478
+ Each entry i of Y correspond to the original class of the ith input (ith row of X).
479
+ X_val: 2D array
480
+ Piece of train dataset used as validation input set. If no validation is intended, set X_val to nan.
481
+ Y_val: 1D array
482
+ Piece of train dataset used as validation target set.
483
+ W: 4D array
484
+ Initial set of weights (see function init_W for more details).
485
+ alpha: float
486
+ Step parameter of the gradient descent algorithm. We must have alpha > 0.
487
+ alpha_decay: float
488
+ The more the program is closer to the optimum, the more it is necessary to take smaller steps. In this case it
489
+ is interesting to decrease alpha gradually. At each epoch we update alpha with alpha = alpha_decay * alpha.
490
+ Default is alpha_decay = 0.5.
491
+ Lambda: float
492
+ Regularization parameter for the cost function. We must have Lambda >= 0. Default is Lambda = 0.1.
493
+ epochs: int
494
+ Desired number of epochs to use in the training stage. Default is 10 epochs.
495
+ batch: int
496
+ Size of the batch in the learning algorithm. After passing through batch inputs and accumulating their costs,
497
+ the corresponding gradient is used to make the next step. Default is 1 batch.
498
+ constraint: float
499
+ Large weights size can be a sign of an unstable network. Ona may force the magnitude of all weights to be below
500
+ a specified value, given by this parameter. At each iteration the program applies the update formula
501
+ W = constraint * W/max(abs(W)). This way the largest magnitude of W is always equal to this parameter. Default
502
+ is constraint = np.inf, which means to not use constraints.
503
+ dropout: float
504
+ This value should be between 1 (no dropout is used) and 0 (drop is used for all nodes). At each iteration the
505
+ program decides, for each node, if it will be used or not, based on this probability. Default is dropout = 1.
506
+ display: bool
507
+ If set to True (default), the program displays some relevant results and plots about the training stage.
508
+
509
+ Outputs
510
+ -------
511
+ W: 4D array
512
+ Optimized set of weights.
513
+ accuracy: list
514
+ Evolution of the accuracy (0 % to 100 %) at each iteration.
515
+ cost_function: list
516
+ Evolution of the cost function at each iteration.
517
+ success: int
518
+ Total number of successes after the training.
519
+ """
520
+
521
+ # Verify if the parameter dropout is consistent.
522
+ if dropout < 0 or dropout > 1:
523
+ sys.exit('The parameter dropout has to be between 0 ans 1.')
524
+ elif dropout == 0:
525
+ sys.exit('The parameter dropout has to be bigger than 0.')
526
+
527
+ # Number of samples.
528
+ num_samples, n = X.shape
529
+
530
+ # Transform target into hot encoded form.
531
+ Y_hot_encoded = hot_encoded_target(Y)
532
+
533
+ # Identity matrix with size of inputs minus bias.
534
+ I = np.identity(n-1)
535
+
536
+ # Create vectors to visualize some information later.
537
+ accuracy = []
538
+ accuracy_val = []
539
+ cost_function = []
540
+
541
+ for ep in range(epochs):
542
+ count = 0
543
+ success = 0
544
+ cum_cost = 0
545
+ cum_grad = np.zeros(W.shape)
546
+
547
+ for j in range(num_samples):
548
+ x = X[j, :]
549
+ y = Y_hot_encoded[j, :]
550
+ dot_prods = dot_products(x, W, dropout)
551
+ grad_J = grad(x, y, W, dot_prods, Lambda)
552
+ cum_grad += grad_J
553
+
554
+ # Update weights after seeing batch inputs.
555
+ if (j % batch == 0) and (j >= batch):
556
+ W = update(alpha, W, cum_grad/batch, constraint)
557
+ cum_grad = 0*cum_grad
558
+ elif j == num_samples-1:
559
+ W = update(alpha, W, cum_grad/batch, constraint)
560
+
561
+ # Predict class of current input.
562
+ x_class = find_class(x, W)
563
+ if x_class == Y[j]:
564
+ success += 1
565
+
566
+ # Save the value of the cost function with respect to this individual input.
567
+ hx, dhx = h(W, dot_prods)
568
+ cost = (1 / 2) * np.linalg.norm(hx - y) ** 2
569
+ cum_cost += cost
570
+
571
+ # Display progress bar.
572
+ if j % (num_samples//80) == 0 and j >= (num_samples//80):
573
+ count += 1
574
+ s = "Epoch " + str(ep+1) + ": [" + min(80, count) * "=" + min((80-count), 80) * " " + "]" + \
575
+ " " + str(np.round(100*j/num_samples, 1)) + "%"
576
+ sys.stdout.write('\r' + s)
577
+
578
+ # Display progress bar at 100%.
579
+ s = "Epoch " + str(ep+1) + ": [" + min(80, count) * "=" + min((80-count), 80) * " " + "]" + " " + "100.0% / acc=" + str(round(100 * success/num_samples, 2)) + '%'
580
+ sys.stdout.write('\r' + s)
581
+
582
+ # Update relevant information.
583
+ acc = 100 * success/num_samples
584
+ accuracy.append(100 * success/num_samples)
585
+ cost_function.append(cum_cost/num_samples)
586
+
587
+ # Update alpha.
588
+ alpha = alpha_decay * alpha
589
+
590
+ # The X_val dataset is supposed to be already normalized.
591
+ if np.sum(np.isnan(X_val)) == 0:
592
+ accuracy_val.append(cpd_test(X_val, Y_val, W, I))
593
+ print()
594
+
595
+ if display:
596
+ disp_results(accuracy, accuracy_val, cost_function, W, epochs)
597
+
598
+ return W, accuracy, accuracy_val, cost_function, success
599
+
600
+
601
+ def disp_results(accuracy, accuracy_val, cost_function, W, epochs):
602
+ """
603
+ Display results of the training stage.
604
+
605
+ Inputs
606
+ ------
607
+ accuracy: list
608
+ Evolution of the accuracy (0 % to 100 %) at each iteration.
609
+ accuracy_val list
610
+ Evolution of the validation accuracy (0 % to 100 %) at each iteration.
611
+ cost_function: list
612
+ Evolution of the cost function at each iteration.
613
+ W: 4D array
614
+ Initial set of weights (see function init_W for more details).
615
+ epochs: int
616
+ Number desired of epochs to use in the training stage.
617
+ """
618
+
619
+ m, L, n, R = W.shape
620
+ print()
621
+ print('Dimensions of input:', n)
622
+ print('Dimensions of target:', m)
623
+ print('Rank:', R)
624
+ print('Number of factor matrices:', L)
625
+ print('Number of epochs:', epochs)
626
+ print('Number of weights:', ((L - 1) * n + m) * R)
627
+ print('Accuracy over training dataset:', np.round(accuracy[-1], 3), '%')
628
+ if len(accuracy_val) > 0:
629
+ print('Accuracy over validation dataset:', np.round(accuracy_val[-1], 3), '%')
630
+ print()
631
+
632
+ plt.figure(figsize=[10, 4])
633
+ plt.plot(np.arange(1, epochs+1), accuracy, markersize=2, label='train')
634
+ if len(accuracy_val) > 0:
635
+ plt.plot(np.arange(1, epochs + 1), accuracy_val, markersize=2, label='validation')
636
+ plt.title('Learning Curve')
637
+ plt.grid()
638
+ plt.legend()
639
+ plt.xlabel('Epoch')
640
+ plt.show()
641
+
642
+ plt.figure(figsize=[10, 4])
643
+ plt.plot(np.arange(1, epochs+1), cost_function, 'r', markersize=2)
644
+ plt.title('Cost function')
645
+ plt.grid()
646
+ plt.xlabel('Epoch')
647
+ plt.show()
648
+
649
+ return
650
+
651
+
652
+ def cpd_test(X_test, Y_test, W, U, mu=0, sigma=1):
653
+ """
654
+ After the training stage it is time to validate the model with the test dataset. This function is responsible for
655
+ that part. The default values for mu and sigma assume that no normalization was performed. Additionally, if no PCA
656
+ was also performed, pass the matrix U as np.identity(X.shape[1]).
657
+
658
+ Inputs
659
+ ------
660
+ X_test: 2D array
661
+ Test data in 2D array format. Each row correspond to a single sample of the test data.
662
+ Y_test:
663
+ Each entry i of Y_test correspond to the class of the ith input of the test dataset (ith row of X_test).
664
+ W: 4D array
665
+ Weights obtained after the training stage.
666
+ U: 2D array
667
+ Left singular vectors of the SVD of the covariance matrix of the normalized X_test. The compressed data is given
668
+ by dot(X_test_new, U), where X_test_new is the normalized version of X_test.
669
+ mu, sigma: float
670
+ mu and sigma are such the normalized data is given by (X_test - mu)/sigma.
671
+
672
+ Outputs
673
+ -------
674
+ accuracy: float
675
+ Accuracy of the model over the test dataset.
676
+ """
677
+
678
+ predictions = cpd_predictions(X_test, W, U, mu=mu, sigma=sigma)
679
+
680
+ num_samples = X_test.shape[0]
681
+ success = 0
682
+ for j in range(num_samples):
683
+ x_class = predictions[j]
684
+ y = Y_test[j]
685
+ if x_class == y:
686
+ success += 1
687
+
688
+ accuracy = 100 * success/num_samples
689
+
690
+ return accuracy
691
+
692
+
693
+ def cpd_predictions(X_test, W, U, mu=0, sigma=1):
694
+ """
695
+ Given set of new inputs X_test, this function makes the corresponding predictions. The default values for mu and
696
+ sigma assume that no normalization was performed. Additionally, if no PCA was also performed, pass the matrix U as
697
+ np.identity(X.shape[1]).
698
+
699
+ Inputs
700
+ ------
701
+ X_test: 2D array
702
+ Test data in 2D array format. Each row correspond to a single sample of the test data.
703
+ W: 4D array
704
+ Weights obtained after the training stage.
705
+ U: 2D array
706
+ Left singular vectors of the SVD of the covariance matrix of the normalized X_test. The compressed data is given
707
+ by dot(X_test_new, U), where X_test_new is the normalized version of X_test.
708
+ mu, sigma: float
709
+ mu and sigma are such the normalized data is given by (X_test - mu)/sigma.
710
+
711
+ Outputs
712
+ -------
713
+ predictions: 1D array
714
+ Predictions of the classes of each sample in X_test.
715
+ """
716
+
717
+ # Transform data to compressed form. This is necessary since the weights W were obtained over compressed data.
718
+ X_test_new = norm_pca(X_test, U, mu, sigma)
719
+ num_samples, n = X_test_new.shape
720
+
721
+ # Add bias
722
+ aux = np.ones((num_samples, n + 1))
723
+ aux[:, 1:] = X_test_new
724
+ X_bias = aux
725
+
726
+ # Create array of predictions, which are integers from 0 to m-1.
727
+ predictions = np.zeros(num_samples, dtype=np.int64)
728
+
729
+ for j in range(num_samples):
730
+ x = X_bias[j, :]
731
+ x_class = find_class(x, W)
732
+ predictions[j] = x_class
733
+
734
+ return predictions
735
+
736
+
737
+ def simplify_model(W, r, options=False):
738
+ """
739
+ With the set of weights W, obtained after training and testing, we can construct the corresponding coordinate
740
+ tensors and compute a rank-r CPD for them. The idea is to simplify the model, using less rank-1 terms.
741
+
742
+ Inputs
743
+ ------
744
+ W: 4D array
745
+ Weights obtained after the training stage.
746
+ r: int
747
+ Desired rank.
748
+ options: class
749
+ Set of options to be used in the CPD computation (see the Tensor Fox documentation for more details).
750
+
751
+ Outputs
752
+ -------
753
+ new_W: float 4D-ndarray
754
+ New model based in rank-r approximations. An array with the same shape as W, except for the last dimension.
755
+ errors: list
756
+ Each errors[k] is the relative error between the rank-r approximation and Tk.
757
+ """
758
+
759
+ # Initialize variables.
760
+ m, L, n, R = W.shape
761
+ new_W = np.zeros((m, L, n, r))
762
+ errors = []
763
+
764
+ for k in range(m):
765
+ # Construct list of factor matrices for Tk.
766
+ factors = []
767
+ for l in range(L):
768
+ factors.append(W[k, l, :, :])
769
+ # Convert factor matrices to coordinates format.
770
+ Tk = tfx.cnv.cpd2tens(factors)
771
+ # Compute rank-r approximation for Tk.
772
+ factors, output = tfx.cpd(Tk, r, options)
773
+ errors.append(output.rel_error)
774
+ for l in range(L):
775
+ new_W[k, l, :, :] = factors[l]
776
+
777
+ return new_W, errors
778
+
779
+
780
+ # MLSVD LEARNING
781
+
782
+
783
+ def create_sets(X, Y, num_samples, p, var_type, display):
784
+ """
785
+ This function separates the inputs by class in a list called samples_per_class. If the number of inputs per
786
+ class is no equal, the program randomly select some inputs to be repeated in a certain class. This is done
787
+ so that all classes have the same number of elements.
788
+ There is the option to add noise to these repeated inputs, so we can have a simple method of data augmentation
789
+ to apply.
790
+
791
+ Inputs
792
+ ------
793
+ X: 2D array
794
+ Train data in 2D array format. Each row correspond to a single sample of the data.
795
+ Y: 1D array
796
+ Each entry i of Y correspond to the class of the ith input (ith row of X).
797
+ num_samples: int
798
+ The desired number of samples per class. This number should be equal or greater than the current maximum number
799
+ of samples in a class.
800
+ p: float
801
+ Proportion of the noise with respect to the original data. For example, if p=0.1, then the noise added has
802
+ size less or equal than 10% of the size of the original data.
803
+ var_type: str
804
+ This variable indicates the type or perturbation. In both cases we use uniform distribution to pick the
805
+ perturbations. The only possibilities considered are 'int' and 'float'. We remark that you should use float
806
+ whenever the data is normalized.
807
+ display: bool
808
+ If True (default), the function shows the number of inputs for each class.
809
+
810
+ Outputs
811
+ -------
812
+ X_new, Y_new: arrays
813
+ New versions of X and Y, maybe with more data.
814
+ inputs: list
815
+ Each element of this list is a list with all inputs with same class.
816
+ """
817
+
818
+ total_num_samples = X.shape[0]
819
+ num_classes = int(max(Y)) + 1
820
+ X_new = []
821
+ Y_new = []
822
+ inputs = [[] for i in range(num_classes)]
823
+
824
+ # Create list with all inputs separated by class.
825
+ for i in range(total_num_samples):
826
+ x = X[i, :]
827
+ y = int(Y[i])
828
+ inputs[y].append(x)
829
+
830
+ # Counting of inputs for each class.
831
+ samples_per_class = []
832
+ for i in range(num_classes):
833
+ samples_per_class.append(len(inputs[i]))
834
+ if display:
835
+ print('Inputs of class', i, '=', len(inputs[i]))
836
+
837
+ # Make each class to have the same number of inputs.
838
+ if num_samples is not None:
839
+ max_class = max(max(samples_per_class), num_samples)
840
+ else:
841
+ max_class = max(samples_per_class)
842
+
843
+ for i in range(num_classes):
844
+ c = samples_per_class[i]
845
+ diff = max_class - c
846
+ if diff > 1:
847
+ for j in range(diff):
848
+ idx = np.random.randint(c)
849
+ x = inputs[i][idx]
850
+
851
+ # Apply data augmentation.
852
+ xp = np.random.uniform(low=min(x), high=1+max(x), size=x.shape[0])
853
+ if var_type == 'float':
854
+ x_new = x + p*xp
855
+ elif var_type == 'int':
856
+ x_new = x + (p*xp).astype(int)
857
+ inputs[i].append(x_new)
858
+
859
+ # Update X_new and Y_new.
860
+ X_new.append(x_new)
861
+ Y_new.append(i)
862
+
863
+ # Convert X_new and Y_new to arrays.
864
+ X_new = np.array(X_new)
865
+ Y_new = np.array(Y_new)
866
+ X_new = np.concatenate([X, X_new])
867
+ Y_new = np.concatenate([Y, Y_new])
868
+
869
+ if display:
870
+ print()
871
+ print('After fixing number of inputs per class:')
872
+ for i in range(num_classes):
873
+ print('Inputs of class', i, '=', len(inputs[i]))
874
+
875
+ return X_new, Y_new, inputs
876
+
877
+
878
+ def data2tens(X, Y, num_samples=None, p=0, var_type='float', display=True):
879
+ """
880
+ Given the input dataset X and the target dataset Y, this function separates the inputs by class so that each
881
+ class has the same number of inputs m. If n is the dimension of the inputs and each class has p inputs, the
882
+ tensor created has shape m x n x p. Each frontal slice contains all inputs corresponding to a certain class.
883
+
884
+ Inputs
885
+ ------
886
+ X: 2D array
887
+ Train data in 2D array format. Each row correspond to a single sample of the data.
888
+ Y: 1D array
889
+ Each entry i of Y correspond to the class of the ith input (ith row of X).
890
+ num_samples: int
891
+ The desired number of samples per class. This number should be equal or greater than the current maximum number
892
+ of samples in a class.
893
+ p: float
894
+ Proportion of the noise with respect to the original data. For example, if p=0.1, then the noise added has
895
+ size less or equal than 10% of the size of the original data. Default is p=0, which means to not perturb.
896
+ var_type: str
897
+ This variable indicates the type or perturbation. In both cases we use uniform distribution to pick the
898
+ perturbations. The only possibilities considered are 'int' and 'float'. Default is 'float'. We remark that you
899
+ should use float whenever the data is normalized.
900
+ display: bool
901
+ If True (default), the function shows the number of inputs for each class.
902
+
903
+ Outputs
904
+ -------
905
+ T: 3D array
906
+ Each vector T[i, :, k] is the ith input of the kth class.
907
+ X_new, Y_new: arrays
908
+ New versions of X and Y, maybe with more data.
909
+ """
910
+
911
+ # Create list with inputs organized by class.
912
+ X, Y, inputs = create_sets(X, Y, num_samples, p, var_type, display)
913
+ num_inputs = len(inputs[0])
914
+ input_size = inputs[0][0].size
915
+ num_classes = len(inputs)
916
+
917
+ # Create tensor.
918
+ T = np.zeros((num_inputs, input_size, num_classes))
919
+ for i in range(num_inputs):
920
+ for k in range(num_classes):
921
+ T[i, :, k] = np.array(inputs[k][i])
922
+
923
+ return T, X, Y
924
+
925
+
926
+ def mlsvd_train(T, r, options=False):
927
+ """
928
+ Compute MLSVD (Multilinear Singular Value Decomposition) of T, a tensor of shape m x n x p. More precisely, we have
929
+ that T = (U1, U2, U3)*S (in fact, S is truncated so this is an approximation), where S is a R1 x R2 x R3 tensor (we
930
+ have that R3 = p, that is, the number of classes can't change), S_energy is the total energy retained after
931
+ truncating the original MLSVD, U1 is a m x R1 matrix, U2 is a n x R2 matrix, U3 is a p x R3 matrix, sigma1 is a
932
+ vector with the first R1 singular values of T1 (first unfolding), sigma2 is a vector with the first R2 singular
933
+ values of T2, sigma3 is a vector with the first R3 singular values of T3, rel_error = | T - (U1, U2, U3)*S | / |T|
934
+ is the relative error of the truncated MLSVD.
935
+
936
+ Inputs
937
+ ------
938
+ T: 3D array
939
+ Each vector T[i, :, k] is the ith input of the kth class.
940
+ r: int
941
+ Rank used to compute the MLSVD. See the function mlsvd in Tensor Fox for more details about this parameter. If a
942
+ specific truncation trunc_dims is passed, the parameter r is irrelevant.
943
+ options: class
944
+ Options to the computation of the MLSVD. See more in the Tensor Fox documentation. If set to False, the default
945
+ options are used.
946
+
947
+ Outputs
948
+ -------
949
+ F: 3D array
950
+ Each vector F[i, :, k] is the compressed ith input of the kth class. We have F = (I, I, U3)*S =
951
+ = (U1.T, U2.T, I)*T, where is the identity matrix.
952
+ U2: 2D array
953
+ Second orthogonal matrix of the MLSVD of T.
954
+ success: bool
955
+ success is False only if the last dimension of T was trancated. In this case the proccess must be repeated.
956
+ """
957
+
958
+ print('Training model...')
959
+ m, n, p = T.shape
960
+ num_samples, num_classes = m, p
961
+ print('Original shape:', T.shape)
962
+
963
+ # Set options
964
+ options = tfx.aux.make_options(options)
965
+
966
+ Tsize = np.linalg.norm(T)
967
+ if options.display == 3:
968
+ S, U, UT, sigmas, rel_error = tfx.cmpr.mlsvd(T, Tsize, r, options)
969
+ else:
970
+ S, U, UT, sigmas = tfx.cmpr.mlsvd(T, Tsize, r, options)
971
+
972
+ R1, R2, R3 = S.shape
973
+ U1, U2, U3 = U
974
+
975
+ # Sometimes the MLSVD may truncate the last dimension p. We have to repeat the training in this case.
976
+ if R3 != num_classes:
977
+ print('The number of classes was truncated. This cannot happen. Try again, but if the problem persists, we '
978
+ 'recommend to manually pass the truncation with the parameter `trunc_dims`.')
979
+ success = False
980
+ F, U2 = 0, 0
981
+ else:
982
+ success = True
983
+ S1 = tfx.cnv.unfold(S, 1)
984
+ F = tfx.mlinalg.multilin_mult([np.identity(R1), np.identity(R2), U3], S1, S.shape)
985
+ print('MLSVD shape:', F.shape)
986
+ if options.display == 3:
987
+ print('Error:', rel_error)
988
+ print('Working with', np.round(100*int(prod(F.shape, dtype=uint64))/int(prod(T.shape, dtype=uint64)), 4), '% of the original size')
989
+ print()
990
+
991
+ return F, U2, success
992
+
993
+
994
+ def mlsvd_test(X_test, Y_test, F, U2, p, mu=0, sigma=1):
995
+ """
996
+ After the training stage it is time to validate the model with the test dataset. This function is responsible for
997
+ that part.
998
+
999
+ Inputs
1000
+ ------
1001
+ X_test: 2D array
1002
+ Test data in 2D array format. Each row correspond to a single sample of the test data.
1003
+ Y_test:
1004
+ Each entry i of Y_test correspond to the class of the ith input of the test dataset (ith row of X_test).
1005
+ F: 3D array
1006
+ Same tensor of mlsvd_train function.
1007
+ U2: 2D array
1008
+ Same matrix of mlsvd_train function.
1009
+ p: Float
1010
+ Float between 0 and 1. The program compress the data so that 100*p % is retained. For example, if
1011
+ p = 0.90, then 90 % of the data is retained after compression. This compression is used in each slice
1012
+ of F to make the predictions.
1013
+ mu, sigma: float
1014
+ mu and sigma are such the normalized data is given by (X_test - mu)/sigma.
1015
+
1016
+ Outputs
1017
+ -------
1018
+ accuracy: float
1019
+ Accuracy of the model over the test dataset.
1020
+ """
1021
+
1022
+ predictions = mlsvd_predictions(X_test, F, U2, p, mu=mu, sigma=sigma)
1023
+
1024
+ num_samples = X_test.shape[0]
1025
+ success = 0
1026
+ for j in range(num_samples):
1027
+ x_class = predictions[j]
1028
+ y = Y_test[j]
1029
+ if x_class == y:
1030
+ success += 1
1031
+
1032
+ accuracy = 100 * success/num_samples
1033
+
1034
+ return accuracy
1035
+
1036
+
1037
+ def mlsvd_predictions(X_test, F, U2, p, mu=0, sigma=1):
1038
+ """
1039
+ In a more geometric interpretation, first we consider the space generated by the columns associated to the digit d,
1040
+ then we check the distance between z_new and this space. This distance is the norm associated with the solution of
1041
+ the least squares problem. After choosing d, this choice means z_new is more close to the space associated to the
1042
+ digit d.
1043
+
1044
+ Inputs
1045
+ ------
1046
+ X_test: 2D array
1047
+ Test data in 2D array format. Each row correspond to a single sample of the test data.
1048
+ F: 3D array
1049
+ Same tensor of mlsvd_train function.
1050
+ U2: 2D array
1051
+ Same matrix of mlsvd_train function.
1052
+ p: Float
1053
+ Float between 0 and 1. The program compress the data so that 100*p % is retained. For example, if
1054
+ p = 0.90, then 90 % of the data is retained after compression. This compression is used in each slice
1055
+ of F to make the predictions.
1056
+ mu, sigma: float
1057
+ mu and sigma are such the normalized data is given by (X_test - mu)/sigma.
1058
+
1059
+ Outputs
1060
+ -------
1061
+ predictions: 1D array
1062
+ Predictions of the classes of each sample in X_test.
1063
+ """
1064
+
1065
+ print('Computing predictions...')
1066
+ X_test_new = (X_test - mu)/sigma
1067
+ num_samples, num_classes = X_test_new.shape[0], F.shape[2]
1068
+ predictions = np.zeros(num_samples, dtype=np.int64)
1069
+
1070
+ # Compute truncated SVD of slices of F.
1071
+ F_slices_svd = []
1072
+ for d in range(num_classes):
1073
+ U, S, Vt = np.linalg.svd(F[:, :, d])
1074
+ # Truncate U
1075
+ n = S.size
1076
+ total_energy = np.sum(S)
1077
+ for k in range(n):
1078
+ energy = np.sum(S[0:k])
1079
+ ratio = energy / total_energy
1080
+ if ratio > p:
1081
+ break
1082
+ F_slices_svd.append(Vt[:k, :].T)
1083
+
1084
+ # Make predictions.
1085
+ for i in range(num_samples):
1086
+ z = X_test_new[i, :]
1087
+ z_new = np.dot(U2.T, z)
1088
+ best_dist = np.inf
1089
+ for d in range(num_classes):
1090
+ x = np.dot(F_slices_svd[d].T, z_new)
1091
+ # Distance between z' and the space associated to the digit d.
1092
+ dist = np.linalg.norm(np.dot(F_slices_svd[d], x) - z_new)
1093
+ if dist < best_dist:
1094
+ best_class = d
1095
+ best_dist = dist
1096
+
1097
+ # Save results.
1098
+ predictions[i] = best_class
1099
+
1100
+ print('Finished')
1101
+
1102
+ return predictions
1103
+