TensorFox 1.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,811 @@
1
+ """
2
+ Gauss-Newton Module
3
+ ===================
4
+ This module implement the damped Gauss-Newton algorithm, with iterations performed with aid of the conjugate gradient
5
+ method.
6
+
7
+ References
8
+ ==========
9
+ - K. Madsen, H. B. Nielsen, and O. Tingleff, Methods for Non-Linear Least Squares Problems, 2nd edition, Informatics
10
+ and Mathematical Modelling, Technical University of Denmark, 2004.
11
+ """
12
+
13
+ # Python modules
14
+ import numpy as np
15
+ from numpy import inf, mean, concatenate, empty, array, zeros, ones, identity, float64, sqrt, dot, nan, diag, exp, sign
16
+ from numpy.linalg import norm, solve, qr, LinAlgError
17
+ from numpy.random import randint
18
+ import sys
19
+ from numba import njit
20
+ from copy import deepcopy
21
+
22
+ # Tensor Fox modules
23
+ import TensorFox.Alternating_Least_Squares as alsq
24
+ import TensorFox.Conversion as cnv
25
+ import TensorFox.Critical as crt
26
+ import TensorFox.MultilinearAlgebra as mlinalg
27
+
28
+
29
+ def dGN(T, factors, R, options):
30
+ """
31
+ This function uses the Damped Gauss-Newton method to compute an approximation of T with rank R. A starting point to
32
+ initiate the iterations must be given. This point is given by the parameter factors.
33
+ The Damped Gauss-Newton method is an iterative method, updating a point x at each iteration. The last computed x is
34
+ gives an approximate CPD in flat form, and from this we have the components to form the actual CPD.
35
+
36
+ Inputs
37
+ ------
38
+ T: float array
39
+ factors: list of 2-D arrays
40
+ The factor matrices used as starting point.
41
+ R: int
42
+ The desired rank of the approximating tensor.
43
+ options: class
44
+ Class with the options. See the Auxiliar module documentation for more information.
45
+
46
+ Outputs
47
+ -------
48
+ best_factors: list of 2-D arrays
49
+ The factor matrices of the approximated CPD of T.
50
+ step_sizes: float 1-D array
51
+ Distance between the computed points at each iteration.
52
+ errors: float 1-D array
53
+ Error of the computed approximating tensor at each iteration.
54
+ improv: float 1-D array
55
+ Improvement of the error at each iteration. More precisely, the difference between the relative error of the
56
+ current iteration and the previous one.
57
+ gradients: float 1-D array
58
+ Gradient of the error function at each iteration.
59
+ stop: 0, 1, 2, 3, 4, 5, 6 or 7
60
+ This value indicates why the dGN function stopped. Below we summarize the cases.
61
+ 0: errors[it] < tol. Relative error is small enough.
62
+ 1: step_sizes[it] < tol_steps. Steps are small enough.
63
+ 2: improv[it] < tol_improv. Improvement in the relative error is small enough.
64
+ 3: gradients[it] < tol_grad. Gradient is small enough (infinity norm).
65
+ 4: mean(abs(errors[it-k : it] - errors[it-k-1 : it-1]))/Tsize < 10*tol_improv. Average of the last
66
+ k = 1 + int(maxiter/10) relative errors is small enough. Keeping track of the averages is useful when the
67
+ errors improvements are just a little above the threshold for a long time. We want them above the threshold
68
+ indeed, but not too close for a long time.
69
+ 5: limit of iterations reached.
70
+ 6: dGN diverged.
71
+ 7: Average improvement is too small compared to the average error.
72
+ 8: no refinement was performed (this is not really a stopping condition, but it is necessary to indicate when
73
+ the program can't give a stopping condition in the refinement stage).
74
+ """
75
+
76
+ # INITIALIZE RELEVANT VARIABLES
77
+
78
+ # Extract all relevant variables from the class of options.
79
+ init_damp = options.init_damp
80
+ maxiter = options.maxiter
81
+ tol = options.tol
82
+ tol_step = options.tol_step
83
+ tol_improv = options.tol_improv
84
+ tol_grad = options.tol_grad
85
+ tol_jump = options.tol_jump
86
+ symm = options.symm
87
+ display = options.display
88
+ factors_norm = options.factors_norm
89
+ inner_method = options.inner_method
90
+ cg_maxiter = options.cg_maxiter
91
+ cg_factor = options.cg_factor
92
+ cg_tol = options.cg_tol
93
+
94
+ # Verify if some factor should be fixed or not. This only happens when the bicpd function was called.
95
+ L = len(factors)
96
+ fix_mode = -1
97
+ orig_factors = [[] for l in range(L)]
98
+ for l in range(L):
99
+ if type(factors[l]) == list:
100
+ fix_mode = l
101
+ orig_factors[l] = deepcopy(factors[l][0])
102
+ factors[l] = factors[l][0]
103
+
104
+ # Set the other variables.
105
+ dims = T.shape
106
+ Tsize = norm(T)
107
+ error = 1
108
+ best_error = inf
109
+ stop = 5
110
+ if type(init_damp) == list:
111
+ damp = init_damp[0]
112
+ else:
113
+ damp = init_damp * mean(np.abs(T))
114
+ const = 1 + int(maxiter / 10)
115
+
116
+ # The program is encouraged to make more iterations for small problems.
117
+ if R * sum(dims) <= 100:
118
+ print("-> Small problem detected (R * sum(dims) <= 100). All the convergence tolerances (tol, tol_step, tol_improv, tol_grad) have been set to zero.", file=sys.stderr)
119
+ tol = 0
120
+ tol_step = 0
121
+ tol_improv = 0
122
+ tol_grad = 0
123
+
124
+ # INITIALIZE RELEVANT ARRAYS
125
+
126
+ x = concatenate([factors[l].flatten('F') for l in range(L)])
127
+ y = zeros(R * sum(dims), dtype=float64)
128
+ step_sizes = zeros(maxiter)
129
+ errors = zeros(maxiter)
130
+ improv = zeros(maxiter)
131
+ gradients = zeros(maxiter)
132
+ best_factors = deepcopy(factors)
133
+
134
+ # Prepare data to use in each Gauss-Newton iteration.
135
+ data = prepare_data(dims, R)
136
+
137
+ # Compute unfoldings.
138
+ Tl = [cnv.unfold_C(T, l+1) for l in range(L)]
139
+ T1_approx = zeros(Tl[0].shape, dtype=float64)
140
+
141
+ if display > 1:
142
+ if display == 4:
143
+ print(' ',
144
+ '{:^9}'.format('Iteration'),
145
+ '| {:^11}'.format('Rel error'),
146
+ '| {:^11}'.format('Step size'),
147
+ '| {:^11}'.format('Improvement'),
148
+ '| {:^11}'.format('norm(grad)'),
149
+ '| {:^11}'.format('Predicted error'),
150
+ '| {:^10}'.format('# Inner iterations'))
151
+ else:
152
+ print(' ',
153
+ '{:^9}'.format('Iteration'),
154
+ '| {:^9}'.format('Rel error'),
155
+ '| {:^11}'.format('Step size'),
156
+ '| {:^10}'.format('Improvement'),
157
+ '| {:^10}'.format('norm(grad)'),
158
+ '| {:^10}'.format('Predicted error'),
159
+ '| {:^10}'.format('# Inner iterations'))
160
+
161
+ # START GAUSS-NEWTON ITERATIONS
162
+
163
+ for it in range(maxiter):
164
+ # Keep the previous value of x and error to compare with the new ones in the next iteration.
165
+ old_x = x
166
+ old_error = error
167
+
168
+ # Computation of the Gauss-Newton iteration formula to obtain the new point x + y, where x is the
169
+ # previous point and y is the new step obtained as the solution of min_y |Ay - b|.
170
+ inner_parameters = damp, inner_method, cg_maxiter, cg_factor, cg_tol, tol_jump, symm, factors_norm, fix_mode
171
+ T1_approx, factors, x, y, grad, itn, residualnorm, error = \
172
+ compute_step(Tsize, Tl, T1_approx, factors, orig_factors, data, x, y, inner_parameters, it, old_error, options)
173
+
174
+ # Update gain ratio and damping parameter.
175
+ gain_ratio = update_gain_ratio(damp, old_error, error, Tsize, old_x, x, grad)
176
+ damp = update_damp(damp, init_damp, gain_ratio, it)
177
+
178
+ # Update best solution.
179
+ if error < best_error:
180
+ best_error = error
181
+ best_factors = deepcopy(factors)
182
+
183
+ # Save relevant information about the current iteration.
184
+ errors[it] = error
185
+ step_sizes[it] = norm(x - old_x) / norm(old_x)
186
+ gradients[it] = norm(grad, inf)
187
+ if it == 0:
188
+ improv[it] = errors[it]
189
+ else:
190
+ improv[it] = np.abs(errors[it] - errors[it-1])
191
+
192
+ # Show information about current iteration.
193
+ if display > 1:
194
+ if display == 4:
195
+ print(' ',
196
+ '{:^8}'.format(it+1),
197
+ '| {:^10.5e}'.format(errors[it]),
198
+ '| {:^10.5e}'.format(step_sizes[it]),
199
+ '| {:^10.5e}'.format(improv[it]),
200
+ '| {:^11.5e}'.format(gradients[it]),
201
+ '| {:^15.5e}'.format(residualnorm),
202
+ '| {:^16}'.format(itn))
203
+ else:
204
+ print(' ',
205
+ '{:^9}'.format(it+1),
206
+ '| {:^9.2e}'.format(errors[it]),
207
+ '| {:^11.2e}'.format(step_sizes[it]),
208
+ '| {:^11.2e}'.format(improv[it]),
209
+ '| {:^10.2e}'.format(gradients[it]),
210
+ '| {:^15.2e}'.format(residualnorm),
211
+ '| {:^16}'.format(itn))
212
+
213
+ # Stopping conditions.
214
+ if it > 1:
215
+ if errors[it] < tol:
216
+ stop = 0
217
+ break
218
+ if step_sizes[it] < tol_step:
219
+ stop = 1
220
+ break
221
+ if improv[it] < tol_improv:
222
+ stop = 2
223
+ break
224
+ if gradients[it] < tol_grad:
225
+ stop = 3
226
+ break
227
+ if it > 2*const and it % const == 0:
228
+ # Let const=1+int(maxiter/10). Comparing the average errors of const consecutive iterations prevents
229
+ # the program to continue iterating when the error starts to oscillate without decreasing.
230
+ mean1 = mean(errors[it - 2*const: it - const])
231
+ mean2 = mean(errors[it - const: it])
232
+ if mean1 - mean2 <= tol_improv:
233
+ stop = 4
234
+ break
235
+ # If the average improvements is too small compared to the average errors, the program stops.
236
+ mean3 = mean(improv[it - const: it])
237
+ if mean3 < 1e-3 * mean2:
238
+ stop = 7
239
+ break
240
+ # Prevent blow ups.
241
+ if error > max(1, Tsize ** 2) / (1e-16 + tol):
242
+ stop = 6
243
+ break
244
+
245
+ # SAVE LAST COMPUTED INFORMATION
246
+
247
+ errors = errors[0: it+1]
248
+ step_sizes = step_sizes[0: it+1]
249
+ improv = improv[0: it+1]
250
+ gradients = gradients[0: it+1]
251
+
252
+ return best_factors, step_sizes, errors, improv, gradients, stop
253
+
254
+
255
+ def compute_step(Tsize, Tl, T1_approx, factors, orig_factors, data, x, y, inner_parameters, it, old_error, options):
256
+ """
257
+ This function uses the chosen inner method to compute the next step.
258
+ """
259
+
260
+ # Initialize first variables.
261
+ L = len(factors)
262
+ damp, inner_method, cg_maxiter, cg_factor, cg_tol, tol_jump, symm, factors_norm, fix_mode = inner_parameters
263
+ if type(inner_method) == list:
264
+ inner_method = inner_method[it]
265
+
266
+ # Call the inner method.
267
+ if inner_method == 'cg':
268
+ cg_maxiter = 1 + (L-2) * int(cg_factor * randint(1 + it**0.4, 2 + it**0.9))
269
+ y, grad, JT_J_grad, itn, residualnorm = cg(Tl, factors, data, y, damp, cg_maxiter, cg_tol, options)
270
+
271
+ elif inner_method == 'cg_static':
272
+ y, grad, JT_J_grad, itn, residualnorm = cg(Tl, factors, data, y, damp, cg_maxiter, cg_tol, options)
273
+
274
+ elif inner_method == 'als':
275
+ factors = alsq.als_iteration(Tl, factors, fix_mode)
276
+ x = concatenate([factors[l].flatten('F') for l in range(L)])
277
+ y *= 0
278
+
279
+ elif inner_method == 'direct':
280
+ y, grad, itn, residualnorm = direct(Tl, factors, data, y, damp)
281
+
282
+ else:
283
+ sys.exit("Wrong inner method name. Must be 'cg', 'cg_static', 'als' or 'direct'.")
284
+
285
+ # Update results.
286
+ x = x + y
287
+
288
+ # Balance and transform factors.
289
+ factors = cnv.x2cpd(x, factors)
290
+ factors = cnv.transform(factors, symm, factors_norm)
291
+
292
+ # Some mode may be fixed when the bicpd is called.
293
+ if L == 3:
294
+ for l in range(L):
295
+ if fix_mode == l:
296
+ factors[l] = deepcopy(orig_factors[l])
297
+
298
+ # Compute error.
299
+ T1_approx = cnv.cpd2unfold1(T1_approx, factors)
300
+ error = crt.fastnorm(Tl[0], T1_approx) / Tsize
301
+
302
+ # Sometimes the step is too bad and increase the error by much. In this case we discard the computed step and
303
+ # use the DogLeg method to compute the next step.
304
+ if it > 3:
305
+ if inner_method == 'cg' or inner_method == 'cg_static':
306
+ if error > tol_jump * old_error:
307
+ x = x - y
308
+ factors, x, y, error = compute_dogleg_steps(Tsize, Tl, T1_approx, factors, grad, JT_J_grad, x, y, error, inner_parameters)
309
+
310
+ if inner_method == 'als':
311
+ return T1_approx, factors, x, y, [nan], '-', Tsize*error, error
312
+
313
+ return T1_approx, factors, x, y, grad, itn, residualnorm, error
314
+
315
+
316
+ def cg(Tl, factors, data, y, damp, maxiter, tol, options):
317
+ """
318
+ Conjugate gradient algorithm specialized to the tensor case.
319
+ """
320
+
321
+ L = len(factors)
322
+ R = factors[0].shape[1]
323
+ dims = array([factors[l].shape[0] for l in range(L)])
324
+ # The program is encouraged to make more CG iterations for small problems.
325
+ if R * sum(dims) > 100:
326
+ maxiter = min(maxiter, R * sum(dims))
327
+ else:
328
+ tol = 0
329
+
330
+ # Give names to the arrays.
331
+ Gr, P1, P2, A, B, P_VT_W, result, result_tmp, Gamma, gamma, sum_dims, M, residual_cg, P, Q, z, g, JT_J_grad, N, gg = data
332
+
333
+ # Compute the values of all arrays.
334
+ Gr, P1, P2 = gramians(factors, Gr, P1, P2)
335
+ Gamma, gamma = regularization(Gamma, gamma, P1, dims, sum_dims)
336
+ M = precond(Gamma, gamma, M, damp, dims, sum_dims)
337
+ y *= 0
338
+
339
+ # Compute grad.
340
+ grad = -compute_grad(Tl, factors, P1, g, N, gg, dims, sum_dims)
341
+
342
+ # Compute J^T*J*grad.
343
+ V = [ grad[sum_dims[l]: sum_dims[l+1]].reshape(R, dims[l]) for l in range(L) ]
344
+ for l in range(L):
345
+ dot(V[l], factors[l], out=A[l])
346
+ dot(V[l].T, P1[l], out=B[l])
347
+ JT_J_grad = matvec(factors, P2, P_VT_W, result, result_tmp, JT_J_grad, A, B, dims, sum_dims)
348
+
349
+ # Compute initial variables for CG.
350
+ residual_cg = M * grad
351
+ P = residual_cg
352
+ residualnorm = dot(residual_cg.T, residual_cg)
353
+ if residualnorm == 0.0:
354
+ residualnorm = 1e-6
355
+
356
+ # CG iterations.
357
+ y, itn, residualnorm = cg_iterations(factors, P1, P2, A, B, P_VT_W, result, result_tmp, M, P,
358
+ Gamma, damp, z, residual_cg, residualnorm, y, tol, maxiter, dims, sum_dims)
359
+
360
+ # Save condition numbers of intermediate matrices. Only for display = 4, since these are heavy computations.
361
+ if options.display == 4 and not options.refine:
362
+ H = hessian(factors, P1, P2, sum_dims)
363
+ Hd = H + damp * diag(Gamma)
364
+ Hp = M[:, None] * Hd * M[None, :]
365
+ cond_H = np.linalg.cond(H)
366
+ cond_Hd = np.linalg.cond(Hd)
367
+ cond_Hp = np.linalg.cond(Hp)
368
+ options.conditioning.append([cond_H, cond_Hd, cond_Hp, damp])
369
+
370
+ return M * y, grad, JT_J_grad, itn + 1, residualnorm
371
+
372
+
373
+ def cg_iterations(factors, P1, P2, A, B, P_VT_W, result, result_tmp, M, P,
374
+ Gamma, damp, z, residual_cg, residualnorm, y, tol, maxiter, dims, sum_dims):
375
+ """
376
+ Conjugate gradient iterations.
377
+ """
378
+
379
+ L = len(dims)
380
+ R = factors[0].shape[1]
381
+
382
+ for itn in range(maxiter):
383
+ Q = M * P
384
+ V = [ Q[sum_dims[l]: sum_dims[l+1]].reshape(R, dims[l]) for l in range(L) ]
385
+
386
+ for l in range(L):
387
+ dot(V[l], factors[l], out=A[l])
388
+ dot(V[l].T, P1[l], out=B[l])
389
+
390
+ z = matvec(factors, P2, P_VT_W, result, result_tmp, z, A, B, dims, sum_dims) + damp * Gamma * Q
391
+ z = M * z
392
+ denominator = dot(P.T, z)
393
+ if denominator == 0.0:
394
+ denominator = 1e-6
395
+
396
+ # Updates.
397
+ alpha = residualnorm / denominator
398
+ y += alpha * P
399
+ residual_cg -= alpha * z
400
+ residualnorm_new = dot(residual_cg.T, residual_cg)
401
+ beta = residualnorm_new / residualnorm
402
+ residualnorm = residualnorm_new
403
+ P = residual_cg + beta * P
404
+
405
+ # Stopping condition.
406
+ if residualnorm <= tol:
407
+ break
408
+
409
+ return y, itn, residualnorm
410
+
411
+
412
+ def compute_grad(Tl, factors, P1, g, N, gg, dims, sum_dims):
413
+ """
414
+ This function computes the gradient of the error function.
415
+ """
416
+
417
+ # Initialize first variables.
418
+ L = len(factors)
419
+ R = factors[0].shape[1]
420
+
421
+ # Main computations.
422
+ for l in range(L):
423
+ itr = [l for l in reversed(range(L))]
424
+ itr.remove(l)
425
+ M = factors[itr[0]]
426
+
427
+ # Compute Khatri-Rao products W^(L) ⊙ ... ⊙ W^(l+1) ⊙ W^(l-1) ⊙ ... ⊙ W^(1).
428
+ for ll in range(L-2):
429
+ tmp = M
430
+ dim1, dim2 = tmp.shape[0], dims[itr[ll+1]]
431
+ M = empty((dim1 * dim2, R), dtype=float64)
432
+ M = mlinalg.khatri_rao(tmp, factors[itr[ll+1]], M)
433
+
434
+ dot(Tl[l], M, out=N[l])
435
+ dot(factors[l], P1[l], out=gg[l])
436
+ g[sum_dims[l]: sum_dims[l+1]] = (gg[l] - N[l]).T.ravel()
437
+
438
+ return g
439
+
440
+
441
+ def prepare_data(dims, R):
442
+ """
443
+ Initialize all necessary matrices to keep the values of several computations during the program.
444
+ """
445
+
446
+ L = len(dims)
447
+
448
+ # Gramians
449
+ Gr = zeros((L, R, R), dtype=float64)
450
+ P1 = ones((L, R, R), dtype=float64)
451
+ P2 = ones((L, L, R, R), dtype=float64)
452
+
453
+ # Initializations of matrices to receive the results of the computations.
454
+ A = zeros((L, R, R), dtype=float64)
455
+ B = [zeros((dims[l], R)) for l in range(L)]
456
+ P_VT_W = zeros((R, R), dtype=float64)
457
+ result = [zeros((dims[l], R), dtype=float64) for l in range(L)]
458
+ result_tmp = zeros((L, R, R), dtype=float64)
459
+
460
+ # Matrices to use when constructing the Tikhonov matrix for regularization.
461
+ Gamma = zeros(R * sum(dims), dtype=float64)
462
+ gamma = zeros((L, R), dtype=float64)
463
+
464
+ # Arrays to be used in the Conjugated Gradient.
465
+ sum_dims = array([R * sum(dims[0:l]) for l in range(L+1)])
466
+ M = ones(R * sum(dims), dtype=float64)
467
+ residual_cg = zeros(R * sum(dims), dtype=float64)
468
+ P = zeros(R * sum(dims), dtype=float64)
469
+ Q = zeros(R * sum(dims), dtype=float64)
470
+ z = zeros(R * sum(dims), dtype=float64)
471
+ JT_J_grad = zeros(R * sum(dims), dtype=float64)
472
+ N = [zeros((dims[l], R), dtype=float64) for l in range(L)]
473
+ gg = [zeros((dims[l], R), dtype=float64) for l in range(L)]
474
+
475
+ # Arrays to be used in the compute_grad function.
476
+ g = zeros(R * sum(dims), dtype=float64)
477
+
478
+ data = [Gr, P1, P2, A, B, P_VT_W, result, result_tmp, Gamma, gamma, sum_dims, M, residual_cg, P, Q, z, g, JT_J_grad, N, gg]
479
+
480
+ return data
481
+
482
+
483
+ def update_gain_ratio(damp, old_error, error, Tsize, old_x, x, grad):
484
+ """
485
+ Update gain ratio.
486
+ """
487
+
488
+ numerator = (Tsize**2) * (old_error**2 - error**2)
489
+ denominator = dot(x - old_x, grad + damp*(x - old_x))
490
+
491
+ if denominator != 0.0:
492
+ gain_ratio = gain_ratio = numerator / denominator
493
+ elif numerator != 0.0 and denominator == 0.0:
494
+ gain_ratio = sign(numerator) * inf
495
+ else:
496
+ gain_ratio = - inf
497
+
498
+ return gain_ratio
499
+
500
+
501
+ def update_damp(damp, init_damp, gain_ratio, it):
502
+ """
503
+ Update damping parameter.
504
+ """
505
+
506
+ if type(init_damp) == list:
507
+ damp = init_damp[it]
508
+ else:
509
+ if gain_ratio < 0.25:
510
+ damp = 3*damp
511
+ elif gain_ratio > 0.75:
512
+ damp = damp/3
513
+
514
+ return damp
515
+
516
+
517
+ def update_delta(delta, gain_ratio, step_size):
518
+ """
519
+ Update trust region radius.
520
+ """
521
+
522
+ if gain_ratio > 0.75:
523
+ delta = max(delta, 2*step_size)
524
+ else:
525
+ sigma = 0.25/(1 + exp(-14*(gain_ratio - 0.75))) + 0.75
526
+ delta = min(sigma*delta, delta)
527
+
528
+ return delta
529
+
530
+
531
+ def gramians(factors, Gr, P1, P2):
532
+ """
533
+ Computes all Gramian matrices of the factor matrices. Also it computes all Hadamard products between the
534
+ different Gramians.
535
+ """
536
+
537
+ L = len(factors)
538
+ R = factors[0].shape[1]
539
+
540
+ for l in range(L):
541
+ Gr[l] = dot(factors[l].T, factors[l], out=Gr[l])
542
+
543
+ for l in range(1, L):
544
+ for ll in range(l):
545
+ P2[l][ll] = ones((R, R), dtype=float64)
546
+ itr = [i for i in range(L)]
547
+ itr.remove(l)
548
+ itr.remove(ll)
549
+ for lll in itr:
550
+ P2[l][ll] = mlinalg.hadamard(P2[l][ll], Gr[lll], P2[l][ll])
551
+ P2[ll][l] = P2[l][ll]
552
+ if l < L-1:
553
+ P1[l] = mlinalg.hadamard(P2[l][ll], Gr[ll], P1[l])
554
+ else:
555
+ P1[l] = mlinalg.hadamard(P2[l][0], Gr[0], P1[l])
556
+
557
+ P1[0] = mlinalg.hadamard(P2[0][1], Gr[1], P1[0])
558
+
559
+ return Gr, P1, P2
560
+
561
+
562
+ def matvec(factors, P2, P_VT_W, result, result_tmp, z, A, B, dims, sum_dims):
563
+ """
564
+ Makes the matrix-vector computation (Jf^T * Jf)*v.
565
+ """
566
+
567
+ L = len(factors)
568
+ R = factors[0].shape[1]
569
+
570
+ result_tmp *= 0
571
+ result_tmp = matvec_inner(A, P2, P_VT_W, result_tmp, L)
572
+
573
+ for l in range(L):
574
+ dot(factors[l], result_tmp[l], out=result[l])
575
+ result[l] += B[l]
576
+ z[sum_dims[l]: sum_dims[l+1]] = cnv.vec(result[l], z[sum_dims[l]: sum_dims[l+1]], dims[l], R)
577
+
578
+ return z
579
+
580
+
581
+ @njit(nogil=True)
582
+ def matvec_inner(A, P2, P_VT_W, result_tmp, L):
583
+ for ll in range(L):
584
+ X = A[ll]
585
+ for l in range(ll):
586
+ P_VT_W = mlinalg.hadamard(P2[l][ll], X, P_VT_W)
587
+ result_tmp[l] += P_VT_W
588
+ for l in range(ll+1, L):
589
+ P_VT_W = mlinalg.hadamard(P2[l][ll], X, P_VT_W)
590
+ result_tmp[l] += P_VT_W
591
+
592
+ return result_tmp
593
+
594
+
595
+ @njit(nogil=True)
596
+ def regularization(Gamma, gamma, P1, dims, sum_dims):
597
+ """
598
+ Computes the Tikhonov matrix Gamma, where Gamma is a diagonal matrix designed specifically to make Jf^T * Jf + Gamma
599
+ diagonally dominant.
600
+ """
601
+
602
+ L, R = gamma.shape
603
+
604
+ s = 0
605
+ for l in range(L):
606
+ tmp = np.max(np.abs(P1[l]))
607
+ if s < tmp:
608
+ s = tmp
609
+ for r in range(R):
610
+ gamma[l, r] = abs(P1[l][r, r])
611
+ gamma = tmp * np.sqrt(gamma)
612
+
613
+ for l in range(L):
614
+ for r in range(R):
615
+ Gamma[sum_dims[l] + r*dims[l]: sum_dims[l] + (r+1)*dims[l]] = gamma[l, r]
616
+
617
+ return Gamma, gamma
618
+
619
+
620
+ @njit(nogil=True)
621
+ def precond(Gamma, gamma, M, damp, dims, sum_dims):
622
+ """
623
+ This function constructs a preconditioner in order to accelerate the Conjugate Gradient function. It is a diagonal
624
+ preconditioner designed to make Jf^T*J + damp*I a unit diagonal matrix. Since the matrix is diagonally dominant,
625
+ the result will be close to the identity matrix (the equalize function does that). Therefore, it will have its
626
+ eigenvalues clustered together.
627
+ """
628
+
629
+ L, R = gamma.shape
630
+
631
+ for l in range(L):
632
+ for r in range(R):
633
+ M[sum_dims[l] + r*dims[l]: sum_dims[l] + (r+1)*dims[l]] = \
634
+ gamma[l, r]**2 + damp**2 * Gamma[sum_dims[l] + r*dims[l]: sum_dims[l] + (r+1)*dims[l]]**2
635
+
636
+ M = 1/sqrt(M)
637
+
638
+ return M
639
+
640
+
641
+ def direct(Tl, factors, data, y, damp):
642
+ """
643
+ This function computes the next dGN step using a direct method. It is very heavy computationally since it
644
+ constructs the full Hessian matrix. Do not use this function for large problems.
645
+ """
646
+
647
+ L = len(factors)
648
+ R = factors[0].shape[1]
649
+ dims = [factors[l].shape[0] for l in range(L)]
650
+
651
+ # Give names to the arrays.
652
+ Gr, P1, P2, A, B, P_VT_W, result, result_tmp, Gamma, gamma, sum_dims, M, residual_cg, P, Q, z, g, JT_J_grad, N, gg = data
653
+
654
+ # Compute the values of all arrays.
655
+ Gr, P1, P2 = gramians(factors, Gr, P1, P2)
656
+ Gamma, gamma = regularization(Gamma, gamma, P1, dims, sum_dims)
657
+ M = precond(Gamma, gamma, M, damp, dims, sum_dims)
658
+ grad = -compute_grad(Tl, factors, P1, g, N, gg, dims, sum_dims)
659
+ H = hessian(factors, P1, P2, sum_dims)
660
+ Hd = H + damp * diag(Gamma)
661
+ MHd = ((Hd.T) * (M**2)).T
662
+ Mgrad = (M**2) * grad
663
+
664
+ # Solve equation MH*y = Mgrad using QR decomposition, followed by a triangular system.
665
+ try:
666
+ q, r = qr(MHd)
667
+ y = dot(q.T, Mgrad)
668
+ y = solve(r, y)
669
+ except LinAlgError:
670
+ y *= 0
671
+
672
+ residualnorm = norm(dot(H, y) - grad)
673
+
674
+ return y, grad, '-', residualnorm
675
+
676
+
677
+ def hessian(factors, P1, P2, sum_dims):
678
+ """
679
+ Approximate Hessian matrix of the error function.
680
+ """
681
+
682
+ L = len(factors)
683
+ R = factors[0].shape[1]
684
+ dims = [factors[l].shape[0] for l in range(L)]
685
+ H = zeros((R * sum(dims), R * sum(dims)))
686
+ vec_factors = [zeros(R*dims[l]) for l in range(L)]
687
+ fortran_factors = [array(factors[l], order='F') for l in range(L)]
688
+ for l in range(L):
689
+ vec_factors[l] = cnv.vec(factors[l], vec_factors[l], dims[l], R)
690
+ vec_factors[l] = vec_factors[l].reshape(R*dims[l], 1).T
691
+
692
+ for l in range(L):
693
+ I = identity(dims[l])
694
+ # Block H_{ll}.
695
+ H[sum_dims[l]:sum_dims[l+1], sum_dims[l]:sum_dims[l+1]] = mlinalg.kronecker(P1[l, :, :], I)
696
+ for ll in range(l):
697
+ I = ones((dims[l], dims[ll]))
698
+ tmp1 = mlinalg.kronecker(P2[l, ll, :, :], I)
699
+ tmp2 = zeros((R*dims[l], R*dims[ll]))
700
+ tmp2 = compute_blocks(tmp2, fortran_factors[l], vec_factors[ll], tuple(dims), R, l, ll)
701
+ # Blocks H_{l, ll} and H_{ll, l}.
702
+ if tmp1.shape[0] == tmp1.shape[1]:
703
+ H[sum_dims[l]:sum_dims[l+1], sum_dims[ll]:sum_dims[ll+1]] = \
704
+ mlinalg.hadamard(tmp1, tmp2, H[sum_dims[l]:sum_dims[l+1], sum_dims[ll]:sum_dims[ll+1]])
705
+ else:
706
+ H[sum_dims[l]:sum_dims[l+1], sum_dims[ll]:sum_dims[ll+1]] = \
707
+ mlinalg.hadamard_rect(tmp1, tmp2, H[sum_dims[l]:sum_dims[l+1], sum_dims[ll]:sum_dims[ll+1]])
708
+
709
+ H[sum_dims[ll]:sum_dims[ll+1], sum_dims[l]:sum_dims[l+1]] = H[sum_dims[l]:sum_dims[l+1], sum_dims[ll]:sum_dims[ll+1]].T
710
+
711
+ return H
712
+
713
+
714
+ @njit(nogil=True)
715
+ def compute_blocks(tmp2, factor, vec, dims, R, l, ll):
716
+ """
717
+ Auxiliary function for the hessian function. The computation of the rank one matrices between the factor matrices
718
+ (factors[l][:, r] * factors[ll][:, rr].T) are done here.
719
+ """
720
+
721
+ for r in range(R):
722
+ tmp = dot(factor[:, r].reshape(dims[l], 1), vec)
723
+ for rr in range(R):
724
+ tmp2[dims[l]*rr:dims[l]*(rr+1), dims[ll]*r:dims[ll]*(r+1)] = tmp[:, dims[ll]*rr:dims[ll]*(rr+1)]
725
+
726
+ return tmp2
727
+
728
+
729
+ def compute_dogleg_steps(Tsize, Tl, T1_approx, factors, grad, JT_J_grad, x, y, error, inner_parameters):
730
+ """
731
+ Compute Dogleg step.
732
+ """
733
+
734
+ count = 0
735
+ best_x = x.copy()
736
+ best_y = y.copy()
737
+ best_error = error
738
+ best_factors = deepcopy(factors)
739
+ gain_ratio = 1
740
+ delta = 1
741
+
742
+ while gain_ratio > 0:
743
+ # Keep the previous value of x and error to compare with the new ones in the next iteration.
744
+ old_x = x
745
+ old_y = y
746
+ old_error = error
747
+ damp, inner_method, cg_maxiter, cg_factor, cg_tol, tol_jump, symm, factors_norm, fix_mode = inner_parameters
748
+
749
+ # Apply dog leg method.
750
+ y = dogleg(y, grad, JT_J_grad, delta)
751
+
752
+ # Update results.
753
+ x = x + y
754
+
755
+ # Balance and transform factors.
756
+ factors = cnv.x2cpd(x, factors, eq=False)
757
+ factors = cnv.transform(factors, symm, factors_norm)
758
+
759
+ # Compute error.
760
+ T1_approx = cnv.cpd2unfold1(T1_approx, factors)
761
+ error = crt.fastnorm(Tl[0], T1_approx) / Tsize
762
+
763
+ # Update gain ratio.
764
+ gain_ratio = update_gain_ratio(damp, old_error, error, Tsize, old_x, x, grad)
765
+
766
+ if error < old_error:
767
+ best_x = x.copy()
768
+ best_y = y.copy()
769
+ best_error = error
770
+ best_factors = deepcopy(factors)
771
+
772
+ # Update delta.
773
+ delta = update_delta(delta, gain_ratio, norm(x - old_x))
774
+
775
+ count += 1
776
+ if count > 10:
777
+ break
778
+
779
+ return best_factors, best_x, best_y, best_error
780
+
781
+
782
+ def dogleg(y, grad, JT_J_grad, delta):
783
+ """
784
+ Subroutine for function compute_dogleg_steps.
785
+ """
786
+
787
+ # Initialize constants.
788
+ alpha = norm(grad)**2 / dot(grad, JT_J_grad)
789
+ y_sd = alpha * grad
790
+ c1 = norm(y_sd)
791
+ c2 = dot(y - y_sd, y_sd)
792
+ c3 = norm(y - y_sd)**2
793
+ c4 = delta**2 - c1**2
794
+ m = c2**2 + c3 * c4
795
+
796
+ # Decide next step.
797
+ if norm(y) < delta:
798
+ y_dl = y
799
+ elif c1 >= delta:
800
+ y_dl = (delta/c1) * y_sd
801
+ elif m < 0:
802
+ y_dl = y
803
+ else:
804
+ c5 = sqrt(m)
805
+ if c2 <= 0:
806
+ beta = (-c2 + c5)/c3
807
+ else:
808
+ beta = c4/(c2 + c5)
809
+ y_dl = y_sd + beta*(y - y_sd)
810
+
811
+ return y_dl