TensorFox 1.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- TensorFox/Alternating_Least_Squares.py +264 -0
- TensorFox/Auxiliar.py +686 -0
- TensorFox/Compression.py +624 -0
- TensorFox/Conversion.py +494 -0
- TensorFox/Critical.py +3320 -0
- TensorFox/Display.py +466 -0
- TensorFox/GaussNewton.py +811 -0
- TensorFox/Initialization.py +386 -0
- TensorFox/MultilinearAlgebra.py +649 -0
- TensorFox/TensorFox.py +1194 -0
- TensorFox/TensorLearning.py +1103 -0
- TensorFox/__init__.py +7 -0
- tensorfox-1.2.2.dist-info/METADATA +257 -0
- tensorfox-1.2.2.dist-info/RECORD +17 -0
- tensorfox-1.2.2.dist-info/WHEEL +5 -0
- tensorfox-1.2.2.dist-info/licenses/LICENSE +674 -0
- tensorfox-1.2.2.dist-info/top_level.txt +1 -0
TensorFox/GaussNewton.py
ADDED
|
@@ -0,0 +1,811 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Gauss-Newton Module
|
|
3
|
+
===================
|
|
4
|
+
This module implement the damped Gauss-Newton algorithm, with iterations performed with aid of the conjugate gradient
|
|
5
|
+
method.
|
|
6
|
+
|
|
7
|
+
References
|
|
8
|
+
==========
|
|
9
|
+
- K. Madsen, H. B. Nielsen, and O. Tingleff, Methods for Non-Linear Least Squares Problems, 2nd edition, Informatics
|
|
10
|
+
and Mathematical Modelling, Technical University of Denmark, 2004.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
# Python modules
|
|
14
|
+
import numpy as np
|
|
15
|
+
from numpy import inf, mean, concatenate, empty, array, zeros, ones, identity, float64, sqrt, dot, nan, diag, exp, sign
|
|
16
|
+
from numpy.linalg import norm, solve, qr, LinAlgError
|
|
17
|
+
from numpy.random import randint
|
|
18
|
+
import sys
|
|
19
|
+
from numba import njit
|
|
20
|
+
from copy import deepcopy
|
|
21
|
+
|
|
22
|
+
# Tensor Fox modules
|
|
23
|
+
import TensorFox.Alternating_Least_Squares as alsq
|
|
24
|
+
import TensorFox.Conversion as cnv
|
|
25
|
+
import TensorFox.Critical as crt
|
|
26
|
+
import TensorFox.MultilinearAlgebra as mlinalg
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def dGN(T, factors, R, options):
|
|
30
|
+
"""
|
|
31
|
+
This function uses the Damped Gauss-Newton method to compute an approximation of T with rank R. A starting point to
|
|
32
|
+
initiate the iterations must be given. This point is given by the parameter factors.
|
|
33
|
+
The Damped Gauss-Newton method is an iterative method, updating a point x at each iteration. The last computed x is
|
|
34
|
+
gives an approximate CPD in flat form, and from this we have the components to form the actual CPD.
|
|
35
|
+
|
|
36
|
+
Inputs
|
|
37
|
+
------
|
|
38
|
+
T: float array
|
|
39
|
+
factors: list of 2-D arrays
|
|
40
|
+
The factor matrices used as starting point.
|
|
41
|
+
R: int
|
|
42
|
+
The desired rank of the approximating tensor.
|
|
43
|
+
options: class
|
|
44
|
+
Class with the options. See the Auxiliar module documentation for more information.
|
|
45
|
+
|
|
46
|
+
Outputs
|
|
47
|
+
-------
|
|
48
|
+
best_factors: list of 2-D arrays
|
|
49
|
+
The factor matrices of the approximated CPD of T.
|
|
50
|
+
step_sizes: float 1-D array
|
|
51
|
+
Distance between the computed points at each iteration.
|
|
52
|
+
errors: float 1-D array
|
|
53
|
+
Error of the computed approximating tensor at each iteration.
|
|
54
|
+
improv: float 1-D array
|
|
55
|
+
Improvement of the error at each iteration. More precisely, the difference between the relative error of the
|
|
56
|
+
current iteration and the previous one.
|
|
57
|
+
gradients: float 1-D array
|
|
58
|
+
Gradient of the error function at each iteration.
|
|
59
|
+
stop: 0, 1, 2, 3, 4, 5, 6 or 7
|
|
60
|
+
This value indicates why the dGN function stopped. Below we summarize the cases.
|
|
61
|
+
0: errors[it] < tol. Relative error is small enough.
|
|
62
|
+
1: step_sizes[it] < tol_steps. Steps are small enough.
|
|
63
|
+
2: improv[it] < tol_improv. Improvement in the relative error is small enough.
|
|
64
|
+
3: gradients[it] < tol_grad. Gradient is small enough (infinity norm).
|
|
65
|
+
4: mean(abs(errors[it-k : it] - errors[it-k-1 : it-1]))/Tsize < 10*tol_improv. Average of the last
|
|
66
|
+
k = 1 + int(maxiter/10) relative errors is small enough. Keeping track of the averages is useful when the
|
|
67
|
+
errors improvements are just a little above the threshold for a long time. We want them above the threshold
|
|
68
|
+
indeed, but not too close for a long time.
|
|
69
|
+
5: limit of iterations reached.
|
|
70
|
+
6: dGN diverged.
|
|
71
|
+
7: Average improvement is too small compared to the average error.
|
|
72
|
+
8: no refinement was performed (this is not really a stopping condition, but it is necessary to indicate when
|
|
73
|
+
the program can't give a stopping condition in the refinement stage).
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
# INITIALIZE RELEVANT VARIABLES
|
|
77
|
+
|
|
78
|
+
# Extract all relevant variables from the class of options.
|
|
79
|
+
init_damp = options.init_damp
|
|
80
|
+
maxiter = options.maxiter
|
|
81
|
+
tol = options.tol
|
|
82
|
+
tol_step = options.tol_step
|
|
83
|
+
tol_improv = options.tol_improv
|
|
84
|
+
tol_grad = options.tol_grad
|
|
85
|
+
tol_jump = options.tol_jump
|
|
86
|
+
symm = options.symm
|
|
87
|
+
display = options.display
|
|
88
|
+
factors_norm = options.factors_norm
|
|
89
|
+
inner_method = options.inner_method
|
|
90
|
+
cg_maxiter = options.cg_maxiter
|
|
91
|
+
cg_factor = options.cg_factor
|
|
92
|
+
cg_tol = options.cg_tol
|
|
93
|
+
|
|
94
|
+
# Verify if some factor should be fixed or not. This only happens when the bicpd function was called.
|
|
95
|
+
L = len(factors)
|
|
96
|
+
fix_mode = -1
|
|
97
|
+
orig_factors = [[] for l in range(L)]
|
|
98
|
+
for l in range(L):
|
|
99
|
+
if type(factors[l]) == list:
|
|
100
|
+
fix_mode = l
|
|
101
|
+
orig_factors[l] = deepcopy(factors[l][0])
|
|
102
|
+
factors[l] = factors[l][0]
|
|
103
|
+
|
|
104
|
+
# Set the other variables.
|
|
105
|
+
dims = T.shape
|
|
106
|
+
Tsize = norm(T)
|
|
107
|
+
error = 1
|
|
108
|
+
best_error = inf
|
|
109
|
+
stop = 5
|
|
110
|
+
if type(init_damp) == list:
|
|
111
|
+
damp = init_damp[0]
|
|
112
|
+
else:
|
|
113
|
+
damp = init_damp * mean(np.abs(T))
|
|
114
|
+
const = 1 + int(maxiter / 10)
|
|
115
|
+
|
|
116
|
+
# The program is encouraged to make more iterations for small problems.
|
|
117
|
+
if R * sum(dims) <= 100:
|
|
118
|
+
print("-> Small problem detected (R * sum(dims) <= 100). All the convergence tolerances (tol, tol_step, tol_improv, tol_grad) have been set to zero.", file=sys.stderr)
|
|
119
|
+
tol = 0
|
|
120
|
+
tol_step = 0
|
|
121
|
+
tol_improv = 0
|
|
122
|
+
tol_grad = 0
|
|
123
|
+
|
|
124
|
+
# INITIALIZE RELEVANT ARRAYS
|
|
125
|
+
|
|
126
|
+
x = concatenate([factors[l].flatten('F') for l in range(L)])
|
|
127
|
+
y = zeros(R * sum(dims), dtype=float64)
|
|
128
|
+
step_sizes = zeros(maxiter)
|
|
129
|
+
errors = zeros(maxiter)
|
|
130
|
+
improv = zeros(maxiter)
|
|
131
|
+
gradients = zeros(maxiter)
|
|
132
|
+
best_factors = deepcopy(factors)
|
|
133
|
+
|
|
134
|
+
# Prepare data to use in each Gauss-Newton iteration.
|
|
135
|
+
data = prepare_data(dims, R)
|
|
136
|
+
|
|
137
|
+
# Compute unfoldings.
|
|
138
|
+
Tl = [cnv.unfold_C(T, l+1) for l in range(L)]
|
|
139
|
+
T1_approx = zeros(Tl[0].shape, dtype=float64)
|
|
140
|
+
|
|
141
|
+
if display > 1:
|
|
142
|
+
if display == 4:
|
|
143
|
+
print(' ',
|
|
144
|
+
'{:^9}'.format('Iteration'),
|
|
145
|
+
'| {:^11}'.format('Rel error'),
|
|
146
|
+
'| {:^11}'.format('Step size'),
|
|
147
|
+
'| {:^11}'.format('Improvement'),
|
|
148
|
+
'| {:^11}'.format('norm(grad)'),
|
|
149
|
+
'| {:^11}'.format('Predicted error'),
|
|
150
|
+
'| {:^10}'.format('# Inner iterations'))
|
|
151
|
+
else:
|
|
152
|
+
print(' ',
|
|
153
|
+
'{:^9}'.format('Iteration'),
|
|
154
|
+
'| {:^9}'.format('Rel error'),
|
|
155
|
+
'| {:^11}'.format('Step size'),
|
|
156
|
+
'| {:^10}'.format('Improvement'),
|
|
157
|
+
'| {:^10}'.format('norm(grad)'),
|
|
158
|
+
'| {:^10}'.format('Predicted error'),
|
|
159
|
+
'| {:^10}'.format('# Inner iterations'))
|
|
160
|
+
|
|
161
|
+
# START GAUSS-NEWTON ITERATIONS
|
|
162
|
+
|
|
163
|
+
for it in range(maxiter):
|
|
164
|
+
# Keep the previous value of x and error to compare with the new ones in the next iteration.
|
|
165
|
+
old_x = x
|
|
166
|
+
old_error = error
|
|
167
|
+
|
|
168
|
+
# Computation of the Gauss-Newton iteration formula to obtain the new point x + y, where x is the
|
|
169
|
+
# previous point and y is the new step obtained as the solution of min_y |Ay - b|.
|
|
170
|
+
inner_parameters = damp, inner_method, cg_maxiter, cg_factor, cg_tol, tol_jump, symm, factors_norm, fix_mode
|
|
171
|
+
T1_approx, factors, x, y, grad, itn, residualnorm, error = \
|
|
172
|
+
compute_step(Tsize, Tl, T1_approx, factors, orig_factors, data, x, y, inner_parameters, it, old_error, options)
|
|
173
|
+
|
|
174
|
+
# Update gain ratio and damping parameter.
|
|
175
|
+
gain_ratio = update_gain_ratio(damp, old_error, error, Tsize, old_x, x, grad)
|
|
176
|
+
damp = update_damp(damp, init_damp, gain_ratio, it)
|
|
177
|
+
|
|
178
|
+
# Update best solution.
|
|
179
|
+
if error < best_error:
|
|
180
|
+
best_error = error
|
|
181
|
+
best_factors = deepcopy(factors)
|
|
182
|
+
|
|
183
|
+
# Save relevant information about the current iteration.
|
|
184
|
+
errors[it] = error
|
|
185
|
+
step_sizes[it] = norm(x - old_x) / norm(old_x)
|
|
186
|
+
gradients[it] = norm(grad, inf)
|
|
187
|
+
if it == 0:
|
|
188
|
+
improv[it] = errors[it]
|
|
189
|
+
else:
|
|
190
|
+
improv[it] = np.abs(errors[it] - errors[it-1])
|
|
191
|
+
|
|
192
|
+
# Show information about current iteration.
|
|
193
|
+
if display > 1:
|
|
194
|
+
if display == 4:
|
|
195
|
+
print(' ',
|
|
196
|
+
'{:^8}'.format(it+1),
|
|
197
|
+
'| {:^10.5e}'.format(errors[it]),
|
|
198
|
+
'| {:^10.5e}'.format(step_sizes[it]),
|
|
199
|
+
'| {:^10.5e}'.format(improv[it]),
|
|
200
|
+
'| {:^11.5e}'.format(gradients[it]),
|
|
201
|
+
'| {:^15.5e}'.format(residualnorm),
|
|
202
|
+
'| {:^16}'.format(itn))
|
|
203
|
+
else:
|
|
204
|
+
print(' ',
|
|
205
|
+
'{:^9}'.format(it+1),
|
|
206
|
+
'| {:^9.2e}'.format(errors[it]),
|
|
207
|
+
'| {:^11.2e}'.format(step_sizes[it]),
|
|
208
|
+
'| {:^11.2e}'.format(improv[it]),
|
|
209
|
+
'| {:^10.2e}'.format(gradients[it]),
|
|
210
|
+
'| {:^15.2e}'.format(residualnorm),
|
|
211
|
+
'| {:^16}'.format(itn))
|
|
212
|
+
|
|
213
|
+
# Stopping conditions.
|
|
214
|
+
if it > 1:
|
|
215
|
+
if errors[it] < tol:
|
|
216
|
+
stop = 0
|
|
217
|
+
break
|
|
218
|
+
if step_sizes[it] < tol_step:
|
|
219
|
+
stop = 1
|
|
220
|
+
break
|
|
221
|
+
if improv[it] < tol_improv:
|
|
222
|
+
stop = 2
|
|
223
|
+
break
|
|
224
|
+
if gradients[it] < tol_grad:
|
|
225
|
+
stop = 3
|
|
226
|
+
break
|
|
227
|
+
if it > 2*const and it % const == 0:
|
|
228
|
+
# Let const=1+int(maxiter/10). Comparing the average errors of const consecutive iterations prevents
|
|
229
|
+
# the program to continue iterating when the error starts to oscillate without decreasing.
|
|
230
|
+
mean1 = mean(errors[it - 2*const: it - const])
|
|
231
|
+
mean2 = mean(errors[it - const: it])
|
|
232
|
+
if mean1 - mean2 <= tol_improv:
|
|
233
|
+
stop = 4
|
|
234
|
+
break
|
|
235
|
+
# If the average improvements is too small compared to the average errors, the program stops.
|
|
236
|
+
mean3 = mean(improv[it - const: it])
|
|
237
|
+
if mean3 < 1e-3 * mean2:
|
|
238
|
+
stop = 7
|
|
239
|
+
break
|
|
240
|
+
# Prevent blow ups.
|
|
241
|
+
if error > max(1, Tsize ** 2) / (1e-16 + tol):
|
|
242
|
+
stop = 6
|
|
243
|
+
break
|
|
244
|
+
|
|
245
|
+
# SAVE LAST COMPUTED INFORMATION
|
|
246
|
+
|
|
247
|
+
errors = errors[0: it+1]
|
|
248
|
+
step_sizes = step_sizes[0: it+1]
|
|
249
|
+
improv = improv[0: it+1]
|
|
250
|
+
gradients = gradients[0: it+1]
|
|
251
|
+
|
|
252
|
+
return best_factors, step_sizes, errors, improv, gradients, stop
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def compute_step(Tsize, Tl, T1_approx, factors, orig_factors, data, x, y, inner_parameters, it, old_error, options):
|
|
256
|
+
"""
|
|
257
|
+
This function uses the chosen inner method to compute the next step.
|
|
258
|
+
"""
|
|
259
|
+
|
|
260
|
+
# Initialize first variables.
|
|
261
|
+
L = len(factors)
|
|
262
|
+
damp, inner_method, cg_maxiter, cg_factor, cg_tol, tol_jump, symm, factors_norm, fix_mode = inner_parameters
|
|
263
|
+
if type(inner_method) == list:
|
|
264
|
+
inner_method = inner_method[it]
|
|
265
|
+
|
|
266
|
+
# Call the inner method.
|
|
267
|
+
if inner_method == 'cg':
|
|
268
|
+
cg_maxiter = 1 + (L-2) * int(cg_factor * randint(1 + it**0.4, 2 + it**0.9))
|
|
269
|
+
y, grad, JT_J_grad, itn, residualnorm = cg(Tl, factors, data, y, damp, cg_maxiter, cg_tol, options)
|
|
270
|
+
|
|
271
|
+
elif inner_method == 'cg_static':
|
|
272
|
+
y, grad, JT_J_grad, itn, residualnorm = cg(Tl, factors, data, y, damp, cg_maxiter, cg_tol, options)
|
|
273
|
+
|
|
274
|
+
elif inner_method == 'als':
|
|
275
|
+
factors = alsq.als_iteration(Tl, factors, fix_mode)
|
|
276
|
+
x = concatenate([factors[l].flatten('F') for l in range(L)])
|
|
277
|
+
y *= 0
|
|
278
|
+
|
|
279
|
+
elif inner_method == 'direct':
|
|
280
|
+
y, grad, itn, residualnorm = direct(Tl, factors, data, y, damp)
|
|
281
|
+
|
|
282
|
+
else:
|
|
283
|
+
sys.exit("Wrong inner method name. Must be 'cg', 'cg_static', 'als' or 'direct'.")
|
|
284
|
+
|
|
285
|
+
# Update results.
|
|
286
|
+
x = x + y
|
|
287
|
+
|
|
288
|
+
# Balance and transform factors.
|
|
289
|
+
factors = cnv.x2cpd(x, factors)
|
|
290
|
+
factors = cnv.transform(factors, symm, factors_norm)
|
|
291
|
+
|
|
292
|
+
# Some mode may be fixed when the bicpd is called.
|
|
293
|
+
if L == 3:
|
|
294
|
+
for l in range(L):
|
|
295
|
+
if fix_mode == l:
|
|
296
|
+
factors[l] = deepcopy(orig_factors[l])
|
|
297
|
+
|
|
298
|
+
# Compute error.
|
|
299
|
+
T1_approx = cnv.cpd2unfold1(T1_approx, factors)
|
|
300
|
+
error = crt.fastnorm(Tl[0], T1_approx) / Tsize
|
|
301
|
+
|
|
302
|
+
# Sometimes the step is too bad and increase the error by much. In this case we discard the computed step and
|
|
303
|
+
# use the DogLeg method to compute the next step.
|
|
304
|
+
if it > 3:
|
|
305
|
+
if inner_method == 'cg' or inner_method == 'cg_static':
|
|
306
|
+
if error > tol_jump * old_error:
|
|
307
|
+
x = x - y
|
|
308
|
+
factors, x, y, error = compute_dogleg_steps(Tsize, Tl, T1_approx, factors, grad, JT_J_grad, x, y, error, inner_parameters)
|
|
309
|
+
|
|
310
|
+
if inner_method == 'als':
|
|
311
|
+
return T1_approx, factors, x, y, [nan], '-', Tsize*error, error
|
|
312
|
+
|
|
313
|
+
return T1_approx, factors, x, y, grad, itn, residualnorm, error
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def cg(Tl, factors, data, y, damp, maxiter, tol, options):
|
|
317
|
+
"""
|
|
318
|
+
Conjugate gradient algorithm specialized to the tensor case.
|
|
319
|
+
"""
|
|
320
|
+
|
|
321
|
+
L = len(factors)
|
|
322
|
+
R = factors[0].shape[1]
|
|
323
|
+
dims = array([factors[l].shape[0] for l in range(L)])
|
|
324
|
+
# The program is encouraged to make more CG iterations for small problems.
|
|
325
|
+
if R * sum(dims) > 100:
|
|
326
|
+
maxiter = min(maxiter, R * sum(dims))
|
|
327
|
+
else:
|
|
328
|
+
tol = 0
|
|
329
|
+
|
|
330
|
+
# Give names to the arrays.
|
|
331
|
+
Gr, P1, P2, A, B, P_VT_W, result, result_tmp, Gamma, gamma, sum_dims, M, residual_cg, P, Q, z, g, JT_J_grad, N, gg = data
|
|
332
|
+
|
|
333
|
+
# Compute the values of all arrays.
|
|
334
|
+
Gr, P1, P2 = gramians(factors, Gr, P1, P2)
|
|
335
|
+
Gamma, gamma = regularization(Gamma, gamma, P1, dims, sum_dims)
|
|
336
|
+
M = precond(Gamma, gamma, M, damp, dims, sum_dims)
|
|
337
|
+
y *= 0
|
|
338
|
+
|
|
339
|
+
# Compute grad.
|
|
340
|
+
grad = -compute_grad(Tl, factors, P1, g, N, gg, dims, sum_dims)
|
|
341
|
+
|
|
342
|
+
# Compute J^T*J*grad.
|
|
343
|
+
V = [ grad[sum_dims[l]: sum_dims[l+1]].reshape(R, dims[l]) for l in range(L) ]
|
|
344
|
+
for l in range(L):
|
|
345
|
+
dot(V[l], factors[l], out=A[l])
|
|
346
|
+
dot(V[l].T, P1[l], out=B[l])
|
|
347
|
+
JT_J_grad = matvec(factors, P2, P_VT_W, result, result_tmp, JT_J_grad, A, B, dims, sum_dims)
|
|
348
|
+
|
|
349
|
+
# Compute initial variables for CG.
|
|
350
|
+
residual_cg = M * grad
|
|
351
|
+
P = residual_cg
|
|
352
|
+
residualnorm = dot(residual_cg.T, residual_cg)
|
|
353
|
+
if residualnorm == 0.0:
|
|
354
|
+
residualnorm = 1e-6
|
|
355
|
+
|
|
356
|
+
# CG iterations.
|
|
357
|
+
y, itn, residualnorm = cg_iterations(factors, P1, P2, A, B, P_VT_W, result, result_tmp, M, P,
|
|
358
|
+
Gamma, damp, z, residual_cg, residualnorm, y, tol, maxiter, dims, sum_dims)
|
|
359
|
+
|
|
360
|
+
# Save condition numbers of intermediate matrices. Only for display = 4, since these are heavy computations.
|
|
361
|
+
if options.display == 4 and not options.refine:
|
|
362
|
+
H = hessian(factors, P1, P2, sum_dims)
|
|
363
|
+
Hd = H + damp * diag(Gamma)
|
|
364
|
+
Hp = M[:, None] * Hd * M[None, :]
|
|
365
|
+
cond_H = np.linalg.cond(H)
|
|
366
|
+
cond_Hd = np.linalg.cond(Hd)
|
|
367
|
+
cond_Hp = np.linalg.cond(Hp)
|
|
368
|
+
options.conditioning.append([cond_H, cond_Hd, cond_Hp, damp])
|
|
369
|
+
|
|
370
|
+
return M * y, grad, JT_J_grad, itn + 1, residualnorm
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def cg_iterations(factors, P1, P2, A, B, P_VT_W, result, result_tmp, M, P,
|
|
374
|
+
Gamma, damp, z, residual_cg, residualnorm, y, tol, maxiter, dims, sum_dims):
|
|
375
|
+
"""
|
|
376
|
+
Conjugate gradient iterations.
|
|
377
|
+
"""
|
|
378
|
+
|
|
379
|
+
L = len(dims)
|
|
380
|
+
R = factors[0].shape[1]
|
|
381
|
+
|
|
382
|
+
for itn in range(maxiter):
|
|
383
|
+
Q = M * P
|
|
384
|
+
V = [ Q[sum_dims[l]: sum_dims[l+1]].reshape(R, dims[l]) for l in range(L) ]
|
|
385
|
+
|
|
386
|
+
for l in range(L):
|
|
387
|
+
dot(V[l], factors[l], out=A[l])
|
|
388
|
+
dot(V[l].T, P1[l], out=B[l])
|
|
389
|
+
|
|
390
|
+
z = matvec(factors, P2, P_VT_W, result, result_tmp, z, A, B, dims, sum_dims) + damp * Gamma * Q
|
|
391
|
+
z = M * z
|
|
392
|
+
denominator = dot(P.T, z)
|
|
393
|
+
if denominator == 0.0:
|
|
394
|
+
denominator = 1e-6
|
|
395
|
+
|
|
396
|
+
# Updates.
|
|
397
|
+
alpha = residualnorm / denominator
|
|
398
|
+
y += alpha * P
|
|
399
|
+
residual_cg -= alpha * z
|
|
400
|
+
residualnorm_new = dot(residual_cg.T, residual_cg)
|
|
401
|
+
beta = residualnorm_new / residualnorm
|
|
402
|
+
residualnorm = residualnorm_new
|
|
403
|
+
P = residual_cg + beta * P
|
|
404
|
+
|
|
405
|
+
# Stopping condition.
|
|
406
|
+
if residualnorm <= tol:
|
|
407
|
+
break
|
|
408
|
+
|
|
409
|
+
return y, itn, residualnorm
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def compute_grad(Tl, factors, P1, g, N, gg, dims, sum_dims):
|
|
413
|
+
"""
|
|
414
|
+
This function computes the gradient of the error function.
|
|
415
|
+
"""
|
|
416
|
+
|
|
417
|
+
# Initialize first variables.
|
|
418
|
+
L = len(factors)
|
|
419
|
+
R = factors[0].shape[1]
|
|
420
|
+
|
|
421
|
+
# Main computations.
|
|
422
|
+
for l in range(L):
|
|
423
|
+
itr = [l for l in reversed(range(L))]
|
|
424
|
+
itr.remove(l)
|
|
425
|
+
M = factors[itr[0]]
|
|
426
|
+
|
|
427
|
+
# Compute Khatri-Rao products W^(L) ⊙ ... ⊙ W^(l+1) ⊙ W^(l-1) ⊙ ... ⊙ W^(1).
|
|
428
|
+
for ll in range(L-2):
|
|
429
|
+
tmp = M
|
|
430
|
+
dim1, dim2 = tmp.shape[0], dims[itr[ll+1]]
|
|
431
|
+
M = empty((dim1 * dim2, R), dtype=float64)
|
|
432
|
+
M = mlinalg.khatri_rao(tmp, factors[itr[ll+1]], M)
|
|
433
|
+
|
|
434
|
+
dot(Tl[l], M, out=N[l])
|
|
435
|
+
dot(factors[l], P1[l], out=gg[l])
|
|
436
|
+
g[sum_dims[l]: sum_dims[l+1]] = (gg[l] - N[l]).T.ravel()
|
|
437
|
+
|
|
438
|
+
return g
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def prepare_data(dims, R):
|
|
442
|
+
"""
|
|
443
|
+
Initialize all necessary matrices to keep the values of several computations during the program.
|
|
444
|
+
"""
|
|
445
|
+
|
|
446
|
+
L = len(dims)
|
|
447
|
+
|
|
448
|
+
# Gramians
|
|
449
|
+
Gr = zeros((L, R, R), dtype=float64)
|
|
450
|
+
P1 = ones((L, R, R), dtype=float64)
|
|
451
|
+
P2 = ones((L, L, R, R), dtype=float64)
|
|
452
|
+
|
|
453
|
+
# Initializations of matrices to receive the results of the computations.
|
|
454
|
+
A = zeros((L, R, R), dtype=float64)
|
|
455
|
+
B = [zeros((dims[l], R)) for l in range(L)]
|
|
456
|
+
P_VT_W = zeros((R, R), dtype=float64)
|
|
457
|
+
result = [zeros((dims[l], R), dtype=float64) for l in range(L)]
|
|
458
|
+
result_tmp = zeros((L, R, R), dtype=float64)
|
|
459
|
+
|
|
460
|
+
# Matrices to use when constructing the Tikhonov matrix for regularization.
|
|
461
|
+
Gamma = zeros(R * sum(dims), dtype=float64)
|
|
462
|
+
gamma = zeros((L, R), dtype=float64)
|
|
463
|
+
|
|
464
|
+
# Arrays to be used in the Conjugated Gradient.
|
|
465
|
+
sum_dims = array([R * sum(dims[0:l]) for l in range(L+1)])
|
|
466
|
+
M = ones(R * sum(dims), dtype=float64)
|
|
467
|
+
residual_cg = zeros(R * sum(dims), dtype=float64)
|
|
468
|
+
P = zeros(R * sum(dims), dtype=float64)
|
|
469
|
+
Q = zeros(R * sum(dims), dtype=float64)
|
|
470
|
+
z = zeros(R * sum(dims), dtype=float64)
|
|
471
|
+
JT_J_grad = zeros(R * sum(dims), dtype=float64)
|
|
472
|
+
N = [zeros((dims[l], R), dtype=float64) for l in range(L)]
|
|
473
|
+
gg = [zeros((dims[l], R), dtype=float64) for l in range(L)]
|
|
474
|
+
|
|
475
|
+
# Arrays to be used in the compute_grad function.
|
|
476
|
+
g = zeros(R * sum(dims), dtype=float64)
|
|
477
|
+
|
|
478
|
+
data = [Gr, P1, P2, A, B, P_VT_W, result, result_tmp, Gamma, gamma, sum_dims, M, residual_cg, P, Q, z, g, JT_J_grad, N, gg]
|
|
479
|
+
|
|
480
|
+
return data
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def update_gain_ratio(damp, old_error, error, Tsize, old_x, x, grad):
|
|
484
|
+
"""
|
|
485
|
+
Update gain ratio.
|
|
486
|
+
"""
|
|
487
|
+
|
|
488
|
+
numerator = (Tsize**2) * (old_error**2 - error**2)
|
|
489
|
+
denominator = dot(x - old_x, grad + damp*(x - old_x))
|
|
490
|
+
|
|
491
|
+
if denominator != 0.0:
|
|
492
|
+
gain_ratio = gain_ratio = numerator / denominator
|
|
493
|
+
elif numerator != 0.0 and denominator == 0.0:
|
|
494
|
+
gain_ratio = sign(numerator) * inf
|
|
495
|
+
else:
|
|
496
|
+
gain_ratio = - inf
|
|
497
|
+
|
|
498
|
+
return gain_ratio
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def update_damp(damp, init_damp, gain_ratio, it):
|
|
502
|
+
"""
|
|
503
|
+
Update damping parameter.
|
|
504
|
+
"""
|
|
505
|
+
|
|
506
|
+
if type(init_damp) == list:
|
|
507
|
+
damp = init_damp[it]
|
|
508
|
+
else:
|
|
509
|
+
if gain_ratio < 0.25:
|
|
510
|
+
damp = 3*damp
|
|
511
|
+
elif gain_ratio > 0.75:
|
|
512
|
+
damp = damp/3
|
|
513
|
+
|
|
514
|
+
return damp
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def update_delta(delta, gain_ratio, step_size):
|
|
518
|
+
"""
|
|
519
|
+
Update trust region radius.
|
|
520
|
+
"""
|
|
521
|
+
|
|
522
|
+
if gain_ratio > 0.75:
|
|
523
|
+
delta = max(delta, 2*step_size)
|
|
524
|
+
else:
|
|
525
|
+
sigma = 0.25/(1 + exp(-14*(gain_ratio - 0.75))) + 0.75
|
|
526
|
+
delta = min(sigma*delta, delta)
|
|
527
|
+
|
|
528
|
+
return delta
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def gramians(factors, Gr, P1, P2):
|
|
532
|
+
"""
|
|
533
|
+
Computes all Gramian matrices of the factor matrices. Also it computes all Hadamard products between the
|
|
534
|
+
different Gramians.
|
|
535
|
+
"""
|
|
536
|
+
|
|
537
|
+
L = len(factors)
|
|
538
|
+
R = factors[0].shape[1]
|
|
539
|
+
|
|
540
|
+
for l in range(L):
|
|
541
|
+
Gr[l] = dot(factors[l].T, factors[l], out=Gr[l])
|
|
542
|
+
|
|
543
|
+
for l in range(1, L):
|
|
544
|
+
for ll in range(l):
|
|
545
|
+
P2[l][ll] = ones((R, R), dtype=float64)
|
|
546
|
+
itr = [i for i in range(L)]
|
|
547
|
+
itr.remove(l)
|
|
548
|
+
itr.remove(ll)
|
|
549
|
+
for lll in itr:
|
|
550
|
+
P2[l][ll] = mlinalg.hadamard(P2[l][ll], Gr[lll], P2[l][ll])
|
|
551
|
+
P2[ll][l] = P2[l][ll]
|
|
552
|
+
if l < L-1:
|
|
553
|
+
P1[l] = mlinalg.hadamard(P2[l][ll], Gr[ll], P1[l])
|
|
554
|
+
else:
|
|
555
|
+
P1[l] = mlinalg.hadamard(P2[l][0], Gr[0], P1[l])
|
|
556
|
+
|
|
557
|
+
P1[0] = mlinalg.hadamard(P2[0][1], Gr[1], P1[0])
|
|
558
|
+
|
|
559
|
+
return Gr, P1, P2
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def matvec(factors, P2, P_VT_W, result, result_tmp, z, A, B, dims, sum_dims):
|
|
563
|
+
"""
|
|
564
|
+
Makes the matrix-vector computation (Jf^T * Jf)*v.
|
|
565
|
+
"""
|
|
566
|
+
|
|
567
|
+
L = len(factors)
|
|
568
|
+
R = factors[0].shape[1]
|
|
569
|
+
|
|
570
|
+
result_tmp *= 0
|
|
571
|
+
result_tmp = matvec_inner(A, P2, P_VT_W, result_tmp, L)
|
|
572
|
+
|
|
573
|
+
for l in range(L):
|
|
574
|
+
dot(factors[l], result_tmp[l], out=result[l])
|
|
575
|
+
result[l] += B[l]
|
|
576
|
+
z[sum_dims[l]: sum_dims[l+1]] = cnv.vec(result[l], z[sum_dims[l]: sum_dims[l+1]], dims[l], R)
|
|
577
|
+
|
|
578
|
+
return z
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
@njit(nogil=True)
|
|
582
|
+
def matvec_inner(A, P2, P_VT_W, result_tmp, L):
|
|
583
|
+
for ll in range(L):
|
|
584
|
+
X = A[ll]
|
|
585
|
+
for l in range(ll):
|
|
586
|
+
P_VT_W = mlinalg.hadamard(P2[l][ll], X, P_VT_W)
|
|
587
|
+
result_tmp[l] += P_VT_W
|
|
588
|
+
for l in range(ll+1, L):
|
|
589
|
+
P_VT_W = mlinalg.hadamard(P2[l][ll], X, P_VT_W)
|
|
590
|
+
result_tmp[l] += P_VT_W
|
|
591
|
+
|
|
592
|
+
return result_tmp
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
@njit(nogil=True)
|
|
596
|
+
def regularization(Gamma, gamma, P1, dims, sum_dims):
|
|
597
|
+
"""
|
|
598
|
+
Computes the Tikhonov matrix Gamma, where Gamma is a diagonal matrix designed specifically to make Jf^T * Jf + Gamma
|
|
599
|
+
diagonally dominant.
|
|
600
|
+
"""
|
|
601
|
+
|
|
602
|
+
L, R = gamma.shape
|
|
603
|
+
|
|
604
|
+
s = 0
|
|
605
|
+
for l in range(L):
|
|
606
|
+
tmp = np.max(np.abs(P1[l]))
|
|
607
|
+
if s < tmp:
|
|
608
|
+
s = tmp
|
|
609
|
+
for r in range(R):
|
|
610
|
+
gamma[l, r] = abs(P1[l][r, r])
|
|
611
|
+
gamma = tmp * np.sqrt(gamma)
|
|
612
|
+
|
|
613
|
+
for l in range(L):
|
|
614
|
+
for r in range(R):
|
|
615
|
+
Gamma[sum_dims[l] + r*dims[l]: sum_dims[l] + (r+1)*dims[l]] = gamma[l, r]
|
|
616
|
+
|
|
617
|
+
return Gamma, gamma
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
@njit(nogil=True)
|
|
621
|
+
def precond(Gamma, gamma, M, damp, dims, sum_dims):
|
|
622
|
+
"""
|
|
623
|
+
This function constructs a preconditioner in order to accelerate the Conjugate Gradient function. It is a diagonal
|
|
624
|
+
preconditioner designed to make Jf^T*J + damp*I a unit diagonal matrix. Since the matrix is diagonally dominant,
|
|
625
|
+
the result will be close to the identity matrix (the equalize function does that). Therefore, it will have its
|
|
626
|
+
eigenvalues clustered together.
|
|
627
|
+
"""
|
|
628
|
+
|
|
629
|
+
L, R = gamma.shape
|
|
630
|
+
|
|
631
|
+
for l in range(L):
|
|
632
|
+
for r in range(R):
|
|
633
|
+
M[sum_dims[l] + r*dims[l]: sum_dims[l] + (r+1)*dims[l]] = \
|
|
634
|
+
gamma[l, r]**2 + damp**2 * Gamma[sum_dims[l] + r*dims[l]: sum_dims[l] + (r+1)*dims[l]]**2
|
|
635
|
+
|
|
636
|
+
M = 1/sqrt(M)
|
|
637
|
+
|
|
638
|
+
return M
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
def direct(Tl, factors, data, y, damp):
|
|
642
|
+
"""
|
|
643
|
+
This function computes the next dGN step using a direct method. It is very heavy computationally since it
|
|
644
|
+
constructs the full Hessian matrix. Do not use this function for large problems.
|
|
645
|
+
"""
|
|
646
|
+
|
|
647
|
+
L = len(factors)
|
|
648
|
+
R = factors[0].shape[1]
|
|
649
|
+
dims = [factors[l].shape[0] for l in range(L)]
|
|
650
|
+
|
|
651
|
+
# Give names to the arrays.
|
|
652
|
+
Gr, P1, P2, A, B, P_VT_W, result, result_tmp, Gamma, gamma, sum_dims, M, residual_cg, P, Q, z, g, JT_J_grad, N, gg = data
|
|
653
|
+
|
|
654
|
+
# Compute the values of all arrays.
|
|
655
|
+
Gr, P1, P2 = gramians(factors, Gr, P1, P2)
|
|
656
|
+
Gamma, gamma = regularization(Gamma, gamma, P1, dims, sum_dims)
|
|
657
|
+
M = precond(Gamma, gamma, M, damp, dims, sum_dims)
|
|
658
|
+
grad = -compute_grad(Tl, factors, P1, g, N, gg, dims, sum_dims)
|
|
659
|
+
H = hessian(factors, P1, P2, sum_dims)
|
|
660
|
+
Hd = H + damp * diag(Gamma)
|
|
661
|
+
MHd = ((Hd.T) * (M**2)).T
|
|
662
|
+
Mgrad = (M**2) * grad
|
|
663
|
+
|
|
664
|
+
# Solve equation MH*y = Mgrad using QR decomposition, followed by a triangular system.
|
|
665
|
+
try:
|
|
666
|
+
q, r = qr(MHd)
|
|
667
|
+
y = dot(q.T, Mgrad)
|
|
668
|
+
y = solve(r, y)
|
|
669
|
+
except LinAlgError:
|
|
670
|
+
y *= 0
|
|
671
|
+
|
|
672
|
+
residualnorm = norm(dot(H, y) - grad)
|
|
673
|
+
|
|
674
|
+
return y, grad, '-', residualnorm
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def hessian(factors, P1, P2, sum_dims):
|
|
678
|
+
"""
|
|
679
|
+
Approximate Hessian matrix of the error function.
|
|
680
|
+
"""
|
|
681
|
+
|
|
682
|
+
L = len(factors)
|
|
683
|
+
R = factors[0].shape[1]
|
|
684
|
+
dims = [factors[l].shape[0] for l in range(L)]
|
|
685
|
+
H = zeros((R * sum(dims), R * sum(dims)))
|
|
686
|
+
vec_factors = [zeros(R*dims[l]) for l in range(L)]
|
|
687
|
+
fortran_factors = [array(factors[l], order='F') for l in range(L)]
|
|
688
|
+
for l in range(L):
|
|
689
|
+
vec_factors[l] = cnv.vec(factors[l], vec_factors[l], dims[l], R)
|
|
690
|
+
vec_factors[l] = vec_factors[l].reshape(R*dims[l], 1).T
|
|
691
|
+
|
|
692
|
+
for l in range(L):
|
|
693
|
+
I = identity(dims[l])
|
|
694
|
+
# Block H_{ll}.
|
|
695
|
+
H[sum_dims[l]:sum_dims[l+1], sum_dims[l]:sum_dims[l+1]] = mlinalg.kronecker(P1[l, :, :], I)
|
|
696
|
+
for ll in range(l):
|
|
697
|
+
I = ones((dims[l], dims[ll]))
|
|
698
|
+
tmp1 = mlinalg.kronecker(P2[l, ll, :, :], I)
|
|
699
|
+
tmp2 = zeros((R*dims[l], R*dims[ll]))
|
|
700
|
+
tmp2 = compute_blocks(tmp2, fortran_factors[l], vec_factors[ll], tuple(dims), R, l, ll)
|
|
701
|
+
# Blocks H_{l, ll} and H_{ll, l}.
|
|
702
|
+
if tmp1.shape[0] == tmp1.shape[1]:
|
|
703
|
+
H[sum_dims[l]:sum_dims[l+1], sum_dims[ll]:sum_dims[ll+1]] = \
|
|
704
|
+
mlinalg.hadamard(tmp1, tmp2, H[sum_dims[l]:sum_dims[l+1], sum_dims[ll]:sum_dims[ll+1]])
|
|
705
|
+
else:
|
|
706
|
+
H[sum_dims[l]:sum_dims[l+1], sum_dims[ll]:sum_dims[ll+1]] = \
|
|
707
|
+
mlinalg.hadamard_rect(tmp1, tmp2, H[sum_dims[l]:sum_dims[l+1], sum_dims[ll]:sum_dims[ll+1]])
|
|
708
|
+
|
|
709
|
+
H[sum_dims[ll]:sum_dims[ll+1], sum_dims[l]:sum_dims[l+1]] = H[sum_dims[l]:sum_dims[l+1], sum_dims[ll]:sum_dims[ll+1]].T
|
|
710
|
+
|
|
711
|
+
return H
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
@njit(nogil=True)
|
|
715
|
+
def compute_blocks(tmp2, factor, vec, dims, R, l, ll):
|
|
716
|
+
"""
|
|
717
|
+
Auxiliary function for the hessian function. The computation of the rank one matrices between the factor matrices
|
|
718
|
+
(factors[l][:, r] * factors[ll][:, rr].T) are done here.
|
|
719
|
+
"""
|
|
720
|
+
|
|
721
|
+
for r in range(R):
|
|
722
|
+
tmp = dot(factor[:, r].reshape(dims[l], 1), vec)
|
|
723
|
+
for rr in range(R):
|
|
724
|
+
tmp2[dims[l]*rr:dims[l]*(rr+1), dims[ll]*r:dims[ll]*(r+1)] = tmp[:, dims[ll]*rr:dims[ll]*(rr+1)]
|
|
725
|
+
|
|
726
|
+
return tmp2
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def compute_dogleg_steps(Tsize, Tl, T1_approx, factors, grad, JT_J_grad, x, y, error, inner_parameters):
|
|
730
|
+
"""
|
|
731
|
+
Compute Dogleg step.
|
|
732
|
+
"""
|
|
733
|
+
|
|
734
|
+
count = 0
|
|
735
|
+
best_x = x.copy()
|
|
736
|
+
best_y = y.copy()
|
|
737
|
+
best_error = error
|
|
738
|
+
best_factors = deepcopy(factors)
|
|
739
|
+
gain_ratio = 1
|
|
740
|
+
delta = 1
|
|
741
|
+
|
|
742
|
+
while gain_ratio > 0:
|
|
743
|
+
# Keep the previous value of x and error to compare with the new ones in the next iteration.
|
|
744
|
+
old_x = x
|
|
745
|
+
old_y = y
|
|
746
|
+
old_error = error
|
|
747
|
+
damp, inner_method, cg_maxiter, cg_factor, cg_tol, tol_jump, symm, factors_norm, fix_mode = inner_parameters
|
|
748
|
+
|
|
749
|
+
# Apply dog leg method.
|
|
750
|
+
y = dogleg(y, grad, JT_J_grad, delta)
|
|
751
|
+
|
|
752
|
+
# Update results.
|
|
753
|
+
x = x + y
|
|
754
|
+
|
|
755
|
+
# Balance and transform factors.
|
|
756
|
+
factors = cnv.x2cpd(x, factors, eq=False)
|
|
757
|
+
factors = cnv.transform(factors, symm, factors_norm)
|
|
758
|
+
|
|
759
|
+
# Compute error.
|
|
760
|
+
T1_approx = cnv.cpd2unfold1(T1_approx, factors)
|
|
761
|
+
error = crt.fastnorm(Tl[0], T1_approx) / Tsize
|
|
762
|
+
|
|
763
|
+
# Update gain ratio.
|
|
764
|
+
gain_ratio = update_gain_ratio(damp, old_error, error, Tsize, old_x, x, grad)
|
|
765
|
+
|
|
766
|
+
if error < old_error:
|
|
767
|
+
best_x = x.copy()
|
|
768
|
+
best_y = y.copy()
|
|
769
|
+
best_error = error
|
|
770
|
+
best_factors = deepcopy(factors)
|
|
771
|
+
|
|
772
|
+
# Update delta.
|
|
773
|
+
delta = update_delta(delta, gain_ratio, norm(x - old_x))
|
|
774
|
+
|
|
775
|
+
count += 1
|
|
776
|
+
if count > 10:
|
|
777
|
+
break
|
|
778
|
+
|
|
779
|
+
return best_factors, best_x, best_y, best_error
|
|
780
|
+
|
|
781
|
+
|
|
782
|
+
def dogleg(y, grad, JT_J_grad, delta):
|
|
783
|
+
"""
|
|
784
|
+
Subroutine for function compute_dogleg_steps.
|
|
785
|
+
"""
|
|
786
|
+
|
|
787
|
+
# Initialize constants.
|
|
788
|
+
alpha = norm(grad)**2 / dot(grad, JT_J_grad)
|
|
789
|
+
y_sd = alpha * grad
|
|
790
|
+
c1 = norm(y_sd)
|
|
791
|
+
c2 = dot(y - y_sd, y_sd)
|
|
792
|
+
c3 = norm(y - y_sd)**2
|
|
793
|
+
c4 = delta**2 - c1**2
|
|
794
|
+
m = c2**2 + c3 * c4
|
|
795
|
+
|
|
796
|
+
# Decide next step.
|
|
797
|
+
if norm(y) < delta:
|
|
798
|
+
y_dl = y
|
|
799
|
+
elif c1 >= delta:
|
|
800
|
+
y_dl = (delta/c1) * y_sd
|
|
801
|
+
elif m < 0:
|
|
802
|
+
y_dl = y
|
|
803
|
+
else:
|
|
804
|
+
c5 = sqrt(m)
|
|
805
|
+
if c2 <= 0:
|
|
806
|
+
beta = (-c2 + c5)/c3
|
|
807
|
+
else:
|
|
808
|
+
beta = c4/(c2 + c5)
|
|
809
|
+
y_dl = y_sd + beta*(y - y_sd)
|
|
810
|
+
|
|
811
|
+
return y_dl
|