ticoi 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ticoi might be problematic. Click here for more details.

ticoi/core.py ADDED
@@ -0,0 +1,1500 @@
1
+ #!/usr/bin/env python
2
+ """
3
+ Main functions to process the temporal inversion of glacier's surface velocity using the TICOI method. The inversion is solved using an Iterative Reweighted Least Square, and a robust downweighted function (Tukey's biweight).
4
+ - mu_regularisation: Build the regularisation matrix
5
+ - weight_for_inversion: Initialisation of the weights used in the IRLS approach
6
+ - inversion_iteration: Compute an iteration of the inversion (weights are updated using the residuals)
7
+ - inversion: Main function to be called, makes the temporal inversion with an IRLS approach using a given solver, it returns leap frog velocities (velcoties between consecutive dates) with an irregular temporal sampling.
8
+ - interpolation_post: Interpolate Irregular Leap Frog time series (result of an inversion) to Regular LF time series using Cumulative Displacement times series.
9
+ - process: Launch the entire process, data loading, inversion and interpolation
10
+ - visualisation: Different figures can be shown in this function, according to what the user wants
11
+
12
+ Author : Laurane Charrier
13
+ Reference:
14
+ Charrier, L., Yan, Y., Koeniguer, E. C., Leinss, S., & Trouvé, E. (2021). Extraction of velocity time series with an optimal temporal sampling from displacement
15
+ observation networks. IEEE Transactions on Geoscience and Remote Sensing.
16
+ Charrier, L., Yan, Y., Colin Koeniguer, E., Mouginot, J., Millan, R., & Trouvé, E. (2022). Fusion of multi-temporal and multi-sensor ice velocity observations.
17
+ ISPRS annals of the photogrammetry, remote sensing and spatial information sciences, 3, 311-318.
18
+ """
19
+
20
+ import asyncio
21
+ import itertools
22
+ import time
23
+ import warnings
24
+ from typing import List, Optional, Union, Tuple
25
+
26
+ import numpy as np
27
+ import pandas as pd
28
+ import scipy.sparse as sp
29
+ import xarray as xr
30
+ from joblib import Parallel, delayed
31
+ from scipy import stats
32
+ from tqdm import tqdm
33
+
34
+ from ticoi.cube_data_classxr import CubeDataClass
35
+ from ticoi.interpolation_functions import (
36
+ reconstruct_common_ref,
37
+ set_function_for_interpolation,
38
+ visualisation_interpolation,
39
+ )
40
+ from ticoi.inversion_functions import (
41
+ TukeyBiweight,
42
+ class_linear_operator,
43
+ construction_a_lf,
44
+ construction_dates_range_np,
45
+ find_date_obs,
46
+ inversion_one_component,
47
+ inversion_two_components,
48
+ mu_regularisation,
49
+ weight_for_inversion,
50
+ )
51
+ from ticoi.pixel_class import PixelClass
52
+
53
+ warnings.filterwarnings("ignore")
54
+
55
+ # %% ======================================================================== #
56
+ # INVERSION #
57
+ # =========================================================================%% #
58
+
59
+
60
+ def inversion_iteration(
61
+ data: np.ndarray,
62
+ A: np.ndarray,
63
+ dates_range: np.ndarray,
64
+ solver: str,
65
+ coef: int,
66
+ Weight: np.ndarray,
67
+ result_dx: np.ndarray,
68
+ result_dy: np.ndarray,
69
+ mu: np.ndarray,
70
+ regu: int | str = 1,
71
+ accel: np.ndarray | None = None,
72
+ linear_operator=Union["class_linear_operator", None],
73
+ result_quality: list | str | None = None,
74
+ ini: np.ndarray | None = None,
75
+ verbose: bool = False,
76
+ ) -> (np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray | None, np.ndarray | None):
77
+ """
78
+ Compute an iteration of the inversion : update the weights using the weights from the previous iteration and the studentized residual, update the results in consequence
79
+ and compute the residu's norm if required.
80
+
81
+ :param data: [np array] --- Data at a given point
82
+ :param A: [np array] --- Design matrix linking X (vector containing the velocity observations) to Y
83
+ :param dates_range: [list] --- Dates of the displacements in X
84
+ :param solver: [str] --- Solver of the inversion: 'LSMR', 'LSMR_ini', 'LS', 'LS_bounded', 'LSQR'
85
+ :param coef: [int] --- Coef of Tikhonov regularisation
86
+ :param Weight: [np array] --- Weight to give to the inversion
87
+ :param result_dx: [np array] --- Estimated time series vx at the given iteration
88
+ :param result_dy: [np array] --- Estimated time series vx at the given iteration
89
+ :param mu: [np array] --- Regularization matrix
90
+ :param regu: [int | str] [default is 1] --- Type of regularization
91
+ :param accel: [np array | None] [default is None] --- Apriori on the acceleration
92
+ :param linear_operator: [bool] [default is False] --- If linear operator, the inversion is performed using a linear operator (https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.linalg.LinearOperator.html)
93
+ :param result_quality: [list | str | None] [default is None] --- Which can contain 'Norm_residual' to determine the L2 norm of the residuals from the last inversion, 'X_contribution' to determine the number of Y observations which have contributed to estimate each value in X (it corresponds to A.dot(weight))
94
+ :param ini: [np array | None]
95
+ :param verbose: [bool] [default is False] --- Print information along the way
96
+
97
+ :return result_dx, result_dy: [np arrays] --- Obtained results (velocities) for this iteration along x and y axis
98
+ :return weightx, weighty: [np arrays] --- Newly computed weights along x and y axis
99
+ :return residu_normx, residu_normy: [np arrays | None] --- Norm of the residu along x and y axis (when showing the L curve)
100
+ """
101
+
102
+ def compute_residual(A: np.ndarray, v: np.ndarray, X: np.ndarray) -> np.ndarray:
103
+ Residu = v - A.dot(X)
104
+ return Residu
105
+
106
+ def weightf(residu: np.ndarray, Weight: np.ndarray) -> np.ndarray:
107
+ """
108
+ Compute weight according to the residual
109
+
110
+ :param residu: [np array] Residual vector
111
+ :param Weight: [np array | None] Apriori weight
112
+
113
+ :return weight: [np array] Weight for the inversion
114
+ """
115
+
116
+ mad = stats.median_abs_deviation(residu) / 0.6745
117
+ if mad != 0.0:
118
+ r_std = residu / mad
119
+ if Weight is not None: # The weight is a combination of apriori weight and the studentized residual
120
+ # Weight = Weight / (stats.median_abs_deviation(Weight) / 0.6745)
121
+ weight = Weight * TukeyBiweight(r_std, 4.685)
122
+ else:
123
+ weight = TukeyBiweight((r_std), 4.685)
124
+ else:
125
+ weight = np.ones(residu.shape[0])
126
+
127
+ return weight
128
+
129
+ weightx = weightf(compute_residual(A, data[:, 0], result_dx), Weight[0])
130
+ weighty = weightf(compute_residual(A, data[:, 1], result_dy), Weight[1])
131
+
132
+ if A.shape[0] < A.shape[1]:
133
+ if verbose:
134
+ print(
135
+ f"[Inversion] If the number of row is lower than the number of columns, the results are not updated {A.shape}"
136
+ )
137
+ return result_dx, result_dy, weightx, weighty, None, None
138
+
139
+ if regu == "directionxy":
140
+ if solver == "LSMR_ini":
141
+ result_dx, result_dy, residu_normx, residu_normy = inversion_two_components(
142
+ A,
143
+ dates_range,
144
+ 0,
145
+ data,
146
+ solver,
147
+ np.concatenate([weightx, weighty]),
148
+ mu,
149
+ coef=coef,
150
+ ini=np.concatenate([result_dx, result_dy]),
151
+ )
152
+ else:
153
+ result_dx, result_dy, residu_normx, residu_normy = inversion_two_components(
154
+ A, dates_range, 0, data, solver, np.concatenate([weightx, weighty]), mu, coef=coef
155
+ )
156
+
157
+ elif solver == "LSMR_ini":
158
+ if ini is None: # Initialization with the result from the previous inversion
159
+ result_dx, residu_normx = inversion_one_component(
160
+ A,
161
+ dates_range,
162
+ 0,
163
+ data,
164
+ solver,
165
+ weightx,
166
+ mu,
167
+ coef=coef,
168
+ ini=result_dx,
169
+ result_quality=result_quality,
170
+ regu=regu,
171
+ accel=accel,
172
+ linear_operator=linear_operator,
173
+ )
174
+ result_dy, residu_normy = inversion_one_component(
175
+ A,
176
+ dates_range,
177
+ 1,
178
+ data,
179
+ solver,
180
+ weighty,
181
+ mu,
182
+ coef=coef,
183
+ ini=result_dy,
184
+ result_quality=result_quality,
185
+ regu=regu,
186
+ accel=accel,
187
+ linear_operator=linear_operator,
188
+ )
189
+ else: # Initialization with the list ini, which can be a moving average
190
+ result_dx, residu_normx = inversion_one_component(
191
+ A,
192
+ dates_range,
193
+ 0,
194
+ data,
195
+ solver,
196
+ weightx,
197
+ mu,
198
+ coef=coef,
199
+ ini=ini[0],
200
+ result_quality=result_quality,
201
+ regu=regu,
202
+ accel=accel,
203
+ linear_operator=linear_operator,
204
+ )
205
+ result_dy, residu_normy = inversion_one_component(
206
+ A,
207
+ dates_range,
208
+ 1,
209
+ data,
210
+ solver,
211
+ weighty,
212
+ mu,
213
+ coef=coef,
214
+ ini=ini[1],
215
+ result_quality=result_quality,
216
+ regu=regu,
217
+ accel=accel,
218
+ linear_operator=linear_operator,
219
+ )
220
+
221
+ else: # No initialization
222
+ result_dx, residu_normx = inversion_one_component(
223
+ A,
224
+ dates_range,
225
+ 0,
226
+ data,
227
+ solver,
228
+ weightx,
229
+ mu,
230
+ coef=coef,
231
+ result_quality=result_quality,
232
+ regu=regu,
233
+ accel=accel,
234
+ linear_operator=linear_operator,
235
+ )
236
+ result_dy, residu_normy = inversion_one_component(
237
+ A,
238
+ dates_range,
239
+ 1,
240
+ data,
241
+ solver,
242
+ weighty,
243
+ mu,
244
+ coef=coef,
245
+ result_quality=result_quality,
246
+ regu=regu,
247
+ accel=accel,
248
+ linear_operator=linear_operator,
249
+ )
250
+
251
+ return result_dx, result_dy, weightx, weighty, residu_normx, residu_normy
252
+
253
+
254
+ def inversion_core(
255
+ data: list,
256
+ i: float | int,
257
+ j: float | int,
258
+ dates_range: np.ndarray | None = None,
259
+ solver: str = "LSMR",
260
+ regu: int | str = "1accelnotnull",
261
+ coef: int = 100,
262
+ apriori_weight: bool = False,
263
+ iteration: bool = True,
264
+ threshold_it: float = 0.1,
265
+ unit: int = 365,
266
+ conf: bool = False,
267
+ mean: list | None = None,
268
+ detect_temporal_decorrelation: bool = True,
269
+ linear_operator: bool = False,
270
+ result_quality: list | str | None = None,
271
+ nb_max_iteration: int = 10,
272
+ apriori_weight_in_second_iteration: bool = False,
273
+ visual: bool = False,
274
+ verbose: bool = False,
275
+ ) -> (np.ndarray, pd.DataFrame, pd.DataFrame): # type: ignore
276
+ """
277
+ Computes A in AX = Y and does the inversion using a given solver and regularization.
278
+
279
+ :param data: [list] --- List of arrays, representing the observations: the first array contain the first and last acquisition dates, the second array contain the displacements values and errors, and optionally the last array contain information about the sensor, author, etc
280
+ :params i, j: [float | int] --- Coordinates of the point in pixel
281
+ :param dates_range: [np array | None] [default is None] --- List of np.datetime64 [D], dates of the estimated displacement in X with an irregular temporal sampling (ILF)
282
+ :param solver: [str] [default is 'LSMR'] --- Solver of the inversion: 'LSMR', 'LSMR_ini', 'LS', 'LS_bounded', 'LSQR'
283
+ :param regu: [int | str] [default is 1] --- Type of regularization
284
+ :param coef: [int] [default is 100] --- Coef of Tikhonov regularisation
285
+ :param apriori_weight: [bool] [default is False] --- If True use of aprori weight, based on the provided observation errors
286
+ :param iteration: [bool] [default is True] --- If True, use of iterations
287
+ :param threshold_it: [float] [default is 0.1] --- Threshold to test the stability of the results between each iteration, use to stop the process
288
+ :param unit: [int] [default is 365] --- 1 for m/d, 365 for m/y
289
+ :param conf: [bool] [default is False] --- If True means that the error corresponds to confidence intervals between 0 and 1, otherwise it corresponds to errors in m/y or m/d
290
+ :param mean: [list | None] [default is None] --- Apriori on the average
291
+ :param detect_temporal_decorrelation: [bool] [default is True] --- If True the first inversion is solved using only velocity observations with small temporal baselines, to detect temporal decorelation
292
+ :param linear_operator: [bool] [default is False] --- If linear operator, the inversion is performed using a linear operator (https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.linalg.LinearOperator.html)
293
+ :param result_quality: [list | str | None] [default is None] --- List which can contain 'Norm_residual' to determine the L2 norm of the residuals from the last inversion, 'X_contribution' to determine the number of Y observations which have contributed to estimate each value in X (it corresponds to A.dot(weight))
294
+ :param nb_max_iteration: [int] [default is 10] --- Maximum number of iterations
295
+ :param apriori_weight_in_second_iteration: [bool] [default is False] --- it True use the error to weight each of the iterations, if not use it only in the first iteration
296
+ :param visual: [bool] [default is True] --- Keep the weights for future plots
297
+ :param verbose: [bool] [default is False] --- Print information along the way
298
+
299
+ :return A: [np array | None] --- Design matrix in AX = Y
300
+ :return result: [pd dataframe | None] --- DF with dates, computed displacements and number of observations used to compute each displacement
301
+ :return dataf: [pd dataframe | None] --- Complete DF with dates, velocities, errors, residus, weights, xcount, normr... for further visual purposes (directly depends on param visual and result_quality)
302
+ """
303
+
304
+ if data[0].size: # If there are available data on this pixel
305
+ # Split the data, with one dtype per array
306
+ if len(data) == 3:
307
+ data_dates, data_values, data_str = data
308
+ else:
309
+ data_dates, data_values = data
310
+ del data
311
+
312
+ if dates_range is None:
313
+ dates_range = construction_dates_range_np(data_dates)
314
+
315
+ #### Build A (design matrix in AX = Y)
316
+ if not linear_operator:
317
+ A = construction_a_lf(data_dates, dates_range)
318
+ linear_operator = None
319
+ else: # use a linear operator to solve the inversion, it is sometimes faster
320
+ linear_operator = class_linear_operator()
321
+ linear_operator.load(
322
+ find_date_obs(data_dates[:, :2], dates_range), dates_range, coef
323
+ ) # load parameter of the linear operator
324
+ A = sp.linalg.LinearOperator(
325
+ (data_values.shape[0], len(dates_range) - 1),
326
+ matvec=linear_operator.matvec,
327
+ rmatvec=linear_operator.rmatvec,
328
+ ) # build A
329
+ mu = None
330
+
331
+ # Set a weight of 0, for large temporal baseline in the first inversion
332
+ weight_temporal_decorrelation = (
333
+ np.where(data_values[:, 4] > 180, 0, 1) if detect_temporal_decorrelation else None
334
+ )
335
+ # First weight of the inversion
336
+ Weightx = weight_for_inversion(
337
+ weight_origine=apriori_weight,
338
+ conf=conf,
339
+ data=data_values,
340
+ pos=2,
341
+ inside_Tukey=False,
342
+ temporal_decorrelation=weight_temporal_decorrelation,
343
+ )
344
+ Weighty = weight_for_inversion(
345
+ weight_origine=apriori_weight,
346
+ conf=conf,
347
+ data=data_values,
348
+ pos=3,
349
+ inside_Tukey=False,
350
+ temporal_decorrelation=weight_temporal_decorrelation,
351
+ )
352
+ del weight_temporal_decorrelation
353
+ if not visual:
354
+ if (
355
+ result_quality is not None
356
+ and not apriori_weight_in_second_iteration
357
+ and "Error_propagation" not in result_quality
358
+ ):
359
+ data_values = np.delete(
360
+ data_values, [2, 3], 1
361
+ ) # Delete quality indicator, which are not needed anymore
362
+ # Compute regularisation matrix
363
+ if not linear_operator:
364
+ if regu == "directionxy":
365
+ # Constrain according to the vectorial product, the magnitude of the vector corresponds to mean2, the magnitude of a rolling mean
366
+ mu = mu_regularisation(regu, A, dates_range, ini=mean)
367
+ else:
368
+ mu = mu_regularisation(regu, A, dates_range, ini=mean)
369
+
370
+ ## Initialisation (depending on apriori and solver)
371
+ # # Apriori on acceleration (following)
372
+ # TODO: we can make it shorter
373
+ if regu == "1accelnotnull":
374
+ accel = [
375
+ np.diff(mean[0]),
376
+ np.diff(mean[1]),
377
+ ] # compute acceleration based on the moving average, computing using a given kernel
378
+ mean_ini = [
379
+ np.multiply(mean[i], np.diff(dates_range) / np.timedelta64(1, "D")) for i in range(len(mean))
380
+ ] # compute what should be the displacement in X according to the moving average, computing using a given kernel
381
+
382
+ elif (
383
+ mean is not None and solver == "LSMR_ini"
384
+ ): # initialization is set according the average of the whole time series
385
+ mean_ini = [
386
+ np.multiply(mean[i], np.diff(dates_range) / np.timedelta64(1, "D") / unit) for i in range(len(mean))
387
+ ]
388
+ accel = None
389
+ else:
390
+ mean_ini = None
391
+ accel = None
392
+
393
+ ## Inversion
394
+ if regu == "directionxy":
395
+ result_dx, result_dy, residu_normx, residu_normy = inversion_two_components(
396
+ A, dates_range, 0, data_values, solver, np.concatenate([Weightx, Weighty]), mu, coef=coef, ini=mean_ini
397
+ )
398
+ else:
399
+ result_dx, residu_normx = inversion_one_component(
400
+ A,
401
+ dates_range,
402
+ 0,
403
+ data_values,
404
+ solver,
405
+ Weightx,
406
+ mu,
407
+ coef=coef,
408
+ ini=mean_ini,
409
+ result_quality=None,
410
+ regu=regu,
411
+ linear_operator=linear_operator,
412
+ accel=accel,
413
+ )
414
+ result_dy, residu_normy = inversion_one_component(
415
+ A,
416
+ dates_range,
417
+ 1,
418
+ data_values,
419
+ solver,
420
+ Weighty,
421
+ mu,
422
+ coef=coef,
423
+ ini=mean_ini,
424
+ result_quality=None,
425
+ regu=regu,
426
+ linear_operator=linear_operator,
427
+ accel=accel,
428
+ )
429
+
430
+ if not visual:
431
+ del Weighty, Weightx
432
+
433
+ if regu == "directionxy":
434
+ mu = mu_regularisation(regu, A, dates_range, ini=[mean[0], mean[1], result_dx, result_dy])
435
+
436
+ # Second Iteration
437
+ if iteration:
438
+ if (
439
+ (apriori_weight_in_second_iteration) and apriori_weight
440
+ ): # use apriori weight based on the error or quality indicator, Tukeybiweight(error/MAD(error)/ 0.6745)
441
+ Weightx2 = weight_for_inversion(
442
+ weight_origine=apriori_weight, conf=conf, data=data_values, pos=2, inside_Tukey=False
443
+ )
444
+ Weighty2 = weight_for_inversion(
445
+ weight_origine=apriori_weight, conf=conf, data=data_values, pos=3, inside_Tukey=False
446
+ )
447
+ else:
448
+ Weightx2, Weighty2 = None, None
449
+
450
+ result_dx_i, result_dy_i, weight_2x, weight_2y, residu_normx, residu_normy = inversion_iteration(
451
+ data_values,
452
+ A,
453
+ dates_range,
454
+ solver,
455
+ coef,
456
+ [Weightx2, Weighty2],
457
+ result_dx,
458
+ result_dy,
459
+ mu=mu,
460
+ verbose=verbose,
461
+ regu=regu,
462
+ linear_operator=linear_operator,
463
+ ini=None,
464
+ accel=accel,
465
+ result_quality=result_quality,
466
+ )
467
+ # Continue to iterate until the difference between two results is lower than threshold_it or the number of iteration larger than 10
468
+ i = 2
469
+ while (
470
+ np.mean(abs(result_dx_i - result_dx)) > threshold_it
471
+ or np.mean(abs(result_dy_i - result_dy)) > threshold_it
472
+ ) and i < nb_max_iteration:
473
+ result_dx = result_dx_i
474
+ result_dy = result_dy_i
475
+ result_dx_i, result_dy_i, weight_ix, weight_iy, residu_normx, residu_normy = inversion_iteration(
476
+ data_values,
477
+ A,
478
+ dates_range,
479
+ solver,
480
+ coef,
481
+ [Weightx2, Weighty2],
482
+ result_dx,
483
+ result_dy,
484
+ mu,
485
+ verbose=verbose,
486
+ regu=regu,
487
+ linear_operator=linear_operator,
488
+ ini=None,
489
+ accel=accel,
490
+ result_quality=result_quality,
491
+ )
492
+
493
+ i += 1
494
+
495
+ if verbose:
496
+ print(
497
+ "[Inversion] ",
498
+ i,
499
+ "dx",
500
+ np.mean(abs(result_dx_i - result_dx)),
501
+ "dy",
502
+ np.mean(abs(result_dy_i - result_dy)),
503
+ )
504
+
505
+ if verbose:
506
+ print("[Inversion] End loop", i, np.mean(abs(result_dy_i - result_dy)))
507
+ print("[Inversion] Nb iteration", i)
508
+
509
+ if i == 2:
510
+ weight_iy = weight_2y
511
+ weight_ix = weight_2x
512
+
513
+ del result_dx, result_dy
514
+ if not visual:
515
+ if result_quality is not None and "Error_propagation" not in result_quality:
516
+ del data_values, data_dates
517
+
518
+ else: # If not iteration
519
+ result_dy_i = result_dy
520
+ result_dx_i = result_dx
521
+
522
+ if np.isnan(result_dx_i).all(): # no results
523
+ return None, None, None
524
+
525
+ if not iteration:
526
+ weight_ix = Weightx
527
+ weight_iy = Weighty
528
+ if not visual:
529
+ del Weighty, Weightx
530
+ # compute the number of observations which have contributed to each estimated displacement
531
+ if result_quality is not None and "X_contribution" in result_quality:
532
+ xcount_x = A.T.dot(weight_ix)
533
+ xcount_y = A.T.dot(weight_iy)
534
+
535
+ else:
536
+ xcount_x = xcount_y = np.ones(result_dx_i.shape[0])
537
+
538
+ # propagate the error
539
+ if result_quality is not None and "Error_propagation" in result_quality:
540
+
541
+ def Prop_weight(F, weight, Residu, error):
542
+ error = np.max([Residu, error], axis=0) # take the maximum between residuals and errors
543
+ W = weight.astype("float32")
544
+ FTWF = np.multiply(F.T, W[np.newaxis, :]) @ F
545
+ N = np.linalg.inv(FTWF + coef * mu.T @ mu)
546
+ Prop_weight = np.multiply(np.multiply(N @ F.T, W[np.newaxis, :]) * error, W[np.newaxis, :]) @ F @ N
547
+ sigma0_weight = np.sum(Residu**2 * weight) / (F.shape[0] - F.shape[1])
548
+ prop_wieght_diag = np.diag(Prop_weight)
549
+ # Compute the confidence intervals
550
+ alpha = 0.05 # Confidence level
551
+ t_value = stats.t.ppf(1 - alpha / 2, df=F.shape[0] - F.shape[1])
552
+
553
+ return prop_wieght_diag, sigma0_weight, t_value
554
+
555
+ Residux = data_values[:, 0] - A @ result_dx_i # has a normal distribution
556
+ prop_wieght_diagx, sigma0_weightx, t_valuex = Prop_weight(
557
+ A, weight_ix, Residux, (data_values[:, 2] * data_values[:, -1] / unit) ** 2
558
+ )
559
+
560
+ Residuy = data_values[:, 1] - A @ result_dy_i # has a normal distribution
561
+ prop_wieght_diagy, sigma0_weighty, t_valuey = Prop_weight(
562
+ A, weight_iy, Residuy, (data_values[:, 3] * data_values[:, -1] / unit) ** 2
563
+ )
564
+
565
+ # If visual, save the velocity observation, the errors, the initial weights (weightini), the last weights (weightlast), the residuals from the last inversion, the sensors, and the authors
566
+ if visual:
567
+ vx = data_values[:, 0] / data_values[:, -1] * unit
568
+ vy = data_values[:, 1] / data_values[:, -1] * unit
569
+ Residux = data_values[:, 0] - A.dot(result_dx_i)
570
+ Residuy = data_values[:, 1] - A.dot(result_dy_i)
571
+ dataf = pd.DataFrame(
572
+ {
573
+ "date1": data_dates[:, 0],
574
+ "date2": data_dates[:, 1],
575
+ "vx": vx,
576
+ "vy": vy,
577
+ "errorx": data_values[:, 2],
578
+ "errory": data_values[:, 3],
579
+ "weightinix": Weightx,
580
+ "weightiniy": Weighty,
581
+ "weightlastx": weight_ix,
582
+ "weightlasty": weight_iy,
583
+ "residux": Residux,
584
+ "residuy": Residuy,
585
+ "sensor": data_str[:, 0],
586
+ "author": data_str[:, 1],
587
+ }
588
+ )
589
+ if (
590
+ residu_normx is not None
591
+ ): # save the L2-norm from the last inversion, of the term AXY and the regularization term for the x- and y-component
592
+ NormR = np.zeros(data_values.shape[0])
593
+ NormR[:4] = np.hstack(
594
+ [residu_normx, residu_normy]
595
+ ) # the order is: AXY and regularization term L2-norm for x-component, and AXY and regularization term L2-norm for y-component
596
+ dataf["NormR"] = NormR
597
+ del NormR
598
+ else:
599
+ dataf, A = None, None
600
+
601
+ else: # If there is no data over this pixel
602
+ if verbose:
603
+ print(f"[Inversion] NO DATA TO INVERSE AT POINT {i, j}")
604
+ return None, None, None
605
+
606
+ # pandas dataframe with the saved results
607
+ result = pd.DataFrame(
608
+ {
609
+ "date1": dates_range[:-1],
610
+ "date2": dates_range[1:],
611
+ "result_dx": result_dx_i,
612
+ "result_dy": result_dy_i,
613
+ "xcount_x": xcount_x,
614
+ "xcount_y": xcount_y,
615
+ }
616
+ )
617
+ if residu_normx is not None: # add the norm of the residual
618
+ normr = np.zeros(result.shape[0])
619
+ if normr.shape[0] > 3:
620
+ normr[:4] = np.hstack([residu_normx, residu_normy])
621
+ else:
622
+ normr[: normr.shape[0]] = np.full(normr.shape[0], np.nan)
623
+ result["NormR"] = normr
624
+ del normr
625
+ if result_quality is not None: # add the error propagation
626
+ if "Error_propagation" in result_quality:
627
+ result["error_x"] = prop_wieght_diagx
628
+ result["error_y"] = prop_wieght_diagy
629
+ sigma = np.zeros(result.shape[0])
630
+ sigma[:4] = np.hstack([sigma0_weightx, sigma0_weighty, t_valuex, t_valuey])
631
+ result["sigma0"] = sigma
632
+
633
+ return A, result, dataf
634
+
635
+
636
+ # %% ======================================================================== #
637
+ # INTERPOLATION #
638
+ # =========================================================================%% #
639
+
640
+
641
+ def interpolation_core(
642
+ result: pd.DataFrame,
643
+ interval_output: int = 30,
644
+ option_interpol: str = "spline",
645
+ first_date_interpol: np.datetime64 | str | None = None,
646
+ last_date_interpol: np.datetime64 | str | None = None,
647
+ unit: int = 365,
648
+ redundancy: int | None = 5,
649
+ result_quality: list | None = None,
650
+ ):
651
+ """
652
+ Interpolate Irregular Leap Frog time series (result of an inversion) to Regular LF time series using Cumulative Displacement times series.
653
+
654
+ :param result: [pd dataframe] --- Leap frog displacement for x-component and y-component
655
+ :param interval_output: [int] --- Period between two dates of the obtained RLF
656
+ :param path_save: [str] --- Where to save the figures
657
+ :param option_interpol: [str] [default is 'spline'] --- Type of interpolation, it can be 'spline', 'spline_smooth' or 'nearest'
658
+ :param first_date_interpol: [np.datetime64 | str | None] [default is None] --- First date of the interpolation
659
+ :param last_date_interpol: [np.datetime64 | str | None] [default is None] --- Last date of the interpolation
660
+ :param unit: [int] [default is 365] --- 1 for m/d, 365 for m/y
661
+ :param redundancy: [int | None] [default is None] --- If None there is no redundancy between two velocity in the interpolated time-series, else the overlap between two velocities is redundancy days
662
+ :param result_quality: [list | str | None] [default is None] --- List which can contain 'Norm_residual' to determine the L2 norm of the residuals from the last inversion, 'X_contribution' to determine the number of Y observations which have contributed to estimate each value in X (it corresponds to A.dot(weight))
663
+
664
+ :return dataf_lp: [pd dataframe] --- Result of the temporal interpolation
665
+ """
666
+ ## Reconstruction of COMMON REF TIME SERIES, e.g. cumulative displacement time series
667
+ dataf = reconstruct_common_ref(result) # Build cumulative displacement time series
668
+ if first_date_interpol is None:
669
+ start_date = dataf["Ref_date"][0] # First date at the considered pixel
670
+ else:
671
+ start_date = pd.to_datetime(first_date_interpol)
672
+
673
+ x = np.array(
674
+ (dataf["Second_date"] - np.datetime64(start_date)).dt.days
675
+ ) # Number of days according to the start_date
676
+ if len(x) <= 1 or (
677
+ np.isin("spline", option_interpol) and len(x) <= 3
678
+ ): # It is not possible to interpolate, because too few estimation
679
+ return pd.DataFrame(
680
+ {
681
+ "date1": [],
682
+ "date2": [],
683
+ "vx": [],
684
+ "vy": [],
685
+ "xcount_x": [],
686
+ "xcount_y": [],
687
+ "dz": [],
688
+ "vz": [],
689
+ "xcount_z": [],
690
+ "NormR": [],
691
+ }
692
+ )
693
+
694
+ # Compute the functions used to interpolate
695
+ fdx, fdy, fdx_xcount, fdy_xcount, fdx_error, fdy_error = set_function_for_interpolation(
696
+ option_interpol, x, dataf, result_quality
697
+ )
698
+
699
+ if redundancy is None: # No redundancy between two interpolated velocity
700
+ x_regu = np.arange(np.min(x) + (interval_output - np.min(x) % interval_output), np.max(x), interval_output)
701
+ else: # The overlap between two velocities corresponds to redundancy
702
+ x_regu = np.arange(
703
+ np.min(x) + (redundancy - np.min(x) % redundancy), np.max(x), redundancy
704
+ ) # To make sure that the first element of x_regu is multiple of redundancy
705
+
706
+ if len(x_regu) <= 1: # No interpolation
707
+ return pd.DataFrame(
708
+ {
709
+ "date1": [],
710
+ "date2": [],
711
+ "vx": [],
712
+ "vy": [],
713
+ "xcount_x": [],
714
+ "xcount_y": [],
715
+ "dz": [],
716
+ "vz": [],
717
+ "xcount_z": [],
718
+ "NormR": [],
719
+ }
720
+ )
721
+
722
+ ## Reconstruct a time series with a given temporal sampling, and a given overlap
723
+ step = interval_output if redundancy is None else int(interval_output / redundancy)
724
+ if step >= len(x_regu):
725
+ return pd.DataFrame(
726
+ {
727
+ "date1": [],
728
+ "date2": [],
729
+ "vx": [],
730
+ "vy": [],
731
+ "xcount_x": [],
732
+ "xcount_y": [],
733
+ "dz": [],
734
+ "vz": [],
735
+ "xcount_z": [],
736
+ "NormR": [],
737
+ }
738
+ )
739
+
740
+ x_shifted = x_regu[step:]
741
+ dx = fdx(x_shifted) - fdx(
742
+ x_regu[:-step]
743
+ ) # Equivalent to [fdx(x_regu[i + step]) - fdx(x_regu[i]) for i in range(len(x_regu) - step)]
744
+ dy = fdy(x_shifted) - fdy(
745
+ x_regu[:-step]
746
+ ) # Equivalent to [fdy(x_regu[i + step]) - fdy(x_regu[i]) for i in range(len(x_regu) - step)]
747
+ if result_quality is not None:
748
+ if "X_contribution" in result_quality:
749
+ xcount_x = fdx_xcount(x_shifted) - fdx_xcount(x_regu[:-step])
750
+ xcount_y = fdy_xcount(x_shifted) - fdy_xcount(x_regu[:-step])
751
+ xcount_x[xcount_x < 0] = 0
752
+ xcount_y[xcount_y < 0] = 0
753
+ if "Error_propagation" in result_quality:
754
+ error_x = fdx_error(x_shifted) - fdx_error(x_regu[:-step])
755
+ error_y = fdy_error(x_shifted) - fdy_error(x_regu[:-step])
756
+ vx = dx * unit / interval_output # Convert to velocity in m/d or m/y
757
+ vy = dy * unit / interval_output # Convert to velocity in m/d or m/
758
+
759
+ First_date = start_date + pd.to_timedelta(
760
+ x_regu[:-step], unit="D"
761
+ ) # Equivalent to [start_date + pd.Timedelta(x_regu[i], 'D') for i in range(len(x_regu) - step)]
762
+ Second_date = start_date + pd.to_timedelta(x_shifted, unit="D")
763
+
764
+ dataf_lp = pd.DataFrame({"date1": First_date, "date2": Second_date, "vx": vx, "vy": vy})
765
+ if result_quality is not None:
766
+ if "X_contribution" in result_quality:
767
+ dataf_lp["xcount_x"] = xcount_x
768
+ dataf_lp["xcount_y"] = xcount_y
769
+ if "Error_propagation" in result_quality:
770
+ dataf_lp["error_x"] = error_x * unit / interval_output
771
+ dataf_lp["error_y"] = error_y * unit / interval_output
772
+ dataf_lp["sigma0"] = np.concatenate([result["sigma0"][:4], np.full(dataf_lp.shape[0] - 4, np.nan)])
773
+ del x_regu, First_date, Second_date, vx, vy
774
+
775
+ # Fill with nan values if the first date of the cube which will be interpolated is lower than the first date interpolated for this pixel
776
+ if first_date_interpol is not None and dataf_lp["date1"].iloc[0] > pd.Timestamp(first_date_interpol):
777
+ first_date = np.arange(first_date_interpol, dataf_lp["date1"].iloc[0], np.timedelta64(redundancy, "D"))
778
+ # dataf_lp = full_with_nan(dataf_lp, first_date=first_date,
779
+ # second_date=first_date + np.timedelta64(interval_output, 'D'))
780
+ nul_df = pd.DataFrame(
781
+ {
782
+ "date1": first_date,
783
+ "date2": first_date + np.timedelta64(interval_output, "D"),
784
+ "vx": np.full(len(first_date), np.nan),
785
+ "vy": np.full(len(first_date), np.nan),
786
+ }
787
+ )
788
+ if result_quality is not None:
789
+ if "X_contribution" in result_quality:
790
+ nul_df["xcount_x"] = np.full(len(first_date), np.nan)
791
+ nul_df["xcount_y"] = np.full(len(first_date), np.nan)
792
+ if "Error_propagation" in result_quality:
793
+ nul_df["error_x"] = np.full(len(first_date), np.nan)
794
+ nul_df["error_y"] = np.full(len(first_date), np.nan)
795
+ dataf_lp = pd.concat([nul_df, dataf_lp], ignore_index=True)
796
+
797
+ # Fill with nan values if the last date of the cube which will be interpolated is higher than the last date interpolated for this pixel
798
+ if last_date_interpol is not None and dataf_lp["date2"].iloc[-1] < pd.Timestamp(last_date_interpol):
799
+ first_date = np.arange(
800
+ dataf_lp["date2"].iloc[-1] + np.timedelta64(redundancy, "D"),
801
+ last_date_interpol + np.timedelta64(redundancy, "D"),
802
+ np.timedelta64(redundancy, "D"),
803
+ )
804
+ nul_df = pd.DataFrame(
805
+ {
806
+ "date1": first_date - np.timedelta64(interval_output, "D"),
807
+ "date2": first_date,
808
+ "vx": np.full(len(first_date), np.nan),
809
+ "vy": np.full(len(first_date), np.nan),
810
+ }
811
+ )
812
+ dataf_lp = pd.concat([dataf_lp, nul_df], ignore_index=True)
813
+
814
+ # print(dataf_lp.shape)
815
+ # if dataf_lp.shape[0]!= 567:
816
+ # print('stop')
817
+ return dataf_lp
818
+
819
+
820
+ def interpolation_to_data(
821
+ result: pd.DataFrame,
822
+ data: pd.DataFrame,
823
+ option_interpol: str = "spline",
824
+ unit: int = 365,
825
+ result_quality: list | None = None,
826
+ ):
827
+ """
828
+ Interpolate Irregular Leap Frog time series (result of an inversion) to the dates of given data (useful to compare
829
+ TICOI results to a "ground truth").
830
+
831
+ :param result: [pd dataframe] --- Leap frog displacement for x-component and y-component
832
+ :param data: [pd dataframe] --- Ground truth data which the interpolation must fit along the temporal axis
833
+ :param option_interpol: [str] [default is 'spline'] --- Type of interpolation, it can be 'spline', 'spline_smooth' or 'nearest'
834
+ :param unit: [int] [default is 365] --- 1 for m/d, 365 for m/y
835
+ :param result_quality: [list | str | None] [default is None] --- List which can contain 'Norm_residual' to determine the L2 norm of the residuals from the last inversion, 'X_contribution' to determine the number of Y observations which have contributed to estimate each value in X (it corresponds to A.dot(weight))
836
+ """
837
+
838
+ ## Reconstruction of COMMON REF TIME SERIES, e.g. cumulative displacement time series
839
+ dataf = reconstruct_common_ref(result) # Build cumulative displacement time series
840
+ start_date = dataf["Ref_date"][0] # First date at the considered pixel
841
+ x = np.array(
842
+ (dataf["Second_date"] - np.datetime64(start_date)).dt.days
843
+ ) # Number of days according to the start_date
844
+
845
+ # Interpolation must be caried out in between the min and max date of the original data
846
+ if data["date1"].min() < result["date2"].min() or data["date2"].max() > result["date2"].max():
847
+ data = data[(data["date1"] > result["date2"].min()) & (data["date2"] < result["date2"].max())]
848
+
849
+ # Ground truth first and second dates
850
+ x_gt_date1 = np.array((data["date1"] - start_date).dt.days)
851
+ x_gt_date2 = np.array((data["date2"] - start_date).dt.days)
852
+
853
+ ## Interpolate the displacements and convert to velocities
854
+ # Compute the functions used to interpolate
855
+ fdx, fdy, fdx_xcount, fdy_xcount, fdx_error, fdy_error = set_function_for_interpolation(
856
+ option_interpol, x, dataf, result_quality
857
+ )
858
+
859
+ # Interpolation
860
+ dx = fdx(x_gt_date2) - fdx(
861
+ x_gt_date1
862
+ ) # Equivalent to [fdx(x_regu[i + step]) - fdx(x_regu[i]) for i in range(len(x_regu) - step)]
863
+ dy = fdy(x_gt_date2) - fdy(
864
+ x_gt_date1
865
+ ) # Equivalent to [fdy(x_regu[i + step]) - fdy(x_regu[i]) for i in range(len(x_regu) - step)]
866
+
867
+ # conversion
868
+ vx = dx * unit / data["temporal_baseline"] # Convert to velocity in m/d or m/y
869
+ vy = dy * unit / data["temporal_baseline"] # Convert to velocity in m/d or m/y
870
+
871
+ # Fill dataframe
872
+ First_date = start_date + pd.to_timedelta(
873
+ x_gt_date1, unit="D"
874
+ ) # Equivalent to [start_date + pd.Timedelta(x_regu[i], 'D') for i in range(len(x_regu) - step)]
875
+ Second_date = start_date + pd.to_timedelta(x_gt_date2, unit="D")
876
+ data_dict = {"date1": First_date, "date2": Second_date, "vx": vx, "vy": vy}
877
+ dataf_lp = pd.DataFrame(data_dict)
878
+
879
+ return dataf_lp
880
+
881
+
882
+ # %% ======================================================================== #
883
+ # GLOBAL PROCESS #
884
+ # =========================================================================%% #
885
+
886
+
887
+ def process(
888
+ cube: CubeDataClass,
889
+ i: float | int,
890
+ j: float | int,
891
+ path_save,
892
+ solver: str = "LSMR_ini",
893
+ regu: int | str = "1accelnotnull",
894
+ coef: int = 100,
895
+ flag: xr.Dataset | None = None,
896
+ apriori_weight: bool = False,
897
+ apriori_weight_in_second_iteration: bool = False,
898
+ returned: list | str = "interp",
899
+ obs_filt: xr.Dataset | None = None,
900
+ interpolation_load_pixel: str = "nearest",
901
+ iteration: bool = True,
902
+ interval_output: int = 1,
903
+ first_date_interpol: np.datetime64 | None = None,
904
+ last_date_interpol: np.datetime64 | None = None,
905
+ proj="EPSG:4326",
906
+ threshold_it: float = 0.1,
907
+ conf: bool = True,
908
+ option_interpol: str = "spline",
909
+ redundancy: int | None = None,
910
+ detect_temporal_decorrelation: bool = True,
911
+ unit: int = 365,
912
+ result_quality: list | str | None = None,
913
+ nb_max_iteration: int = 10,
914
+ delete_outliers: int | str | None = None,
915
+ linear_operator: bool = False,
916
+ visual: bool = False,
917
+ verbose: bool = False,
918
+ ):
919
+ """
920
+ :params i, j: [float | int] --- Coordinates of the point in pixel
921
+ :param solver: [str] [default is 'LSMR'] --- Solver of the inversion: 'LSMR', 'LSMR_ini', 'LS', 'LSQR'
922
+ :param regu: [int | str] [default is 1] --- Type of regularization
923
+ :param coef: [int] [default is 100] --- Coef of Tikhonov regularisation
924
+ :param flag: [xr dataset | None] [default is None] --- If not None, the values of the coefficient used for stable areas, surge glacier and non surge glacier
925
+ :param apriori_weight: [bool] [default is False] --- If True use of aprori weight
926
+ :param returned: [list | str] [default is 'interp'] --- What results must be returned ('raw', 'invert' and/or 'interp')
927
+ :param obs_filt: [xr dataset | None] [default is None] --- Filtered dataset (e.g. rolling mean)
928
+ :param interpolation_load_pixel: [str] [default is 'nearest'] --- Type of interpolation to load the previous pixel in the temporal interpolation ('nearest' or 'linear')
929
+ :param iteration: [bool] [default is True] --- If True, use of iterations
930
+ :param interval_output: [int] [default is 1] --- Temporal sampling of the leap frog time series
931
+ :param first_date_interpol: [np.datetime64 | None] --- First date at which the time series are interpolated
932
+ :param last_date_interpol: [np.datetime64 | None] --- Last date at which the time series are interpolated
933
+ :param proj: [str] [default is 'EPSG:4326'] --- Projection of the cube
934
+ :param threshold_it: [float] [default is 0.1] --- Threshold to test the stability of the results between each iteration, use to stop the process
935
+ :param conf: [bool] [default is False] --- If True means that the error corresponds to confidence intervals between 0 and 1, otherwise it corresponds to errors in m/y or m/d
936
+ :param option_interpol: [str] [default is 'spline'] --- Type of interpolation, it can be 'spline', 'spline_smooth' or 'nearest'
937
+ :param redundancy: [int | None] [default is None] --- If None there is no redundancy between two velocity in the interpolated time-series, else the overlap between two velocities is redundancy days
938
+ :param detect_temporal_decorrelation: [bool] [default is True] --- If True the first inversion is solved using only velocity observations with small temporal baselines, to detect temporal decorelation
939
+ :param unit: [int] [default is 365] --- 1 for m/d, 365 for m/y
940
+ :param result_quality: [list | str | None] [default is None] --- List which can contain 'Norm_residual' to determine the L2 norm of the residuals from the last inversion, 'X_contribution' to determine the number of Y observations which have contributed to estimate each value in X (it corresponds to A.dot(weight))
941
+ :param nb_max_iteration: [int] [default is 10] --- Maximum number of iterations
942
+ :param delete_outliers: [int | str | None] [default is None] --- Delete data with a poor quality indicator (if int), or with aberrant direction ('vvc_angle')
943
+ :param linear_operator: [bool] [default is False] --- If linear operator, the inversion is performed using a linear operator (https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.linalg.LinearOperator.html)
944
+ :param visual: [bool] [default is False] --- Keep the weights for future plots
945
+ :param verbose: [bool] [default is False] --- Print information along the way
946
+
947
+ :return dataf_list: [pd dataframe] Result of the temporal inversion + interpolation at point (i, j) if inversion was successful, an empty dataframe if not
948
+ """
949
+
950
+ returned_list = []
951
+
952
+ # Loading data at pixel location
953
+ data = cube.load_pixel(
954
+ i,
955
+ j,
956
+ proj=proj,
957
+ interp=interpolation_load_pixel,
958
+ solver=solver,
959
+ coef=coef,
960
+ regu=regu,
961
+ rolling_mean=obs_filt,
962
+ flag=flag,
963
+ )
964
+
965
+ if "raw" in returned: # return the raw data
966
+ returned_list.append(data)
967
+
968
+ if "invert" in returned or "interp" in returned:
969
+ if flag is not None: # set regu and coef for every flags
970
+ regu, coef = data[3], data[4]
971
+
972
+ # Inversion
973
+ # TODO: to check that!
974
+ if delete_outliers == "median_angle":
975
+ conf = True # Set conf to True, because the errors have been replaced by confidence indicators based on the cos of the angle between the vector of each observation and the median vector
976
+
977
+ result = inversion_core(
978
+ data[0],
979
+ i,
980
+ j,
981
+ dates_range=data[2],
982
+ solver=solver,
983
+ coef=coef,
984
+ apriori_weight=apriori_weight,
985
+ apriori_weight_in_second_iteration=apriori_weight_in_second_iteration,
986
+ unit=unit,
987
+ conf=conf,
988
+ regu=regu,
989
+ mean=data[1],
990
+ iteration=iteration,
991
+ threshold_it=threshold_it,
992
+ detect_temporal_decorrelation=detect_temporal_decorrelation,
993
+ linear_operator=linear_operator,
994
+ result_quality=result_quality,
995
+ nb_max_iteration=nb_max_iteration,
996
+ visual=visual,
997
+ verbose=verbose,
998
+ )
999
+
1000
+ if "invert" in returned:
1001
+ if result[1] is not None:
1002
+ returned_list.append(result[1])
1003
+ else:
1004
+ if result_quality is not None and "X_contribution" in result_quality:
1005
+ variables = ["result_dx", "result_dy", "xcount_x", "xcount_y"]
1006
+ else:
1007
+ variables = ["result_dx", "result_dy"]
1008
+ returned_list.append(pd.DataFrame({"date1": [], "date2": [], **{col: [] for col in variables}}))
1009
+
1010
+ if "interp" in returned:
1011
+ # Interpolation
1012
+ if result[1] is not None: # If inversion have been performed
1013
+ dataf_list = interpolation_core(
1014
+ result[1],
1015
+ interval_output,
1016
+ option_interpol=option_interpol,
1017
+ first_date_interpol=first_date_interpol,
1018
+ last_date_interpol=last_date_interpol,
1019
+ unit=unit,
1020
+ redundancy=redundancy,
1021
+ result_quality=result_quality,
1022
+ )
1023
+
1024
+ if result_quality is not None and "Norm_residual" in result_quality:
1025
+ dataf_list["NormR"] = result[1]["NormR"] # Store norm of the residual from the inversion
1026
+ returned_list.append(dataf_list)
1027
+ else:
1028
+ if result_quality is not None and "Norm_residual" in result_quality:
1029
+ returned_list.append(
1030
+ pd.DataFrame(
1031
+ {"date1": [], "date2": [], "vx": [], "vy": [], "xcount_x": [], "xcount_y": [], "NormR": []}
1032
+ )
1033
+ )
1034
+ else:
1035
+ returned_list.append(
1036
+ pd.DataFrame({"date1": [], "date2": [], "vx": [], "vy": [], "xcount_x": [], "xcount_y": []})
1037
+ )
1038
+
1039
+ if len(returned_list) == 1:
1040
+ return returned_list[0]
1041
+ return returned_list if len(returned_list) > 0 else None
1042
+
1043
+
1044
+ def chunk_to_block(cube: CubeDataClass, block_size: float = 1, verbose: bool = False):
1045
+ """
1046
+ Split a dataset in blocks of a given size (maximum).
1047
+
1048
+ :param cube: [cube_data_class] --- Cube to be splited in blocks
1049
+ :param block_size: [float] [default is 1] --- Maximum size (in GB) of the blocks
1050
+ :param verbose: [bool] [default is False] --- Print information along the way
1051
+
1052
+ :return blocks: [list] --- List of the boundaries of each blocks (x_start, x_end, y_start, y_end)
1053
+ """
1054
+
1055
+ GB = 1073741824
1056
+ blocks = []
1057
+ if cube.ds.nbytes > block_size * GB:
1058
+ try:
1059
+ num_elements = np.prod([cube.ds.chunks[dim][0] for dim in cube.ds.chunks.keys()])
1060
+ except ValueError:
1061
+ cube = cube.ds.unify_chunks() # ValueError: Object has inconsistent chunks along dimension x. This can be fixed by calling unify_chunks().
1062
+
1063
+ chunk_bytes = num_elements * cube.ds["vx"].dtype.itemsize
1064
+
1065
+ nchunks_block = int(block_size * GB // chunk_bytes)
1066
+
1067
+ x_step = int(np.sqrt(nchunks_block))
1068
+ y_step = nchunks_block // x_step
1069
+
1070
+ nblocks_x = int(np.ceil(len(cube.ds.chunks["x"]) / x_step))
1071
+ nblocks_y = int(np.ceil(len(cube.ds.chunks["y"]) / y_step))
1072
+
1073
+ nblocks = nblocks_x * nblocks_y
1074
+ if verbose:
1075
+ print(
1076
+ f"[Block process] Divide into {nblocks} blocks\n blocks size: {x_step * cube.ds.chunks['x'][0]} x {y_step * cube.ds.chunks['y'][0]}"
1077
+ )
1078
+
1079
+ for i in range(nblocks_y):
1080
+ for j in range(nblocks_x):
1081
+ x_start = j * x_step * cube.ds.chunks["x"][0]
1082
+ y_start = i * y_step * cube.ds.chunks["y"][0]
1083
+ x_end = x_start + x_step * cube.ds.chunks["x"][0] if j != nblocks_x - 1 else cube.ds.dims["x"]
1084
+ y_end = y_start + y_step * cube.ds.chunks["y"][0] if i != nblocks_y - 1 else cube.ds.dims["y"]
1085
+ blocks.append([x_start, x_end, y_start, y_end])
1086
+ else:
1087
+ blocks.append([0, cube.ds.dims["x"], 0, cube.ds.dims["y"]])
1088
+ if verbose:
1089
+ print(f"[Block process] Cube size smaller than {block_size}GB, no need to divide")
1090
+
1091
+ return blocks
1092
+
1093
+
1094
+ def load_block(cube: CubeDataClass, x_start: int, x_end: int, y_start: int, y_end: int, flag: xr.Dataset | None = None):
1095
+ """
1096
+ Persist a block in memory, i.e. load it in a distributed way.
1097
+
1098
+ :param cube: [cube_data_class] --- Cube splited in blocks
1099
+ :params x_start, x_end, y_start, y_end: [int] --- Boundaries of the block
1100
+
1101
+ :return block: [cube_data_class] --- Sub-cube of cube according to the boundaries (block)
1102
+ :return duration: [float] --- Duration of the block loading
1103
+ """
1104
+
1105
+ start = time.time()
1106
+ block = CubeDataClass()
1107
+ block.ds = cube.ds.isel(x=slice(x_start, x_end), y=slice(y_start, y_end))
1108
+ block.ds = block.ds.persist()
1109
+ block.update_dimension()
1110
+ if flag is not None:
1111
+ block_flag = flag.isel(x=slice(x_start, x_end), y=slice(y_start, y_end))
1112
+ block_flag = block_flag.persist()
1113
+ else:
1114
+ block_flag = None
1115
+ duration = time.time() - start
1116
+
1117
+ return block, block_flag, duration
1118
+
1119
+
1120
+ def process_blocks_refine(
1121
+ cube: CubeDataClass,
1122
+ nb_cpu: int = 8,
1123
+ block_size: float = 0.5,
1124
+ returned: list | str = "interp",
1125
+ preData_kwargs: dict = None,
1126
+ inversion_kwargs: dict | None = None,
1127
+ verbose: bool = False,
1128
+ ):
1129
+ """
1130
+ Separate the cube in several blocks computed synchronously one after the other by loading one block while the other is computed (with
1131
+ parallelization) in order to avoid memory overconsumption and kernel crashing, and benefit from smaller computation time.
1132
+
1133
+ :param cube: [cube_data_class] --- Cube of raw data to be processed
1134
+ :param nb_cpu: [int] [default is 8] --- Number of processing unit to use for parallel processing
1135
+ :param block_size: [float] [default is 0.5] --- Maximum size of the blocks (in GB)
1136
+ :param returned: [list | str] [default is 'interp'] --- What results must be returned ('raw', 'invert' and/or 'interp')
1137
+ :param preData_kwargs: [dict] [default is None] --- Pre-processing parameters (see cube_data_classxr.filter_cube)
1138
+ :param inversion_kwargs: [dict] [default is None] --- Inversion (and interpolation) parameters (see core.process)
1139
+ :param verbose: [bool] [default is False] --- Print information along the way
1140
+
1141
+ :return: [pd dataframe] Resulting estimated time series after inversion (and interpolation)
1142
+ """
1143
+
1144
+ async def process_block(
1145
+ block: CubeDataClass, returned: list | str = "interp", nb_cpu: int = 8, verbose: bool = False
1146
+ ):
1147
+ xy_values = itertools.product(block.ds["x"].values, block.ds["y"].values)
1148
+ # Return only raw data => no need to filter the cube
1149
+ if "raw" in returned and (isinstance(returned, str) or len(returned) == 1): # Only load the raw data
1150
+ xy_values_tqdm = tqdm(xy_values, total=(block.nx * block.ny))
1151
+ result_block = Parallel(n_jobs=nb_cpu, verbose=0)(
1152
+ delayed(block.load_pixel)(
1153
+ i,
1154
+ j,
1155
+ proj=inversion_kwargs["proj"],
1156
+ interp=inversion_kwargs["interpolation_load_pixel"],
1157
+ solver=inversion_kwargs["solver"],
1158
+ regu=inversion_kwargs["regu"],
1159
+ rolling_mean=None,
1160
+ visual=inversion_kwargs["visual"],
1161
+ )
1162
+ for i, j in xy_values_tqdm
1163
+ )
1164
+ return result_block
1165
+
1166
+ # Filter the cube
1167
+ obs_filt, flag_block = block.filter_cube_before_inversion(**preData_kwargs)
1168
+ if isinstance(inversion_kwargs, dict):
1169
+ inversion_kwargs.update({"flag": flag_block})
1170
+
1171
+ # There is no data on the whole block (masked data)
1172
+ if obs_filt is None and "interp" in returned:
1173
+ if inversion_kwargs["result_quality"] is not None and "Norm_residual" in inversion_kwargs["result_quality"]:
1174
+ return [
1175
+ pd.DataFrame(
1176
+ {"date1": [], "date2": [], "vx": [], "vy": [], "xcount_x": [], "xcount_y": [], "NormR": []}
1177
+ )
1178
+ ]
1179
+ else:
1180
+ return [
1181
+ pd.DataFrame(
1182
+ {"First_date": [], "Second_date": [], "vx": [], "vy": [], "xcount_x": [], "xcount_y": []}
1183
+ )
1184
+ ]
1185
+
1186
+ xy_values_tqdm = tqdm(xy_values, total=(obs_filt["x"].shape[0] * obs_filt["y"].shape[0]))
1187
+ result_block = Parallel(n_jobs=nb_cpu, verbose=0)(
1188
+ delayed(process)(block, i, j, obs_filt=obs_filt, returned=returned, **inversion_kwargs)
1189
+ for i, j in xy_values_tqdm
1190
+ )
1191
+
1192
+ return result_block
1193
+
1194
+ async def process_blocks_main(cube, nb_cpu=8, block_size=0.5, returned="interp", verbose=False):
1195
+ if isinstance(preData_kwargs, dict) and "flag" in preData_kwargs.keys():
1196
+ flag = preData_kwargs["flag"]
1197
+ if flag is not None:
1198
+ flag = cube.create_flag(flag)
1199
+ else:
1200
+ flag = None
1201
+
1202
+ blocks = chunk_to_block(cube, block_size=block_size, verbose=True) # Split the cube in smaller blocks
1203
+
1204
+ dataf_list = [None] * (cube.nx * cube.ny)
1205
+
1206
+ loop = asyncio.get_event_loop()
1207
+ for n in range(len(blocks)):
1208
+ print(f"[Block process] Processing block {n + 1}/{len(blocks)}")
1209
+
1210
+ # Load the first block and start the loop
1211
+ if n == 0:
1212
+ x_start, x_end, y_start, y_end = blocks[0]
1213
+ future = loop.run_in_executor(None, load_block, cube, x_start, x_end, y_start, y_end, flag)
1214
+
1215
+ block, block_flag, duration = await future
1216
+ print(f"Block {n + 1} loaded in {duration:.2f} s")
1217
+
1218
+ if n < len(blocks) - 1:
1219
+ # Load the next block while processing the current block
1220
+ x_start, x_end, y_start, y_end = blocks[n + 1]
1221
+ future = loop.run_in_executor(None, load_block, cube, x_start, x_end, y_start, y_end, flag)
1222
+
1223
+ # need to change the flag back...
1224
+ if flag is not None:
1225
+ preData_kwargs.update({"flag": block_flag})
1226
+
1227
+ block_result = await process_block(
1228
+ block, returned=returned, nb_cpu=nb_cpu, verbose=verbose
1229
+ ) # Process TICOI
1230
+
1231
+ # Transform to list
1232
+ for i in range(len(block_result)):
1233
+ row = i % block.ny + blocks[n][2]
1234
+ col = np.floor(i / block.ny) + blocks[n][0]
1235
+ idx = int(col * cube.ny + row)
1236
+
1237
+ dataf_list[idx] = block_result[i]
1238
+
1239
+ del block_result, block
1240
+
1241
+ if isinstance(returned, list) and len(returned) > 1:
1242
+ dataf_list = {returned[r]: [dataf_list[i][r] for i in range(len(dataf_list))] for r in range(len(returned))}
1243
+
1244
+ return dataf_list
1245
+
1246
+ # /!\ The use of asyncio can cause problems when the code is launched from an IDE if it has its own event loop
1247
+ # (leads to RuntimeError), you must launch it in an external terminal (IDEs generally offer this option)
1248
+ return asyncio.run(
1249
+ process_blocks_main(cube, nb_cpu=nb_cpu, block_size=block_size, returned=returned, verbose=verbose)
1250
+ )
1251
+
1252
+
1253
+ # %% ======================================================================== #
1254
+ # VISUALISATION #
1255
+ # =========================================================================%% #
1256
+
1257
+
1258
+ def visualization_core(
1259
+ list_dataf: pd.DataFrame,
1260
+ option_visual: List,
1261
+ type_data: List = ["obs", "invert"],
1262
+ save: bool = False,
1263
+ show: bool = True,
1264
+ path_save: Optional[str] = None,
1265
+ A: Optional[np.array] = None,
1266
+ log_scale: bool = False,
1267
+ cmap: str = "viridis",
1268
+ colors: List[str] = ["blueviolet", "orange"],
1269
+ figsize: tuple[int, int] = (10, 6),
1270
+ vminmax: List[int] | None = None,
1271
+ ):
1272
+ r"""
1273
+ Visualization function for the output of pixel_ticoi
1274
+ /!\ Many figures can be plotted
1275
+
1276
+ :param list_dataf: [pd.DataFrame] --- cube dataset, which could be the observations, the inverted results (ILF), and the filtered observations. Make sure that the order is coherent with type_data.
1277
+ :param option_visual: [list] --- list of options for visualization
1278
+ :param type_data: [list] --- type of data: 'obs' for the observations, 'invert' for the inverted results, and 'obs_filt' for the filtered observations. Make sure that this is coherent with list_dataf.
1279
+ :param save:[bool] [default is False] --- if True, save the figures
1280
+ :param show: [bool] [default is True] --- if True, show the figures
1281
+ :param path_save: [str|None] [default is None] --- path where to save the figures
1282
+ :param A: [np.array] [default is None] --- design matrix
1283
+ :param log_scale: [bool] [default is False] --- if True, plot the figures into log scale
1284
+ :param cmap: [str] [default is 'viridis''] --- color map used in the plots
1285
+ :param colors: [list of str] [default is ['blueviolet', 'orange']] --- List of colors to used for plotting the time series
1286
+ :param figsize: tuple[int, int] [default is (10,6)] --- Size of the figures
1287
+ :param vminmax: List[int] [default is None] --- Min and max values for the y-axis of the plots
1288
+ """
1289
+
1290
+ pixel_object = PixelClass()
1291
+ pixel_object.load(list_dataf, save=save, show=show, A=A, path_save=path_save, figsize=figsize, type_data=type_data)
1292
+
1293
+ dico_visual = {
1294
+ "obs_xy": (lambda pix: pix.plot_vx_vy(color=colors[0], type_data="obs")),
1295
+ "obs_magnitude": (lambda pix: pix.plot_vv(color=colors[0], type_data="obs", vminmax=vminmax)),
1296
+ "obs_vxvy_quality": (lambda pix: pix.plot_vx_vy_quality(cmap=cmap, type_data="obs")),
1297
+ "invertxy": (lambda pix: pix.plot_vx_vy(color=colors[1])),
1298
+ "invertxy_overlaid": (lambda pix: pix.plot_vx_vy_overlaid(colors=colors)),
1299
+ "obsfiltxy_overlaid": (lambda pix: pix.plot_vx_vy_overlaid(colors=colors, type_data="obs_filt")),
1300
+ "obsfiltvv_overlaid": (lambda pix: pix.plot_vv_overlaid(colors=colors, type_data="obs_filt", vminmax=vminmax)),
1301
+ "invertvv_overlaid": (lambda pix: pix.plot_vv_overlaid(colors=colors, vminmax=vminmax)),
1302
+ "invertvv": (lambda pix: pix.plot_vv(color=colors[1], vminmax=vminmax)),
1303
+ "invert_vv_quality": (lambda pix: pix.plot_vv_quality(cmap=cmap, type_data="invert")),
1304
+ "residuals": (lambda pix: pix.plot_residuals(log_scale=log_scale)),
1305
+ "xcount_xy": (lambda pix: pix.plot_xcount_vx_vy(cmap=cmap)),
1306
+ "xcount_vv": (lambda pix: pix.plot_xcount_vv(cmap=cmap)),
1307
+ "invert_weight": (lambda pix: pix.plot_weights_inversion()),
1308
+ "direction": (lambda pix: pix.plot_direction()),
1309
+ }
1310
+
1311
+ for option in option_visual:
1312
+ if option in dico_visual.keys():
1313
+ dico_visual[option](pixel_object)
1314
+
1315
+
1316
+ def save_cube_parameters(
1317
+ cube: "CubeDataClass",
1318
+ load_kwargs: dict,
1319
+ preData_kwargs: dict,
1320
+ inversion_kwargs: dict,
1321
+ returned: list | None = None,
1322
+ ) -> Tuple[str, str]:
1323
+ """
1324
+
1325
+ :param cube: [cube_data_class] --- cube dataset
1326
+ :param load_kwargs: [dict] --- parameters used to load the cube
1327
+ :param prep_kwargs: [dict] --- parameters used to pre the cube
1328
+ :param inversion_kwargs: [dict] --- parameters used to load the cube
1329
+ :return:
1330
+ """
1331
+ sensor_array = np.unique(cube.ds["sensor"])
1332
+ sensor_strings = [str(sensor) for sensor in sensor_array]
1333
+ sensor = ", ".join(sensor_strings)
1334
+
1335
+ source = f"Temporal inversion on cube {cube.filename} using TICOI"
1336
+ source += (
1337
+ f" with a selection of dates among {load_kwargs['pick_date']},"
1338
+ if load_kwargs["pick_date"] is not None
1339
+ else "" + f" with a selection of the temporal baselines among {load_kwargs['pick_temp_bas']}"
1340
+ if load_kwargs["pick_temp_bas"] is not None
1341
+ else ("" + f" with a subset of {load_kwargs['subset']}")
1342
+ if load_kwargs["subset"] is not None
1343
+ else ""
1344
+ )
1345
+
1346
+ if inversion_kwargs["apriori_weight"]:
1347
+ source += " and apriori weight"
1348
+ source += f". The regularisation coefficient is {inversion_kwargs['coef']}."
1349
+ if "interp" in returned:
1350
+ source += f"The interpolation method used is {inversion_kwargs['option_interpol']}."
1351
+ source += f"The interpolation baseline is {inversion_kwargs['interval_output']} days."
1352
+ source += f"The temporal spacing (redundancy) is {inversion_kwargs['redundancy']} days."
1353
+
1354
+ source += f"The preparation are argument are: {preData_kwargs}"
1355
+ return source, sensor
1356
+
1357
+
1358
+ def ticoi_one_pixel(
1359
+ cube_name: str,
1360
+ i: int,
1361
+ j: int,
1362
+ save: bool = False,
1363
+ path_save: str | None = None,
1364
+ show: bool = True,
1365
+ option_visual: List = ["invertvv_overlaid"],
1366
+ verbose: bool = False,
1367
+ load_kwargs: dict = {},
1368
+ load_pixel_kwargs: dict = {},
1369
+ preData_kwargs: dict = {},
1370
+ inversion_kwargs: dict = {},
1371
+ interpolation_kwargs: dict = {},
1372
+ already_loaded: pd.DataFrame | None = None,
1373
+ ):
1374
+ """
1375
+ :param cube_name: [string] --- name of the cube dataset
1376
+ :param i: [int] --- pixel index
1377
+ :param j: [int] --- pixel index
1378
+ :param save: [bool] --- whether to save the figures or not
1379
+ :param path_save: [string] --- path to save the figures
1380
+ :param show: [bool] --- whether to show the figures or not
1381
+ :param option_visual: [list] --- option visual
1382
+ :param verbose: [bool] --- whether to plot some text
1383
+ :param load_kwargs: [dict] --- parameters used to load the cube
1384
+ :param load_pixel_kwargs: [dict] --- parameters used to load the pixel
1385
+ :param preData_kwargs: [dict] --- parameters used to prepare the cube
1386
+ :param inversion_kwargs: [dict] --- parameters used for the inversion
1387
+ :param interpolation_kwargs: [dict] --- parameters used for the interpolation
1388
+ :param already_loaded: [pd.Dataframe or None] --- whether the dataframe of the pixel is already loaded or not
1389
+ :return:
1390
+ """
1391
+ # %% ======================================================================== #
1392
+ # DATA LOADING #
1393
+ # =========================================================================%% #
1394
+
1395
+ if verbose:
1396
+ start = [time.time()]
1397
+
1398
+ if already_loaded is None:
1399
+ # Load the main cube
1400
+ cube = CubeDataClass()
1401
+ cube.load(cube_name, **load_kwargs)
1402
+
1403
+ if verbose:
1404
+ stop = [time.time()]
1405
+ print(f"[Data loading] Loading the data cube.s took {round((stop[0] - start[0]), 4)} s")
1406
+ print(f"[Data loading] Cube of dimension (nz,nx,ny) : ({cube.nz}, {cube.nx}, {cube.ny}) ")
1407
+
1408
+ start.append(time.time())
1409
+
1410
+ # Filter the cube (compute rolling_mean for regu=1accelnotnull)
1411
+ obs_filt, flag = cube.filter_cube_before_inversion(**preData_kwargs)
1412
+
1413
+ # Load pixel data
1414
+ data, mean, dates_range = cube.load_pixel(i, j, rolling_mean=obs_filt, **load_pixel_kwargs)
1415
+ # Prepare interpolation dates
1416
+ first_date_interpol, last_date_interpol = cube.prepare_interpolation_date()
1417
+ interpolation_kwargs.update(
1418
+ {"first_date_interpol": first_date_interpol, "last_date_interpol": last_date_interpol}
1419
+ )
1420
+
1421
+ else:
1422
+ data, mean, dates_range = already_loaded
1423
+ cube_date1 = data["date1"].tolist()
1424
+ cube_date1 = cube_date1 + data["date2"].tolist()
1425
+ cube_date1.remove(np.min(cube_date1))
1426
+ first_date_interpol = np.min(cube_date1)
1427
+ last_date_interpol = np.max(data["date2"])
1428
+ interpolation_kwargs.update(
1429
+ {"first_date_interpol": first_date_interpol, "last_date_interpol": last_date_interpol}
1430
+ )
1431
+ data = [
1432
+ data[["date1", "date2"]].to_numpy(),
1433
+ data[["dx", "dy", "errorx", "errory", "temporal_baseline"]].to_numpy(),
1434
+ data[["sensor", "author"]].to_numpy(),
1435
+ ]
1436
+
1437
+ if verbose:
1438
+ stop.append(time.time())
1439
+ print(f"[Data loading] Loading the pixel took {round((stop[1] - start[1]), 4)} s")
1440
+
1441
+ # %% ======================================================================== #
1442
+ # INVERSION #
1443
+ # =========================================================================%% #
1444
+
1445
+ if verbose:
1446
+ start.append(time.time())
1447
+
1448
+ # Proceed to inversion
1449
+ A, result, dataf = inversion_core(data, i, j, dates_range=dates_range, mean=mean, **inversion_kwargs)
1450
+
1451
+ if verbose:
1452
+ stop.append(time.time())
1453
+ print(f"[Inversion] Inversion took {round((stop[2] - start[2]), 4)} s")
1454
+ if save:
1455
+ result.to_csv(f"{path_save}/ILF_result.csv")
1456
+
1457
+ # %% ======================================================================== #
1458
+ # INTERPOLATION #
1459
+ # =========================================================================%% #
1460
+
1461
+ if verbose:
1462
+ start.append(time.time())
1463
+
1464
+ # Proceed to interpolation
1465
+ dataf_lp = interpolation_core(result, **interpolation_kwargs)
1466
+
1467
+ if save:
1468
+ dataf_lp.to_csv(f"{path_save}/ILF_result.csv")
1469
+
1470
+ if verbose:
1471
+ stop.append(time.time())
1472
+ print(f"[Interpolation] Interpolation took {round((stop[3] - start[3]), 4)} s")
1473
+
1474
+ if save:
1475
+ dataf_lp.to_csv(f"{path_save}/RLF_result.csv")
1476
+ if show or save: # plot some figures
1477
+ visualization_core(
1478
+ [dataf, result],
1479
+ option_visual=option_visual,
1480
+ save=save,
1481
+ show=show,
1482
+ path_save=path_save,
1483
+ A=A,
1484
+ log_scale=False,
1485
+ cmap="rainbow",
1486
+ colors=["orange", "blue"],
1487
+ )
1488
+ visualisation_interpolation(
1489
+ [dataf, dataf_lp],
1490
+ option_visual=option_visual,
1491
+ save=save,
1492
+ show=show,
1493
+ path_save=path_save,
1494
+ colors=["orange", "blue"],
1495
+ )
1496
+
1497
+ if verbose:
1498
+ print(f"[Overall] Overall processing took {round((stop[3] - start[0]), 4)} s")
1499
+
1500
+ return data, dataf, dataf_lp