ticoi 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ticoi might be problematic. Click here for more details.
- ticoi/__about__.py +1 -0
- ticoi/__init__.py +0 -0
- ticoi/core.py +1500 -0
- ticoi/cube_data_classxr.py +2204 -0
- ticoi/cube_writer.py +741 -0
- ticoi/example.py +81 -0
- ticoi/filtering_functions.py +676 -0
- ticoi/interpolation_functions.py +236 -0
- ticoi/inversion_functions.py +1015 -0
- ticoi/mjd2date.py +31 -0
- ticoi/optimize_coefficient_functions.py +264 -0
- ticoi/pixel_class.py +1830 -0
- ticoi/seasonality_functions.py +209 -0
- ticoi/utils.py +725 -0
- ticoi-0.0.1.dist-info/METADATA +152 -0
- ticoi-0.0.1.dist-info/RECORD +18 -0
- ticoi-0.0.1.dist-info/WHEEL +4 -0
- ticoi-0.0.1.dist-info/licenses/LICENSE +165 -0
ticoi/core.py
ADDED
|
@@ -0,0 +1,1500 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
"""
|
|
3
|
+
Main functions to process the temporal inversion of glacier's surface velocity using the TICOI method. The inversion is solved using an Iterative Reweighted Least Square, and a robust downweighted function (Tukey's biweight).
|
|
4
|
+
- mu_regularisation: Build the regularisation matrix
|
|
5
|
+
- weight_for_inversion: Initialisation of the weights used in the IRLS approach
|
|
6
|
+
- inversion_iteration: Compute an iteration of the inversion (weights are updated using the residuals)
|
|
7
|
+
- inversion: Main function to be called, makes the temporal inversion with an IRLS approach using a given solver, it returns leap frog velocities (velcoties between consecutive dates) with an irregular temporal sampling.
|
|
8
|
+
- interpolation_post: Interpolate Irregular Leap Frog time series (result of an inversion) to Regular LF time series using Cumulative Displacement times series.
|
|
9
|
+
- process: Launch the entire process, data loading, inversion and interpolation
|
|
10
|
+
- visualisation: Different figures can be shown in this function, according to what the user wants
|
|
11
|
+
|
|
12
|
+
Author : Laurane Charrier
|
|
13
|
+
Reference:
|
|
14
|
+
Charrier, L., Yan, Y., Koeniguer, E. C., Leinss, S., & Trouvé, E. (2021). Extraction of velocity time series with an optimal temporal sampling from displacement
|
|
15
|
+
observation networks. IEEE Transactions on Geoscience and Remote Sensing.
|
|
16
|
+
Charrier, L., Yan, Y., Colin Koeniguer, E., Mouginot, J., Millan, R., & Trouvé, E. (2022). Fusion of multi-temporal and multi-sensor ice velocity observations.
|
|
17
|
+
ISPRS annals of the photogrammetry, remote sensing and spatial information sciences, 3, 311-318.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import asyncio
|
|
21
|
+
import itertools
|
|
22
|
+
import time
|
|
23
|
+
import warnings
|
|
24
|
+
from typing import List, Optional, Union, Tuple
|
|
25
|
+
|
|
26
|
+
import numpy as np
|
|
27
|
+
import pandas as pd
|
|
28
|
+
import scipy.sparse as sp
|
|
29
|
+
import xarray as xr
|
|
30
|
+
from joblib import Parallel, delayed
|
|
31
|
+
from scipy import stats
|
|
32
|
+
from tqdm import tqdm
|
|
33
|
+
|
|
34
|
+
from ticoi.cube_data_classxr import CubeDataClass
|
|
35
|
+
from ticoi.interpolation_functions import (
|
|
36
|
+
reconstruct_common_ref,
|
|
37
|
+
set_function_for_interpolation,
|
|
38
|
+
visualisation_interpolation,
|
|
39
|
+
)
|
|
40
|
+
from ticoi.inversion_functions import (
|
|
41
|
+
TukeyBiweight,
|
|
42
|
+
class_linear_operator,
|
|
43
|
+
construction_a_lf,
|
|
44
|
+
construction_dates_range_np,
|
|
45
|
+
find_date_obs,
|
|
46
|
+
inversion_one_component,
|
|
47
|
+
inversion_two_components,
|
|
48
|
+
mu_regularisation,
|
|
49
|
+
weight_for_inversion,
|
|
50
|
+
)
|
|
51
|
+
from ticoi.pixel_class import PixelClass
|
|
52
|
+
|
|
53
|
+
warnings.filterwarnings("ignore")
|
|
54
|
+
|
|
55
|
+
# %% ======================================================================== #
|
|
56
|
+
# INVERSION #
|
|
57
|
+
# =========================================================================%% #
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def inversion_iteration(
|
|
61
|
+
data: np.ndarray,
|
|
62
|
+
A: np.ndarray,
|
|
63
|
+
dates_range: np.ndarray,
|
|
64
|
+
solver: str,
|
|
65
|
+
coef: int,
|
|
66
|
+
Weight: np.ndarray,
|
|
67
|
+
result_dx: np.ndarray,
|
|
68
|
+
result_dy: np.ndarray,
|
|
69
|
+
mu: np.ndarray,
|
|
70
|
+
regu: int | str = 1,
|
|
71
|
+
accel: np.ndarray | None = None,
|
|
72
|
+
linear_operator=Union["class_linear_operator", None],
|
|
73
|
+
result_quality: list | str | None = None,
|
|
74
|
+
ini: np.ndarray | None = None,
|
|
75
|
+
verbose: bool = False,
|
|
76
|
+
) -> (np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray | None, np.ndarray | None):
|
|
77
|
+
"""
|
|
78
|
+
Compute an iteration of the inversion : update the weights using the weights from the previous iteration and the studentized residual, update the results in consequence
|
|
79
|
+
and compute the residu's norm if required.
|
|
80
|
+
|
|
81
|
+
:param data: [np array] --- Data at a given point
|
|
82
|
+
:param A: [np array] --- Design matrix linking X (vector containing the velocity observations) to Y
|
|
83
|
+
:param dates_range: [list] --- Dates of the displacements in X
|
|
84
|
+
:param solver: [str] --- Solver of the inversion: 'LSMR', 'LSMR_ini', 'LS', 'LS_bounded', 'LSQR'
|
|
85
|
+
:param coef: [int] --- Coef of Tikhonov regularisation
|
|
86
|
+
:param Weight: [np array] --- Weight to give to the inversion
|
|
87
|
+
:param result_dx: [np array] --- Estimated time series vx at the given iteration
|
|
88
|
+
:param result_dy: [np array] --- Estimated time series vx at the given iteration
|
|
89
|
+
:param mu: [np array] --- Regularization matrix
|
|
90
|
+
:param regu: [int | str] [default is 1] --- Type of regularization
|
|
91
|
+
:param accel: [np array | None] [default is None] --- Apriori on the acceleration
|
|
92
|
+
:param linear_operator: [bool] [default is False] --- If linear operator, the inversion is performed using a linear operator (https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.linalg.LinearOperator.html)
|
|
93
|
+
:param result_quality: [list | str | None] [default is None] --- Which can contain 'Norm_residual' to determine the L2 norm of the residuals from the last inversion, 'X_contribution' to determine the number of Y observations which have contributed to estimate each value in X (it corresponds to A.dot(weight))
|
|
94
|
+
:param ini: [np array | None]
|
|
95
|
+
:param verbose: [bool] [default is False] --- Print information along the way
|
|
96
|
+
|
|
97
|
+
:return result_dx, result_dy: [np arrays] --- Obtained results (velocities) for this iteration along x and y axis
|
|
98
|
+
:return weightx, weighty: [np arrays] --- Newly computed weights along x and y axis
|
|
99
|
+
:return residu_normx, residu_normy: [np arrays | None] --- Norm of the residu along x and y axis (when showing the L curve)
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
def compute_residual(A: np.ndarray, v: np.ndarray, X: np.ndarray) -> np.ndarray:
|
|
103
|
+
Residu = v - A.dot(X)
|
|
104
|
+
return Residu
|
|
105
|
+
|
|
106
|
+
def weightf(residu: np.ndarray, Weight: np.ndarray) -> np.ndarray:
|
|
107
|
+
"""
|
|
108
|
+
Compute weight according to the residual
|
|
109
|
+
|
|
110
|
+
:param residu: [np array] Residual vector
|
|
111
|
+
:param Weight: [np array | None] Apriori weight
|
|
112
|
+
|
|
113
|
+
:return weight: [np array] Weight for the inversion
|
|
114
|
+
"""
|
|
115
|
+
|
|
116
|
+
mad = stats.median_abs_deviation(residu) / 0.6745
|
|
117
|
+
if mad != 0.0:
|
|
118
|
+
r_std = residu / mad
|
|
119
|
+
if Weight is not None: # The weight is a combination of apriori weight and the studentized residual
|
|
120
|
+
# Weight = Weight / (stats.median_abs_deviation(Weight) / 0.6745)
|
|
121
|
+
weight = Weight * TukeyBiweight(r_std, 4.685)
|
|
122
|
+
else:
|
|
123
|
+
weight = TukeyBiweight((r_std), 4.685)
|
|
124
|
+
else:
|
|
125
|
+
weight = np.ones(residu.shape[0])
|
|
126
|
+
|
|
127
|
+
return weight
|
|
128
|
+
|
|
129
|
+
weightx = weightf(compute_residual(A, data[:, 0], result_dx), Weight[0])
|
|
130
|
+
weighty = weightf(compute_residual(A, data[:, 1], result_dy), Weight[1])
|
|
131
|
+
|
|
132
|
+
if A.shape[0] < A.shape[1]:
|
|
133
|
+
if verbose:
|
|
134
|
+
print(
|
|
135
|
+
f"[Inversion] If the number of row is lower than the number of columns, the results are not updated {A.shape}"
|
|
136
|
+
)
|
|
137
|
+
return result_dx, result_dy, weightx, weighty, None, None
|
|
138
|
+
|
|
139
|
+
if regu == "directionxy":
|
|
140
|
+
if solver == "LSMR_ini":
|
|
141
|
+
result_dx, result_dy, residu_normx, residu_normy = inversion_two_components(
|
|
142
|
+
A,
|
|
143
|
+
dates_range,
|
|
144
|
+
0,
|
|
145
|
+
data,
|
|
146
|
+
solver,
|
|
147
|
+
np.concatenate([weightx, weighty]),
|
|
148
|
+
mu,
|
|
149
|
+
coef=coef,
|
|
150
|
+
ini=np.concatenate([result_dx, result_dy]),
|
|
151
|
+
)
|
|
152
|
+
else:
|
|
153
|
+
result_dx, result_dy, residu_normx, residu_normy = inversion_two_components(
|
|
154
|
+
A, dates_range, 0, data, solver, np.concatenate([weightx, weighty]), mu, coef=coef
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
elif solver == "LSMR_ini":
|
|
158
|
+
if ini is None: # Initialization with the result from the previous inversion
|
|
159
|
+
result_dx, residu_normx = inversion_one_component(
|
|
160
|
+
A,
|
|
161
|
+
dates_range,
|
|
162
|
+
0,
|
|
163
|
+
data,
|
|
164
|
+
solver,
|
|
165
|
+
weightx,
|
|
166
|
+
mu,
|
|
167
|
+
coef=coef,
|
|
168
|
+
ini=result_dx,
|
|
169
|
+
result_quality=result_quality,
|
|
170
|
+
regu=regu,
|
|
171
|
+
accel=accel,
|
|
172
|
+
linear_operator=linear_operator,
|
|
173
|
+
)
|
|
174
|
+
result_dy, residu_normy = inversion_one_component(
|
|
175
|
+
A,
|
|
176
|
+
dates_range,
|
|
177
|
+
1,
|
|
178
|
+
data,
|
|
179
|
+
solver,
|
|
180
|
+
weighty,
|
|
181
|
+
mu,
|
|
182
|
+
coef=coef,
|
|
183
|
+
ini=result_dy,
|
|
184
|
+
result_quality=result_quality,
|
|
185
|
+
regu=regu,
|
|
186
|
+
accel=accel,
|
|
187
|
+
linear_operator=linear_operator,
|
|
188
|
+
)
|
|
189
|
+
else: # Initialization with the list ini, which can be a moving average
|
|
190
|
+
result_dx, residu_normx = inversion_one_component(
|
|
191
|
+
A,
|
|
192
|
+
dates_range,
|
|
193
|
+
0,
|
|
194
|
+
data,
|
|
195
|
+
solver,
|
|
196
|
+
weightx,
|
|
197
|
+
mu,
|
|
198
|
+
coef=coef,
|
|
199
|
+
ini=ini[0],
|
|
200
|
+
result_quality=result_quality,
|
|
201
|
+
regu=regu,
|
|
202
|
+
accel=accel,
|
|
203
|
+
linear_operator=linear_operator,
|
|
204
|
+
)
|
|
205
|
+
result_dy, residu_normy = inversion_one_component(
|
|
206
|
+
A,
|
|
207
|
+
dates_range,
|
|
208
|
+
1,
|
|
209
|
+
data,
|
|
210
|
+
solver,
|
|
211
|
+
weighty,
|
|
212
|
+
mu,
|
|
213
|
+
coef=coef,
|
|
214
|
+
ini=ini[1],
|
|
215
|
+
result_quality=result_quality,
|
|
216
|
+
regu=regu,
|
|
217
|
+
accel=accel,
|
|
218
|
+
linear_operator=linear_operator,
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
else: # No initialization
|
|
222
|
+
result_dx, residu_normx = inversion_one_component(
|
|
223
|
+
A,
|
|
224
|
+
dates_range,
|
|
225
|
+
0,
|
|
226
|
+
data,
|
|
227
|
+
solver,
|
|
228
|
+
weightx,
|
|
229
|
+
mu,
|
|
230
|
+
coef=coef,
|
|
231
|
+
result_quality=result_quality,
|
|
232
|
+
regu=regu,
|
|
233
|
+
accel=accel,
|
|
234
|
+
linear_operator=linear_operator,
|
|
235
|
+
)
|
|
236
|
+
result_dy, residu_normy = inversion_one_component(
|
|
237
|
+
A,
|
|
238
|
+
dates_range,
|
|
239
|
+
1,
|
|
240
|
+
data,
|
|
241
|
+
solver,
|
|
242
|
+
weighty,
|
|
243
|
+
mu,
|
|
244
|
+
coef=coef,
|
|
245
|
+
result_quality=result_quality,
|
|
246
|
+
regu=regu,
|
|
247
|
+
accel=accel,
|
|
248
|
+
linear_operator=linear_operator,
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
return result_dx, result_dy, weightx, weighty, residu_normx, residu_normy
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def inversion_core(
|
|
255
|
+
data: list,
|
|
256
|
+
i: float | int,
|
|
257
|
+
j: float | int,
|
|
258
|
+
dates_range: np.ndarray | None = None,
|
|
259
|
+
solver: str = "LSMR",
|
|
260
|
+
regu: int | str = "1accelnotnull",
|
|
261
|
+
coef: int = 100,
|
|
262
|
+
apriori_weight: bool = False,
|
|
263
|
+
iteration: bool = True,
|
|
264
|
+
threshold_it: float = 0.1,
|
|
265
|
+
unit: int = 365,
|
|
266
|
+
conf: bool = False,
|
|
267
|
+
mean: list | None = None,
|
|
268
|
+
detect_temporal_decorrelation: bool = True,
|
|
269
|
+
linear_operator: bool = False,
|
|
270
|
+
result_quality: list | str | None = None,
|
|
271
|
+
nb_max_iteration: int = 10,
|
|
272
|
+
apriori_weight_in_second_iteration: bool = False,
|
|
273
|
+
visual: bool = False,
|
|
274
|
+
verbose: bool = False,
|
|
275
|
+
) -> (np.ndarray, pd.DataFrame, pd.DataFrame): # type: ignore
|
|
276
|
+
"""
|
|
277
|
+
Computes A in AX = Y and does the inversion using a given solver and regularization.
|
|
278
|
+
|
|
279
|
+
:param data: [list] --- List of arrays, representing the observations: the first array contain the first and last acquisition dates, the second array contain the displacements values and errors, and optionally the last array contain information about the sensor, author, etc
|
|
280
|
+
:params i, j: [float | int] --- Coordinates of the point in pixel
|
|
281
|
+
:param dates_range: [np array | None] [default is None] --- List of np.datetime64 [D], dates of the estimated displacement in X with an irregular temporal sampling (ILF)
|
|
282
|
+
:param solver: [str] [default is 'LSMR'] --- Solver of the inversion: 'LSMR', 'LSMR_ini', 'LS', 'LS_bounded', 'LSQR'
|
|
283
|
+
:param regu: [int | str] [default is 1] --- Type of regularization
|
|
284
|
+
:param coef: [int] [default is 100] --- Coef of Tikhonov regularisation
|
|
285
|
+
:param apriori_weight: [bool] [default is False] --- If True use of aprori weight, based on the provided observation errors
|
|
286
|
+
:param iteration: [bool] [default is True] --- If True, use of iterations
|
|
287
|
+
:param threshold_it: [float] [default is 0.1] --- Threshold to test the stability of the results between each iteration, use to stop the process
|
|
288
|
+
:param unit: [int] [default is 365] --- 1 for m/d, 365 for m/y
|
|
289
|
+
:param conf: [bool] [default is False] --- If True means that the error corresponds to confidence intervals between 0 and 1, otherwise it corresponds to errors in m/y or m/d
|
|
290
|
+
:param mean: [list | None] [default is None] --- Apriori on the average
|
|
291
|
+
:param detect_temporal_decorrelation: [bool] [default is True] --- If True the first inversion is solved using only velocity observations with small temporal baselines, to detect temporal decorelation
|
|
292
|
+
:param linear_operator: [bool] [default is False] --- If linear operator, the inversion is performed using a linear operator (https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.linalg.LinearOperator.html)
|
|
293
|
+
:param result_quality: [list | str | None] [default is None] --- List which can contain 'Norm_residual' to determine the L2 norm of the residuals from the last inversion, 'X_contribution' to determine the number of Y observations which have contributed to estimate each value in X (it corresponds to A.dot(weight))
|
|
294
|
+
:param nb_max_iteration: [int] [default is 10] --- Maximum number of iterations
|
|
295
|
+
:param apriori_weight_in_second_iteration: [bool] [default is False] --- it True use the error to weight each of the iterations, if not use it only in the first iteration
|
|
296
|
+
:param visual: [bool] [default is True] --- Keep the weights for future plots
|
|
297
|
+
:param verbose: [bool] [default is False] --- Print information along the way
|
|
298
|
+
|
|
299
|
+
:return A: [np array | None] --- Design matrix in AX = Y
|
|
300
|
+
:return result: [pd dataframe | None] --- DF with dates, computed displacements and number of observations used to compute each displacement
|
|
301
|
+
:return dataf: [pd dataframe | None] --- Complete DF with dates, velocities, errors, residus, weights, xcount, normr... for further visual purposes (directly depends on param visual and result_quality)
|
|
302
|
+
"""
|
|
303
|
+
|
|
304
|
+
if data[0].size: # If there are available data on this pixel
|
|
305
|
+
# Split the data, with one dtype per array
|
|
306
|
+
if len(data) == 3:
|
|
307
|
+
data_dates, data_values, data_str = data
|
|
308
|
+
else:
|
|
309
|
+
data_dates, data_values = data
|
|
310
|
+
del data
|
|
311
|
+
|
|
312
|
+
if dates_range is None:
|
|
313
|
+
dates_range = construction_dates_range_np(data_dates)
|
|
314
|
+
|
|
315
|
+
#### Build A (design matrix in AX = Y)
|
|
316
|
+
if not linear_operator:
|
|
317
|
+
A = construction_a_lf(data_dates, dates_range)
|
|
318
|
+
linear_operator = None
|
|
319
|
+
else: # use a linear operator to solve the inversion, it is sometimes faster
|
|
320
|
+
linear_operator = class_linear_operator()
|
|
321
|
+
linear_operator.load(
|
|
322
|
+
find_date_obs(data_dates[:, :2], dates_range), dates_range, coef
|
|
323
|
+
) # load parameter of the linear operator
|
|
324
|
+
A = sp.linalg.LinearOperator(
|
|
325
|
+
(data_values.shape[0], len(dates_range) - 1),
|
|
326
|
+
matvec=linear_operator.matvec,
|
|
327
|
+
rmatvec=linear_operator.rmatvec,
|
|
328
|
+
) # build A
|
|
329
|
+
mu = None
|
|
330
|
+
|
|
331
|
+
# Set a weight of 0, for large temporal baseline in the first inversion
|
|
332
|
+
weight_temporal_decorrelation = (
|
|
333
|
+
np.where(data_values[:, 4] > 180, 0, 1) if detect_temporal_decorrelation else None
|
|
334
|
+
)
|
|
335
|
+
# First weight of the inversion
|
|
336
|
+
Weightx = weight_for_inversion(
|
|
337
|
+
weight_origine=apriori_weight,
|
|
338
|
+
conf=conf,
|
|
339
|
+
data=data_values,
|
|
340
|
+
pos=2,
|
|
341
|
+
inside_Tukey=False,
|
|
342
|
+
temporal_decorrelation=weight_temporal_decorrelation,
|
|
343
|
+
)
|
|
344
|
+
Weighty = weight_for_inversion(
|
|
345
|
+
weight_origine=apriori_weight,
|
|
346
|
+
conf=conf,
|
|
347
|
+
data=data_values,
|
|
348
|
+
pos=3,
|
|
349
|
+
inside_Tukey=False,
|
|
350
|
+
temporal_decorrelation=weight_temporal_decorrelation,
|
|
351
|
+
)
|
|
352
|
+
del weight_temporal_decorrelation
|
|
353
|
+
if not visual:
|
|
354
|
+
if (
|
|
355
|
+
result_quality is not None
|
|
356
|
+
and not apriori_weight_in_second_iteration
|
|
357
|
+
and "Error_propagation" not in result_quality
|
|
358
|
+
):
|
|
359
|
+
data_values = np.delete(
|
|
360
|
+
data_values, [2, 3], 1
|
|
361
|
+
) # Delete quality indicator, which are not needed anymore
|
|
362
|
+
# Compute regularisation matrix
|
|
363
|
+
if not linear_operator:
|
|
364
|
+
if regu == "directionxy":
|
|
365
|
+
# Constrain according to the vectorial product, the magnitude of the vector corresponds to mean2, the magnitude of a rolling mean
|
|
366
|
+
mu = mu_regularisation(regu, A, dates_range, ini=mean)
|
|
367
|
+
else:
|
|
368
|
+
mu = mu_regularisation(regu, A, dates_range, ini=mean)
|
|
369
|
+
|
|
370
|
+
## Initialisation (depending on apriori and solver)
|
|
371
|
+
# # Apriori on acceleration (following)
|
|
372
|
+
# TODO: we can make it shorter
|
|
373
|
+
if regu == "1accelnotnull":
|
|
374
|
+
accel = [
|
|
375
|
+
np.diff(mean[0]),
|
|
376
|
+
np.diff(mean[1]),
|
|
377
|
+
] # compute acceleration based on the moving average, computing using a given kernel
|
|
378
|
+
mean_ini = [
|
|
379
|
+
np.multiply(mean[i], np.diff(dates_range) / np.timedelta64(1, "D")) for i in range(len(mean))
|
|
380
|
+
] # compute what should be the displacement in X according to the moving average, computing using a given kernel
|
|
381
|
+
|
|
382
|
+
elif (
|
|
383
|
+
mean is not None and solver == "LSMR_ini"
|
|
384
|
+
): # initialization is set according the average of the whole time series
|
|
385
|
+
mean_ini = [
|
|
386
|
+
np.multiply(mean[i], np.diff(dates_range) / np.timedelta64(1, "D") / unit) for i in range(len(mean))
|
|
387
|
+
]
|
|
388
|
+
accel = None
|
|
389
|
+
else:
|
|
390
|
+
mean_ini = None
|
|
391
|
+
accel = None
|
|
392
|
+
|
|
393
|
+
## Inversion
|
|
394
|
+
if regu == "directionxy":
|
|
395
|
+
result_dx, result_dy, residu_normx, residu_normy = inversion_two_components(
|
|
396
|
+
A, dates_range, 0, data_values, solver, np.concatenate([Weightx, Weighty]), mu, coef=coef, ini=mean_ini
|
|
397
|
+
)
|
|
398
|
+
else:
|
|
399
|
+
result_dx, residu_normx = inversion_one_component(
|
|
400
|
+
A,
|
|
401
|
+
dates_range,
|
|
402
|
+
0,
|
|
403
|
+
data_values,
|
|
404
|
+
solver,
|
|
405
|
+
Weightx,
|
|
406
|
+
mu,
|
|
407
|
+
coef=coef,
|
|
408
|
+
ini=mean_ini,
|
|
409
|
+
result_quality=None,
|
|
410
|
+
regu=regu,
|
|
411
|
+
linear_operator=linear_operator,
|
|
412
|
+
accel=accel,
|
|
413
|
+
)
|
|
414
|
+
result_dy, residu_normy = inversion_one_component(
|
|
415
|
+
A,
|
|
416
|
+
dates_range,
|
|
417
|
+
1,
|
|
418
|
+
data_values,
|
|
419
|
+
solver,
|
|
420
|
+
Weighty,
|
|
421
|
+
mu,
|
|
422
|
+
coef=coef,
|
|
423
|
+
ini=mean_ini,
|
|
424
|
+
result_quality=None,
|
|
425
|
+
regu=regu,
|
|
426
|
+
linear_operator=linear_operator,
|
|
427
|
+
accel=accel,
|
|
428
|
+
)
|
|
429
|
+
|
|
430
|
+
if not visual:
|
|
431
|
+
del Weighty, Weightx
|
|
432
|
+
|
|
433
|
+
if regu == "directionxy":
|
|
434
|
+
mu = mu_regularisation(regu, A, dates_range, ini=[mean[0], mean[1], result_dx, result_dy])
|
|
435
|
+
|
|
436
|
+
# Second Iteration
|
|
437
|
+
if iteration:
|
|
438
|
+
if (
|
|
439
|
+
(apriori_weight_in_second_iteration) and apriori_weight
|
|
440
|
+
): # use apriori weight based on the error or quality indicator, Tukeybiweight(error/MAD(error)/ 0.6745)
|
|
441
|
+
Weightx2 = weight_for_inversion(
|
|
442
|
+
weight_origine=apriori_weight, conf=conf, data=data_values, pos=2, inside_Tukey=False
|
|
443
|
+
)
|
|
444
|
+
Weighty2 = weight_for_inversion(
|
|
445
|
+
weight_origine=apriori_weight, conf=conf, data=data_values, pos=3, inside_Tukey=False
|
|
446
|
+
)
|
|
447
|
+
else:
|
|
448
|
+
Weightx2, Weighty2 = None, None
|
|
449
|
+
|
|
450
|
+
result_dx_i, result_dy_i, weight_2x, weight_2y, residu_normx, residu_normy = inversion_iteration(
|
|
451
|
+
data_values,
|
|
452
|
+
A,
|
|
453
|
+
dates_range,
|
|
454
|
+
solver,
|
|
455
|
+
coef,
|
|
456
|
+
[Weightx2, Weighty2],
|
|
457
|
+
result_dx,
|
|
458
|
+
result_dy,
|
|
459
|
+
mu=mu,
|
|
460
|
+
verbose=verbose,
|
|
461
|
+
regu=regu,
|
|
462
|
+
linear_operator=linear_operator,
|
|
463
|
+
ini=None,
|
|
464
|
+
accel=accel,
|
|
465
|
+
result_quality=result_quality,
|
|
466
|
+
)
|
|
467
|
+
# Continue to iterate until the difference between two results is lower than threshold_it or the number of iteration larger than 10
|
|
468
|
+
i = 2
|
|
469
|
+
while (
|
|
470
|
+
np.mean(abs(result_dx_i - result_dx)) > threshold_it
|
|
471
|
+
or np.mean(abs(result_dy_i - result_dy)) > threshold_it
|
|
472
|
+
) and i < nb_max_iteration:
|
|
473
|
+
result_dx = result_dx_i
|
|
474
|
+
result_dy = result_dy_i
|
|
475
|
+
result_dx_i, result_dy_i, weight_ix, weight_iy, residu_normx, residu_normy = inversion_iteration(
|
|
476
|
+
data_values,
|
|
477
|
+
A,
|
|
478
|
+
dates_range,
|
|
479
|
+
solver,
|
|
480
|
+
coef,
|
|
481
|
+
[Weightx2, Weighty2],
|
|
482
|
+
result_dx,
|
|
483
|
+
result_dy,
|
|
484
|
+
mu,
|
|
485
|
+
verbose=verbose,
|
|
486
|
+
regu=regu,
|
|
487
|
+
linear_operator=linear_operator,
|
|
488
|
+
ini=None,
|
|
489
|
+
accel=accel,
|
|
490
|
+
result_quality=result_quality,
|
|
491
|
+
)
|
|
492
|
+
|
|
493
|
+
i += 1
|
|
494
|
+
|
|
495
|
+
if verbose:
|
|
496
|
+
print(
|
|
497
|
+
"[Inversion] ",
|
|
498
|
+
i,
|
|
499
|
+
"dx",
|
|
500
|
+
np.mean(abs(result_dx_i - result_dx)),
|
|
501
|
+
"dy",
|
|
502
|
+
np.mean(abs(result_dy_i - result_dy)),
|
|
503
|
+
)
|
|
504
|
+
|
|
505
|
+
if verbose:
|
|
506
|
+
print("[Inversion] End loop", i, np.mean(abs(result_dy_i - result_dy)))
|
|
507
|
+
print("[Inversion] Nb iteration", i)
|
|
508
|
+
|
|
509
|
+
if i == 2:
|
|
510
|
+
weight_iy = weight_2y
|
|
511
|
+
weight_ix = weight_2x
|
|
512
|
+
|
|
513
|
+
del result_dx, result_dy
|
|
514
|
+
if not visual:
|
|
515
|
+
if result_quality is not None and "Error_propagation" not in result_quality:
|
|
516
|
+
del data_values, data_dates
|
|
517
|
+
|
|
518
|
+
else: # If not iteration
|
|
519
|
+
result_dy_i = result_dy
|
|
520
|
+
result_dx_i = result_dx
|
|
521
|
+
|
|
522
|
+
if np.isnan(result_dx_i).all(): # no results
|
|
523
|
+
return None, None, None
|
|
524
|
+
|
|
525
|
+
if not iteration:
|
|
526
|
+
weight_ix = Weightx
|
|
527
|
+
weight_iy = Weighty
|
|
528
|
+
if not visual:
|
|
529
|
+
del Weighty, Weightx
|
|
530
|
+
# compute the number of observations which have contributed to each estimated displacement
|
|
531
|
+
if result_quality is not None and "X_contribution" in result_quality:
|
|
532
|
+
xcount_x = A.T.dot(weight_ix)
|
|
533
|
+
xcount_y = A.T.dot(weight_iy)
|
|
534
|
+
|
|
535
|
+
else:
|
|
536
|
+
xcount_x = xcount_y = np.ones(result_dx_i.shape[0])
|
|
537
|
+
|
|
538
|
+
# propagate the error
|
|
539
|
+
if result_quality is not None and "Error_propagation" in result_quality:
|
|
540
|
+
|
|
541
|
+
def Prop_weight(F, weight, Residu, error):
|
|
542
|
+
error = np.max([Residu, error], axis=0) # take the maximum between residuals and errors
|
|
543
|
+
W = weight.astype("float32")
|
|
544
|
+
FTWF = np.multiply(F.T, W[np.newaxis, :]) @ F
|
|
545
|
+
N = np.linalg.inv(FTWF + coef * mu.T @ mu)
|
|
546
|
+
Prop_weight = np.multiply(np.multiply(N @ F.T, W[np.newaxis, :]) * error, W[np.newaxis, :]) @ F @ N
|
|
547
|
+
sigma0_weight = np.sum(Residu**2 * weight) / (F.shape[0] - F.shape[1])
|
|
548
|
+
prop_wieght_diag = np.diag(Prop_weight)
|
|
549
|
+
# Compute the confidence intervals
|
|
550
|
+
alpha = 0.05 # Confidence level
|
|
551
|
+
t_value = stats.t.ppf(1 - alpha / 2, df=F.shape[0] - F.shape[1])
|
|
552
|
+
|
|
553
|
+
return prop_wieght_diag, sigma0_weight, t_value
|
|
554
|
+
|
|
555
|
+
Residux = data_values[:, 0] - A @ result_dx_i # has a normal distribution
|
|
556
|
+
prop_wieght_diagx, sigma0_weightx, t_valuex = Prop_weight(
|
|
557
|
+
A, weight_ix, Residux, (data_values[:, 2] * data_values[:, -1] / unit) ** 2
|
|
558
|
+
)
|
|
559
|
+
|
|
560
|
+
Residuy = data_values[:, 1] - A @ result_dy_i # has a normal distribution
|
|
561
|
+
prop_wieght_diagy, sigma0_weighty, t_valuey = Prop_weight(
|
|
562
|
+
A, weight_iy, Residuy, (data_values[:, 3] * data_values[:, -1] / unit) ** 2
|
|
563
|
+
)
|
|
564
|
+
|
|
565
|
+
# If visual, save the velocity observation, the errors, the initial weights (weightini), the last weights (weightlast), the residuals from the last inversion, the sensors, and the authors
|
|
566
|
+
if visual:
|
|
567
|
+
vx = data_values[:, 0] / data_values[:, -1] * unit
|
|
568
|
+
vy = data_values[:, 1] / data_values[:, -1] * unit
|
|
569
|
+
Residux = data_values[:, 0] - A.dot(result_dx_i)
|
|
570
|
+
Residuy = data_values[:, 1] - A.dot(result_dy_i)
|
|
571
|
+
dataf = pd.DataFrame(
|
|
572
|
+
{
|
|
573
|
+
"date1": data_dates[:, 0],
|
|
574
|
+
"date2": data_dates[:, 1],
|
|
575
|
+
"vx": vx,
|
|
576
|
+
"vy": vy,
|
|
577
|
+
"errorx": data_values[:, 2],
|
|
578
|
+
"errory": data_values[:, 3],
|
|
579
|
+
"weightinix": Weightx,
|
|
580
|
+
"weightiniy": Weighty,
|
|
581
|
+
"weightlastx": weight_ix,
|
|
582
|
+
"weightlasty": weight_iy,
|
|
583
|
+
"residux": Residux,
|
|
584
|
+
"residuy": Residuy,
|
|
585
|
+
"sensor": data_str[:, 0],
|
|
586
|
+
"author": data_str[:, 1],
|
|
587
|
+
}
|
|
588
|
+
)
|
|
589
|
+
if (
|
|
590
|
+
residu_normx is not None
|
|
591
|
+
): # save the L2-norm from the last inversion, of the term AXY and the regularization term for the x- and y-component
|
|
592
|
+
NormR = np.zeros(data_values.shape[0])
|
|
593
|
+
NormR[:4] = np.hstack(
|
|
594
|
+
[residu_normx, residu_normy]
|
|
595
|
+
) # the order is: AXY and regularization term L2-norm for x-component, and AXY and regularization term L2-norm for y-component
|
|
596
|
+
dataf["NormR"] = NormR
|
|
597
|
+
del NormR
|
|
598
|
+
else:
|
|
599
|
+
dataf, A = None, None
|
|
600
|
+
|
|
601
|
+
else: # If there is no data over this pixel
|
|
602
|
+
if verbose:
|
|
603
|
+
print(f"[Inversion] NO DATA TO INVERSE AT POINT {i, j}")
|
|
604
|
+
return None, None, None
|
|
605
|
+
|
|
606
|
+
# pandas dataframe with the saved results
|
|
607
|
+
result = pd.DataFrame(
|
|
608
|
+
{
|
|
609
|
+
"date1": dates_range[:-1],
|
|
610
|
+
"date2": dates_range[1:],
|
|
611
|
+
"result_dx": result_dx_i,
|
|
612
|
+
"result_dy": result_dy_i,
|
|
613
|
+
"xcount_x": xcount_x,
|
|
614
|
+
"xcount_y": xcount_y,
|
|
615
|
+
}
|
|
616
|
+
)
|
|
617
|
+
if residu_normx is not None: # add the norm of the residual
|
|
618
|
+
normr = np.zeros(result.shape[0])
|
|
619
|
+
if normr.shape[0] > 3:
|
|
620
|
+
normr[:4] = np.hstack([residu_normx, residu_normy])
|
|
621
|
+
else:
|
|
622
|
+
normr[: normr.shape[0]] = np.full(normr.shape[0], np.nan)
|
|
623
|
+
result["NormR"] = normr
|
|
624
|
+
del normr
|
|
625
|
+
if result_quality is not None: # add the error propagation
|
|
626
|
+
if "Error_propagation" in result_quality:
|
|
627
|
+
result["error_x"] = prop_wieght_diagx
|
|
628
|
+
result["error_y"] = prop_wieght_diagy
|
|
629
|
+
sigma = np.zeros(result.shape[0])
|
|
630
|
+
sigma[:4] = np.hstack([sigma0_weightx, sigma0_weighty, t_valuex, t_valuey])
|
|
631
|
+
result["sigma0"] = sigma
|
|
632
|
+
|
|
633
|
+
return A, result, dataf
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
# %% ======================================================================== #
|
|
637
|
+
# INTERPOLATION #
|
|
638
|
+
# =========================================================================%% #
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
def interpolation_core(
|
|
642
|
+
result: pd.DataFrame,
|
|
643
|
+
interval_output: int = 30,
|
|
644
|
+
option_interpol: str = "spline",
|
|
645
|
+
first_date_interpol: np.datetime64 | str | None = None,
|
|
646
|
+
last_date_interpol: np.datetime64 | str | None = None,
|
|
647
|
+
unit: int = 365,
|
|
648
|
+
redundancy: int | None = 5,
|
|
649
|
+
result_quality: list | None = None,
|
|
650
|
+
):
|
|
651
|
+
"""
|
|
652
|
+
Interpolate Irregular Leap Frog time series (result of an inversion) to Regular LF time series using Cumulative Displacement times series.
|
|
653
|
+
|
|
654
|
+
:param result: [pd dataframe] --- Leap frog displacement for x-component and y-component
|
|
655
|
+
:param interval_output: [int] --- Period between two dates of the obtained RLF
|
|
656
|
+
:param path_save: [str] --- Where to save the figures
|
|
657
|
+
:param option_interpol: [str] [default is 'spline'] --- Type of interpolation, it can be 'spline', 'spline_smooth' or 'nearest'
|
|
658
|
+
:param first_date_interpol: [np.datetime64 | str | None] [default is None] --- First date of the interpolation
|
|
659
|
+
:param last_date_interpol: [np.datetime64 | str | None] [default is None] --- Last date of the interpolation
|
|
660
|
+
:param unit: [int] [default is 365] --- 1 for m/d, 365 for m/y
|
|
661
|
+
:param redundancy: [int | None] [default is None] --- If None there is no redundancy between two velocity in the interpolated time-series, else the overlap between two velocities is redundancy days
|
|
662
|
+
:param result_quality: [list | str | None] [default is None] --- List which can contain 'Norm_residual' to determine the L2 norm of the residuals from the last inversion, 'X_contribution' to determine the number of Y observations which have contributed to estimate each value in X (it corresponds to A.dot(weight))
|
|
663
|
+
|
|
664
|
+
:return dataf_lp: [pd dataframe] --- Result of the temporal interpolation
|
|
665
|
+
"""
|
|
666
|
+
## Reconstruction of COMMON REF TIME SERIES, e.g. cumulative displacement time series
|
|
667
|
+
dataf = reconstruct_common_ref(result) # Build cumulative displacement time series
|
|
668
|
+
if first_date_interpol is None:
|
|
669
|
+
start_date = dataf["Ref_date"][0] # First date at the considered pixel
|
|
670
|
+
else:
|
|
671
|
+
start_date = pd.to_datetime(first_date_interpol)
|
|
672
|
+
|
|
673
|
+
x = np.array(
|
|
674
|
+
(dataf["Second_date"] - np.datetime64(start_date)).dt.days
|
|
675
|
+
) # Number of days according to the start_date
|
|
676
|
+
if len(x) <= 1 or (
|
|
677
|
+
np.isin("spline", option_interpol) and len(x) <= 3
|
|
678
|
+
): # It is not possible to interpolate, because too few estimation
|
|
679
|
+
return pd.DataFrame(
|
|
680
|
+
{
|
|
681
|
+
"date1": [],
|
|
682
|
+
"date2": [],
|
|
683
|
+
"vx": [],
|
|
684
|
+
"vy": [],
|
|
685
|
+
"xcount_x": [],
|
|
686
|
+
"xcount_y": [],
|
|
687
|
+
"dz": [],
|
|
688
|
+
"vz": [],
|
|
689
|
+
"xcount_z": [],
|
|
690
|
+
"NormR": [],
|
|
691
|
+
}
|
|
692
|
+
)
|
|
693
|
+
|
|
694
|
+
# Compute the functions used to interpolate
|
|
695
|
+
fdx, fdy, fdx_xcount, fdy_xcount, fdx_error, fdy_error = set_function_for_interpolation(
|
|
696
|
+
option_interpol, x, dataf, result_quality
|
|
697
|
+
)
|
|
698
|
+
|
|
699
|
+
if redundancy is None: # No redundancy between two interpolated velocity
|
|
700
|
+
x_regu = np.arange(np.min(x) + (interval_output - np.min(x) % interval_output), np.max(x), interval_output)
|
|
701
|
+
else: # The overlap between two velocities corresponds to redundancy
|
|
702
|
+
x_regu = np.arange(
|
|
703
|
+
np.min(x) + (redundancy - np.min(x) % redundancy), np.max(x), redundancy
|
|
704
|
+
) # To make sure that the first element of x_regu is multiple of redundancy
|
|
705
|
+
|
|
706
|
+
if len(x_regu) <= 1: # No interpolation
|
|
707
|
+
return pd.DataFrame(
|
|
708
|
+
{
|
|
709
|
+
"date1": [],
|
|
710
|
+
"date2": [],
|
|
711
|
+
"vx": [],
|
|
712
|
+
"vy": [],
|
|
713
|
+
"xcount_x": [],
|
|
714
|
+
"xcount_y": [],
|
|
715
|
+
"dz": [],
|
|
716
|
+
"vz": [],
|
|
717
|
+
"xcount_z": [],
|
|
718
|
+
"NormR": [],
|
|
719
|
+
}
|
|
720
|
+
)
|
|
721
|
+
|
|
722
|
+
## Reconstruct a time series with a given temporal sampling, and a given overlap
|
|
723
|
+
step = interval_output if redundancy is None else int(interval_output / redundancy)
|
|
724
|
+
if step >= len(x_regu):
|
|
725
|
+
return pd.DataFrame(
|
|
726
|
+
{
|
|
727
|
+
"date1": [],
|
|
728
|
+
"date2": [],
|
|
729
|
+
"vx": [],
|
|
730
|
+
"vy": [],
|
|
731
|
+
"xcount_x": [],
|
|
732
|
+
"xcount_y": [],
|
|
733
|
+
"dz": [],
|
|
734
|
+
"vz": [],
|
|
735
|
+
"xcount_z": [],
|
|
736
|
+
"NormR": [],
|
|
737
|
+
}
|
|
738
|
+
)
|
|
739
|
+
|
|
740
|
+
x_shifted = x_regu[step:]
|
|
741
|
+
dx = fdx(x_shifted) - fdx(
|
|
742
|
+
x_regu[:-step]
|
|
743
|
+
) # Equivalent to [fdx(x_regu[i + step]) - fdx(x_regu[i]) for i in range(len(x_regu) - step)]
|
|
744
|
+
dy = fdy(x_shifted) - fdy(
|
|
745
|
+
x_regu[:-step]
|
|
746
|
+
) # Equivalent to [fdy(x_regu[i + step]) - fdy(x_regu[i]) for i in range(len(x_regu) - step)]
|
|
747
|
+
if result_quality is not None:
|
|
748
|
+
if "X_contribution" in result_quality:
|
|
749
|
+
xcount_x = fdx_xcount(x_shifted) - fdx_xcount(x_regu[:-step])
|
|
750
|
+
xcount_y = fdy_xcount(x_shifted) - fdy_xcount(x_regu[:-step])
|
|
751
|
+
xcount_x[xcount_x < 0] = 0
|
|
752
|
+
xcount_y[xcount_y < 0] = 0
|
|
753
|
+
if "Error_propagation" in result_quality:
|
|
754
|
+
error_x = fdx_error(x_shifted) - fdx_error(x_regu[:-step])
|
|
755
|
+
error_y = fdy_error(x_shifted) - fdy_error(x_regu[:-step])
|
|
756
|
+
vx = dx * unit / interval_output # Convert to velocity in m/d or m/y
|
|
757
|
+
vy = dy * unit / interval_output # Convert to velocity in m/d or m/
|
|
758
|
+
|
|
759
|
+
First_date = start_date + pd.to_timedelta(
|
|
760
|
+
x_regu[:-step], unit="D"
|
|
761
|
+
) # Equivalent to [start_date + pd.Timedelta(x_regu[i], 'D') for i in range(len(x_regu) - step)]
|
|
762
|
+
Second_date = start_date + pd.to_timedelta(x_shifted, unit="D")
|
|
763
|
+
|
|
764
|
+
dataf_lp = pd.DataFrame({"date1": First_date, "date2": Second_date, "vx": vx, "vy": vy})
|
|
765
|
+
if result_quality is not None:
|
|
766
|
+
if "X_contribution" in result_quality:
|
|
767
|
+
dataf_lp["xcount_x"] = xcount_x
|
|
768
|
+
dataf_lp["xcount_y"] = xcount_y
|
|
769
|
+
if "Error_propagation" in result_quality:
|
|
770
|
+
dataf_lp["error_x"] = error_x * unit / interval_output
|
|
771
|
+
dataf_lp["error_y"] = error_y * unit / interval_output
|
|
772
|
+
dataf_lp["sigma0"] = np.concatenate([result["sigma0"][:4], np.full(dataf_lp.shape[0] - 4, np.nan)])
|
|
773
|
+
del x_regu, First_date, Second_date, vx, vy
|
|
774
|
+
|
|
775
|
+
# Fill with nan values if the first date of the cube which will be interpolated is lower than the first date interpolated for this pixel
|
|
776
|
+
if first_date_interpol is not None and dataf_lp["date1"].iloc[0] > pd.Timestamp(first_date_interpol):
|
|
777
|
+
first_date = np.arange(first_date_interpol, dataf_lp["date1"].iloc[0], np.timedelta64(redundancy, "D"))
|
|
778
|
+
# dataf_lp = full_with_nan(dataf_lp, first_date=first_date,
|
|
779
|
+
# second_date=first_date + np.timedelta64(interval_output, 'D'))
|
|
780
|
+
nul_df = pd.DataFrame(
|
|
781
|
+
{
|
|
782
|
+
"date1": first_date,
|
|
783
|
+
"date2": first_date + np.timedelta64(interval_output, "D"),
|
|
784
|
+
"vx": np.full(len(first_date), np.nan),
|
|
785
|
+
"vy": np.full(len(first_date), np.nan),
|
|
786
|
+
}
|
|
787
|
+
)
|
|
788
|
+
if result_quality is not None:
|
|
789
|
+
if "X_contribution" in result_quality:
|
|
790
|
+
nul_df["xcount_x"] = np.full(len(first_date), np.nan)
|
|
791
|
+
nul_df["xcount_y"] = np.full(len(first_date), np.nan)
|
|
792
|
+
if "Error_propagation" in result_quality:
|
|
793
|
+
nul_df["error_x"] = np.full(len(first_date), np.nan)
|
|
794
|
+
nul_df["error_y"] = np.full(len(first_date), np.nan)
|
|
795
|
+
dataf_lp = pd.concat([nul_df, dataf_lp], ignore_index=True)
|
|
796
|
+
|
|
797
|
+
# Fill with nan values if the last date of the cube which will be interpolated is higher than the last date interpolated for this pixel
|
|
798
|
+
if last_date_interpol is not None and dataf_lp["date2"].iloc[-1] < pd.Timestamp(last_date_interpol):
|
|
799
|
+
first_date = np.arange(
|
|
800
|
+
dataf_lp["date2"].iloc[-1] + np.timedelta64(redundancy, "D"),
|
|
801
|
+
last_date_interpol + np.timedelta64(redundancy, "D"),
|
|
802
|
+
np.timedelta64(redundancy, "D"),
|
|
803
|
+
)
|
|
804
|
+
nul_df = pd.DataFrame(
|
|
805
|
+
{
|
|
806
|
+
"date1": first_date - np.timedelta64(interval_output, "D"),
|
|
807
|
+
"date2": first_date,
|
|
808
|
+
"vx": np.full(len(first_date), np.nan),
|
|
809
|
+
"vy": np.full(len(first_date), np.nan),
|
|
810
|
+
}
|
|
811
|
+
)
|
|
812
|
+
dataf_lp = pd.concat([dataf_lp, nul_df], ignore_index=True)
|
|
813
|
+
|
|
814
|
+
# print(dataf_lp.shape)
|
|
815
|
+
# if dataf_lp.shape[0]!= 567:
|
|
816
|
+
# print('stop')
|
|
817
|
+
return dataf_lp
|
|
818
|
+
|
|
819
|
+
|
|
820
|
+
def interpolation_to_data(
|
|
821
|
+
result: pd.DataFrame,
|
|
822
|
+
data: pd.DataFrame,
|
|
823
|
+
option_interpol: str = "spline",
|
|
824
|
+
unit: int = 365,
|
|
825
|
+
result_quality: list | None = None,
|
|
826
|
+
):
|
|
827
|
+
"""
|
|
828
|
+
Interpolate Irregular Leap Frog time series (result of an inversion) to the dates of given data (useful to compare
|
|
829
|
+
TICOI results to a "ground truth").
|
|
830
|
+
|
|
831
|
+
:param result: [pd dataframe] --- Leap frog displacement for x-component and y-component
|
|
832
|
+
:param data: [pd dataframe] --- Ground truth data which the interpolation must fit along the temporal axis
|
|
833
|
+
:param option_interpol: [str] [default is 'spline'] --- Type of interpolation, it can be 'spline', 'spline_smooth' or 'nearest'
|
|
834
|
+
:param unit: [int] [default is 365] --- 1 for m/d, 365 for m/y
|
|
835
|
+
:param result_quality: [list | str | None] [default is None] --- List which can contain 'Norm_residual' to determine the L2 norm of the residuals from the last inversion, 'X_contribution' to determine the number of Y observations which have contributed to estimate each value in X (it corresponds to A.dot(weight))
|
|
836
|
+
"""
|
|
837
|
+
|
|
838
|
+
## Reconstruction of COMMON REF TIME SERIES, e.g. cumulative displacement time series
|
|
839
|
+
dataf = reconstruct_common_ref(result) # Build cumulative displacement time series
|
|
840
|
+
start_date = dataf["Ref_date"][0] # First date at the considered pixel
|
|
841
|
+
x = np.array(
|
|
842
|
+
(dataf["Second_date"] - np.datetime64(start_date)).dt.days
|
|
843
|
+
) # Number of days according to the start_date
|
|
844
|
+
|
|
845
|
+
# Interpolation must be caried out in between the min and max date of the original data
|
|
846
|
+
if data["date1"].min() < result["date2"].min() or data["date2"].max() > result["date2"].max():
|
|
847
|
+
data = data[(data["date1"] > result["date2"].min()) & (data["date2"] < result["date2"].max())]
|
|
848
|
+
|
|
849
|
+
# Ground truth first and second dates
|
|
850
|
+
x_gt_date1 = np.array((data["date1"] - start_date).dt.days)
|
|
851
|
+
x_gt_date2 = np.array((data["date2"] - start_date).dt.days)
|
|
852
|
+
|
|
853
|
+
## Interpolate the displacements and convert to velocities
|
|
854
|
+
# Compute the functions used to interpolate
|
|
855
|
+
fdx, fdy, fdx_xcount, fdy_xcount, fdx_error, fdy_error = set_function_for_interpolation(
|
|
856
|
+
option_interpol, x, dataf, result_quality
|
|
857
|
+
)
|
|
858
|
+
|
|
859
|
+
# Interpolation
|
|
860
|
+
dx = fdx(x_gt_date2) - fdx(
|
|
861
|
+
x_gt_date1
|
|
862
|
+
) # Equivalent to [fdx(x_regu[i + step]) - fdx(x_regu[i]) for i in range(len(x_regu) - step)]
|
|
863
|
+
dy = fdy(x_gt_date2) - fdy(
|
|
864
|
+
x_gt_date1
|
|
865
|
+
) # Equivalent to [fdy(x_regu[i + step]) - fdy(x_regu[i]) for i in range(len(x_regu) - step)]
|
|
866
|
+
|
|
867
|
+
# conversion
|
|
868
|
+
vx = dx * unit / data["temporal_baseline"] # Convert to velocity in m/d or m/y
|
|
869
|
+
vy = dy * unit / data["temporal_baseline"] # Convert to velocity in m/d or m/y
|
|
870
|
+
|
|
871
|
+
# Fill dataframe
|
|
872
|
+
First_date = start_date + pd.to_timedelta(
|
|
873
|
+
x_gt_date1, unit="D"
|
|
874
|
+
) # Equivalent to [start_date + pd.Timedelta(x_regu[i], 'D') for i in range(len(x_regu) - step)]
|
|
875
|
+
Second_date = start_date + pd.to_timedelta(x_gt_date2, unit="D")
|
|
876
|
+
data_dict = {"date1": First_date, "date2": Second_date, "vx": vx, "vy": vy}
|
|
877
|
+
dataf_lp = pd.DataFrame(data_dict)
|
|
878
|
+
|
|
879
|
+
return dataf_lp
|
|
880
|
+
|
|
881
|
+
|
|
882
|
+
# %% ======================================================================== #
|
|
883
|
+
# GLOBAL PROCESS #
|
|
884
|
+
# =========================================================================%% #
|
|
885
|
+
|
|
886
|
+
|
|
887
|
+
def process(
|
|
888
|
+
cube: CubeDataClass,
|
|
889
|
+
i: float | int,
|
|
890
|
+
j: float | int,
|
|
891
|
+
path_save,
|
|
892
|
+
solver: str = "LSMR_ini",
|
|
893
|
+
regu: int | str = "1accelnotnull",
|
|
894
|
+
coef: int = 100,
|
|
895
|
+
flag: xr.Dataset | None = None,
|
|
896
|
+
apriori_weight: bool = False,
|
|
897
|
+
apriori_weight_in_second_iteration: bool = False,
|
|
898
|
+
returned: list | str = "interp",
|
|
899
|
+
obs_filt: xr.Dataset | None = None,
|
|
900
|
+
interpolation_load_pixel: str = "nearest",
|
|
901
|
+
iteration: bool = True,
|
|
902
|
+
interval_output: int = 1,
|
|
903
|
+
first_date_interpol: np.datetime64 | None = None,
|
|
904
|
+
last_date_interpol: np.datetime64 | None = None,
|
|
905
|
+
proj="EPSG:4326",
|
|
906
|
+
threshold_it: float = 0.1,
|
|
907
|
+
conf: bool = True,
|
|
908
|
+
option_interpol: str = "spline",
|
|
909
|
+
redundancy: int | None = None,
|
|
910
|
+
detect_temporal_decorrelation: bool = True,
|
|
911
|
+
unit: int = 365,
|
|
912
|
+
result_quality: list | str | None = None,
|
|
913
|
+
nb_max_iteration: int = 10,
|
|
914
|
+
delete_outliers: int | str | None = None,
|
|
915
|
+
linear_operator: bool = False,
|
|
916
|
+
visual: bool = False,
|
|
917
|
+
verbose: bool = False,
|
|
918
|
+
):
|
|
919
|
+
"""
|
|
920
|
+
:params i, j: [float | int] --- Coordinates of the point in pixel
|
|
921
|
+
:param solver: [str] [default is 'LSMR'] --- Solver of the inversion: 'LSMR', 'LSMR_ini', 'LS', 'LSQR'
|
|
922
|
+
:param regu: [int | str] [default is 1] --- Type of regularization
|
|
923
|
+
:param coef: [int] [default is 100] --- Coef of Tikhonov regularisation
|
|
924
|
+
:param flag: [xr dataset | None] [default is None] --- If not None, the values of the coefficient used for stable areas, surge glacier and non surge glacier
|
|
925
|
+
:param apriori_weight: [bool] [default is False] --- If True use of aprori weight
|
|
926
|
+
:param returned: [list | str] [default is 'interp'] --- What results must be returned ('raw', 'invert' and/or 'interp')
|
|
927
|
+
:param obs_filt: [xr dataset | None] [default is None] --- Filtered dataset (e.g. rolling mean)
|
|
928
|
+
:param interpolation_load_pixel: [str] [default is 'nearest'] --- Type of interpolation to load the previous pixel in the temporal interpolation ('nearest' or 'linear')
|
|
929
|
+
:param iteration: [bool] [default is True] --- If True, use of iterations
|
|
930
|
+
:param interval_output: [int] [default is 1] --- Temporal sampling of the leap frog time series
|
|
931
|
+
:param first_date_interpol: [np.datetime64 | None] --- First date at which the time series are interpolated
|
|
932
|
+
:param last_date_interpol: [np.datetime64 | None] --- Last date at which the time series are interpolated
|
|
933
|
+
:param proj: [str] [default is 'EPSG:4326'] --- Projection of the cube
|
|
934
|
+
:param threshold_it: [float] [default is 0.1] --- Threshold to test the stability of the results between each iteration, use to stop the process
|
|
935
|
+
:param conf: [bool] [default is False] --- If True means that the error corresponds to confidence intervals between 0 and 1, otherwise it corresponds to errors in m/y or m/d
|
|
936
|
+
:param option_interpol: [str] [default is 'spline'] --- Type of interpolation, it can be 'spline', 'spline_smooth' or 'nearest'
|
|
937
|
+
:param redundancy: [int | None] [default is None] --- If None there is no redundancy between two velocity in the interpolated time-series, else the overlap between two velocities is redundancy days
|
|
938
|
+
:param detect_temporal_decorrelation: [bool] [default is True] --- If True the first inversion is solved using only velocity observations with small temporal baselines, to detect temporal decorelation
|
|
939
|
+
:param unit: [int] [default is 365] --- 1 for m/d, 365 for m/y
|
|
940
|
+
:param result_quality: [list | str | None] [default is None] --- List which can contain 'Norm_residual' to determine the L2 norm of the residuals from the last inversion, 'X_contribution' to determine the number of Y observations which have contributed to estimate each value in X (it corresponds to A.dot(weight))
|
|
941
|
+
:param nb_max_iteration: [int] [default is 10] --- Maximum number of iterations
|
|
942
|
+
:param delete_outliers: [int | str | None] [default is None] --- Delete data with a poor quality indicator (if int), or with aberrant direction ('vvc_angle')
|
|
943
|
+
:param linear_operator: [bool] [default is False] --- If linear operator, the inversion is performed using a linear operator (https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.linalg.LinearOperator.html)
|
|
944
|
+
:param visual: [bool] [default is False] --- Keep the weights for future plots
|
|
945
|
+
:param verbose: [bool] [default is False] --- Print information along the way
|
|
946
|
+
|
|
947
|
+
:return dataf_list: [pd dataframe] Result of the temporal inversion + interpolation at point (i, j) if inversion was successful, an empty dataframe if not
|
|
948
|
+
"""
|
|
949
|
+
|
|
950
|
+
returned_list = []
|
|
951
|
+
|
|
952
|
+
# Loading data at pixel location
|
|
953
|
+
data = cube.load_pixel(
|
|
954
|
+
i,
|
|
955
|
+
j,
|
|
956
|
+
proj=proj,
|
|
957
|
+
interp=interpolation_load_pixel,
|
|
958
|
+
solver=solver,
|
|
959
|
+
coef=coef,
|
|
960
|
+
regu=regu,
|
|
961
|
+
rolling_mean=obs_filt,
|
|
962
|
+
flag=flag,
|
|
963
|
+
)
|
|
964
|
+
|
|
965
|
+
if "raw" in returned: # return the raw data
|
|
966
|
+
returned_list.append(data)
|
|
967
|
+
|
|
968
|
+
if "invert" in returned or "interp" in returned:
|
|
969
|
+
if flag is not None: # set regu and coef for every flags
|
|
970
|
+
regu, coef = data[3], data[4]
|
|
971
|
+
|
|
972
|
+
# Inversion
|
|
973
|
+
# TODO: to check that!
|
|
974
|
+
if delete_outliers == "median_angle":
|
|
975
|
+
conf = True # Set conf to True, because the errors have been replaced by confidence indicators based on the cos of the angle between the vector of each observation and the median vector
|
|
976
|
+
|
|
977
|
+
result = inversion_core(
|
|
978
|
+
data[0],
|
|
979
|
+
i,
|
|
980
|
+
j,
|
|
981
|
+
dates_range=data[2],
|
|
982
|
+
solver=solver,
|
|
983
|
+
coef=coef,
|
|
984
|
+
apriori_weight=apriori_weight,
|
|
985
|
+
apriori_weight_in_second_iteration=apriori_weight_in_second_iteration,
|
|
986
|
+
unit=unit,
|
|
987
|
+
conf=conf,
|
|
988
|
+
regu=regu,
|
|
989
|
+
mean=data[1],
|
|
990
|
+
iteration=iteration,
|
|
991
|
+
threshold_it=threshold_it,
|
|
992
|
+
detect_temporal_decorrelation=detect_temporal_decorrelation,
|
|
993
|
+
linear_operator=linear_operator,
|
|
994
|
+
result_quality=result_quality,
|
|
995
|
+
nb_max_iteration=nb_max_iteration,
|
|
996
|
+
visual=visual,
|
|
997
|
+
verbose=verbose,
|
|
998
|
+
)
|
|
999
|
+
|
|
1000
|
+
if "invert" in returned:
|
|
1001
|
+
if result[1] is not None:
|
|
1002
|
+
returned_list.append(result[1])
|
|
1003
|
+
else:
|
|
1004
|
+
if result_quality is not None and "X_contribution" in result_quality:
|
|
1005
|
+
variables = ["result_dx", "result_dy", "xcount_x", "xcount_y"]
|
|
1006
|
+
else:
|
|
1007
|
+
variables = ["result_dx", "result_dy"]
|
|
1008
|
+
returned_list.append(pd.DataFrame({"date1": [], "date2": [], **{col: [] for col in variables}}))
|
|
1009
|
+
|
|
1010
|
+
if "interp" in returned:
|
|
1011
|
+
# Interpolation
|
|
1012
|
+
if result[1] is not None: # If inversion have been performed
|
|
1013
|
+
dataf_list = interpolation_core(
|
|
1014
|
+
result[1],
|
|
1015
|
+
interval_output,
|
|
1016
|
+
option_interpol=option_interpol,
|
|
1017
|
+
first_date_interpol=first_date_interpol,
|
|
1018
|
+
last_date_interpol=last_date_interpol,
|
|
1019
|
+
unit=unit,
|
|
1020
|
+
redundancy=redundancy,
|
|
1021
|
+
result_quality=result_quality,
|
|
1022
|
+
)
|
|
1023
|
+
|
|
1024
|
+
if result_quality is not None and "Norm_residual" in result_quality:
|
|
1025
|
+
dataf_list["NormR"] = result[1]["NormR"] # Store norm of the residual from the inversion
|
|
1026
|
+
returned_list.append(dataf_list)
|
|
1027
|
+
else:
|
|
1028
|
+
if result_quality is not None and "Norm_residual" in result_quality:
|
|
1029
|
+
returned_list.append(
|
|
1030
|
+
pd.DataFrame(
|
|
1031
|
+
{"date1": [], "date2": [], "vx": [], "vy": [], "xcount_x": [], "xcount_y": [], "NormR": []}
|
|
1032
|
+
)
|
|
1033
|
+
)
|
|
1034
|
+
else:
|
|
1035
|
+
returned_list.append(
|
|
1036
|
+
pd.DataFrame({"date1": [], "date2": [], "vx": [], "vy": [], "xcount_x": [], "xcount_y": []})
|
|
1037
|
+
)
|
|
1038
|
+
|
|
1039
|
+
if len(returned_list) == 1:
|
|
1040
|
+
return returned_list[0]
|
|
1041
|
+
return returned_list if len(returned_list) > 0 else None
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
def chunk_to_block(cube: CubeDataClass, block_size: float = 1, verbose: bool = False):
|
|
1045
|
+
"""
|
|
1046
|
+
Split a dataset in blocks of a given size (maximum).
|
|
1047
|
+
|
|
1048
|
+
:param cube: [cube_data_class] --- Cube to be splited in blocks
|
|
1049
|
+
:param block_size: [float] [default is 1] --- Maximum size (in GB) of the blocks
|
|
1050
|
+
:param verbose: [bool] [default is False] --- Print information along the way
|
|
1051
|
+
|
|
1052
|
+
:return blocks: [list] --- List of the boundaries of each blocks (x_start, x_end, y_start, y_end)
|
|
1053
|
+
"""
|
|
1054
|
+
|
|
1055
|
+
GB = 1073741824
|
|
1056
|
+
blocks = []
|
|
1057
|
+
if cube.ds.nbytes > block_size * GB:
|
|
1058
|
+
try:
|
|
1059
|
+
num_elements = np.prod([cube.ds.chunks[dim][0] for dim in cube.ds.chunks.keys()])
|
|
1060
|
+
except ValueError:
|
|
1061
|
+
cube = cube.ds.unify_chunks() # ValueError: Object has inconsistent chunks along dimension x. This can be fixed by calling unify_chunks().
|
|
1062
|
+
|
|
1063
|
+
chunk_bytes = num_elements * cube.ds["vx"].dtype.itemsize
|
|
1064
|
+
|
|
1065
|
+
nchunks_block = int(block_size * GB // chunk_bytes)
|
|
1066
|
+
|
|
1067
|
+
x_step = int(np.sqrt(nchunks_block))
|
|
1068
|
+
y_step = nchunks_block // x_step
|
|
1069
|
+
|
|
1070
|
+
nblocks_x = int(np.ceil(len(cube.ds.chunks["x"]) / x_step))
|
|
1071
|
+
nblocks_y = int(np.ceil(len(cube.ds.chunks["y"]) / y_step))
|
|
1072
|
+
|
|
1073
|
+
nblocks = nblocks_x * nblocks_y
|
|
1074
|
+
if verbose:
|
|
1075
|
+
print(
|
|
1076
|
+
f"[Block process] Divide into {nblocks} blocks\n blocks size: {x_step * cube.ds.chunks['x'][0]} x {y_step * cube.ds.chunks['y'][0]}"
|
|
1077
|
+
)
|
|
1078
|
+
|
|
1079
|
+
for i in range(nblocks_y):
|
|
1080
|
+
for j in range(nblocks_x):
|
|
1081
|
+
x_start = j * x_step * cube.ds.chunks["x"][0]
|
|
1082
|
+
y_start = i * y_step * cube.ds.chunks["y"][0]
|
|
1083
|
+
x_end = x_start + x_step * cube.ds.chunks["x"][0] if j != nblocks_x - 1 else cube.ds.dims["x"]
|
|
1084
|
+
y_end = y_start + y_step * cube.ds.chunks["y"][0] if i != nblocks_y - 1 else cube.ds.dims["y"]
|
|
1085
|
+
blocks.append([x_start, x_end, y_start, y_end])
|
|
1086
|
+
else:
|
|
1087
|
+
blocks.append([0, cube.ds.dims["x"], 0, cube.ds.dims["y"]])
|
|
1088
|
+
if verbose:
|
|
1089
|
+
print(f"[Block process] Cube size smaller than {block_size}GB, no need to divide")
|
|
1090
|
+
|
|
1091
|
+
return blocks
|
|
1092
|
+
|
|
1093
|
+
|
|
1094
|
+
def load_block(cube: CubeDataClass, x_start: int, x_end: int, y_start: int, y_end: int, flag: xr.Dataset | None = None):
|
|
1095
|
+
"""
|
|
1096
|
+
Persist a block in memory, i.e. load it in a distributed way.
|
|
1097
|
+
|
|
1098
|
+
:param cube: [cube_data_class] --- Cube splited in blocks
|
|
1099
|
+
:params x_start, x_end, y_start, y_end: [int] --- Boundaries of the block
|
|
1100
|
+
|
|
1101
|
+
:return block: [cube_data_class] --- Sub-cube of cube according to the boundaries (block)
|
|
1102
|
+
:return duration: [float] --- Duration of the block loading
|
|
1103
|
+
"""
|
|
1104
|
+
|
|
1105
|
+
start = time.time()
|
|
1106
|
+
block = CubeDataClass()
|
|
1107
|
+
block.ds = cube.ds.isel(x=slice(x_start, x_end), y=slice(y_start, y_end))
|
|
1108
|
+
block.ds = block.ds.persist()
|
|
1109
|
+
block.update_dimension()
|
|
1110
|
+
if flag is not None:
|
|
1111
|
+
block_flag = flag.isel(x=slice(x_start, x_end), y=slice(y_start, y_end))
|
|
1112
|
+
block_flag = block_flag.persist()
|
|
1113
|
+
else:
|
|
1114
|
+
block_flag = None
|
|
1115
|
+
duration = time.time() - start
|
|
1116
|
+
|
|
1117
|
+
return block, block_flag, duration
|
|
1118
|
+
|
|
1119
|
+
|
|
1120
|
+
def process_blocks_refine(
|
|
1121
|
+
cube: CubeDataClass,
|
|
1122
|
+
nb_cpu: int = 8,
|
|
1123
|
+
block_size: float = 0.5,
|
|
1124
|
+
returned: list | str = "interp",
|
|
1125
|
+
preData_kwargs: dict = None,
|
|
1126
|
+
inversion_kwargs: dict | None = None,
|
|
1127
|
+
verbose: bool = False,
|
|
1128
|
+
):
|
|
1129
|
+
"""
|
|
1130
|
+
Separate the cube in several blocks computed synchronously one after the other by loading one block while the other is computed (with
|
|
1131
|
+
parallelization) in order to avoid memory overconsumption and kernel crashing, and benefit from smaller computation time.
|
|
1132
|
+
|
|
1133
|
+
:param cube: [cube_data_class] --- Cube of raw data to be processed
|
|
1134
|
+
:param nb_cpu: [int] [default is 8] --- Number of processing unit to use for parallel processing
|
|
1135
|
+
:param block_size: [float] [default is 0.5] --- Maximum size of the blocks (in GB)
|
|
1136
|
+
:param returned: [list | str] [default is 'interp'] --- What results must be returned ('raw', 'invert' and/or 'interp')
|
|
1137
|
+
:param preData_kwargs: [dict] [default is None] --- Pre-processing parameters (see cube_data_classxr.filter_cube)
|
|
1138
|
+
:param inversion_kwargs: [dict] [default is None] --- Inversion (and interpolation) parameters (see core.process)
|
|
1139
|
+
:param verbose: [bool] [default is False] --- Print information along the way
|
|
1140
|
+
|
|
1141
|
+
:return: [pd dataframe] Resulting estimated time series after inversion (and interpolation)
|
|
1142
|
+
"""
|
|
1143
|
+
|
|
1144
|
+
async def process_block(
|
|
1145
|
+
block: CubeDataClass, returned: list | str = "interp", nb_cpu: int = 8, verbose: bool = False
|
|
1146
|
+
):
|
|
1147
|
+
xy_values = itertools.product(block.ds["x"].values, block.ds["y"].values)
|
|
1148
|
+
# Return only raw data => no need to filter the cube
|
|
1149
|
+
if "raw" in returned and (isinstance(returned, str) or len(returned) == 1): # Only load the raw data
|
|
1150
|
+
xy_values_tqdm = tqdm(xy_values, total=(block.nx * block.ny))
|
|
1151
|
+
result_block = Parallel(n_jobs=nb_cpu, verbose=0)(
|
|
1152
|
+
delayed(block.load_pixel)(
|
|
1153
|
+
i,
|
|
1154
|
+
j,
|
|
1155
|
+
proj=inversion_kwargs["proj"],
|
|
1156
|
+
interp=inversion_kwargs["interpolation_load_pixel"],
|
|
1157
|
+
solver=inversion_kwargs["solver"],
|
|
1158
|
+
regu=inversion_kwargs["regu"],
|
|
1159
|
+
rolling_mean=None,
|
|
1160
|
+
visual=inversion_kwargs["visual"],
|
|
1161
|
+
)
|
|
1162
|
+
for i, j in xy_values_tqdm
|
|
1163
|
+
)
|
|
1164
|
+
return result_block
|
|
1165
|
+
|
|
1166
|
+
# Filter the cube
|
|
1167
|
+
obs_filt, flag_block = block.filter_cube_before_inversion(**preData_kwargs)
|
|
1168
|
+
if isinstance(inversion_kwargs, dict):
|
|
1169
|
+
inversion_kwargs.update({"flag": flag_block})
|
|
1170
|
+
|
|
1171
|
+
# There is no data on the whole block (masked data)
|
|
1172
|
+
if obs_filt is None and "interp" in returned:
|
|
1173
|
+
if inversion_kwargs["result_quality"] is not None and "Norm_residual" in inversion_kwargs["result_quality"]:
|
|
1174
|
+
return [
|
|
1175
|
+
pd.DataFrame(
|
|
1176
|
+
{"date1": [], "date2": [], "vx": [], "vy": [], "xcount_x": [], "xcount_y": [], "NormR": []}
|
|
1177
|
+
)
|
|
1178
|
+
]
|
|
1179
|
+
else:
|
|
1180
|
+
return [
|
|
1181
|
+
pd.DataFrame(
|
|
1182
|
+
{"First_date": [], "Second_date": [], "vx": [], "vy": [], "xcount_x": [], "xcount_y": []}
|
|
1183
|
+
)
|
|
1184
|
+
]
|
|
1185
|
+
|
|
1186
|
+
xy_values_tqdm = tqdm(xy_values, total=(obs_filt["x"].shape[0] * obs_filt["y"].shape[0]))
|
|
1187
|
+
result_block = Parallel(n_jobs=nb_cpu, verbose=0)(
|
|
1188
|
+
delayed(process)(block, i, j, obs_filt=obs_filt, returned=returned, **inversion_kwargs)
|
|
1189
|
+
for i, j in xy_values_tqdm
|
|
1190
|
+
)
|
|
1191
|
+
|
|
1192
|
+
return result_block
|
|
1193
|
+
|
|
1194
|
+
async def process_blocks_main(cube, nb_cpu=8, block_size=0.5, returned="interp", verbose=False):
|
|
1195
|
+
if isinstance(preData_kwargs, dict) and "flag" in preData_kwargs.keys():
|
|
1196
|
+
flag = preData_kwargs["flag"]
|
|
1197
|
+
if flag is not None:
|
|
1198
|
+
flag = cube.create_flag(flag)
|
|
1199
|
+
else:
|
|
1200
|
+
flag = None
|
|
1201
|
+
|
|
1202
|
+
blocks = chunk_to_block(cube, block_size=block_size, verbose=True) # Split the cube in smaller blocks
|
|
1203
|
+
|
|
1204
|
+
dataf_list = [None] * (cube.nx * cube.ny)
|
|
1205
|
+
|
|
1206
|
+
loop = asyncio.get_event_loop()
|
|
1207
|
+
for n in range(len(blocks)):
|
|
1208
|
+
print(f"[Block process] Processing block {n + 1}/{len(blocks)}")
|
|
1209
|
+
|
|
1210
|
+
# Load the first block and start the loop
|
|
1211
|
+
if n == 0:
|
|
1212
|
+
x_start, x_end, y_start, y_end = blocks[0]
|
|
1213
|
+
future = loop.run_in_executor(None, load_block, cube, x_start, x_end, y_start, y_end, flag)
|
|
1214
|
+
|
|
1215
|
+
block, block_flag, duration = await future
|
|
1216
|
+
print(f"Block {n + 1} loaded in {duration:.2f} s")
|
|
1217
|
+
|
|
1218
|
+
if n < len(blocks) - 1:
|
|
1219
|
+
# Load the next block while processing the current block
|
|
1220
|
+
x_start, x_end, y_start, y_end = blocks[n + 1]
|
|
1221
|
+
future = loop.run_in_executor(None, load_block, cube, x_start, x_end, y_start, y_end, flag)
|
|
1222
|
+
|
|
1223
|
+
# need to change the flag back...
|
|
1224
|
+
if flag is not None:
|
|
1225
|
+
preData_kwargs.update({"flag": block_flag})
|
|
1226
|
+
|
|
1227
|
+
block_result = await process_block(
|
|
1228
|
+
block, returned=returned, nb_cpu=nb_cpu, verbose=verbose
|
|
1229
|
+
) # Process TICOI
|
|
1230
|
+
|
|
1231
|
+
# Transform to list
|
|
1232
|
+
for i in range(len(block_result)):
|
|
1233
|
+
row = i % block.ny + blocks[n][2]
|
|
1234
|
+
col = np.floor(i / block.ny) + blocks[n][0]
|
|
1235
|
+
idx = int(col * cube.ny + row)
|
|
1236
|
+
|
|
1237
|
+
dataf_list[idx] = block_result[i]
|
|
1238
|
+
|
|
1239
|
+
del block_result, block
|
|
1240
|
+
|
|
1241
|
+
if isinstance(returned, list) and len(returned) > 1:
|
|
1242
|
+
dataf_list = {returned[r]: [dataf_list[i][r] for i in range(len(dataf_list))] for r in range(len(returned))}
|
|
1243
|
+
|
|
1244
|
+
return dataf_list
|
|
1245
|
+
|
|
1246
|
+
# /!\ The use of asyncio can cause problems when the code is launched from an IDE if it has its own event loop
|
|
1247
|
+
# (leads to RuntimeError), you must launch it in an external terminal (IDEs generally offer this option)
|
|
1248
|
+
return asyncio.run(
|
|
1249
|
+
process_blocks_main(cube, nb_cpu=nb_cpu, block_size=block_size, returned=returned, verbose=verbose)
|
|
1250
|
+
)
|
|
1251
|
+
|
|
1252
|
+
|
|
1253
|
+
# %% ======================================================================== #
|
|
1254
|
+
# VISUALISATION #
|
|
1255
|
+
# =========================================================================%% #
|
|
1256
|
+
|
|
1257
|
+
|
|
1258
|
+
def visualization_core(
|
|
1259
|
+
list_dataf: pd.DataFrame,
|
|
1260
|
+
option_visual: List,
|
|
1261
|
+
type_data: List = ["obs", "invert"],
|
|
1262
|
+
save: bool = False,
|
|
1263
|
+
show: bool = True,
|
|
1264
|
+
path_save: Optional[str] = None,
|
|
1265
|
+
A: Optional[np.array] = None,
|
|
1266
|
+
log_scale: bool = False,
|
|
1267
|
+
cmap: str = "viridis",
|
|
1268
|
+
colors: List[str] = ["blueviolet", "orange"],
|
|
1269
|
+
figsize: tuple[int, int] = (10, 6),
|
|
1270
|
+
vminmax: List[int] | None = None,
|
|
1271
|
+
):
|
|
1272
|
+
r"""
|
|
1273
|
+
Visualization function for the output of pixel_ticoi
|
|
1274
|
+
/!\ Many figures can be plotted
|
|
1275
|
+
|
|
1276
|
+
:param list_dataf: [pd.DataFrame] --- cube dataset, which could be the observations, the inverted results (ILF), and the filtered observations. Make sure that the order is coherent with type_data.
|
|
1277
|
+
:param option_visual: [list] --- list of options for visualization
|
|
1278
|
+
:param type_data: [list] --- type of data: 'obs' for the observations, 'invert' for the inverted results, and 'obs_filt' for the filtered observations. Make sure that this is coherent with list_dataf.
|
|
1279
|
+
:param save:[bool] [default is False] --- if True, save the figures
|
|
1280
|
+
:param show: [bool] [default is True] --- if True, show the figures
|
|
1281
|
+
:param path_save: [str|None] [default is None] --- path where to save the figures
|
|
1282
|
+
:param A: [np.array] [default is None] --- design matrix
|
|
1283
|
+
:param log_scale: [bool] [default is False] --- if True, plot the figures into log scale
|
|
1284
|
+
:param cmap: [str] [default is 'viridis''] --- color map used in the plots
|
|
1285
|
+
:param colors: [list of str] [default is ['blueviolet', 'orange']] --- List of colors to used for plotting the time series
|
|
1286
|
+
:param figsize: tuple[int, int] [default is (10,6)] --- Size of the figures
|
|
1287
|
+
:param vminmax: List[int] [default is None] --- Min and max values for the y-axis of the plots
|
|
1288
|
+
"""
|
|
1289
|
+
|
|
1290
|
+
pixel_object = PixelClass()
|
|
1291
|
+
pixel_object.load(list_dataf, save=save, show=show, A=A, path_save=path_save, figsize=figsize, type_data=type_data)
|
|
1292
|
+
|
|
1293
|
+
dico_visual = {
|
|
1294
|
+
"obs_xy": (lambda pix: pix.plot_vx_vy(color=colors[0], type_data="obs")),
|
|
1295
|
+
"obs_magnitude": (lambda pix: pix.plot_vv(color=colors[0], type_data="obs", vminmax=vminmax)),
|
|
1296
|
+
"obs_vxvy_quality": (lambda pix: pix.plot_vx_vy_quality(cmap=cmap, type_data="obs")),
|
|
1297
|
+
"invertxy": (lambda pix: pix.plot_vx_vy(color=colors[1])),
|
|
1298
|
+
"invertxy_overlaid": (lambda pix: pix.plot_vx_vy_overlaid(colors=colors)),
|
|
1299
|
+
"obsfiltxy_overlaid": (lambda pix: pix.plot_vx_vy_overlaid(colors=colors, type_data="obs_filt")),
|
|
1300
|
+
"obsfiltvv_overlaid": (lambda pix: pix.plot_vv_overlaid(colors=colors, type_data="obs_filt", vminmax=vminmax)),
|
|
1301
|
+
"invertvv_overlaid": (lambda pix: pix.plot_vv_overlaid(colors=colors, vminmax=vminmax)),
|
|
1302
|
+
"invertvv": (lambda pix: pix.plot_vv(color=colors[1], vminmax=vminmax)),
|
|
1303
|
+
"invert_vv_quality": (lambda pix: pix.plot_vv_quality(cmap=cmap, type_data="invert")),
|
|
1304
|
+
"residuals": (lambda pix: pix.plot_residuals(log_scale=log_scale)),
|
|
1305
|
+
"xcount_xy": (lambda pix: pix.plot_xcount_vx_vy(cmap=cmap)),
|
|
1306
|
+
"xcount_vv": (lambda pix: pix.plot_xcount_vv(cmap=cmap)),
|
|
1307
|
+
"invert_weight": (lambda pix: pix.plot_weights_inversion()),
|
|
1308
|
+
"direction": (lambda pix: pix.plot_direction()),
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
for option in option_visual:
|
|
1312
|
+
if option in dico_visual.keys():
|
|
1313
|
+
dico_visual[option](pixel_object)
|
|
1314
|
+
|
|
1315
|
+
|
|
1316
|
+
def save_cube_parameters(
|
|
1317
|
+
cube: "CubeDataClass",
|
|
1318
|
+
load_kwargs: dict,
|
|
1319
|
+
preData_kwargs: dict,
|
|
1320
|
+
inversion_kwargs: dict,
|
|
1321
|
+
returned: list | None = None,
|
|
1322
|
+
) -> Tuple[str, str]:
|
|
1323
|
+
"""
|
|
1324
|
+
|
|
1325
|
+
:param cube: [cube_data_class] --- cube dataset
|
|
1326
|
+
:param load_kwargs: [dict] --- parameters used to load the cube
|
|
1327
|
+
:param prep_kwargs: [dict] --- parameters used to pre the cube
|
|
1328
|
+
:param inversion_kwargs: [dict] --- parameters used to load the cube
|
|
1329
|
+
:return:
|
|
1330
|
+
"""
|
|
1331
|
+
sensor_array = np.unique(cube.ds["sensor"])
|
|
1332
|
+
sensor_strings = [str(sensor) for sensor in sensor_array]
|
|
1333
|
+
sensor = ", ".join(sensor_strings)
|
|
1334
|
+
|
|
1335
|
+
source = f"Temporal inversion on cube {cube.filename} using TICOI"
|
|
1336
|
+
source += (
|
|
1337
|
+
f" with a selection of dates among {load_kwargs['pick_date']},"
|
|
1338
|
+
if load_kwargs["pick_date"] is not None
|
|
1339
|
+
else "" + f" with a selection of the temporal baselines among {load_kwargs['pick_temp_bas']}"
|
|
1340
|
+
if load_kwargs["pick_temp_bas"] is not None
|
|
1341
|
+
else ("" + f" with a subset of {load_kwargs['subset']}")
|
|
1342
|
+
if load_kwargs["subset"] is not None
|
|
1343
|
+
else ""
|
|
1344
|
+
)
|
|
1345
|
+
|
|
1346
|
+
if inversion_kwargs["apriori_weight"]:
|
|
1347
|
+
source += " and apriori weight"
|
|
1348
|
+
source += f". The regularisation coefficient is {inversion_kwargs['coef']}."
|
|
1349
|
+
if "interp" in returned:
|
|
1350
|
+
source += f"The interpolation method used is {inversion_kwargs['option_interpol']}."
|
|
1351
|
+
source += f"The interpolation baseline is {inversion_kwargs['interval_output']} days."
|
|
1352
|
+
source += f"The temporal spacing (redundancy) is {inversion_kwargs['redundancy']} days."
|
|
1353
|
+
|
|
1354
|
+
source += f"The preparation are argument are: {preData_kwargs}"
|
|
1355
|
+
return source, sensor
|
|
1356
|
+
|
|
1357
|
+
|
|
1358
|
+
def ticoi_one_pixel(
|
|
1359
|
+
cube_name: str,
|
|
1360
|
+
i: int,
|
|
1361
|
+
j: int,
|
|
1362
|
+
save: bool = False,
|
|
1363
|
+
path_save: str | None = None,
|
|
1364
|
+
show: bool = True,
|
|
1365
|
+
option_visual: List = ["invertvv_overlaid"],
|
|
1366
|
+
verbose: bool = False,
|
|
1367
|
+
load_kwargs: dict = {},
|
|
1368
|
+
load_pixel_kwargs: dict = {},
|
|
1369
|
+
preData_kwargs: dict = {},
|
|
1370
|
+
inversion_kwargs: dict = {},
|
|
1371
|
+
interpolation_kwargs: dict = {},
|
|
1372
|
+
already_loaded: pd.DataFrame | None = None,
|
|
1373
|
+
):
|
|
1374
|
+
"""
|
|
1375
|
+
:param cube_name: [string] --- name of the cube dataset
|
|
1376
|
+
:param i: [int] --- pixel index
|
|
1377
|
+
:param j: [int] --- pixel index
|
|
1378
|
+
:param save: [bool] --- whether to save the figures or not
|
|
1379
|
+
:param path_save: [string] --- path to save the figures
|
|
1380
|
+
:param show: [bool] --- whether to show the figures or not
|
|
1381
|
+
:param option_visual: [list] --- option visual
|
|
1382
|
+
:param verbose: [bool] --- whether to plot some text
|
|
1383
|
+
:param load_kwargs: [dict] --- parameters used to load the cube
|
|
1384
|
+
:param load_pixel_kwargs: [dict] --- parameters used to load the pixel
|
|
1385
|
+
:param preData_kwargs: [dict] --- parameters used to prepare the cube
|
|
1386
|
+
:param inversion_kwargs: [dict] --- parameters used for the inversion
|
|
1387
|
+
:param interpolation_kwargs: [dict] --- parameters used for the interpolation
|
|
1388
|
+
:param already_loaded: [pd.Dataframe or None] --- whether the dataframe of the pixel is already loaded or not
|
|
1389
|
+
:return:
|
|
1390
|
+
"""
|
|
1391
|
+
# %% ======================================================================== #
|
|
1392
|
+
# DATA LOADING #
|
|
1393
|
+
# =========================================================================%% #
|
|
1394
|
+
|
|
1395
|
+
if verbose:
|
|
1396
|
+
start = [time.time()]
|
|
1397
|
+
|
|
1398
|
+
if already_loaded is None:
|
|
1399
|
+
# Load the main cube
|
|
1400
|
+
cube = CubeDataClass()
|
|
1401
|
+
cube.load(cube_name, **load_kwargs)
|
|
1402
|
+
|
|
1403
|
+
if verbose:
|
|
1404
|
+
stop = [time.time()]
|
|
1405
|
+
print(f"[Data loading] Loading the data cube.s took {round((stop[0] - start[0]), 4)} s")
|
|
1406
|
+
print(f"[Data loading] Cube of dimension (nz,nx,ny) : ({cube.nz}, {cube.nx}, {cube.ny}) ")
|
|
1407
|
+
|
|
1408
|
+
start.append(time.time())
|
|
1409
|
+
|
|
1410
|
+
# Filter the cube (compute rolling_mean for regu=1accelnotnull)
|
|
1411
|
+
obs_filt, flag = cube.filter_cube_before_inversion(**preData_kwargs)
|
|
1412
|
+
|
|
1413
|
+
# Load pixel data
|
|
1414
|
+
data, mean, dates_range = cube.load_pixel(i, j, rolling_mean=obs_filt, **load_pixel_kwargs)
|
|
1415
|
+
# Prepare interpolation dates
|
|
1416
|
+
first_date_interpol, last_date_interpol = cube.prepare_interpolation_date()
|
|
1417
|
+
interpolation_kwargs.update(
|
|
1418
|
+
{"first_date_interpol": first_date_interpol, "last_date_interpol": last_date_interpol}
|
|
1419
|
+
)
|
|
1420
|
+
|
|
1421
|
+
else:
|
|
1422
|
+
data, mean, dates_range = already_loaded
|
|
1423
|
+
cube_date1 = data["date1"].tolist()
|
|
1424
|
+
cube_date1 = cube_date1 + data["date2"].tolist()
|
|
1425
|
+
cube_date1.remove(np.min(cube_date1))
|
|
1426
|
+
first_date_interpol = np.min(cube_date1)
|
|
1427
|
+
last_date_interpol = np.max(data["date2"])
|
|
1428
|
+
interpolation_kwargs.update(
|
|
1429
|
+
{"first_date_interpol": first_date_interpol, "last_date_interpol": last_date_interpol}
|
|
1430
|
+
)
|
|
1431
|
+
data = [
|
|
1432
|
+
data[["date1", "date2"]].to_numpy(),
|
|
1433
|
+
data[["dx", "dy", "errorx", "errory", "temporal_baseline"]].to_numpy(),
|
|
1434
|
+
data[["sensor", "author"]].to_numpy(),
|
|
1435
|
+
]
|
|
1436
|
+
|
|
1437
|
+
if verbose:
|
|
1438
|
+
stop.append(time.time())
|
|
1439
|
+
print(f"[Data loading] Loading the pixel took {round((stop[1] - start[1]), 4)} s")
|
|
1440
|
+
|
|
1441
|
+
# %% ======================================================================== #
|
|
1442
|
+
# INVERSION #
|
|
1443
|
+
# =========================================================================%% #
|
|
1444
|
+
|
|
1445
|
+
if verbose:
|
|
1446
|
+
start.append(time.time())
|
|
1447
|
+
|
|
1448
|
+
# Proceed to inversion
|
|
1449
|
+
A, result, dataf = inversion_core(data, i, j, dates_range=dates_range, mean=mean, **inversion_kwargs)
|
|
1450
|
+
|
|
1451
|
+
if verbose:
|
|
1452
|
+
stop.append(time.time())
|
|
1453
|
+
print(f"[Inversion] Inversion took {round((stop[2] - start[2]), 4)} s")
|
|
1454
|
+
if save:
|
|
1455
|
+
result.to_csv(f"{path_save}/ILF_result.csv")
|
|
1456
|
+
|
|
1457
|
+
# %% ======================================================================== #
|
|
1458
|
+
# INTERPOLATION #
|
|
1459
|
+
# =========================================================================%% #
|
|
1460
|
+
|
|
1461
|
+
if verbose:
|
|
1462
|
+
start.append(time.time())
|
|
1463
|
+
|
|
1464
|
+
# Proceed to interpolation
|
|
1465
|
+
dataf_lp = interpolation_core(result, **interpolation_kwargs)
|
|
1466
|
+
|
|
1467
|
+
if save:
|
|
1468
|
+
dataf_lp.to_csv(f"{path_save}/ILF_result.csv")
|
|
1469
|
+
|
|
1470
|
+
if verbose:
|
|
1471
|
+
stop.append(time.time())
|
|
1472
|
+
print(f"[Interpolation] Interpolation took {round((stop[3] - start[3]), 4)} s")
|
|
1473
|
+
|
|
1474
|
+
if save:
|
|
1475
|
+
dataf_lp.to_csv(f"{path_save}/RLF_result.csv")
|
|
1476
|
+
if show or save: # plot some figures
|
|
1477
|
+
visualization_core(
|
|
1478
|
+
[dataf, result],
|
|
1479
|
+
option_visual=option_visual,
|
|
1480
|
+
save=save,
|
|
1481
|
+
show=show,
|
|
1482
|
+
path_save=path_save,
|
|
1483
|
+
A=A,
|
|
1484
|
+
log_scale=False,
|
|
1485
|
+
cmap="rainbow",
|
|
1486
|
+
colors=["orange", "blue"],
|
|
1487
|
+
)
|
|
1488
|
+
visualisation_interpolation(
|
|
1489
|
+
[dataf, dataf_lp],
|
|
1490
|
+
option_visual=option_visual,
|
|
1491
|
+
save=save,
|
|
1492
|
+
show=show,
|
|
1493
|
+
path_save=path_save,
|
|
1494
|
+
colors=["orange", "blue"],
|
|
1495
|
+
)
|
|
1496
|
+
|
|
1497
|
+
if verbose:
|
|
1498
|
+
print(f"[Overall] Overall processing took {round((stop[3] - start[0]), 4)} s")
|
|
1499
|
+
|
|
1500
|
+
return data, dataf, dataf_lp
|