romda 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- romda/__init__.py +28 -0
- romda/bias_estimators/__init__.py +12 -0
- romda/bias_estimators/aux.py +526 -0
- romda/bias_estimators/bias.py +526 -0
- romda/bias_estimators/constantbias.py +105 -0
- romda/bias_estimators/esn.py +420 -0
- romda/data_assimilation.py +68 -0
- romda/estimators/__init__.py +71 -0
- romda/estimators/base.py +431 -0
- romda/estimators/deterministic.py +288 -0
- romda/estimators/ensembles.py +984 -0
- romda/estimators/inflation.py +49 -0
- romda/models/__init__.py +51 -0
- romda/models/data_driven/__init__.py +20 -0
- romda/models/data_driven/autoencoders/__init__.py +320 -0
- romda/models/data_driven/autoencoders/pod.py +492 -0
- romda/models/data_driven/autoencoders/pod_utils.py +576 -0
- romda/models/data_driven/esn.py +1212 -0
- romda/models/data_driven/esn_config.py +652 -0
- romda/models/data_driven/linear_model.py +212 -0
- romda/models/data_driven/pod_esn.py +581 -0
- romda/observations.py +778 -0
- romda/plotting/__init__.py +4 -0
- romda/plotting/annular.py +557 -0
- romda/plotting/cylinder.py +664 -0
- romda/plotting/plot_results.py +272 -0
- romda/plotting/pod.py +318 -0
- romda/plotting/utils_plotting.py +145 -0
- romda/utils.py +1085 -0
- romda-3.0.0.dist-info/METADATA +197 -0
- romda-3.0.0.dist-info/RECORD +34 -0
- romda-3.0.0.dist-info/WHEEL +5 -0
- romda-3.0.0.dist-info/licenses/LICENSE +21 -0
- romda-3.0.0.dist-info/top_level.txt +1 -0
romda/__init__.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""
|
|
2
|
+
romda — real-time reduced-order modelling and bias-aware data assimilation.
|
|
3
|
+
|
|
4
|
+
Main entry points
|
|
5
|
+
-----------------
|
|
6
|
+
- ``romda.models`` : physical and data-driven forecast models
|
|
7
|
+
- ``romda.estimators`` : Estimator hierarchy — EnKF, EnSRKF, rBA_EnKF,
|
|
8
|
+
the deterministic filters and the smoothers
|
|
9
|
+
- ``romda.bias_estimators`` : ESN, constant, drift-linear and no-bias estimators
|
|
10
|
+
- ``romda.observations`` : truth/observation generation and loading
|
|
11
|
+
|
|
12
|
+
The ML building blocks (POD / SPOD / autoencoders) live in
|
|
13
|
+
``romda.models.data_driven`` (``autoencoders``); the ESN reservoir core comes
|
|
14
|
+
from the external ``echostatenetwork`` package.
|
|
15
|
+
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
__version__ = "3.0.0"
|
|
19
|
+
|
|
20
|
+
from romda import bias_estimators, estimators, models
|
|
21
|
+
from romda.observations import Observations
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"models",
|
|
25
|
+
"bias_estimators",
|
|
26
|
+
"estimators",
|
|
27
|
+
"Observations",
|
|
28
|
+
]
|
|
@@ -0,0 +1,526 @@
|
|
|
1
|
+
"""Helpers for building, caching, and plotting the training dataset used by data-driven bias estimators (e.g. `ESN_bias`)."""
|
|
2
|
+
|
|
3
|
+
from typing import Optional, Union
|
|
4
|
+
|
|
5
|
+
import matplotlib.pyplot as plt
|
|
6
|
+
import numpy as np
|
|
7
|
+
from matplotlib.colors import Normalize
|
|
8
|
+
from romda.models import Model
|
|
9
|
+
from romda.observations import Observations
|
|
10
|
+
from romda.utils import check_valid_file, correlation, load_from_pickle_file, mean_vector_to_ensemble
|
|
11
|
+
from typeguard import typechecked
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@typechecked
|
|
15
|
+
def prepare_reference_data(reference_data: Union[Observations, list[Observations]],
|
|
16
|
+
minimum_training_steps: int) -> tuple[list[np.ndarray], list[np.ndarray]]:
|
|
17
|
+
"""Prepare reference data for training the bias model.
|
|
18
|
+
|
|
19
|
+
Extracts the raw observations and true values from the reference data, trimmed
|
|
20
|
+
to the trailing `minimum_training_steps` time steps.
|
|
21
|
+
|
|
22
|
+
Parameters
|
|
23
|
+
----------
|
|
24
|
+
reference_data : Observations or list of Observations
|
|
25
|
+
The reference data to prepare.
|
|
26
|
+
minimum_training_steps : int
|
|
27
|
+
Minimum number of training steps required; each returned array is trimmed to
|
|
28
|
+
this length.
|
|
29
|
+
|
|
30
|
+
Returns
|
|
31
|
+
-------
|
|
32
|
+
y_raw : list of ndarray, each shape (minimum_training_steps, Nq, 1)
|
|
33
|
+
Raw observations from the reference data.
|
|
34
|
+
y_true : list of ndarray, each shape (minimum_training_steps, Nq, 1)
|
|
35
|
+
True (noise-free) values from the reference data.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
if isinstance(reference_data, Observations):
|
|
40
|
+
reference_data = [reference_data]
|
|
41
|
+
|
|
42
|
+
y_raw, y_true = [], []
|
|
43
|
+
for rfd in reference_data:
|
|
44
|
+
yr, yt = rfd.y_raw.copy(), rfd.y_true.copy()
|
|
45
|
+
if yr.ndim == 2:
|
|
46
|
+
yr = yr[:, :, np.newaxis]
|
|
47
|
+
if yt.ndim == 2:
|
|
48
|
+
yt = yt[:, :, np.newaxis]
|
|
49
|
+
|
|
50
|
+
y_raw.append(yr[-minimum_training_steps:])
|
|
51
|
+
y_true.append(yt[-minimum_training_steps:])
|
|
52
|
+
|
|
53
|
+
return y_raw, y_true
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@typechecked
|
|
57
|
+
def correlate_data(y_L_model: np.ndarray,
|
|
58
|
+
y_raw: np.ndarray,
|
|
59
|
+
augment_len: int,
|
|
60
|
+
minimum_training_steps: int,
|
|
61
|
+
) -> np.ndarray:
|
|
62
|
+
|
|
63
|
+
"""Correlate model-generated data with raw observations to build a training set.
|
|
64
|
+
|
|
65
|
+
1. Computes the correlation between the model-generated data and the raw
|
|
66
|
+
observations.
|
|
67
|
+
2. Identifies the best lag for each observed variable.
|
|
68
|
+
3. Constructs a training dataset from the model data at these optimal lags,
|
|
69
|
+
optionally augmented with a mid-point and/or worst-lag sample.
|
|
70
|
+
|
|
71
|
+
Parameters
|
|
72
|
+
----------
|
|
73
|
+
y_L_model : ndarray, shape (Nt, Nq, L)
|
|
74
|
+
Model-generated data (`L` independent trajectories).
|
|
75
|
+
y_raw : ndarray, shape (Nt, Nq, 1)
|
|
76
|
+
Raw observations.
|
|
77
|
+
augment_len : int
|
|
78
|
+
Number of augmented samples per observed variable: 1 uses only the best
|
|
79
|
+
lag; 2 adds a mid-point lag; 3 also adds the worst lag.
|
|
80
|
+
minimum_training_steps : int
|
|
81
|
+
Minimum number of training steps required.
|
|
82
|
+
|
|
83
|
+
Returns
|
|
84
|
+
-------
|
|
85
|
+
ndarray, shape (minimum_training_steps, Nq, L * augment_len)
|
|
86
|
+
Training data assembled from the best- (and optionally mid-/worst-) lag
|
|
87
|
+
model trajectories.
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
Nt_min = minimum_training_steps
|
|
91
|
+
Nt, Nq, L = y_L_model.shape
|
|
92
|
+
N_corr = Nt - Nt_min
|
|
93
|
+
|
|
94
|
+
train_data_model = np.zeros([Nt_min, Nq, L * augment_len])
|
|
95
|
+
lags = np.linspace(start=0, stop=N_corr, num=N_corr, dtype=int)
|
|
96
|
+
|
|
97
|
+
# Use only the first Ncorr time steps for computation time and to ensure we have enough data for training after lagging
|
|
98
|
+
y_raw_c = y_raw[:N_corr, ..., 0] - np.mean(y_raw[:N_corr, ..., 0], axis=0, keepdims=True)
|
|
99
|
+
epsilon = 1e-8
|
|
100
|
+
y_raw_c /= (np.max(np.abs(y_raw_c), axis=0, keepdims=True) + epsilon)
|
|
101
|
+
|
|
102
|
+
shifted_y_model_list = [y_L_model[lag: N_corr + lag] for lag in range(N_corr)]
|
|
103
|
+
correlations = np.array([correlation(y_raw_c, yy) for yy in shifted_y_model_list])
|
|
104
|
+
|
|
105
|
+
for ii in range(L):
|
|
106
|
+
corr_values = correlations[:, ii]
|
|
107
|
+
best_lag = lags[np.argmax(corr_values)]
|
|
108
|
+
|
|
109
|
+
base_col = augment_len * ii
|
|
110
|
+
train_data_model[:, :, base_col] = y_L_model[best_lag:best_lag + Nt_min, :, ii]
|
|
111
|
+
|
|
112
|
+
if augment_len >= 2:
|
|
113
|
+
worst_lag = lags[np.argmin(corr_values)]
|
|
114
|
+
mid_lag = int(np.mean([best_lag, worst_lag]))
|
|
115
|
+
train_data_model[:, :, base_col + 1] = y_L_model[mid_lag:mid_lag + Nt_min, :, ii]
|
|
116
|
+
if augment_len >= 3:
|
|
117
|
+
train_data_model[:, :, base_col + 2] = y_L_model[worst_lag:worst_lag + Nt_min, :, ii]
|
|
118
|
+
|
|
119
|
+
return train_data_model
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@typechecked
|
|
123
|
+
def sample_model_states(rom: Model,
|
|
124
|
+
L: int,
|
|
125
|
+
minimum_training_steps: int,
|
|
126
|
+
std_phi: Optional[Union[float, np.ndarray]] = None,
|
|
127
|
+
std_alpha: Optional[Union[float, dict[str, Union[float, list[float], tuple[float, float]]]]] = None,
|
|
128
|
+
) -> np.ndarray:
|
|
129
|
+
"""Sample model states from the ROM to build a training dataset for the bias model.
|
|
130
|
+
|
|
131
|
+
1. Initializes the ROM with an ensemble of states (sampled initial conditions and,
|
|
132
|
+
if specified, parameters).
|
|
133
|
+
2. Integrates the ROM forward in time to generate model data.
|
|
134
|
+
3. Processes this data into a training dataset for the bias model.
|
|
135
|
+
|
|
136
|
+
Parameters
|
|
137
|
+
----------
|
|
138
|
+
rom : Model
|
|
139
|
+
The reduced-order model to sample states from.
|
|
140
|
+
L : int
|
|
141
|
+
Number of samples (ensemble members) to generate.
|
|
142
|
+
minimum_training_steps : int
|
|
143
|
+
Minimum number of time steps to integrate the ROM for, to generate enough
|
|
144
|
+
data for training the bias model.
|
|
145
|
+
std_phi : float, optional
|
|
146
|
+
Standard deviation for sampling the initial conditions. If None (default),
|
|
147
|
+
the standard deviation of the ROM's current state is used.
|
|
148
|
+
std_alpha : float or dict, optional
|
|
149
|
+
Standard deviation for sampling the parameters. A float is used as a
|
|
150
|
+
multiplier on the current-state standard deviation, giving a
|
|
151
|
+
``[mean - std_alpha*std, mean + std_alpha*std]`` range for each estimated
|
|
152
|
+
parameter; a dict should have parameter names as keys and per-parameter
|
|
153
|
+
standard deviations as values. If None (default), the range is taken from
|
|
154
|
+
the current state of the ROM.
|
|
155
|
+
|
|
156
|
+
Returns
|
|
157
|
+
-------
|
|
158
|
+
ndarray
|
|
159
|
+
Observable history of the (re-)integrated ROM ensemble, used as the model
|
|
160
|
+
data for training.
|
|
161
|
+
"""
|
|
162
|
+
|
|
163
|
+
model = rom.copy()
|
|
164
|
+
|
|
165
|
+
if std_phi is None:
|
|
166
|
+
std_phi = np.std(model.current_state[:model.Nphi, :], axis=-1)
|
|
167
|
+
|
|
168
|
+
if std_alpha is None:
|
|
169
|
+
std_alpha = {}
|
|
170
|
+
for i, key in enumerate(model.est_alpha):
|
|
171
|
+
param = model.current_state[model.Nphi + i, :]
|
|
172
|
+
std_alpha[key] = [min(param), max(param)]
|
|
173
|
+
elif isinstance(std_alpha, float):
|
|
174
|
+
# A scalar std_alpha is a multiplier on the ensemble standard deviation:
|
|
175
|
+
# build a [mean - const*std, mean + const*std] range for each estimated
|
|
176
|
+
# parameter from the current state of the ROM.
|
|
177
|
+
const = std_alpha
|
|
178
|
+
std_alpha = {}
|
|
179
|
+
for i, key in enumerate(model.est_alpha):
|
|
180
|
+
param = model.current_state[model.Nphi + i, :]
|
|
181
|
+
mean_param = np.mean(param)
|
|
182
|
+
std_alpha[key] = [mean_param - const * np.std(param), mean_param + const * np.std(param)]
|
|
183
|
+
|
|
184
|
+
assert std_phi is not None, "std_phi must be specified or computed from the model state."
|
|
185
|
+
assert std_alpha is not None, "std_alpha must be specified or computed from the model state."
|
|
186
|
+
|
|
187
|
+
def sample_ensemble(psi0_mean, ensemble_size):
|
|
188
|
+
new_phi = mean_vector_to_ensemble(
|
|
189
|
+
rng=model.rng,
|
|
190
|
+
mean_vec=psi0_mean[:model.Nphi],
|
|
191
|
+
std=std_phi,
|
|
192
|
+
m=ensemble_size,
|
|
193
|
+
method='uniform',
|
|
194
|
+
)
|
|
195
|
+
if std_alpha:
|
|
196
|
+
|
|
197
|
+
new_alpha = mean_vector_to_ensemble(
|
|
198
|
+
rng=model.rng,
|
|
199
|
+
mean_vec=psi0_mean[model.Nphi:model.Nphi + model.Na],
|
|
200
|
+
std=std_alpha,
|
|
201
|
+
m=ensemble_size,
|
|
202
|
+
method='uniform',
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
return np.concatenate([new_phi, new_alpha], axis=0)
|
|
206
|
+
else:
|
|
207
|
+
return new_phi
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
psi0 = np.mean(model.current_state.copy(), axis=-1)
|
|
211
|
+
Nt = int(np.round(model.t_transient / model.dt, model.precision_t)) - 1
|
|
212
|
+
|
|
213
|
+
# Add parameters to the state vector if parameter uncertainty is given
|
|
214
|
+
if std_alpha and psi0.shape[0] == model.Nphi:
|
|
215
|
+
assert isinstance(std_alpha, dict), "std_alpha must be a dict if parameter uncertainty is specified."
|
|
216
|
+
model.ensemble_cfg = dict(Na=len(std_alpha), est_alpha=list(std_alpha.keys()), m=L)
|
|
217
|
+
psi0 = np.hstack([psi0, np.zeros((len(std_alpha),))])
|
|
218
|
+
for ii, key in enumerate(std_alpha.keys()):
|
|
219
|
+
psi0[model.Nphi + ii] = getattr(model, key)
|
|
220
|
+
psi0_ens = sample_ensemble(psi0, L)
|
|
221
|
+
model.update_history(psi=psi0_ens[np.newaxis, :, :], t=0.0, reset=True)
|
|
222
|
+
|
|
223
|
+
elif model.m != L:
|
|
224
|
+
psi0_ens = sample_ensemble(psi0, L)
|
|
225
|
+
model.update_history(psi=psi0_ens[np.newaxis, :, :], t=0.0, reset=True)
|
|
226
|
+
|
|
227
|
+
psi, t = model.time_integrate(Nt=Nt)
|
|
228
|
+
model.update_history(psi=psi, t=t, reset=True)
|
|
229
|
+
|
|
230
|
+
y_L_model = model.get_observable_hist()
|
|
231
|
+
psi_last = psi[-1, :, :]
|
|
232
|
+
|
|
233
|
+
tol = 1e-1
|
|
234
|
+
N_CR = int(round(model.t_CR / model.dt))
|
|
235
|
+
range_y = np.max(np.max(y_L_model[-N_CR:], axis=0) - np.min(y_L_model[-N_CR:], axis=0), axis=0)
|
|
236
|
+
idx_fixed = range_y < tol
|
|
237
|
+
|
|
238
|
+
if len(np.flatnonzero(idx_fixed)) / len(idx_fixed) >= 0.2:
|
|
239
|
+
idx_fixed[np.flatnonzero(idx_fixed)[0]] = 0
|
|
240
|
+
psi0 = psi_last[:, ~idx_fixed]
|
|
241
|
+
new_psi0 = sample_ensemble(np.mean(psi0, axis=-1), len(np.flatnonzero(idx_fixed)))
|
|
242
|
+
psi0 = np.concatenate([psi0, new_psi0], axis=-1)
|
|
243
|
+
else:
|
|
244
|
+
psi0 = psi_last
|
|
245
|
+
|
|
246
|
+
model.update_history(psi=psi0[np.newaxis, :, :], reset=True)
|
|
247
|
+
psi, t = model.time_integrate(Nt=minimum_training_steps + N_CR)
|
|
248
|
+
model.update_history(psi=psi, t=t, reset=True)
|
|
249
|
+
model.close()
|
|
250
|
+
|
|
251
|
+
return model.get_observable_hist()
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def load_bias_training_dataset(
|
|
255
|
+
filename,
|
|
256
|
+
necessary_properties: dict,
|
|
257
|
+
minimum_training_steps: int,
|
|
258
|
+
augment_data_length: int,
|
|
259
|
+
L: int,
|
|
260
|
+
expected_Ndim: Optional[int] = None,
|
|
261
|
+
):
|
|
262
|
+
"""Load a cached bias-training dataset from disk, if valid.
|
|
263
|
+
|
|
264
|
+
Returns None (triggering a fresh dataset build) if `filename` is None, the file
|
|
265
|
+
is missing/invalid, its configuration doesn't match `necessary_properties`, or
|
|
266
|
+
its shape doesn't match `minimum_training_steps` / `augment_data_length` / `L` /
|
|
267
|
+
`expected_Ndim`.
|
|
268
|
+
|
|
269
|
+
Parameters
|
|
270
|
+
----------
|
|
271
|
+
filename : str or None
|
|
272
|
+
Path to the cached training-dataset pickle file.
|
|
273
|
+
necessary_properties : dict
|
|
274
|
+
Configuration the cached dataset must match (see `check_valid_file`).
|
|
275
|
+
minimum_training_steps : int
|
|
276
|
+
Minimum number of time steps the cached data must contain.
|
|
277
|
+
augment_data_length : int
|
|
278
|
+
Expected augmentation factor of the cached data.
|
|
279
|
+
L : int
|
|
280
|
+
Expected number of samples of the cached data.
|
|
281
|
+
expected_Ndim : int, optional
|
|
282
|
+
Expected trailing (observable) dimension of the cached data.
|
|
283
|
+
|
|
284
|
+
Returns
|
|
285
|
+
-------
|
|
286
|
+
dict or None
|
|
287
|
+
The cached training-data dictionary, or None if unavailable/invalid.
|
|
288
|
+
"""
|
|
289
|
+
if filename is None:
|
|
290
|
+
return None
|
|
291
|
+
|
|
292
|
+
try: #Check if file exists...
|
|
293
|
+
loaded_train_data = load_from_pickle_file(filename)
|
|
294
|
+
except FileNotFoundError or AssertionError:
|
|
295
|
+
print(f'Run multi-parameter training data: file {filename} not found or does not contain a dictionary')
|
|
296
|
+
return None
|
|
297
|
+
|
|
298
|
+
#Check if loaded file is valid...
|
|
299
|
+
if not isinstance(loaded_train_data, dict):
|
|
300
|
+
print(f'Loaded file is invalid: {type(loaded_train_data)}: {loaded_train_data}')
|
|
301
|
+
return None
|
|
302
|
+
elif not check_valid_file(loaded_train_data, necessary_properties):
|
|
303
|
+
return None
|
|
304
|
+
else:
|
|
305
|
+
# Check if the data has enough time steps and matches the expected shape based on augment_data_length and L
|
|
306
|
+
data = loaded_train_data['data']
|
|
307
|
+
if data.shape[1] < minimum_training_steps:
|
|
308
|
+
print('Re-run multi-parameter training data: Increase the length of the training data')
|
|
309
|
+
return None
|
|
310
|
+
if augment_data_length > 1 and data.shape[0] != L * augment_data_length:
|
|
311
|
+
print('Re-run multi-parameter training data: augment_data_length does not match the number of samples in the loaded training data')
|
|
312
|
+
return None
|
|
313
|
+
if expected_Ndim is not None and data.shape[-1] != expected_Ndim:
|
|
314
|
+
print(f'Re-run multi-parameter training data: cached output dimension {data.shape[-1]} does not match expected {expected_Ndim}')
|
|
315
|
+
return None
|
|
316
|
+
|
|
317
|
+
print('OK: Loaded training dataset for bias model.')
|
|
318
|
+
return loaded_train_data
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def create_bias_training_dataset(config: dict,
|
|
323
|
+
rom: Model,
|
|
324
|
+
reference_data,
|
|
325
|
+
minimum_training_steps: int,
|
|
326
|
+
L: int,
|
|
327
|
+
correlation_based_training: bool,
|
|
328
|
+
augment_data_length: int,
|
|
329
|
+
biased_observations: bool,
|
|
330
|
+
std_phi: Optional[float] = None,
|
|
331
|
+
std_alpha: Optional[Union[float, dict[str, Union[float, list[float]]]]] = None,
|
|
332
|
+
) -> dict:
|
|
333
|
+
|
|
334
|
+
"""Build a training dataset for the bias model from ROM samples and reference data.
|
|
335
|
+
|
|
336
|
+
Parameters
|
|
337
|
+
----------
|
|
338
|
+
config : dict
|
|
339
|
+
Configuration for creating the training dataset; echoed into the returned
|
|
340
|
+
dictionary.
|
|
341
|
+
rom : Model
|
|
342
|
+
The reduced-order model to sample states from.
|
|
343
|
+
reference_data : Observations or list of Observations
|
|
344
|
+
Reference data to prepare for training.
|
|
345
|
+
minimum_training_steps : int
|
|
346
|
+
Minimum number of time steps required for training the bias model.
|
|
347
|
+
L : int
|
|
348
|
+
Number of samples to generate from the ROM.
|
|
349
|
+
correlation_based_training : bool
|
|
350
|
+
If True, correlate the model-generated data with the raw observations (via
|
|
351
|
+
`correlate_data`) rather than pairing them directly at matching time steps.
|
|
352
|
+
augment_data_length : int
|
|
353
|
+
Number of augmented samples per observed variable: 1 uses only the best
|
|
354
|
+
lag/direct pairing; 2 adds a mid-point (or scaled) sample; 3 also adds the
|
|
355
|
+
worst lag (or oppositely-scaled) sample.
|
|
356
|
+
biased_observations : bool
|
|
357
|
+
Whether to also include the model bias (true minus model-generated data) in
|
|
358
|
+
the training dataset, alongside the innovations.
|
|
359
|
+
std_phi : float, optional
|
|
360
|
+
Standard deviation for sampling initial conditions; see `sample_model_states`.
|
|
361
|
+
std_alpha : float or dict, optional
|
|
362
|
+
Standard deviation for sampling parameters; see `sample_model_states`.
|
|
363
|
+
|
|
364
|
+
Returns
|
|
365
|
+
-------
|
|
366
|
+
dict
|
|
367
|
+
Training data for the bias model, plus the entries of `config`:
|
|
368
|
+
|
|
369
|
+
- ``'data'`` : ndarray, shape (L * augment_data_length, minimum_training_steps, Ndim)
|
|
370
|
+
where ``Ndim = Nq`` if not `biased_observations`, else ``2 * Nq``.
|
|
371
|
+
- ``'y_model'`` : ndarray, shape (L, minimum_training_steps, Nq) — the
|
|
372
|
+
model-generated data used for training.
|
|
373
|
+
"""
|
|
374
|
+
|
|
375
|
+
y_model_L = sample_model_states(rom=rom,
|
|
376
|
+
L=L,
|
|
377
|
+
minimum_training_steps=minimum_training_steps,
|
|
378
|
+
std_phi=std_phi,
|
|
379
|
+
std_alpha=std_alpha)
|
|
380
|
+
|
|
381
|
+
print('\n\n Preparing reference data for training...')
|
|
382
|
+
print('y_model_L shape:', y_model_L.shape)
|
|
383
|
+
|
|
384
|
+
y_raw, y_true = prepare_reference_data(reference_data, minimum_training_steps=minimum_training_steps)
|
|
385
|
+
augment = augment_data_length > 1
|
|
386
|
+
|
|
387
|
+
if not correlation_based_training:
|
|
388
|
+
innovations_all, model_bias_all = [], []
|
|
389
|
+
for yr, yt in zip(y_raw, y_true):
|
|
390
|
+
innovations = (yr - y_model_L[-minimum_training_steps:]).transpose((2, 0, 1))
|
|
391
|
+
innovations_all.append(innovations)
|
|
392
|
+
|
|
393
|
+
if augment:
|
|
394
|
+
innovations_all.append(innovations * 1e-1)
|
|
395
|
+
innovations_all.append(innovations * -1e-2)
|
|
396
|
+
|
|
397
|
+
if biased_observations:
|
|
398
|
+
model_bias = (yt - y_model_L[-minimum_training_steps:]).transpose((2, 0, 1))
|
|
399
|
+
model_bias_all.append(model_bias)
|
|
400
|
+
if augment:
|
|
401
|
+
model_bias_all.append(model_bias * 1e-1)
|
|
402
|
+
model_bias_all.append(model_bias * -1e-2)
|
|
403
|
+
else:
|
|
404
|
+
innovations_all, model_bias_all = [], []
|
|
405
|
+
y_model = []
|
|
406
|
+
for yr, yt in zip(y_raw, y_true):
|
|
407
|
+
ym_L = correlate_data(y_model_L, yr, augment_data_length, minimum_training_steps)
|
|
408
|
+
y_model.append(ym_L.copy())
|
|
409
|
+
|
|
410
|
+
innovations = (yr - ym_L).transpose((2, 0, 1)) #shape (L, minimum_training_steps, Nq)
|
|
411
|
+
innovations_all.append(innovations)
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
if biased_observations:
|
|
415
|
+
model_bias = (yt - ym_L).transpose((2, 0, 1))
|
|
416
|
+
model_bias_all.append(model_bias)
|
|
417
|
+
y_model_L = np.concatenate(y_model, axis=0)
|
|
418
|
+
|
|
419
|
+
if not biased_observations:
|
|
420
|
+
train_data = np.concatenate(innovations_all, axis=0)
|
|
421
|
+
else:
|
|
422
|
+
innovations_all = np.concatenate(innovations_all, axis=0)
|
|
423
|
+
model_bias_all = np.concatenate(model_bias_all, axis=0)
|
|
424
|
+
train_data = np.concatenate([model_bias_all, innovations_all], axis=2)
|
|
425
|
+
# Save to dictionary
|
|
426
|
+
train_data_dict = {key: val for key, val in config.items()}
|
|
427
|
+
train_data_dict.update(data=train_data,
|
|
428
|
+
y_model=y_model_L)
|
|
429
|
+
return train_data_dict
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def plot_train_data(truth, bias_data, t_CR):
|
|
435
|
+
"""Plot the observable and bias training samples against the truth.
|
|
436
|
+
|
|
437
|
+
Shows one window before the first observation, colouring each training sample
|
|
438
|
+
(of `bias_data`) by its bias RMS.
|
|
439
|
+
|
|
440
|
+
Parameters
|
|
441
|
+
----------
|
|
442
|
+
truth : Observations
|
|
443
|
+
Reference truth used to select the plotting window and overlay the true
|
|
444
|
+
observable/bias signals.
|
|
445
|
+
bias_data : dict
|
|
446
|
+
Training-data dictionary as returned by `create_bias_training_dataset`.
|
|
447
|
+
t_CR : float
|
|
448
|
+
Characteristic (e.g. oscillation) time scale, used to set the plotting
|
|
449
|
+
window length.
|
|
450
|
+
"""
|
|
451
|
+
L, _, _ = bias_data['data'].shape
|
|
452
|
+
|
|
453
|
+
Nt = int(t_CR / truth.dt)
|
|
454
|
+
i0_t = np.argmin(np.abs(truth.t_true - truth.t_obs[0]))
|
|
455
|
+
|
|
456
|
+
# Build a common valid time window and select the segment before first observation.
|
|
457
|
+
n_common = min(
|
|
458
|
+
len(truth.t_true),
|
|
459
|
+
truth.y_true.shape[0],
|
|
460
|
+
truth.b_true.shape[0],
|
|
461
|
+
bias_data['y_model'].shape[0],
|
|
462
|
+
bias_data['data'].shape[1],
|
|
463
|
+
)
|
|
464
|
+
i_end = min(max(i0_t, 1), n_common)
|
|
465
|
+
i_start = max(i_end - Nt, 0)
|
|
466
|
+
if i_end - i_start < 2:
|
|
467
|
+
i_end = n_common
|
|
468
|
+
i_start = max(i_end - Nt, 0)
|
|
469
|
+
|
|
470
|
+
yt = truth.y_true[i_start:i_end]
|
|
471
|
+
bt = truth.b_true[i_start:i_end]
|
|
472
|
+
yr = bias_data['y_model'][i_start:i_end].transpose(2, 0, 1)
|
|
473
|
+
Nq = yt.shape[1]
|
|
474
|
+
br = bias_data['data'][:, i_start:i_end, :Nq]
|
|
475
|
+
tt = truth.t_true[i_start:i_end]
|
|
476
|
+
|
|
477
|
+
if len(tt) == 0:
|
|
478
|
+
raise ValueError('Selected plotting window is empty. Check t_CR and training_data dimensions.')
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
RS = []
|
|
482
|
+
for ii in range(L):
|
|
483
|
+
RS.append(np.linalg.norm(br[ii][:, 0]) / np.sqrt(len(yt)))
|
|
484
|
+
|
|
485
|
+
RS = np.asarray(RS, dtype=float)
|
|
486
|
+
true_RMS = np.linalg.norm(bt[:, 0]) / np.sqrt(len(yt))
|
|
487
|
+
|
|
488
|
+
# Plot training data (single row) --------------------------
|
|
489
|
+
fig = plt.figure(figsize =[12, 2.7], layout='constrained')
|
|
490
|
+
axs = fig.subplots(1, 2)
|
|
491
|
+
|
|
492
|
+
# Robust color mapping: clip outliers; if RMS are nearly equal, force distinct member colors.
|
|
493
|
+
if np.ptp(RS) < 1e-12:
|
|
494
|
+
color_values = np.linspace(0.0, 1.0, L)
|
|
495
|
+
norm = Normalize(vmin=0.0, vmax=1.0)
|
|
496
|
+
cmap = plt.cm.ScalarMappable(norm=norm, cmap=plt.get_cmap('viridis'))
|
|
497
|
+
cbar_extend = 'neither'
|
|
498
|
+
cbar_title = 'Member'
|
|
499
|
+
else:
|
|
500
|
+
lo, hi = np.percentile(RS, [5, 95])
|
|
501
|
+
if np.isclose(lo, hi):
|
|
502
|
+
lo = float(np.min(RS))
|
|
503
|
+
hi = float(np.max(RS))
|
|
504
|
+
color_values = np.clip(RS, lo, hi)
|
|
505
|
+
norm = Normalize(vmin=float(lo), vmax=float(hi))
|
|
506
|
+
cmap = plt.cm.ScalarMappable(norm=norm, cmap=plt.get_cmap('viridis'))
|
|
507
|
+
cbar_extend = 'both'
|
|
508
|
+
cbar_title = '$\\mathrm{RMS}$'
|
|
509
|
+
|
|
510
|
+
xlim = [tt[0], tt[-1]]
|
|
511
|
+
|
|
512
|
+
axs[0].plot(tt, yt[:, 0], color='silver', linewidth=6, alpha=.8)
|
|
513
|
+
axs[1].plot(tt, bt[:, 0], color='silver', linewidth=4, alpha=.8)
|
|
514
|
+
|
|
515
|
+
for ii in range(L):
|
|
516
|
+
clr = cmap.to_rgba(color_values[ii])
|
|
517
|
+
axs[0].plot(tt, yr[ii][:, 0], color=clr)
|
|
518
|
+
axs[1].plot(tt, br[ii][:, 0], color=clr)
|
|
519
|
+
|
|
520
|
+
axs[0].legend(['Truth'], bbox_to_anchor=(0., 0.25), loc='upper left')
|
|
521
|
+
axs[1].legend([f'True RMS $={true_RMS:.3f}$'], bbox_to_anchor=(0., 0.25), loc='upper left')
|
|
522
|
+
axs[0].set(xlabel='$t$', ylabel='$\\eta$', xlim=xlim)
|
|
523
|
+
axs[1].set(xlabel='$t$', ylabel='$b$', xlim=xlim)
|
|
524
|
+
|
|
525
|
+
clb = fig.colorbar(cmap, ax=axs, orientation='vertical', extend=cbar_extend)
|
|
526
|
+
clb.ax.set_title(cbar_title)
|