grid-process 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- grid_process/__init__.py +14 -0
- grid_process/aggreg.py +447 -0
- grid_process/blending.py +206 -0
- grid_process/config/__init__.py +5 -0
- grid_process/config/grid_process.yaml +20 -0
- grid_process/config/library_settings.py +68 -0
- grid_process/da_utils.py +104 -0
- grid_process/data_model.py +531 -0
- grid_process/ds_utils.py +61 -0
- grid_process/ens_stats.py +396 -0
- grid_process/grid_interp.py +2016 -0
- grid_process/grid_interp_utils.py +208 -0
- grid_process/helpers.py +174 -0
- grid_process/optimization.py +50 -0
- grid_process/sun.py +502 -0
- grid_process/time_utils.py +110 -0
- grid_process/timeseries.py +547 -0
- grid_process/units_conv.py +209 -0
- grid_process-1.0.0.dist-info/METADATA +155 -0
- grid_process-1.0.0.dist-info/RECORD +23 -0
- grid_process-1.0.0.dist-info/WHEEL +4 -0
- grid_process-1.0.0.dist-info/licenses/AUTHORS +11 -0
- grid_process-1.0.0.dist-info/licenses/LICENSE +29 -0
grid_process/__init__.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Copyright (c) 2026 MeteoSwiss, contributors listed in AUTHORS
|
|
2
|
+
#
|
|
3
|
+
# Distributed under the terms of the BSD 3-Clause License.
|
|
4
|
+
#
|
|
5
|
+
# SPDX-License-Identifier: BSD-3-Clause
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
grid-process provides functionalities to process xarray Datasets from weather
|
|
9
|
+
forecast grids, following configurable data models and conventions.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from grid_process.config.library_settings import LibrarySettings
|
|
13
|
+
|
|
14
|
+
CONFIG = LibrarySettings()
|
grid_process/aggreg.py
ADDED
|
@@ -0,0 +1,447 @@
|
|
|
1
|
+
# Copyright (c) 2026 MeteoSwiss, contributors listed in AUTHORS
|
|
2
|
+
#
|
|
3
|
+
# Distributed under the terms of the BSD 3-Clause License.
|
|
4
|
+
#
|
|
5
|
+
# SPDX-License-Identifier: BSD-3-Clause
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
Module to perform temporal aggregations
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import logging
|
|
12
|
+
from functools import partial
|
|
13
|
+
from typing import Any
|
|
14
|
+
from collections.abc import Callable, Iterable, Hashable
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
import pandas as pd
|
|
18
|
+
import xarray as xr
|
|
19
|
+
|
|
20
|
+
from grid_process.helpers import make_accessor, keepattrs
|
|
21
|
+
from grid_process.data_model import get_validtime
|
|
22
|
+
from grid_process.ds_utils import cast_dtype
|
|
23
|
+
from grid_process.timeseries import interptogranularity, InterpMethods
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# Register xarray accessor so module functions can also be called as methods
|
|
29
|
+
# on xr.Dataset objects
|
|
30
|
+
@xr.register_dataset_accessor("aggreg")
|
|
31
|
+
class Aggreg:
|
|
32
|
+
"""
|
|
33
|
+
Xarray accessors class for aggregations
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def __init__(self, ds: xr.Dataset):
|
|
37
|
+
self._data = ds
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
dataset_accessor = partial(make_accessor, accessor_class=Aggreg)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataset_accessor
|
|
44
|
+
def reshape4intervals_wtst(ds: xr.Dataset, granularity: int, interval: int, offset: int = 0) -> xr.Dataset:
|
|
45
|
+
"""
|
|
46
|
+
Wrapper for :func:`reshape4intervals` to reshape a dataset into intervals,
|
|
47
|
+
additionally handling local time conversion and producing interval
|
|
48
|
+
duplicates for summer and winter time.
|
|
49
|
+
|
|
50
|
+
Parameters
|
|
51
|
+
----------
|
|
52
|
+
ds : xarray.Dataset
|
|
53
|
+
dataset with a valid time dimension to be reshaped. Must contain one constant
|
|
54
|
+
time resolution
|
|
55
|
+
granularity : int
|
|
56
|
+
time in hours in between two consecutive interval starts
|
|
57
|
+
interval : int
|
|
58
|
+
time in hours of interval duration
|
|
59
|
+
offset : int, optional
|
|
60
|
+
time in hours by which interval starts should be shifted from 00:00 am.
|
|
61
|
+
E.g. if 6, intervals include the one starting at 6am (4 and 5 am UTC)
|
|
62
|
+
|
|
63
|
+
Returns
|
|
64
|
+
-------
|
|
65
|
+
ds_out : xarray.Dataset
|
|
66
|
+
reshaped dataset with dimensions valid time x `window`. Valid time contains
|
|
67
|
+
timestamps for respective interval, `window` has length of the interval
|
|
68
|
+
and contains all timesteps belonging to the interval
|
|
69
|
+
|
|
70
|
+
See also
|
|
71
|
+
--------
|
|
72
|
+
:func:`reshape4intervals`, :func:`interval_stat`
|
|
73
|
+
"""
|
|
74
|
+
if granularity > 1:
|
|
75
|
+
offset_utc_w = offset - 1
|
|
76
|
+
ds_w = reshape4intervals(ds, granularity, interval, offset_utc_w)
|
|
77
|
+
offset_utc_s = offset - 2
|
|
78
|
+
ds_s = reshape4intervals(ds, granularity, interval, offset_utc_s)
|
|
79
|
+
return xr.merge([ds_w, ds_s], combine_attrs="override", compat="no_conflicts", join="outer")
|
|
80
|
+
|
|
81
|
+
return reshape4intervals(ds, granularity, interval, 0)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataset_accessor
|
|
85
|
+
def reshape4intervals(ds: xr.Dataset, granularity: int, interval: int, offset: int = 0) -> xr.Dataset:
|
|
86
|
+
"""
|
|
87
|
+
Reshape a dataset into time intervals. The intervals are represented by
|
|
88
|
+
the `window` dimension.
|
|
89
|
+
|
|
90
|
+
Parameters
|
|
91
|
+
----------
|
|
92
|
+
ds : xarray.Dataset
|
|
93
|
+
dataset with valid time dimension to be reshaped. Must contain one constant
|
|
94
|
+
time resolution
|
|
95
|
+
granularity : int
|
|
96
|
+
time in hours in between two consecutive interval starts
|
|
97
|
+
interval : int
|
|
98
|
+
time in hours of interval duration
|
|
99
|
+
offset : int, optional
|
|
100
|
+
time in hours by which interval starts should be shifted from 00:00 am.
|
|
101
|
+
E.g. if 6, intervals include the one starting at 6am
|
|
102
|
+
|
|
103
|
+
Returns
|
|
104
|
+
-------
|
|
105
|
+
ds_out : xarray.Dataset
|
|
106
|
+
reshaped dataset with dimensions valid time x `window`. Valid time
|
|
107
|
+
contains timestamps for respective interval, `window` has length of the
|
|
108
|
+
interval and contains all timesteps belonging to the interval
|
|
109
|
+
|
|
110
|
+
See also
|
|
111
|
+
--------
|
|
112
|
+
:func:`reshape4intervals_wtst`, :func:`interval_stat`
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
t_dim = get_validtime(ds, raise_notfound=True)
|
|
116
|
+
|
|
117
|
+
try:
|
|
118
|
+
ds_ind = _get_valid_start(ds, granularity, offset, t_dim)
|
|
119
|
+
interval_ind, granularity_ind = _adjust_resolution(ds_ind, interval, granularity, t_dim)
|
|
120
|
+
|
|
121
|
+
indices = _get_intervalindices(ds_ind, interval_ind, granularity_ind, t_dim)
|
|
122
|
+
return ds_ind.isel({t_dim: indices}).drop_vars(t_dim).rename({"time2": t_dim})
|
|
123
|
+
|
|
124
|
+
except (IndexError, ValueError):
|
|
125
|
+
logger.debug("Not enough timestamps to cover at least one aggregation interval")
|
|
126
|
+
return ds.isel({t_dim: slice(None, 0)}).expand_dims("window")
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _get_valid_start(ds: xr.Dataset, granularity: int, offset: int, t_dim: str) -> xr.Dataset:
|
|
130
|
+
"""
|
|
131
|
+
Get valid starting timestamp for reshaping a dataset into intervals, based
|
|
132
|
+
on the granularity of the desired intervals, the offset and the initial
|
|
133
|
+
time resolution of the data (does not require hourly resolution, just
|
|
134
|
+
constant one throughout). Use to drop timestamps in the beginning belonging
|
|
135
|
+
to 'incomplete' intervals.
|
|
136
|
+
|
|
137
|
+
Parameters
|
|
138
|
+
----------
|
|
139
|
+
ds : xarray.Dataset
|
|
140
|
+
dataset to be reshaped into intervals. Must contain one constant time
|
|
141
|
+
resolution
|
|
142
|
+
granularity : int
|
|
143
|
+
time in hours in between two consecutive interval starts
|
|
144
|
+
offset : int
|
|
145
|
+
time in hours by which interval starts should be shifted from 00:00 am.
|
|
146
|
+
E.g. if 6, intervals include the one starting at 6am
|
|
147
|
+
t_dim : str
|
|
148
|
+
name of the valid time dimension
|
|
149
|
+
|
|
150
|
+
Returns
|
|
151
|
+
-------
|
|
152
|
+
ds_out : xarray.Dataset
|
|
153
|
+
dataset cut at the first valid time from which one interval can be formed
|
|
154
|
+
"""
|
|
155
|
+
|
|
156
|
+
time_res = ds[t_dim][1].values - ds[t_dim][0].values
|
|
157
|
+
offset_td = np.timedelta64(offset, "h")
|
|
158
|
+
start_t = pd.to_datetime(min(ds[t_dim].values)) - time_res - offset_td
|
|
159
|
+
start_t = start_t.ceil(str(granularity) + "h") + time_res + offset_td
|
|
160
|
+
|
|
161
|
+
return ds.sel({t_dim: slice(start_t, None)})
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _adjust_resolution(ds_ind: xr.Dataset, interval: int, granularity: int, t_dim: str) -> tuple[int, int]:
|
|
165
|
+
"""
|
|
166
|
+
Adjust indexing lengths of granularity and interval before reshaping a
|
|
167
|
+
dataset to required form for interval aggregations. This allow cases where
|
|
168
|
+
time resolution of a dataset is not 1hr (e.g. if desired interval is 3hrs
|
|
169
|
+
but time-resolution of the dataset is 10mins, index-length for aggregation
|
|
170
|
+
should be 18 instead of 3).
|
|
171
|
+
|
|
172
|
+
Parameters
|
|
173
|
+
----------
|
|
174
|
+
ds_ind : xarray.Dataset
|
|
175
|
+
dataset to be reshaped. Must contain one constant time resolution
|
|
176
|
+
interval : int
|
|
177
|
+
time in hours of interval duration
|
|
178
|
+
granularity : int
|
|
179
|
+
time in hours in between two consecutive interval starts
|
|
180
|
+
t_dim : str
|
|
181
|
+
name of the valid time dimension
|
|
182
|
+
|
|
183
|
+
Returns
|
|
184
|
+
-------
|
|
185
|
+
interval_ind : int
|
|
186
|
+
interval duration in amount of corresponding timesteps given the time
|
|
187
|
+
resolution of ds_ind
|
|
188
|
+
granularity_ind : int
|
|
189
|
+
amount of corresponding timesteps in between two consecutive intervals,
|
|
190
|
+
given the time resolution of ds_ind and the specified granularity
|
|
191
|
+
"""
|
|
192
|
+
|
|
193
|
+
time_res = ds_ind[t_dim][1].values - ds_ind[t_dim][0].values
|
|
194
|
+
interval_ind = np.timedelta64(interval, "h") / time_res
|
|
195
|
+
granularity_ind = np.timedelta64(granularity, "h") / time_res
|
|
196
|
+
|
|
197
|
+
if interval_ind % 1 != 0:
|
|
198
|
+
raise ValueError("Interval must be a multiple of the dataset resolution")
|
|
199
|
+
|
|
200
|
+
return int(interval_ind), int(granularity_ind)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _get_intervalindices(ds_ind: xr.Dataset, interval_ind: int, granularity_ind: int, t_dim: str) -> xr.DataArray:
|
|
204
|
+
"""
|
|
205
|
+
Get an array of indices referring to positions of timestamps to reshape
|
|
206
|
+
a dataset according to desired interval and granularity.
|
|
207
|
+
|
|
208
|
+
Parameters
|
|
209
|
+
----------
|
|
210
|
+
ds_ind : xarray.Dataset
|
|
211
|
+
dataset to be reshaped. Must contain one constant time-resolution and
|
|
212
|
+
be cut to correct start time
|
|
213
|
+
interval_ind : int
|
|
214
|
+
interval duration in amount of corresponding timesteps given the time
|
|
215
|
+
resolution of ds_ind
|
|
216
|
+
granularity_ind : int
|
|
217
|
+
amount of corresponding timesteps in between two consecutive intervals,
|
|
218
|
+
given the time resolution of ds_ind and specified granularity
|
|
219
|
+
t_dim : str
|
|
220
|
+
name of the valid time dimension
|
|
221
|
+
|
|
222
|
+
Returns
|
|
223
|
+
-------
|
|
224
|
+
da_ind : xarray.DataArray
|
|
225
|
+
array of indices for reshaping with dims `time2` x `window` (newly formed
|
|
226
|
+
intervals, labelled by their end-point) x (timesteps per interval)
|
|
227
|
+
"""
|
|
228
|
+
|
|
229
|
+
indices = np.stack(
|
|
230
|
+
[
|
|
231
|
+
np.arange(ind, ind + interval_ind)
|
|
232
|
+
for ind in np.arange(ds_ind.sizes[t_dim] - interval_ind + 1, step=granularity_ind)
|
|
233
|
+
]
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
coordinates = ds_ind[t_dim][indices[:, -1]].values
|
|
237
|
+
return xr.DataArray(indices, dims=("time2", "window"), coords={"time2": coordinates})
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def split_by_timeres(ds: xr.Dataset) -> list[xr.Dataset]:
|
|
241
|
+
"""
|
|
242
|
+
Split a dataset where the time granularity changes. In case several
|
|
243
|
+
switches of granularity are present, the split is done at the last
|
|
244
|
+
granularity switch.
|
|
245
|
+
|
|
246
|
+
Parameters
|
|
247
|
+
----------
|
|
248
|
+
ds : xarray.Dataset
|
|
249
|
+
dataset with valid time dimension and non constant time resolution
|
|
250
|
+
|
|
251
|
+
Returns
|
|
252
|
+
-------
|
|
253
|
+
out : list of two xarray.Dataset
|
|
254
|
+
First element contains the first part of the dataset (with potentially
|
|
255
|
+
another granularity switch hidden). Second element contains the last
|
|
256
|
+
part of the dataset with constant granularity. In case the dataset is
|
|
257
|
+
already at constant granularity or is of length 1 in valid time dimension,
|
|
258
|
+
the output list has length 1
|
|
259
|
+
"""
|
|
260
|
+
|
|
261
|
+
t_dim = get_validtime(ds, raise_notfound=True)
|
|
262
|
+
|
|
263
|
+
if ds.sizes[t_dim] <= 1:
|
|
264
|
+
return [ds]
|
|
265
|
+
|
|
266
|
+
time_delta = ds[t_dim].diff(t_dim).values
|
|
267
|
+
switch_gran = np.nonzero(time_delta != time_delta[-1])[0]
|
|
268
|
+
|
|
269
|
+
if switch_gran.size > 0:
|
|
270
|
+
switch = switch_gran[-1] + 1
|
|
271
|
+
return [
|
|
272
|
+
ds.isel({t_dim: slice(None, switch)}),
|
|
273
|
+
ds.isel({t_dim: slice(switch, None)}),
|
|
274
|
+
]
|
|
275
|
+
|
|
276
|
+
return [ds]
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
@dataset_accessor
|
|
280
|
+
@keepattrs
|
|
281
|
+
def interval_stat(
|
|
282
|
+
ds: xr.Dataset,
|
|
283
|
+
granularity: int,
|
|
284
|
+
interval: int,
|
|
285
|
+
offset: int,
|
|
286
|
+
agg_func: Callable,
|
|
287
|
+
lt_duplicates: bool = True,
|
|
288
|
+
interp_method: InterpMethods | None = None,
|
|
289
|
+
**agg_func_kwargs: Any,
|
|
290
|
+
) -> xr.Dataset:
|
|
291
|
+
"""
|
|
292
|
+
Aggregate values in time over a moving window defined by a time
|
|
293
|
+
``granularity``, ``interval`` and ``offset``. Aggregation is based on the
|
|
294
|
+
aggregating function ``agg_func``. The timeseries must have a granularity
|
|
295
|
+
of maximum one hour or an ``interp_method`` must be specified for prior
|
|
296
|
+
interpolation. The granularity does not need to be constant.
|
|
297
|
+
|
|
298
|
+
Parameters
|
|
299
|
+
----------
|
|
300
|
+
ds : xarray.Dataset
|
|
301
|
+
Must have a valid time dimension, potentially with non-constant time resolution.
|
|
302
|
+
Typical valid time dimension names are supported, and can be extended via environment variables:
|
|
303
|
+
|
|
304
|
+
- Valid time: export GRID_PROCESS__data_model__validtime (e.g., '["time", "valid_time", "dummy_time"]')
|
|
305
|
+
- When the coordinates variables have a CF compliant `standard_name` attribute, arbitrary names can be
|
|
306
|
+
used directly.
|
|
307
|
+
granularity : int
|
|
308
|
+
time in hours in between two consecutive interval starts
|
|
309
|
+
interval : int
|
|
310
|
+
time in hours of interval duration
|
|
311
|
+
offset : int
|
|
312
|
+
time in hours by which interval starts should be shifted from 00:00 am.
|
|
313
|
+
E.g. if 6, intervals include the one starting after 6am
|
|
314
|
+
agg_func : callable
|
|
315
|
+
specify temporal aggregation function to use. The function must operate
|
|
316
|
+
on `window` dimension. E.g. ``agg_func = lambda x: x.mean('window')``
|
|
317
|
+
lt_duplicates : bool, optional
|
|
318
|
+
if True, aggregations are done in the DWH compliant style, which means
|
|
319
|
+
intervals are calculated with respect to both UTC+1 and UTC+2. Default
|
|
320
|
+
is True
|
|
321
|
+
interp_method : str, optional
|
|
322
|
+
specify the temporal interpolation method to use if part of the
|
|
323
|
+
timeseries has a greater time resolution than one hour. Default is no
|
|
324
|
+
prior temporal interpolation (thus assuming the timeseries has max
|
|
325
|
+
granularity of 1hr). Must be a method accepted by
|
|
326
|
+
:func:`~grid_process.timeseries.interptogranularity`
|
|
327
|
+
**agg_func_kwargs : any
|
|
328
|
+
keyword arguments passed to the ``agg_func``
|
|
329
|
+
|
|
330
|
+
Returns
|
|
331
|
+
-------
|
|
332
|
+
ds_out : xarray.Dataset
|
|
333
|
+
dataset with aggregated values. Uncomplete intervals are exluded.
|
|
334
|
+
|
|
335
|
+
See also
|
|
336
|
+
--------
|
|
337
|
+
:func:`reshape4intervals_wtst`, :func:`reshape4intervals`,
|
|
338
|
+
:func:`~grid_process.timeseries.interptogranularity`
|
|
339
|
+
|
|
340
|
+
Note
|
|
341
|
+
----
|
|
342
|
+
Alternatively, one can use xarray.Dataset.resample in some cases,
|
|
343
|
+
especially when aggregating over uncomplete intervals is preferred.
|
|
344
|
+
However, this behaviour can be problematic for cumulative aggregations
|
|
345
|
+
(i.e. sum). Note that xarray.Dataset.resample has many more interesting
|
|
346
|
+
options than this function although it does not allow to specify an
|
|
347
|
+
interval and granularity separately.
|
|
348
|
+
"""
|
|
349
|
+
|
|
350
|
+
if interp_method is not None:
|
|
351
|
+
ds = interptogranularity(ds, "1h", interp_method)
|
|
352
|
+
|
|
353
|
+
list_ds = split_by_timeres(ds)
|
|
354
|
+
if len(list_ds) == 2:
|
|
355
|
+
list_ds = [
|
|
356
|
+
xr.merge(
|
|
357
|
+
[interval_stat(ds, 1, 1, 0, agg_func, False, **agg_func_kwargs) for ds in list_ds],
|
|
358
|
+
compat="no_conflicts",
|
|
359
|
+
join="outer",
|
|
360
|
+
)
|
|
361
|
+
]
|
|
362
|
+
|
|
363
|
+
func = reshape4intervals_wtst if lt_duplicates else reshape4intervals
|
|
364
|
+
ds_out = agg_func(func(list_ds[0], granularity, interval, offset), **agg_func_kwargs)
|
|
365
|
+
|
|
366
|
+
cast_dtype(ds_out, ds)
|
|
367
|
+
return ds_out
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
@dataset_accessor
|
|
371
|
+
def mode(reshaped_ds: xr.Dataset, ordered_mode: dict[Hashable, list[Any]] | None = None) -> xr.Dataset:
|
|
372
|
+
"""
|
|
373
|
+
Take the most frequent category (mode) among categorical variables along
|
|
374
|
+
`window` dimension. Could be used as input parameter ``agg_func`` for
|
|
375
|
+
:func:`interval_stat`.
|
|
376
|
+
|
|
377
|
+
Parameters
|
|
378
|
+
----------
|
|
379
|
+
reshaped_ds : xarray.Dataset
|
|
380
|
+
dataset with dimension `window` along which the mode is taken
|
|
381
|
+
ordered_mode : dict {varname: list[any]}, optional
|
|
382
|
+
ordered list of possible catergories for each data variable in the
|
|
383
|
+
input dataset. If set, the returned category will match the priority
|
|
384
|
+
order of this list in case of a tie between 2 or more categories
|
|
385
|
+
|
|
386
|
+
Returns
|
|
387
|
+
-------
|
|
388
|
+
ds_out : xarray.Dataset
|
|
389
|
+
dataset with `window` dimension reduced
|
|
390
|
+
|
|
391
|
+
See also
|
|
392
|
+
--------
|
|
393
|
+
:func:`interval_stat`, :func:`count`
|
|
394
|
+
"""
|
|
395
|
+
|
|
396
|
+
ds = []
|
|
397
|
+
for name, da in reshaped_ds.items():
|
|
398
|
+
categ = None if ordered_mode is None else ordered_mode[name]
|
|
399
|
+
counts = count(da, categ)
|
|
400
|
+
cat_arr = np.array(list(counts))
|
|
401
|
+
stacked_da = np.stack(list(counts.values()))
|
|
402
|
+
|
|
403
|
+
ds.append(
|
|
404
|
+
xr.DataArray(
|
|
405
|
+
cat_arr[np.argmax(stacked_da, axis=0)],
|
|
406
|
+
dims=[dim for dim in counts[cat_arr[0]].dims if dim != "window"],
|
|
407
|
+
)
|
|
408
|
+
.assign_coords(counts[cat_arr[0]].coords)
|
|
409
|
+
.rename(name)
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
return xr.merge(ds).assign_attrs(reshaped_ds.attrs)
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def count(reshaped_da: xr.DataArray, categ: Iterable | None = None) -> dict[Any, xr.DataArray]:
|
|
416
|
+
"""
|
|
417
|
+
Count the number of occurences along the `window` dimension for each
|
|
418
|
+
element present in the input DataArray or in the ``categ`` list.
|
|
419
|
+
|
|
420
|
+
Parameters
|
|
421
|
+
----------
|
|
422
|
+
reshaped_da : xarray.DataArray
|
|
423
|
+
array with dimension `window` along which the count is done
|
|
424
|
+
categ : Iterable, optional
|
|
425
|
+
ordered iterable of categories for which to count the occurences. If not
|
|
426
|
+
set, ``categ`` is the unique categories in the input DataArray
|
|
427
|
+
|
|
428
|
+
Returns
|
|
429
|
+
-------
|
|
430
|
+
counts : dict of xarray.DataArray
|
|
431
|
+
the keys are the different elements found in the array and the values
|
|
432
|
+
are arrays with the `window` dimension reduced and containing the
|
|
433
|
+
respective count
|
|
434
|
+
|
|
435
|
+
See also
|
|
436
|
+
--------
|
|
437
|
+
:func:`mode`
|
|
438
|
+
"""
|
|
439
|
+
|
|
440
|
+
counts = {}
|
|
441
|
+
|
|
442
|
+
if categ is None:
|
|
443
|
+
categ = np.unique(reshaped_da)
|
|
444
|
+
|
|
445
|
+
for ct in categ:
|
|
446
|
+
counts[ct] = (reshaped_da == ct).sum(dim="window")
|
|
447
|
+
return counts
|
grid_process/blending.py
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
# Copyright (c) 2026 MeteoSwiss, contributors listed in AUTHORS
|
|
2
|
+
#
|
|
3
|
+
# Distributed under the terms of the BSD 3-Clause License.
|
|
4
|
+
#
|
|
5
|
+
# SPDX-License-Identifier: BSD-3-Clause
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
Module for blending datasets together
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import logging
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
import pandas as pd
|
|
16
|
+
import xarray as xr
|
|
17
|
+
|
|
18
|
+
from grid_process.helpers import replace_vars_attrs
|
|
19
|
+
from grid_process.data_model import get_t_dim, get_validtime, get_reftime
|
|
20
|
+
from grid_process.units_conv import consistency_check
|
|
21
|
+
from grid_process.timeseries import (
|
|
22
|
+
setindex_validtime,
|
|
23
|
+
interptogranularity,
|
|
24
|
+
InterpMethods,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
logger = logging.getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def combine_models(datasets: Sequence[xr.Dataset], strict_start: bool = True) -> xr.Dataset:
|
|
31
|
+
"""
|
|
32
|
+
Combine datasets in valid time dimension. The new coordinates follow the normal
|
|
33
|
+
broadcasting and alignment rules of ``join='outer'``. Vacant cells in the
|
|
34
|
+
expanded coordinates are filled with numpy.nan .
|
|
35
|
+
|
|
36
|
+
Parameters
|
|
37
|
+
----------
|
|
38
|
+
datasets : sequence of xarray.Dataset
|
|
39
|
+
Datasets with valid time or lead time dimension to be combined. The sequence
|
|
40
|
+
must reflect priority order, so where several datasets overlap in time, the
|
|
41
|
+
data of the first one is kept
|
|
42
|
+
Typical temporal dimensions names are supported, and can be extended via environment variables:
|
|
43
|
+
|
|
44
|
+
- Valid time: export GRID_PROCESS__data_model__validtime (e.g., '["time", "valid_time", "dummy_time"]')
|
|
45
|
+
- Lead time: export GRID_PROCESS__data_model__leadtime (e.g., '["lead_time", "step", "dummy_leadtime"]')
|
|
46
|
+
- When the temporal coordinates variables have a CF compliant `standard_name` attribute, arbitrary names can
|
|
47
|
+
be used directly.
|
|
48
|
+
strict_start : bool, optional
|
|
49
|
+
If True (default), the resulting timeseries strictly starts with the
|
|
50
|
+
dataset having highest priority (i.e. no older timestamps from lower
|
|
51
|
+
priority datasets can be prepended)
|
|
52
|
+
|
|
53
|
+
Returns
|
|
54
|
+
-------
|
|
55
|
+
combined_ds : xarray.Dataset
|
|
56
|
+
contains the full timeseries resulting from the datasets combination.
|
|
57
|
+
Valid time dimension is labelled by numpy.datetime64 although
|
|
58
|
+
inputs could have lead time dimension (numpy.timedelta64)
|
|
59
|
+
|
|
60
|
+
See also
|
|
61
|
+
--------
|
|
62
|
+
xarray.Dataset.combine_first,
|
|
63
|
+
:func:`~grid_process.timeseries.setindex_validtime`
|
|
64
|
+
|
|
65
|
+
Note
|
|
66
|
+
----
|
|
67
|
+
This function is not aimed to combine models that do not have the same grid
|
|
68
|
+
and members, but to combine them after regridding/interpolation and ensemble
|
|
69
|
+
statistic computations (quantiles, probabilities). A typical use case for this
|
|
70
|
+
function could be to combine past deterministic analyses to all members of a NWP
|
|
71
|
+
model as it will automatically broadcast it to the ensemble members dimension and
|
|
72
|
+
pioritize analyses over the NWP forecast for common timestamps in both datasets.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
units_attrs = consistency_check(datasets, check_vars=False)
|
|
76
|
+
datasets = [setindex_validtime(ds) for ds in datasets if ds.sizes[get_t_dim(ds)]]
|
|
77
|
+
combined_attrs = {"source": blend_src_attr(datasets)}
|
|
78
|
+
combined_ds = datasets[0].copy()
|
|
79
|
+
combined_ds = combined_ds.drop_vars(get_reftime(combined_ds, "coord"), errors="ignore") # type: ignore[arg-type]
|
|
80
|
+
vt_dim = get_validtime(combined_ds)
|
|
81
|
+
vt_start = combined_ds[vt_dim][0].values if strict_start else None
|
|
82
|
+
|
|
83
|
+
for ds in datasets[1:]:
|
|
84
|
+
vt_dim_ds = get_validtime(ds)
|
|
85
|
+
if vt_dim_ds != vt_dim:
|
|
86
|
+
ds = ds.rename({vt_dim_ds: vt_dim})
|
|
87
|
+
if vt_start is not None:
|
|
88
|
+
ds = ds.sel({vt_dim: slice(vt_start, None)})
|
|
89
|
+
ds = ds.drop_vars(get_reftime(ds, "coord"), errors="ignore") # type: ignore[arg-type]
|
|
90
|
+
combined_ds = combined_ds.combine_first(ds)
|
|
91
|
+
|
|
92
|
+
replace_vars_attrs(combined_ds, units_attrs)
|
|
93
|
+
combined_ds.attrs = combined_attrs
|
|
94
|
+
|
|
95
|
+
return combined_ds
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def cleaver( # pylint: disable=too-many-locals
|
|
99
|
+
ds_tail: xr.Dataset,
|
|
100
|
+
ds_head: xr.Dataset,
|
|
101
|
+
h: int = 6,
|
|
102
|
+
interp_method: InterpMethods = "cubic",
|
|
103
|
+
) -> xr.Dataset:
|
|
104
|
+
"""
|
|
105
|
+
Combine two timeseries by applying a linear seamless transition from the
|
|
106
|
+
last values of the tail to the remaining of the head.
|
|
107
|
+
|
|
108
|
+
Parameters
|
|
109
|
+
----------
|
|
110
|
+
ds_tail : xarray.Dataset
|
|
111
|
+
dataset with a valid time or lead time dimension. Contains the tail of
|
|
112
|
+
the newly produced timeseries
|
|
113
|
+
ds_head : xarray.Dataset
|
|
114
|
+
dataset with a valid time or lead time dimension. Contains the head of
|
|
115
|
+
the newly produced timeseries
|
|
116
|
+
h : int, optional
|
|
117
|
+
number of hours over which the seamless transition is done. Default is
|
|
118
|
+
6
|
|
119
|
+
interp_method : str
|
|
120
|
+
arg passed directly to
|
|
121
|
+
:func:`~grid_process.timeseries.interptogranularity` in case the
|
|
122
|
+
tail and the head do not overlap. Default is 'cubic'
|
|
123
|
+
|
|
124
|
+
Returns
|
|
125
|
+
-------
|
|
126
|
+
seamless_ds : xarray.Dataset
|
|
127
|
+
newly produced seamless timeseries from the two input datasets. Has a
|
|
128
|
+
valid time dimension labelled by numpy.datetime64 although inputs could
|
|
129
|
+
have a lead time dimension (numpy.timedelta64)
|
|
130
|
+
|
|
131
|
+
See also
|
|
132
|
+
--------
|
|
133
|
+
:func:`~grid_process.timeseries.interptogranularity`
|
|
134
|
+
:func:`~grid_process.timeseries.setindex_validtime`
|
|
135
|
+
"""
|
|
136
|
+
|
|
137
|
+
units_attrs = consistency_check((ds_tail, ds_head))
|
|
138
|
+
combined_attrs = {"source": blend_src_attr([ds_tail, ds_head])}
|
|
139
|
+
ds_tail = setindex_validtime(ds_tail)
|
|
140
|
+
ds_tail = ds_tail.drop_vars(get_reftime(ds_tail, "coord"), errors="ignore") # type: ignore[arg-type]
|
|
141
|
+
ds_head = setindex_validtime(ds_head)
|
|
142
|
+
ds_head = ds_head.drop_vars(get_reftime(ds_head, "coord"), errors="ignore") # type: ignore[arg-type]
|
|
143
|
+
|
|
144
|
+
vt_dim = get_validtime(ds_tail)
|
|
145
|
+
vt_dim_head = get_validtime(ds_head)
|
|
146
|
+
|
|
147
|
+
if vt_dim_head != vt_dim:
|
|
148
|
+
ds_head = ds_head.rename({vt_dim_head: vt_dim})
|
|
149
|
+
|
|
150
|
+
overlap_t = ds_tail[vt_dim].where(ds_tail[vt_dim].isin(ds_head[vt_dim]), drop=True).values
|
|
151
|
+
|
|
152
|
+
if overlap_t.size == 0:
|
|
153
|
+
logger.warning(
|
|
154
|
+
"No seamless transition is performed as no overlapping timestamps was found between the tail and the head"
|
|
155
|
+
)
|
|
156
|
+
combined_ds = ds_tail.combine_first(ds_head)
|
|
157
|
+
combined_ds.attrs = combined_attrs
|
|
158
|
+
return interptogranularity(combined_ds, method=interp_method)
|
|
159
|
+
|
|
160
|
+
last_t = overlap_t.max()
|
|
161
|
+
h_td = np.timedelta64(h, "h")
|
|
162
|
+
incr = np.timedelta64(1, "s")
|
|
163
|
+
ds_clever = ds_head.sel({vt_dim: slice(last_t + incr, last_t + h_td - incr)})
|
|
164
|
+
factor = (1 - (ds_clever[vt_dim] - last_t) / h_td).astype(np.float32)
|
|
165
|
+
last_t_pd = pd.to_datetime(last_t) # hack for bug with current xarray version
|
|
166
|
+
|
|
167
|
+
delta_last_t = ds_tail.sel({vt_dim: last_t_pd}) - ds_head.sel({vt_dim: last_t_pd})
|
|
168
|
+
ds_clever += factor * delta_last_t
|
|
169
|
+
|
|
170
|
+
seamless_ds = xr.concat(
|
|
171
|
+
(
|
|
172
|
+
ds_tail.sel({vt_dim: slice(None, last_t)}),
|
|
173
|
+
ds_clever,
|
|
174
|
+
ds_head.sel({vt_dim: slice(last_t + h_td, None)}),
|
|
175
|
+
),
|
|
176
|
+
dim=vt_dim,
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
replace_vars_attrs(seamless_ds, units_attrs)
|
|
180
|
+
seamless_ds.attrs = combined_attrs
|
|
181
|
+
|
|
182
|
+
return seamless_ds
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def blend_src_attr(datasets: Sequence[xr.Dataset]) -> dict[str, str]:
|
|
186
|
+
"""
|
|
187
|
+
Returns a dict with keys being the sources of the input datasets and
|
|
188
|
+
values the corresponding reftimes. This dict can become the output
|
|
189
|
+
``source`` attribute when blending datasets together.
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
src_dict = {}
|
|
193
|
+
for ds in datasets:
|
|
194
|
+
reftime_coord = get_reftime(ds, "coord")
|
|
195
|
+
if isinstance(ds.source, dict):
|
|
196
|
+
src_dict.update(ds.source)
|
|
197
|
+
elif reftime_coord is not None:
|
|
198
|
+
if len(ds[reftime_coord].dims) == 0:
|
|
199
|
+
reftime = ds[reftime_coord].values
|
|
200
|
+
else:
|
|
201
|
+
reftime = ds[reftime_coord].values[-1]
|
|
202
|
+
src_dict.update({ds.source: pd.to_datetime(reftime).isoformat()})
|
|
203
|
+
elif ds.source not in src_dict:
|
|
204
|
+
src_dict.update({ds.source: None})
|
|
205
|
+
|
|
206
|
+
return src_dict
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
cache_size:
|
|
2
|
+
nn_lookup: 10
|
|
3
|
+
nn_lookup_regular: 10
|
|
4
|
+
reproject_poi: 6
|
|
5
|
+
reproject_grid: 6
|
|
6
|
+
|
|
7
|
+
data_model: # supported names
|
|
8
|
+
grid:
|
|
9
|
+
unstructured_dim: ["cell", "points"]
|
|
10
|
+
y: ["y", "latitude", "lat"]
|
|
11
|
+
x: ["x", "longitude", "lon"]
|
|
12
|
+
surf_alt: ["surface_altitude", "HSURF", "altitude"]
|
|
13
|
+
poi:
|
|
14
|
+
dim: "poi" # dimension name given to point of interests after interpolation
|
|
15
|
+
coord_from: "name" # geodataframe column to become coordinate labels
|
|
16
|
+
alt_from: ["height_masl", "height", "altitude", "elevation"] # geodataframe column that represents altitude in meters
|
|
17
|
+
validtime: ["time", "valid_time"]
|
|
18
|
+
leadtime: ["lead_time", "step"]
|
|
19
|
+
reftime: ["forecast_reference_time", "ref_time", "reftime"]
|
|
20
|
+
ens_dim: ["realization", "eps", "epsd_1", "epsm", "number"]
|