disdrodb 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +68 -34
- disdrodb/_config.py +5 -4
- disdrodb/_version.py +16 -3
- disdrodb/accessor/__init__.py +20 -0
- disdrodb/accessor/methods.py +125 -0
- disdrodb/api/checks.py +177 -24
- disdrodb/api/configs.py +3 -3
- disdrodb/api/info.py +13 -13
- disdrodb/api/io.py +281 -22
- disdrodb/api/path.py +184 -195
- disdrodb/api/search.py +18 -9
- disdrodb/cli/disdrodb_create_summary.py +103 -0
- disdrodb/cli/disdrodb_create_summary_station.py +91 -0
- disdrodb/cli/disdrodb_run_l0.py +1 -1
- disdrodb/cli/disdrodb_run_l0_station.py +1 -1
- disdrodb/cli/disdrodb_run_l0a_station.py +1 -1
- disdrodb/cli/disdrodb_run_l0b.py +1 -1
- disdrodb/cli/disdrodb_run_l0b_station.py +3 -3
- disdrodb/cli/disdrodb_run_l0c.py +1 -1
- disdrodb/cli/disdrodb_run_l0c_station.py +3 -3
- disdrodb/cli/disdrodb_run_l1_station.py +2 -2
- disdrodb/cli/disdrodb_run_l2e_station.py +2 -2
- disdrodb/cli/disdrodb_run_l2m_station.py +2 -2
- disdrodb/configs.py +149 -4
- disdrodb/constants.py +61 -0
- disdrodb/data_transfer/download_data.py +127 -11
- disdrodb/etc/configs/attributes.yaml +339 -0
- disdrodb/etc/configs/encodings.yaml +473 -0
- disdrodb/etc/products/L1/global.yaml +13 -0
- disdrodb/etc/products/L2E/10MIN.yaml +12 -0
- disdrodb/etc/products/L2E/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/global.yaml +22 -0
- disdrodb/etc/products/L2M/10MIN.yaml +12 -0
- disdrodb/etc/products/L2M/GAMMA_ML.yaml +8 -0
- disdrodb/etc/products/L2M/NGAMMA_GS_LOG_ND_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/NGAMMA_GS_ND_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/NGAMMA_GS_Z_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/global.yaml +26 -0
- disdrodb/issue/writer.py +2 -0
- disdrodb/l0/__init__.py +13 -0
- disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +4 -4
- disdrodb/l0/configs/PARSIVEL/l0b_cf_attrs.yml +1 -1
- disdrodb/l0/configs/PARSIVEL/l0b_encodings.yml +3 -3
- disdrodb/l0/configs/PARSIVEL/raw_data_format.yml +1 -1
- disdrodb/l0/configs/PARSIVEL2/l0b_cf_attrs.yml +5 -5
- disdrodb/l0/configs/PARSIVEL2/l0b_encodings.yml +3 -3
- disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml +1 -1
- disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +4 -4
- disdrodb/l0/configs/PWS100/raw_data_format.yml +1 -1
- disdrodb/l0/l0a_processing.py +37 -32
- disdrodb/l0/l0b_nc_processing.py +118 -8
- disdrodb/l0/l0b_processing.py +30 -65
- disdrodb/l0/l0c_processing.py +369 -259
- disdrodb/l0/readers/LPM/ARM/ARM_LPM.py +7 -0
- disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_LPM_NC.py +66 -0
- disdrodb/l0/readers/LPM/SLOVENIA/{CRNI_VRH.py → UL.py} +3 -0
- disdrodb/l0/readers/LPM/SWITZERLAND/INNERERIZ_LPM.py +195 -0
- disdrodb/l0/readers/PARSIVEL/GPM/PIERS.py +0 -2
- disdrodb/l0/readers/PARSIVEL/JAPAN/JMA.py +4 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/ARM/ARM_PARSIVEL2.py +4 -0
- disdrodb/l0/readers/PARSIVEL2/BELGIUM/ILVO.py +168 -0
- disdrodb/l0/readers/PARSIVEL2/CANADA/UQAM_NC.py +69 -0
- disdrodb/l0/readers/PARSIVEL2/DENMARK/DTU.py +165 -0
- disdrodb/l0/readers/PARSIVEL2/FINLAND/FMI_PARSIVEL2.py +69 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +255 -134
- disdrodb/l0/readers/PARSIVEL2/FRANCE/OSUG.py +525 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +9 -7
- disdrodb/l0/readers/PARSIVEL2/KIT/BURKINA_FASO.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/KIT/TEAMX.py +123 -0
- disdrodb/l0/readers/PARSIVEL2/{NETHERLANDS/DELFT.py → MPI/BCO_PARSIVEL2.py} +41 -71
- disdrodb/l0/readers/PARSIVEL2/MPI/BOWTIE.py +220 -0
- disdrodb/l0/readers/PARSIVEL2/NASA/APU.py +120 -0
- disdrodb/l0/readers/PARSIVEL2/NASA/LPVEX.py +109 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/FARM_PARSIVEL2.py +1 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_MIPS.py +126 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_PIPS.py +165 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +20 -12
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT_NC.py +5 -0
- disdrodb/l0/readers/PARSIVEL2/SPAIN/CENER.py +144 -0
- disdrodb/l0/readers/PARSIVEL2/SPAIN/CR1000DL.py +201 -0
- disdrodb/l0/readers/PARSIVEL2/SPAIN/LIAISE.py +137 -0
- disdrodb/l0/readers/PARSIVEL2/USA/C3WE.py +146 -0
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +105 -99
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100_SIRTA.py +151 -0
- disdrodb/l1/__init__.py +5 -0
- disdrodb/l1/fall_velocity.py +46 -0
- disdrodb/l1/filters.py +34 -20
- disdrodb/l1/processing.py +46 -45
- disdrodb/l1/resampling.py +77 -66
- disdrodb/l1_env/routines.py +18 -3
- disdrodb/l2/__init__.py +7 -0
- disdrodb/l2/empirical_dsd.py +58 -10
- disdrodb/l2/processing.py +268 -117
- disdrodb/metadata/checks.py +132 -125
- disdrodb/metadata/standards.py +3 -1
- disdrodb/psd/fitting.py +631 -345
- disdrodb/psd/models.py +9 -6
- disdrodb/routines/__init__.py +54 -0
- disdrodb/{l0/routines.py → routines/l0.py} +316 -355
- disdrodb/{l1/routines.py → routines/l1.py} +76 -116
- disdrodb/routines/l2.py +1019 -0
- disdrodb/{routines.py → routines/wrappers.py} +98 -10
- disdrodb/scattering/__init__.py +16 -4
- disdrodb/scattering/axis_ratio.py +61 -37
- disdrodb/scattering/permittivity.py +504 -0
- disdrodb/scattering/routines.py +746 -184
- disdrodb/summary/__init__.py +17 -0
- disdrodb/summary/routines.py +4196 -0
- disdrodb/utils/archiving.py +434 -0
- disdrodb/utils/attrs.py +68 -125
- disdrodb/utils/cli.py +5 -5
- disdrodb/utils/compression.py +30 -1
- disdrodb/utils/dask.py +121 -9
- disdrodb/utils/dataframe.py +61 -7
- disdrodb/utils/decorators.py +31 -0
- disdrodb/utils/directories.py +35 -15
- disdrodb/utils/encoding.py +37 -19
- disdrodb/{l2 → utils}/event.py +15 -173
- disdrodb/utils/logger.py +14 -7
- disdrodb/utils/manipulations.py +81 -0
- disdrodb/utils/routines.py +166 -0
- disdrodb/utils/subsetting.py +214 -0
- disdrodb/utils/time.py +35 -177
- disdrodb/utils/writer.py +20 -7
- disdrodb/utils/xarray.py +5 -4
- disdrodb/viz/__init__.py +13 -0
- disdrodb/viz/plots.py +398 -0
- {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/METADATA +4 -3
- {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/RECORD +139 -98
- {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/entry_points.txt +2 -0
- disdrodb/l1/encoding_attrs.py +0 -642
- disdrodb/l2/processing_options.py +0 -213
- disdrodb/l2/routines.py +0 -868
- /disdrodb/l0/readers/PARSIVEL/SLOVENIA/{UL_FGG.py → UL.py} +0 -0
- {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/WHEEL +0 -0
- {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/licenses/LICENSE +0 -0
- {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
# -----------------------------------------------------------------------------.
|
|
2
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
3
|
+
#
|
|
4
|
+
# This program is free software: you can redistribute it and/or modify
|
|
5
|
+
# it under the terms of the GNU General Public License as published by
|
|
6
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
7
|
+
# (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This program is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
12
|
+
# GNU General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU General Public License
|
|
15
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
16
|
+
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
"""This module contains functions for subsetting and aligning DISDRODB products."""
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
from xarray.core.utils import either_dict_or_kwargs
|
|
22
|
+
|
|
23
|
+
from disdrodb.constants import DIAMETER_DIMENSION, VELOCITY_DIMENSION
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def is_1d_non_dimensional_coord(xr_obj, coord):
|
|
27
|
+
"""Checks if a coordinate is a 1d, non-dimensional coordinate."""
|
|
28
|
+
if coord not in xr_obj.coords:
|
|
29
|
+
return False
|
|
30
|
+
if xr_obj[coord].ndim != 1:
|
|
31
|
+
return False
|
|
32
|
+
is_1d_dim_coord = xr_obj[coord].dims[0] == coord
|
|
33
|
+
return not is_1d_dim_coord
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _get_dim_of_1d_non_dimensional_coord(xr_obj, coord):
|
|
37
|
+
"""Get the dimension of a 1D non-dimension coordinate."""
|
|
38
|
+
if not is_1d_non_dimensional_coord(xr_obj, coord):
|
|
39
|
+
raise ValueError(f"'{coord}' is not a dimension or a 1D non-dimensional coordinate.")
|
|
40
|
+
dim = xr_obj[coord].dims[0]
|
|
41
|
+
return dim
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _get_dim_isel_on_non_dim_coord_from_isel(xr_obj, coord, isel_indices):
|
|
45
|
+
"""Get dimension and isel_indices related to a 1D non-dimension coordinate.
|
|
46
|
+
|
|
47
|
+
Parameters
|
|
48
|
+
----------
|
|
49
|
+
xr_obj : (xr.Dataset, xr.DataArray)
|
|
50
|
+
A xarray object.
|
|
51
|
+
coord : str
|
|
52
|
+
Name of the coordinate wishing to subset with .sel
|
|
53
|
+
isel_indices : (str, int, float, list, np.array)
|
|
54
|
+
Coordinate indices wishing to be selected.
|
|
55
|
+
|
|
56
|
+
Returns
|
|
57
|
+
-------
|
|
58
|
+
dim : str
|
|
59
|
+
Dimension related to the 1D non-dimension coordinate.
|
|
60
|
+
isel_indices : (int, list, slice)
|
|
61
|
+
Indices for index-based selection.
|
|
62
|
+
"""
|
|
63
|
+
dim = _get_dim_of_1d_non_dimensional_coord(xr_obj, coord)
|
|
64
|
+
return dim, isel_indices
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _get_dim_isel_indices_from_isel_indices(xr_obj, key, indices, method="dummy"): # noqa
|
|
68
|
+
"""Return the dimension and isel_indices related to the dimension position indices of a coordinate."""
|
|
69
|
+
# Non-dimensional coordinate case
|
|
70
|
+
if key not in xr_obj.dims:
|
|
71
|
+
key, indices = _get_dim_isel_on_non_dim_coord_from_isel(xr_obj, coord=key, isel_indices=indices)
|
|
72
|
+
return key, indices
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _get_isel_indices_from_sel_indices(xr_obj, coord, sel_indices, method):
|
|
76
|
+
"""Get isel_indices corresponding to sel_indices."""
|
|
77
|
+
da_coord = xr_obj[coord]
|
|
78
|
+
dim = da_coord.dims[0]
|
|
79
|
+
da_coord = da_coord.assign_coords({"isel_indices": (dim, np.arange(0, da_coord.size))})
|
|
80
|
+
da_subset = da_coord.swap_dims({dim: coord}).sel({coord: sel_indices}, method=method)
|
|
81
|
+
isel_indices = da_subset["isel_indices"].data
|
|
82
|
+
return isel_indices
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _get_dim_isel_on_non_dim_coord_from_sel(xr_obj, coord, sel_indices, method):
|
|
86
|
+
"""
|
|
87
|
+
Return the dimension and isel_indices related to a 1D non-dimension coordinate.
|
|
88
|
+
|
|
89
|
+
Parameters
|
|
90
|
+
----------
|
|
91
|
+
xr_obj : (xr.Dataset, xr.DataArray)
|
|
92
|
+
A xarray object.
|
|
93
|
+
coord : str
|
|
94
|
+
Name of the coordinate wishing to subset with .sel
|
|
95
|
+
sel_indices : (str, int, float, list, np.array)
|
|
96
|
+
Coordinate values wishing to be selected.
|
|
97
|
+
|
|
98
|
+
Returns
|
|
99
|
+
-------
|
|
100
|
+
dim : str
|
|
101
|
+
Dimension related to the 1D non-dimension coordinate.
|
|
102
|
+
isel_indices : np.ndarray
|
|
103
|
+
Indices for index-based selection.
|
|
104
|
+
"""
|
|
105
|
+
dim = _get_dim_of_1d_non_dimensional_coord(xr_obj, coord)
|
|
106
|
+
isel_indices = _get_isel_indices_from_sel_indices(xr_obj, coord=coord, sel_indices=sel_indices, method=method)
|
|
107
|
+
return dim, isel_indices
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _get_dim_isel_indices_from_sel_indices(xr_obj, key, indices, method):
|
|
111
|
+
"""Return the dimension and isel_indices related to values of a coordinate."""
|
|
112
|
+
# Dimension case
|
|
113
|
+
if key in xr_obj.dims:
|
|
114
|
+
if key not in xr_obj.coords:
|
|
115
|
+
raise ValueError(f"Can not subset with disdrodb.sel the dimension '{key}' if it is not also a coordinate.")
|
|
116
|
+
isel_indices = _get_isel_indices_from_sel_indices(xr_obj, coord=key, sel_indices=indices, method=method)
|
|
117
|
+
# Non-dimensional coordinate case
|
|
118
|
+
else:
|
|
119
|
+
key, isel_indices = _get_dim_isel_on_non_dim_coord_from_sel(
|
|
120
|
+
xr_obj,
|
|
121
|
+
coord=key,
|
|
122
|
+
sel_indices=indices,
|
|
123
|
+
method=method,
|
|
124
|
+
)
|
|
125
|
+
return key, isel_indices
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _get_dim_isel_indices_function(func):
|
|
129
|
+
func_dict = {
|
|
130
|
+
"sel": _get_dim_isel_indices_from_sel_indices,
|
|
131
|
+
"isel": _get_dim_isel_indices_from_isel_indices,
|
|
132
|
+
}
|
|
133
|
+
return func_dict[func]
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _subset(xr_obj, indexers=None, func="isel", drop=False, method=None, **indexers_kwargs):
|
|
137
|
+
"""Perform selection with isel or isel."""
|
|
138
|
+
# Retrieve indexers
|
|
139
|
+
indexers = either_dict_or_kwargs(indexers, indexers_kwargs, func)
|
|
140
|
+
# Get function returning isel_indices
|
|
141
|
+
get_dim_isel_indices = _get_dim_isel_indices_function(func)
|
|
142
|
+
# Define isel_dict
|
|
143
|
+
isel_dict = {}
|
|
144
|
+
for key, indices in indexers.items():
|
|
145
|
+
key, isel_indices = get_dim_isel_indices(xr_obj, key=key, indices=indices, method=method)
|
|
146
|
+
if key in isel_dict:
|
|
147
|
+
raise ValueError(f"Multiple indexers point to the '{key}' dimension.")
|
|
148
|
+
isel_dict[key] = isel_indices
|
|
149
|
+
|
|
150
|
+
# Subset and update area
|
|
151
|
+
xr_obj = xr_obj.isel(isel_dict, drop=drop)
|
|
152
|
+
return xr_obj
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def isel(xr_obj, indexers=None, drop=False, **indexers_kwargs):
|
|
156
|
+
"""Perform index-based dimension selection."""
|
|
157
|
+
return _subset(xr_obj, indexers=indexers, func="isel", drop=drop, **indexers_kwargs)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def sel(xr_obj, indexers=None, drop=False, method=None, **indexers_kwargs):
|
|
161
|
+
"""Perform value-based coordinate selection.
|
|
162
|
+
|
|
163
|
+
Slices are treated as inclusive of both the start and stop values, unlike normal Python indexing.
|
|
164
|
+
The disdrodb `sel` method is empowered to:
|
|
165
|
+
|
|
166
|
+
- slice by disdrodb-id strings !
|
|
167
|
+
- slice by any xarray coordinate value !
|
|
168
|
+
|
|
169
|
+
You can use string shortcuts for datetime coordinates (e.g., '2000-01' to select all values in January 2000).
|
|
170
|
+
"""
|
|
171
|
+
return _subset(xr_obj, indexers=indexers, func="sel", drop=drop, method=method, **indexers_kwargs)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def align(*args):
|
|
175
|
+
"""Align DISDRODB products over time, velocity and diameter dimensions."""
|
|
176
|
+
list_xr_obj = args
|
|
177
|
+
|
|
178
|
+
# Check input
|
|
179
|
+
if len(list_xr_obj) <= 1:
|
|
180
|
+
raise ValueError("At least two xarray object are required for alignment.")
|
|
181
|
+
|
|
182
|
+
# Define dimensions used for alignment
|
|
183
|
+
dims_to_align = ["time", DIAMETER_DIMENSION, VELOCITY_DIMENSION]
|
|
184
|
+
|
|
185
|
+
# Check which dimensions and coordinates are available across all datasets
|
|
186
|
+
coords = [coord for coord in dims_to_align if all(coord in xr_obj.coords for xr_obj in list_xr_obj)]
|
|
187
|
+
if not coords:
|
|
188
|
+
raise ValueError("No common coordinates found among the input datasets for alignment.")
|
|
189
|
+
|
|
190
|
+
# Start with the input datasets
|
|
191
|
+
list_aligned = list(list_xr_obj)
|
|
192
|
+
|
|
193
|
+
# Loop over the dimensions which are available
|
|
194
|
+
for coord in coords:
|
|
195
|
+
# Retrieve list of coordinate values
|
|
196
|
+
list_coord_values = [xr_obj[coord].data for xr_obj in list_aligned]
|
|
197
|
+
|
|
198
|
+
# Retrieve intersection of coordinates values
|
|
199
|
+
# - np.atleast_1d ensure that the dimension is not dropped if only 1 value
|
|
200
|
+
# - np.intersect1d returns the sorted array of common unique elements
|
|
201
|
+
common_values = list_coord_values[0]
|
|
202
|
+
for coord_values in list_coord_values[1:]:
|
|
203
|
+
common_values = np.intersect1d(common_values, coord_values)
|
|
204
|
+
sel_indices = np.atleast_1d(common_values)
|
|
205
|
+
|
|
206
|
+
# Check there are common coordinate values
|
|
207
|
+
if len(sel_indices) == 0:
|
|
208
|
+
raise ValueError(f"No common {coord} values across input objects.")
|
|
209
|
+
|
|
210
|
+
# Subset dataset
|
|
211
|
+
new_list_aligned = [sel(xr_obj, {coord: sel_indices}) for xr_obj in list_aligned]
|
|
212
|
+
list_aligned = new_list_aligned
|
|
213
|
+
|
|
214
|
+
return list_aligned
|
disdrodb/utils/time.py
CHANGED
|
@@ -29,11 +29,12 @@ from disdrodb.utils.xarray import define_fill_value_dictionary
|
|
|
29
29
|
|
|
30
30
|
logger = logging.getLogger(__name__)
|
|
31
31
|
|
|
32
|
+
|
|
32
33
|
####------------------------------------------------------------------------------------.
|
|
33
34
|
#### Sampling Interval Acronyms
|
|
34
35
|
|
|
35
36
|
|
|
36
|
-
def
|
|
37
|
+
def seconds_to_temporal_resolution(seconds):
|
|
37
38
|
"""
|
|
38
39
|
Convert a duration in seconds to a readable string format (e.g., "1H30", "1D2H").
|
|
39
40
|
|
|
@@ -57,27 +58,27 @@ def seconds_to_acronym(seconds):
|
|
|
57
58
|
parts.append(f"{components.minutes}MIN")
|
|
58
59
|
if components.seconds > 0:
|
|
59
60
|
parts.append(f"{components.seconds}S")
|
|
60
|
-
|
|
61
|
-
return
|
|
61
|
+
temporal_resolution = "".join(parts)
|
|
62
|
+
return temporal_resolution
|
|
62
63
|
|
|
63
64
|
|
|
64
|
-
def get_resampling_information(
|
|
65
|
+
def get_resampling_information(temporal_resolution):
|
|
65
66
|
"""
|
|
66
|
-
Extract resampling information from the
|
|
67
|
+
Extract resampling information from the temporal_resolution string.
|
|
67
68
|
|
|
68
69
|
Parameters
|
|
69
70
|
----------
|
|
70
|
-
|
|
71
|
-
A string representing the
|
|
71
|
+
temporal_resolution: str
|
|
72
|
+
A string representing the product temporal resolution: e.g., "1H30MIN", "ROLL1H30MIN".
|
|
72
73
|
|
|
73
74
|
Returns
|
|
74
75
|
-------
|
|
75
76
|
sample_interval_seconds, rolling: tuple
|
|
76
77
|
Sample_interval in seconds and whether rolling is enabled.
|
|
77
78
|
"""
|
|
78
|
-
rolling =
|
|
79
|
+
rolling = temporal_resolution.startswith("ROLL")
|
|
79
80
|
if rolling:
|
|
80
|
-
|
|
81
|
+
temporal_resolution = temporal_resolution[4:] # Remove "ROLL"
|
|
81
82
|
|
|
82
83
|
# Allowed pattern: one or more occurrences of "<number><unit>"
|
|
83
84
|
# where unit is exactly one of D, H, MIN, or S.
|
|
@@ -85,15 +86,15 @@ def get_resampling_information(sample_interval_acronym):
|
|
|
85
86
|
pattern = r"^(\d+(?:D|H|MIN|S))+$"
|
|
86
87
|
|
|
87
88
|
# Check if the entire string matches the pattern
|
|
88
|
-
if not re.match(pattern,
|
|
89
|
+
if not re.match(pattern, temporal_resolution):
|
|
89
90
|
raise ValueError(
|
|
90
|
-
f"Invalid
|
|
91
|
+
f"Invalid temporal resolution '{temporal_resolution}'. "
|
|
91
92
|
"Must be composed of one or more <number><unit> groups, where unit is D, H, MIN, or S.",
|
|
92
93
|
)
|
|
93
94
|
|
|
94
95
|
# Regular expression to match duration components and extract all (value, unit) pairs
|
|
95
96
|
pattern = r"(\d+)(D|H|MIN|S)"
|
|
96
|
-
matches = re.findall(pattern,
|
|
97
|
+
matches = re.findall(pattern, temporal_resolution)
|
|
97
98
|
|
|
98
99
|
# Conversion factors for each unit
|
|
99
100
|
unit_to_seconds = {
|
|
@@ -112,21 +113,21 @@ def get_resampling_information(sample_interval_acronym):
|
|
|
112
113
|
return sample_interval, rolling
|
|
113
114
|
|
|
114
115
|
|
|
115
|
-
def
|
|
116
|
+
def temporal_resolution_to_seconds(temporal_resolution):
|
|
116
117
|
"""
|
|
117
|
-
Extract the interval in seconds from the
|
|
118
|
+
Extract the measurement interval in seconds from the temporal resolution string.
|
|
118
119
|
|
|
119
120
|
Parameters
|
|
120
121
|
----------
|
|
121
|
-
|
|
122
|
-
A string representing
|
|
122
|
+
temporal_resolution: str
|
|
123
|
+
A string representing the product measurement interval: e.g., "1H30MIN", "ROLL1H30MIN".
|
|
123
124
|
|
|
124
125
|
Returns
|
|
125
126
|
-------
|
|
126
127
|
seconds
|
|
127
128
|
Duration in seconds.
|
|
128
129
|
"""
|
|
129
|
-
seconds, _ = get_resampling_information(
|
|
130
|
+
seconds, _ = get_resampling_information(temporal_resolution)
|
|
130
131
|
return seconds
|
|
131
132
|
|
|
132
133
|
|
|
@@ -262,6 +263,7 @@ def regularize_dataset(
|
|
|
262
263
|
Regularized dataset.
|
|
263
264
|
|
|
264
265
|
"""
|
|
266
|
+
attrs = xr_obj.attrs.copy()
|
|
265
267
|
xr_obj = _check_time_sorted(xr_obj, time_dim=time_dim)
|
|
266
268
|
start_time, end_time = get_dataset_start_end_time(xr_obj, time_dim=time_dim)
|
|
267
269
|
|
|
@@ -289,11 +291,14 @@ def regularize_dataset(
|
|
|
289
291
|
# tolerance=tolerance, # mismatch in seconds
|
|
290
292
|
fill_value=fill_value,
|
|
291
293
|
)
|
|
294
|
+
|
|
295
|
+
# Ensure attributes are preserved
|
|
296
|
+
xr_obj.attrs = attrs
|
|
292
297
|
return xr_obj
|
|
293
298
|
|
|
294
299
|
|
|
295
300
|
####------------------------------------------
|
|
296
|
-
####
|
|
301
|
+
#### Interval utilities
|
|
297
302
|
|
|
298
303
|
|
|
299
304
|
def ensure_sample_interval_in_seconds(sample_interval): # noqa: PLR0911
|
|
@@ -376,7 +381,7 @@ def ensure_sample_interval_in_seconds(sample_interval): # noqa: PLR0911
|
|
|
376
381
|
raise TypeError("Float array sample_interval must contain only whole numbers.")
|
|
377
382
|
return sample_interval.astype(int)
|
|
378
383
|
|
|
379
|
-
# Deal with xarray.
|
|
384
|
+
# Deal with xarray.DataArray of floats that are all integer-valued (with optionally some NaN)
|
|
380
385
|
if isinstance(sample_interval, xr.DataArray) and np.issubdtype(sample_interval.dtype, np.floating):
|
|
381
386
|
arr = sample_interval.copy()
|
|
382
387
|
data = arr.data
|
|
@@ -397,6 +402,17 @@ def ensure_sample_interval_in_seconds(sample_interval): # noqa: PLR0911
|
|
|
397
402
|
)
|
|
398
403
|
|
|
399
404
|
|
|
405
|
+
def ensure_timedelta_seconds(interval):
|
|
406
|
+
"""Return an a scalar value/array in seconds or timedelta object as numpy.timedelta64 in seconds."""
|
|
407
|
+
if isinstance(interval, (xr.DataArray, np.ndarray)):
|
|
408
|
+
return ensure_sample_interval_in_seconds(interval).astype("m8[s]")
|
|
409
|
+
return np.array(ensure_sample_interval_in_seconds(interval), dtype="m8[s]")
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
####------------------------------------------
|
|
413
|
+
#### Sample Interval Utilities
|
|
414
|
+
|
|
415
|
+
|
|
400
416
|
def infer_sample_interval(ds, robust=False, verbose=False, logger=None):
|
|
401
417
|
"""Infer the sample interval of a dataset.
|
|
402
418
|
|
|
@@ -497,161 +513,3 @@ def infer_sample_interval(ds, robust=False, verbose=False, logger=None):
|
|
|
497
513
|
)
|
|
498
514
|
log_warning(logger=logger, msg=msg, verbose=verbose)
|
|
499
515
|
return int(sample_interval)
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
####---------------------------------------------------------------------------------
|
|
503
|
-
#### Timesteps regularization
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
def get_problematic_timestep_indices(timesteps, sample_interval):
|
|
507
|
-
"""Identify timesteps with missing previous or following timesteps."""
|
|
508
|
-
previous_time = timesteps - pd.Timedelta(seconds=sample_interval)
|
|
509
|
-
next_time = timesteps + pd.Timedelta(seconds=sample_interval)
|
|
510
|
-
idx_previous_missing = np.where(~np.isin(previous_time, timesteps))[0][1:]
|
|
511
|
-
idx_next_missing = np.where(~np.isin(next_time, timesteps))[0][:-1]
|
|
512
|
-
idx_isolated_missing = np.intersect1d(idx_previous_missing, idx_next_missing)
|
|
513
|
-
idx_previous_missing = idx_previous_missing[np.isin(idx_previous_missing, idx_isolated_missing, invert=True)]
|
|
514
|
-
idx_next_missing = idx_next_missing[np.isin(idx_next_missing, idx_isolated_missing, invert=True)]
|
|
515
|
-
return idx_previous_missing, idx_next_missing, idx_isolated_missing
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
def regularize_timesteps(ds, sample_interval, robust=False, add_quality_flag=True, logger=None, verbose=True):
|
|
519
|
-
"""Ensure timesteps match with the sample_interval.
|
|
520
|
-
|
|
521
|
-
This function:
|
|
522
|
-
- drop dataset indices with duplicated timesteps,
|
|
523
|
-
- but does not add missing timesteps to the dataset.
|
|
524
|
-
"""
|
|
525
|
-
# Check sorted by time and sort if necessary
|
|
526
|
-
ds = ensure_sorted_by_time(ds)
|
|
527
|
-
|
|
528
|
-
# Convert time to pandas.DatetimeIndex for easier manipulation
|
|
529
|
-
times = pd.to_datetime(ds["time"].to_numpy())
|
|
530
|
-
|
|
531
|
-
# Determine the start and end times
|
|
532
|
-
start_time = times[0].floor(f"{sample_interval}s")
|
|
533
|
-
end_time = times[-1].ceil(f"{sample_interval}s")
|
|
534
|
-
|
|
535
|
-
# Create the expected time grid
|
|
536
|
-
expected_times = pd.date_range(start=start_time, end=end_time, freq=f"{sample_interval}s")
|
|
537
|
-
|
|
538
|
-
# Convert to numpy arrays
|
|
539
|
-
times = times.to_numpy(dtype="M8[s]")
|
|
540
|
-
expected_times = expected_times.to_numpy(dtype="M8[s]")
|
|
541
|
-
|
|
542
|
-
# Map original times to the nearest expected times
|
|
543
|
-
# Calculate the difference between original times and expected times
|
|
544
|
-
time_deltas = np.abs(times - expected_times[:, None]).astype(int)
|
|
545
|
-
|
|
546
|
-
# Find the index of the closest expected time for each original time
|
|
547
|
-
nearest_indices = np.argmin(time_deltas, axis=0)
|
|
548
|
-
adjusted_times = expected_times[nearest_indices]
|
|
549
|
-
|
|
550
|
-
# Check for duplicates in adjusted times
|
|
551
|
-
unique_times, counts = np.unique(adjusted_times, return_counts=True)
|
|
552
|
-
duplicates = unique_times[counts > 1]
|
|
553
|
-
|
|
554
|
-
# Initialize time quality flag
|
|
555
|
-
# - 0 when ok or just rounded to closest 00
|
|
556
|
-
# - 1 if previous timestep is missing
|
|
557
|
-
# - 2 if next timestep is missing
|
|
558
|
-
# - 3 if previous and next timestep is missing
|
|
559
|
-
# - 4 if solved duplicated timesteps
|
|
560
|
-
# - 5 if needed to drop duplicated timesteps and select the last
|
|
561
|
-
flag_previous_missing = 1
|
|
562
|
-
flag_next_missing = 2
|
|
563
|
-
flag_isolated_timestep = 3
|
|
564
|
-
flag_solved_duplicated_timestep = 4
|
|
565
|
-
flag_dropped_duplicated_timestep = 5
|
|
566
|
-
qc_flag = np.zeros(adjusted_times.shape)
|
|
567
|
-
|
|
568
|
-
# Initialize list with the duplicated timesteps index to drop
|
|
569
|
-
# - We drop the first occurrence because is likely the shortest interval
|
|
570
|
-
idx_to_drop = []
|
|
571
|
-
|
|
572
|
-
# Attempt to resolve for duplicates
|
|
573
|
-
if duplicates.size > 0:
|
|
574
|
-
# Handle duplicates
|
|
575
|
-
for dup_time in duplicates:
|
|
576
|
-
# Indices of duplicates
|
|
577
|
-
dup_indices = np.where(adjusted_times == dup_time)[0]
|
|
578
|
-
n_duplicates = len(dup_indices)
|
|
579
|
-
# Define previous and following timestep
|
|
580
|
-
prev_time = dup_time - pd.Timedelta(seconds=sample_interval)
|
|
581
|
-
next_time = dup_time + pd.Timedelta(seconds=sample_interval)
|
|
582
|
-
# Try to find missing slots before and after
|
|
583
|
-
# - If more than 3 duplicates, impossible to solve !
|
|
584
|
-
count_solved = 0
|
|
585
|
-
# If the previous timestep is available, set that one
|
|
586
|
-
if n_duplicates == 2:
|
|
587
|
-
if prev_time not in adjusted_times:
|
|
588
|
-
adjusted_times[dup_indices[0]] = prev_time
|
|
589
|
-
qc_flag[dup_indices[0]] = flag_solved_duplicated_timestep
|
|
590
|
-
count_solved += 1
|
|
591
|
-
elif next_time not in adjusted_times:
|
|
592
|
-
adjusted_times[dup_indices[-1]] = next_time
|
|
593
|
-
qc_flag[dup_indices[-1]] = flag_solved_duplicated_timestep
|
|
594
|
-
count_solved += 1
|
|
595
|
-
else:
|
|
596
|
-
pass
|
|
597
|
-
elif n_duplicates == 3:
|
|
598
|
-
if prev_time not in adjusted_times:
|
|
599
|
-
adjusted_times[dup_indices[0]] = prev_time
|
|
600
|
-
qc_flag[dup_indices[0]] = flag_solved_duplicated_timestep
|
|
601
|
-
count_solved += 1
|
|
602
|
-
if next_time not in adjusted_times:
|
|
603
|
-
adjusted_times[dup_indices[-1]] = next_time
|
|
604
|
-
qc_flag[dup_indices[-1]] = flag_solved_duplicated_timestep
|
|
605
|
-
count_solved += 1
|
|
606
|
-
if count_solved != n_duplicates - 1:
|
|
607
|
-
idx_to_drop = np.append(idx_to_drop, dup_indices[0:-1])
|
|
608
|
-
qc_flag[dup_indices[-1]] = flag_dropped_duplicated_timestep
|
|
609
|
-
msg = (
|
|
610
|
-
f"Cannot resolve {n_duplicates} duplicated timesteps "
|
|
611
|
-
f"(after trailing seconds correction) around {dup_time}."
|
|
612
|
-
)
|
|
613
|
-
log_warning(logger=logger, msg=msg, verbose=verbose)
|
|
614
|
-
if robust:
|
|
615
|
-
raise ValueError(msg)
|
|
616
|
-
|
|
617
|
-
# Update the time coordinate (Convert to ns for xarray compatibility)
|
|
618
|
-
ds = ds.assign_coords({"time": adjusted_times.astype("datetime64[ns]")})
|
|
619
|
-
|
|
620
|
-
# Update quality flag values for next and previous timestep is missing
|
|
621
|
-
if add_quality_flag:
|
|
622
|
-
idx_previous_missing, idx_next_missing, idx_isolated_missing = get_problematic_timestep_indices(
|
|
623
|
-
adjusted_times,
|
|
624
|
-
sample_interval,
|
|
625
|
-
)
|
|
626
|
-
qc_flag[idx_previous_missing] = np.maximum(qc_flag[idx_previous_missing], flag_previous_missing)
|
|
627
|
-
qc_flag[idx_next_missing] = np.maximum(qc_flag[idx_next_missing], flag_next_missing)
|
|
628
|
-
qc_flag[idx_isolated_missing] = np.maximum(qc_flag[idx_isolated_missing], flag_isolated_timestep)
|
|
629
|
-
|
|
630
|
-
# If the first timestep is at 00:00 and currently flagged as previous missing (1), reset to 0
|
|
631
|
-
# first_time = pd.to_datetime(adjusted_times[0]).time()
|
|
632
|
-
# first_expected_time = pd.Timestamp("00:00:00").time()
|
|
633
|
-
# if first_time == first_expected_time and qc_flag[0] == flag_previous_missing:
|
|
634
|
-
# qc_flag[0] = 0
|
|
635
|
-
|
|
636
|
-
# # If the last timestep is flagged and currently flagged as next missing (2), reset it to 0
|
|
637
|
-
# last_time = pd.to_datetime(adjusted_times[-1]).time()
|
|
638
|
-
# last_time_expected = (pd.Timestamp("00:00:00") - pd.Timedelta(30, unit="seconds")).time()
|
|
639
|
-
# # Check if adding one interval would go beyond the end_time
|
|
640
|
-
# if last_time == last_time_expected and qc_flag[-1] == flag_next_missing:
|
|
641
|
-
# qc_flag[-1] = 0
|
|
642
|
-
|
|
643
|
-
# Assign time quality flag coordinate
|
|
644
|
-
ds["time_qc"] = xr.DataArray(qc_flag, dims="time")
|
|
645
|
-
ds = ds.set_coords("time_qc")
|
|
646
|
-
|
|
647
|
-
# Drop duplicated timesteps
|
|
648
|
-
# - Using ds = ds.drop_isel({"time": idx_to_drop.astype(int)}) raise:
|
|
649
|
-
# --> pandas.errors.InvalidIndexError: Reindexing only valid with uniquely valued Index objects
|
|
650
|
-
# --> https://github.com/pydata/xarray/issues/6605
|
|
651
|
-
if len(idx_to_drop) > 0:
|
|
652
|
-
idx_to_drop = idx_to_drop.astype(int)
|
|
653
|
-
idx_valid_timesteps = np.arange(0, ds["time"].size)
|
|
654
|
-
idx_valid_timesteps = np.delete(idx_valid_timesteps, idx_to_drop)
|
|
655
|
-
ds = ds.isel(time=idx_valid_timesteps)
|
|
656
|
-
# Return dataset
|
|
657
|
-
return ds
|
disdrodb/utils/writer.py
CHANGED
|
@@ -22,11 +22,29 @@ import os
|
|
|
22
22
|
|
|
23
23
|
import xarray as xr
|
|
24
24
|
|
|
25
|
-
from disdrodb.utils.attrs import set_disdrodb_attrs
|
|
25
|
+
from disdrodb.utils.attrs import get_attrs_dict, set_attrs, set_disdrodb_attrs
|
|
26
26
|
from disdrodb.utils.directories import create_directory, remove_if_exists
|
|
27
|
+
from disdrodb.utils.encoding import get_encodings_dict, set_encodings
|
|
27
28
|
|
|
28
29
|
|
|
29
|
-
def
|
|
30
|
+
def finalize_product(ds, product=None) -> xr.Dataset:
|
|
31
|
+
"""Finalize DISDRODB product."""
|
|
32
|
+
# Add variables attributes
|
|
33
|
+
attrs_dict = get_attrs_dict()
|
|
34
|
+
ds = set_attrs(ds, attrs_dict=attrs_dict)
|
|
35
|
+
|
|
36
|
+
# Add variables encoding
|
|
37
|
+
encodings_dict = get_encodings_dict()
|
|
38
|
+
ds = set_encodings(ds, encodings_dict=encodings_dict)
|
|
39
|
+
|
|
40
|
+
# Add DISDRODB global attributes
|
|
41
|
+
# - e.g. in generate_l2_radar it inherit from input dataset !
|
|
42
|
+
if product is not None:
|
|
43
|
+
ds = set_disdrodb_attrs(ds, product=product)
|
|
44
|
+
return ds
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def write_product(ds: xr.Dataset, filepath: str, force: bool = False) -> None:
|
|
30
48
|
"""Save the xarray dataset into a NetCDF file.
|
|
31
49
|
|
|
32
50
|
Parameters
|
|
@@ -35,8 +53,6 @@ def write_product(ds: xr.Dataset, filepath: str, product: str, force: bool = Fal
|
|
|
35
53
|
Input xarray dataset.
|
|
36
54
|
filepath : str
|
|
37
55
|
Output file path.
|
|
38
|
-
product: str
|
|
39
|
-
DISDRODB product name.
|
|
40
56
|
force : bool, optional
|
|
41
57
|
Whether to overwrite existing data.
|
|
42
58
|
If ``True``, overwrite existing data into destination directories.
|
|
@@ -50,8 +66,5 @@ def write_product(ds: xr.Dataset, filepath: str, product: str, force: bool = Fal
|
|
|
50
66
|
# - If force=False --> Raise error
|
|
51
67
|
remove_if_exists(filepath, force=force)
|
|
52
68
|
|
|
53
|
-
# Update attributes
|
|
54
|
-
ds = set_disdrodb_attrs(ds, product=product)
|
|
55
|
-
|
|
56
69
|
# Write netcdf
|
|
57
70
|
ds.to_netcdf(filepath, engine="netcdf4")
|
disdrodb/utils/xarray.py
CHANGED
|
@@ -21,6 +21,8 @@ import numpy as np
|
|
|
21
21
|
import xarray as xr
|
|
22
22
|
from xarray.core import dtypes
|
|
23
23
|
|
|
24
|
+
from disdrodb.constants import DIAMETER_COORDS, VELOCITY_COORDS
|
|
25
|
+
|
|
24
26
|
|
|
25
27
|
def xr_get_last_valid_idx(da_condition, dim, fill_value=None):
|
|
26
28
|
"""
|
|
@@ -104,6 +106,7 @@ def xr_get_last_valid_idx(da_condition, dim, fill_value=None):
|
|
|
104
106
|
def _check_coord_handling(coord_handling):
|
|
105
107
|
if coord_handling not in {"keep", "drop", "unstack"}:
|
|
106
108
|
raise ValueError("coord_handling must be one of 'keep', 'drop', or 'unstack'.")
|
|
109
|
+
return coord_handling
|
|
107
110
|
|
|
108
111
|
|
|
109
112
|
def _unstack_coordinates(xr_obj, dim, prefix, suffix):
|
|
@@ -161,6 +164,8 @@ def unstack_datarray_dimension(da, dim, coord_handling="keep", prefix="", suffix
|
|
|
161
164
|
"""
|
|
162
165
|
# Retrieve DataArray name
|
|
163
166
|
name = da.name
|
|
167
|
+
coord_handling = _check_coord_handling(coord_handling)
|
|
168
|
+
|
|
164
169
|
# Unstack variables
|
|
165
170
|
ds = da.to_dataset(dim=dim)
|
|
166
171
|
rename_dict = {dim_value: f"{prefix}{name}{suffix}{dim_value}" for dim_value in list(ds.data_vars)}
|
|
@@ -246,13 +251,9 @@ def define_fill_value_dictionary(xr_obj):
|
|
|
246
251
|
|
|
247
252
|
def remove_diameter_coordinates(xr_obj):
|
|
248
253
|
"""Drop diameter coordinates from xarray object."""
|
|
249
|
-
from disdrodb import DIAMETER_COORDS
|
|
250
|
-
|
|
251
254
|
return xr_obj.drop_vars(DIAMETER_COORDS, errors="ignore")
|
|
252
255
|
|
|
253
256
|
|
|
254
257
|
def remove_velocity_coordinates(xr_obj):
|
|
255
258
|
"""Drop velocity coordinates from xarray object."""
|
|
256
|
-
from disdrodb import VELOCITY_COORDS
|
|
257
|
-
|
|
258
259
|
return xr_obj.drop_vars(VELOCITY_COORDS, errors="ignore")
|
disdrodb/viz/__init__.py
CHANGED
|
@@ -15,3 +15,16 @@
|
|
|
15
15
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
16
16
|
# -----------------------------------------------------------------------------.
|
|
17
17
|
"""DISDRODB Visualization Module."""
|
|
18
|
+
from disdrodb.viz.plots import (
|
|
19
|
+
compute_dense_lines,
|
|
20
|
+
max_blend_images,
|
|
21
|
+
plot_nd,
|
|
22
|
+
to_rgba,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"compute_dense_lines",
|
|
27
|
+
"max_blend_images",
|
|
28
|
+
"plot_nd",
|
|
29
|
+
"to_rgba",
|
|
30
|
+
]
|