disdrodb 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. disdrodb/__init__.py +68 -34
  2. disdrodb/_config.py +5 -4
  3. disdrodb/_version.py +16 -3
  4. disdrodb/accessor/__init__.py +20 -0
  5. disdrodb/accessor/methods.py +125 -0
  6. disdrodb/api/checks.py +177 -24
  7. disdrodb/api/configs.py +3 -3
  8. disdrodb/api/info.py +13 -13
  9. disdrodb/api/io.py +281 -22
  10. disdrodb/api/path.py +184 -195
  11. disdrodb/api/search.py +18 -9
  12. disdrodb/cli/disdrodb_create_summary.py +103 -0
  13. disdrodb/cli/disdrodb_create_summary_station.py +91 -0
  14. disdrodb/cli/disdrodb_run_l0.py +1 -1
  15. disdrodb/cli/disdrodb_run_l0_station.py +1 -1
  16. disdrodb/cli/disdrodb_run_l0a_station.py +1 -1
  17. disdrodb/cli/disdrodb_run_l0b.py +1 -1
  18. disdrodb/cli/disdrodb_run_l0b_station.py +3 -3
  19. disdrodb/cli/disdrodb_run_l0c.py +1 -1
  20. disdrodb/cli/disdrodb_run_l0c_station.py +3 -3
  21. disdrodb/cli/disdrodb_run_l1_station.py +2 -2
  22. disdrodb/cli/disdrodb_run_l2e_station.py +2 -2
  23. disdrodb/cli/disdrodb_run_l2m_station.py +2 -2
  24. disdrodb/configs.py +149 -4
  25. disdrodb/constants.py +61 -0
  26. disdrodb/data_transfer/download_data.py +127 -11
  27. disdrodb/etc/configs/attributes.yaml +339 -0
  28. disdrodb/etc/configs/encodings.yaml +473 -0
  29. disdrodb/etc/products/L1/global.yaml +13 -0
  30. disdrodb/etc/products/L2E/10MIN.yaml +12 -0
  31. disdrodb/etc/products/L2E/1MIN.yaml +1 -0
  32. disdrodb/etc/products/L2E/global.yaml +22 -0
  33. disdrodb/etc/products/L2M/10MIN.yaml +12 -0
  34. disdrodb/etc/products/L2M/GAMMA_ML.yaml +8 -0
  35. disdrodb/etc/products/L2M/NGAMMA_GS_LOG_ND_MAE.yaml +6 -0
  36. disdrodb/etc/products/L2M/NGAMMA_GS_ND_MAE.yaml +6 -0
  37. disdrodb/etc/products/L2M/NGAMMA_GS_Z_MAE.yaml +6 -0
  38. disdrodb/etc/products/L2M/global.yaml +26 -0
  39. disdrodb/issue/writer.py +2 -0
  40. disdrodb/l0/__init__.py +13 -0
  41. disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +4 -4
  42. disdrodb/l0/configs/PARSIVEL/l0b_cf_attrs.yml +1 -1
  43. disdrodb/l0/configs/PARSIVEL/l0b_encodings.yml +3 -3
  44. disdrodb/l0/configs/PARSIVEL/raw_data_format.yml +1 -1
  45. disdrodb/l0/configs/PARSIVEL2/l0b_cf_attrs.yml +5 -5
  46. disdrodb/l0/configs/PARSIVEL2/l0b_encodings.yml +3 -3
  47. disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml +1 -1
  48. disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +4 -4
  49. disdrodb/l0/configs/PWS100/raw_data_format.yml +1 -1
  50. disdrodb/l0/l0a_processing.py +37 -32
  51. disdrodb/l0/l0b_nc_processing.py +118 -8
  52. disdrodb/l0/l0b_processing.py +30 -65
  53. disdrodb/l0/l0c_processing.py +369 -259
  54. disdrodb/l0/readers/LPM/ARM/ARM_LPM.py +7 -0
  55. disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_LPM_NC.py +66 -0
  56. disdrodb/l0/readers/LPM/SLOVENIA/{CRNI_VRH.py → UL.py} +3 -0
  57. disdrodb/l0/readers/LPM/SWITZERLAND/INNERERIZ_LPM.py +195 -0
  58. disdrodb/l0/readers/PARSIVEL/GPM/PIERS.py +0 -2
  59. disdrodb/l0/readers/PARSIVEL/JAPAN/JMA.py +4 -1
  60. disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +1 -1
  61. disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +1 -1
  62. disdrodb/l0/readers/PARSIVEL2/ARM/ARM_PARSIVEL2.py +4 -0
  63. disdrodb/l0/readers/PARSIVEL2/BELGIUM/ILVO.py +168 -0
  64. disdrodb/l0/readers/PARSIVEL2/CANADA/UQAM_NC.py +69 -0
  65. disdrodb/l0/readers/PARSIVEL2/DENMARK/DTU.py +165 -0
  66. disdrodb/l0/readers/PARSIVEL2/FINLAND/FMI_PARSIVEL2.py +69 -0
  67. disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +255 -134
  68. disdrodb/l0/readers/PARSIVEL2/FRANCE/OSUG.py +525 -0
  69. disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +1 -1
  70. disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +9 -7
  71. disdrodb/l0/readers/PARSIVEL2/KIT/BURKINA_FASO.py +1 -1
  72. disdrodb/l0/readers/PARSIVEL2/KIT/TEAMX.py +123 -0
  73. disdrodb/l0/readers/PARSIVEL2/{NETHERLANDS/DELFT.py → MPI/BCO_PARSIVEL2.py} +41 -71
  74. disdrodb/l0/readers/PARSIVEL2/MPI/BOWTIE.py +220 -0
  75. disdrodb/l0/readers/PARSIVEL2/NASA/APU.py +120 -0
  76. disdrodb/l0/readers/PARSIVEL2/NASA/LPVEX.py +109 -0
  77. disdrodb/l0/readers/PARSIVEL2/NCAR/FARM_PARSIVEL2.py +1 -0
  78. disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +1 -1
  79. disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_MIPS.py +126 -0
  80. disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_PIPS.py +165 -0
  81. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +1 -1
  82. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +20 -12
  83. disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT_NC.py +5 -0
  84. disdrodb/l0/readers/PARSIVEL2/SPAIN/CENER.py +144 -0
  85. disdrodb/l0/readers/PARSIVEL2/SPAIN/CR1000DL.py +201 -0
  86. disdrodb/l0/readers/PARSIVEL2/SPAIN/LIAISE.py +137 -0
  87. disdrodb/l0/readers/PARSIVEL2/USA/C3WE.py +146 -0
  88. disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +105 -99
  89. disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100_SIRTA.py +151 -0
  90. disdrodb/l1/__init__.py +5 -0
  91. disdrodb/l1/fall_velocity.py +46 -0
  92. disdrodb/l1/filters.py +34 -20
  93. disdrodb/l1/processing.py +46 -45
  94. disdrodb/l1/resampling.py +77 -66
  95. disdrodb/l1_env/routines.py +18 -3
  96. disdrodb/l2/__init__.py +7 -0
  97. disdrodb/l2/empirical_dsd.py +58 -10
  98. disdrodb/l2/processing.py +268 -117
  99. disdrodb/metadata/checks.py +132 -125
  100. disdrodb/metadata/standards.py +3 -1
  101. disdrodb/psd/fitting.py +631 -345
  102. disdrodb/psd/models.py +9 -6
  103. disdrodb/routines/__init__.py +54 -0
  104. disdrodb/{l0/routines.py → routines/l0.py} +316 -355
  105. disdrodb/{l1/routines.py → routines/l1.py} +76 -116
  106. disdrodb/routines/l2.py +1019 -0
  107. disdrodb/{routines.py → routines/wrappers.py} +98 -10
  108. disdrodb/scattering/__init__.py +16 -4
  109. disdrodb/scattering/axis_ratio.py +61 -37
  110. disdrodb/scattering/permittivity.py +504 -0
  111. disdrodb/scattering/routines.py +746 -184
  112. disdrodb/summary/__init__.py +17 -0
  113. disdrodb/summary/routines.py +4196 -0
  114. disdrodb/utils/archiving.py +434 -0
  115. disdrodb/utils/attrs.py +68 -125
  116. disdrodb/utils/cli.py +5 -5
  117. disdrodb/utils/compression.py +30 -1
  118. disdrodb/utils/dask.py +121 -9
  119. disdrodb/utils/dataframe.py +61 -7
  120. disdrodb/utils/decorators.py +31 -0
  121. disdrodb/utils/directories.py +35 -15
  122. disdrodb/utils/encoding.py +37 -19
  123. disdrodb/{l2 → utils}/event.py +15 -173
  124. disdrodb/utils/logger.py +14 -7
  125. disdrodb/utils/manipulations.py +81 -0
  126. disdrodb/utils/routines.py +166 -0
  127. disdrodb/utils/subsetting.py +214 -0
  128. disdrodb/utils/time.py +35 -177
  129. disdrodb/utils/writer.py +20 -7
  130. disdrodb/utils/xarray.py +5 -4
  131. disdrodb/viz/__init__.py +13 -0
  132. disdrodb/viz/plots.py +398 -0
  133. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/METADATA +4 -3
  134. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/RECORD +139 -98
  135. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/entry_points.txt +2 -0
  136. disdrodb/l1/encoding_attrs.py +0 -642
  137. disdrodb/l2/processing_options.py +0 -213
  138. disdrodb/l2/routines.py +0 -868
  139. /disdrodb/l0/readers/PARSIVEL/SLOVENIA/{UL_FGG.py → UL.py} +0 -0
  140. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/WHEEL +0 -0
  141. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/licenses/LICENSE +0 -0
  142. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,214 @@
1
+ # -----------------------------------------------------------------------------.
2
+ # Copyright (c) 2021-2023 DISDRODB developers
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU General Public License
15
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
16
+
17
+ # -----------------------------------------------------------------------------.
18
+ """This module contains functions for subsetting and aligning DISDRODB products."""
19
+
20
+ import numpy as np
21
+ from xarray.core.utils import either_dict_or_kwargs
22
+
23
+ from disdrodb.constants import DIAMETER_DIMENSION, VELOCITY_DIMENSION
24
+
25
+
26
+ def is_1d_non_dimensional_coord(xr_obj, coord):
27
+ """Checks if a coordinate is a 1d, non-dimensional coordinate."""
28
+ if coord not in xr_obj.coords:
29
+ return False
30
+ if xr_obj[coord].ndim != 1:
31
+ return False
32
+ is_1d_dim_coord = xr_obj[coord].dims[0] == coord
33
+ return not is_1d_dim_coord
34
+
35
+
36
+ def _get_dim_of_1d_non_dimensional_coord(xr_obj, coord):
37
+ """Get the dimension of a 1D non-dimension coordinate."""
38
+ if not is_1d_non_dimensional_coord(xr_obj, coord):
39
+ raise ValueError(f"'{coord}' is not a dimension or a 1D non-dimensional coordinate.")
40
+ dim = xr_obj[coord].dims[0]
41
+ return dim
42
+
43
+
44
+ def _get_dim_isel_on_non_dim_coord_from_isel(xr_obj, coord, isel_indices):
45
+ """Get dimension and isel_indices related to a 1D non-dimension coordinate.
46
+
47
+ Parameters
48
+ ----------
49
+ xr_obj : (xr.Dataset, xr.DataArray)
50
+ A xarray object.
51
+ coord : str
52
+ Name of the coordinate wishing to subset with .sel
53
+ isel_indices : (str, int, float, list, np.array)
54
+ Coordinate indices wishing to be selected.
55
+
56
+ Returns
57
+ -------
58
+ dim : str
59
+ Dimension related to the 1D non-dimension coordinate.
60
+ isel_indices : (int, list, slice)
61
+ Indices for index-based selection.
62
+ """
63
+ dim = _get_dim_of_1d_non_dimensional_coord(xr_obj, coord)
64
+ return dim, isel_indices
65
+
66
+
67
+ def _get_dim_isel_indices_from_isel_indices(xr_obj, key, indices, method="dummy"): # noqa
68
+ """Return the dimension and isel_indices related to the dimension position indices of a coordinate."""
69
+ # Non-dimensional coordinate case
70
+ if key not in xr_obj.dims:
71
+ key, indices = _get_dim_isel_on_non_dim_coord_from_isel(xr_obj, coord=key, isel_indices=indices)
72
+ return key, indices
73
+
74
+
75
+ def _get_isel_indices_from_sel_indices(xr_obj, coord, sel_indices, method):
76
+ """Get isel_indices corresponding to sel_indices."""
77
+ da_coord = xr_obj[coord]
78
+ dim = da_coord.dims[0]
79
+ da_coord = da_coord.assign_coords({"isel_indices": (dim, np.arange(0, da_coord.size))})
80
+ da_subset = da_coord.swap_dims({dim: coord}).sel({coord: sel_indices}, method=method)
81
+ isel_indices = da_subset["isel_indices"].data
82
+ return isel_indices
83
+
84
+
85
+ def _get_dim_isel_on_non_dim_coord_from_sel(xr_obj, coord, sel_indices, method):
86
+ """
87
+ Return the dimension and isel_indices related to a 1D non-dimension coordinate.
88
+
89
+ Parameters
90
+ ----------
91
+ xr_obj : (xr.Dataset, xr.DataArray)
92
+ A xarray object.
93
+ coord : str
94
+ Name of the coordinate wishing to subset with .sel
95
+ sel_indices : (str, int, float, list, np.array)
96
+ Coordinate values wishing to be selected.
97
+
98
+ Returns
99
+ -------
100
+ dim : str
101
+ Dimension related to the 1D non-dimension coordinate.
102
+ isel_indices : np.ndarray
103
+ Indices for index-based selection.
104
+ """
105
+ dim = _get_dim_of_1d_non_dimensional_coord(xr_obj, coord)
106
+ isel_indices = _get_isel_indices_from_sel_indices(xr_obj, coord=coord, sel_indices=sel_indices, method=method)
107
+ return dim, isel_indices
108
+
109
+
110
+ def _get_dim_isel_indices_from_sel_indices(xr_obj, key, indices, method):
111
+ """Return the dimension and isel_indices related to values of a coordinate."""
112
+ # Dimension case
113
+ if key in xr_obj.dims:
114
+ if key not in xr_obj.coords:
115
+ raise ValueError(f"Can not subset with disdrodb.sel the dimension '{key}' if it is not also a coordinate.")
116
+ isel_indices = _get_isel_indices_from_sel_indices(xr_obj, coord=key, sel_indices=indices, method=method)
117
+ # Non-dimensional coordinate case
118
+ else:
119
+ key, isel_indices = _get_dim_isel_on_non_dim_coord_from_sel(
120
+ xr_obj,
121
+ coord=key,
122
+ sel_indices=indices,
123
+ method=method,
124
+ )
125
+ return key, isel_indices
126
+
127
+
128
+ def _get_dim_isel_indices_function(func):
129
+ func_dict = {
130
+ "sel": _get_dim_isel_indices_from_sel_indices,
131
+ "isel": _get_dim_isel_indices_from_isel_indices,
132
+ }
133
+ return func_dict[func]
134
+
135
+
136
+ def _subset(xr_obj, indexers=None, func="isel", drop=False, method=None, **indexers_kwargs):
137
+ """Perform selection with isel or isel."""
138
+ # Retrieve indexers
139
+ indexers = either_dict_or_kwargs(indexers, indexers_kwargs, func)
140
+ # Get function returning isel_indices
141
+ get_dim_isel_indices = _get_dim_isel_indices_function(func)
142
+ # Define isel_dict
143
+ isel_dict = {}
144
+ for key, indices in indexers.items():
145
+ key, isel_indices = get_dim_isel_indices(xr_obj, key=key, indices=indices, method=method)
146
+ if key in isel_dict:
147
+ raise ValueError(f"Multiple indexers point to the '{key}' dimension.")
148
+ isel_dict[key] = isel_indices
149
+
150
+ # Subset and update area
151
+ xr_obj = xr_obj.isel(isel_dict, drop=drop)
152
+ return xr_obj
153
+
154
+
155
+ def isel(xr_obj, indexers=None, drop=False, **indexers_kwargs):
156
+ """Perform index-based dimension selection."""
157
+ return _subset(xr_obj, indexers=indexers, func="isel", drop=drop, **indexers_kwargs)
158
+
159
+
160
+ def sel(xr_obj, indexers=None, drop=False, method=None, **indexers_kwargs):
161
+ """Perform value-based coordinate selection.
162
+
163
+ Slices are treated as inclusive of both the start and stop values, unlike normal Python indexing.
164
+ The disdrodb `sel` method is empowered to:
165
+
166
+ - slice by disdrodb-id strings !
167
+ - slice by any xarray coordinate value !
168
+
169
+ You can use string shortcuts for datetime coordinates (e.g., '2000-01' to select all values in January 2000).
170
+ """
171
+ return _subset(xr_obj, indexers=indexers, func="sel", drop=drop, method=method, **indexers_kwargs)
172
+
173
+
174
+ def align(*args):
175
+ """Align DISDRODB products over time, velocity and diameter dimensions."""
176
+ list_xr_obj = args
177
+
178
+ # Check input
179
+ if len(list_xr_obj) <= 1:
180
+ raise ValueError("At least two xarray object are required for alignment.")
181
+
182
+ # Define dimensions used for alignment
183
+ dims_to_align = ["time", DIAMETER_DIMENSION, VELOCITY_DIMENSION]
184
+
185
+ # Check which dimensions and coordinates are available across all datasets
186
+ coords = [coord for coord in dims_to_align if all(coord in xr_obj.coords for xr_obj in list_xr_obj)]
187
+ if not coords:
188
+ raise ValueError("No common coordinates found among the input datasets for alignment.")
189
+
190
+ # Start with the input datasets
191
+ list_aligned = list(list_xr_obj)
192
+
193
+ # Loop over the dimensions which are available
194
+ for coord in coords:
195
+ # Retrieve list of coordinate values
196
+ list_coord_values = [xr_obj[coord].data for xr_obj in list_aligned]
197
+
198
+ # Retrieve intersection of coordinates values
199
+ # - np.atleast_1d ensure that the dimension is not dropped if only 1 value
200
+ # - np.intersect1d returns the sorted array of common unique elements
201
+ common_values = list_coord_values[0]
202
+ for coord_values in list_coord_values[1:]:
203
+ common_values = np.intersect1d(common_values, coord_values)
204
+ sel_indices = np.atleast_1d(common_values)
205
+
206
+ # Check there are common coordinate values
207
+ if len(sel_indices) == 0:
208
+ raise ValueError(f"No common {coord} values across input objects.")
209
+
210
+ # Subset dataset
211
+ new_list_aligned = [sel(xr_obj, {coord: sel_indices}) for xr_obj in list_aligned]
212
+ list_aligned = new_list_aligned
213
+
214
+ return list_aligned
disdrodb/utils/time.py CHANGED
@@ -29,11 +29,12 @@ from disdrodb.utils.xarray import define_fill_value_dictionary
29
29
 
30
30
  logger = logging.getLogger(__name__)
31
31
 
32
+
32
33
  ####------------------------------------------------------------------------------------.
33
34
  #### Sampling Interval Acronyms
34
35
 
35
36
 
36
- def seconds_to_acronym(seconds):
37
+ def seconds_to_temporal_resolution(seconds):
37
38
  """
38
39
  Convert a duration in seconds to a readable string format (e.g., "1H30", "1D2H").
39
40
 
@@ -57,27 +58,27 @@ def seconds_to_acronym(seconds):
57
58
  parts.append(f"{components.minutes}MIN")
58
59
  if components.seconds > 0:
59
60
  parts.append(f"{components.seconds}S")
60
- acronym = "".join(parts)
61
- return acronym
61
+ temporal_resolution = "".join(parts)
62
+ return temporal_resolution
62
63
 
63
64
 
64
- def get_resampling_information(sample_interval_acronym):
65
+ def get_resampling_information(temporal_resolution):
65
66
  """
66
- Extract resampling information from the sample interval acronym.
67
+ Extract resampling information from the temporal_resolution string.
67
68
 
68
69
  Parameters
69
70
  ----------
70
- sample_interval_acronym: str
71
- A string representing the sample interval: e.g., "1H30MIN", "ROLL1H30MIN".
71
+ temporal_resolution: str
72
+ A string representing the product temporal resolution: e.g., "1H30MIN", "ROLL1H30MIN".
72
73
 
73
74
  Returns
74
75
  -------
75
76
  sample_interval_seconds, rolling: tuple
76
77
  Sample_interval in seconds and whether rolling is enabled.
77
78
  """
78
- rolling = sample_interval_acronym.startswith("ROLL")
79
+ rolling = temporal_resolution.startswith("ROLL")
79
80
  if rolling:
80
- sample_interval_acronym = sample_interval_acronym[4:] # Remove "ROLL"
81
+ temporal_resolution = temporal_resolution[4:] # Remove "ROLL"
81
82
 
82
83
  # Allowed pattern: one or more occurrences of "<number><unit>"
83
84
  # where unit is exactly one of D, H, MIN, or S.
@@ -85,15 +86,15 @@ def get_resampling_information(sample_interval_acronym):
85
86
  pattern = r"^(\d+(?:D|H|MIN|S))+$"
86
87
 
87
88
  # Check if the entire string matches the pattern
88
- if not re.match(pattern, sample_interval_acronym):
89
+ if not re.match(pattern, temporal_resolution):
89
90
  raise ValueError(
90
- f"Invalid sample interval acronym '{sample_interval_acronym}'. "
91
+ f"Invalid temporal resolution '{temporal_resolution}'. "
91
92
  "Must be composed of one or more <number><unit> groups, where unit is D, H, MIN, or S.",
92
93
  )
93
94
 
94
95
  # Regular expression to match duration components and extract all (value, unit) pairs
95
96
  pattern = r"(\d+)(D|H|MIN|S)"
96
- matches = re.findall(pattern, sample_interval_acronym)
97
+ matches = re.findall(pattern, temporal_resolution)
97
98
 
98
99
  # Conversion factors for each unit
99
100
  unit_to_seconds = {
@@ -112,21 +113,21 @@ def get_resampling_information(sample_interval_acronym):
112
113
  return sample_interval, rolling
113
114
 
114
115
 
115
- def acronym_to_seconds(acronym):
116
+ def temporal_resolution_to_seconds(temporal_resolution):
116
117
  """
117
- Extract the interval in seconds from the duration acronym.
118
+ Extract the measurement interval in seconds from the temporal resolution string.
118
119
 
119
120
  Parameters
120
121
  ----------
121
- acronym: str
122
- A string representing a duration: e.g., "1H30MIN", "ROLL1H30MIN".
122
+ temporal_resolution: str
123
+ A string representing the product measurement interval: e.g., "1H30MIN", "ROLL1H30MIN".
123
124
 
124
125
  Returns
125
126
  -------
126
127
  seconds
127
128
  Duration in seconds.
128
129
  """
129
- seconds, _ = get_resampling_information(acronym)
130
+ seconds, _ = get_resampling_information(temporal_resolution)
130
131
  return seconds
131
132
 
132
133
 
@@ -262,6 +263,7 @@ def regularize_dataset(
262
263
  Regularized dataset.
263
264
 
264
265
  """
266
+ attrs = xr_obj.attrs.copy()
265
267
  xr_obj = _check_time_sorted(xr_obj, time_dim=time_dim)
266
268
  start_time, end_time = get_dataset_start_end_time(xr_obj, time_dim=time_dim)
267
269
 
@@ -289,11 +291,14 @@ def regularize_dataset(
289
291
  # tolerance=tolerance, # mismatch in seconds
290
292
  fill_value=fill_value,
291
293
  )
294
+
295
+ # Ensure attributes are preserved
296
+ xr_obj.attrs = attrs
292
297
  return xr_obj
293
298
 
294
299
 
295
300
  ####------------------------------------------
296
- #### Sampling interval utilities
301
+ #### Interval utilities
297
302
 
298
303
 
299
304
  def ensure_sample_interval_in_seconds(sample_interval): # noqa: PLR0911
@@ -376,7 +381,7 @@ def ensure_sample_interval_in_seconds(sample_interval): # noqa: PLR0911
376
381
  raise TypeError("Float array sample_interval must contain only whole numbers.")
377
382
  return sample_interval.astype(int)
378
383
 
379
- # Deal with xarray.DataArrayy of floats that are all integer-valued (with optionally some NaN)
384
+ # Deal with xarray.DataArray of floats that are all integer-valued (with optionally some NaN)
380
385
  if isinstance(sample_interval, xr.DataArray) and np.issubdtype(sample_interval.dtype, np.floating):
381
386
  arr = sample_interval.copy()
382
387
  data = arr.data
@@ -397,6 +402,17 @@ def ensure_sample_interval_in_seconds(sample_interval): # noqa: PLR0911
397
402
  )
398
403
 
399
404
 
405
+ def ensure_timedelta_seconds(interval):
406
+ """Return an a scalar value/array in seconds or timedelta object as numpy.timedelta64 in seconds."""
407
+ if isinstance(interval, (xr.DataArray, np.ndarray)):
408
+ return ensure_sample_interval_in_seconds(interval).astype("m8[s]")
409
+ return np.array(ensure_sample_interval_in_seconds(interval), dtype="m8[s]")
410
+
411
+
412
+ ####------------------------------------------
413
+ #### Sample Interval Utilities
414
+
415
+
400
416
  def infer_sample_interval(ds, robust=False, verbose=False, logger=None):
401
417
  """Infer the sample interval of a dataset.
402
418
 
@@ -497,161 +513,3 @@ def infer_sample_interval(ds, robust=False, verbose=False, logger=None):
497
513
  )
498
514
  log_warning(logger=logger, msg=msg, verbose=verbose)
499
515
  return int(sample_interval)
500
-
501
-
502
- ####---------------------------------------------------------------------------------
503
- #### Timesteps regularization
504
-
505
-
506
- def get_problematic_timestep_indices(timesteps, sample_interval):
507
- """Identify timesteps with missing previous or following timesteps."""
508
- previous_time = timesteps - pd.Timedelta(seconds=sample_interval)
509
- next_time = timesteps + pd.Timedelta(seconds=sample_interval)
510
- idx_previous_missing = np.where(~np.isin(previous_time, timesteps))[0][1:]
511
- idx_next_missing = np.where(~np.isin(next_time, timesteps))[0][:-1]
512
- idx_isolated_missing = np.intersect1d(idx_previous_missing, idx_next_missing)
513
- idx_previous_missing = idx_previous_missing[np.isin(idx_previous_missing, idx_isolated_missing, invert=True)]
514
- idx_next_missing = idx_next_missing[np.isin(idx_next_missing, idx_isolated_missing, invert=True)]
515
- return idx_previous_missing, idx_next_missing, idx_isolated_missing
516
-
517
-
518
- def regularize_timesteps(ds, sample_interval, robust=False, add_quality_flag=True, logger=None, verbose=True):
519
- """Ensure timesteps match with the sample_interval.
520
-
521
- This function:
522
- - drop dataset indices with duplicated timesteps,
523
- - but does not add missing timesteps to the dataset.
524
- """
525
- # Check sorted by time and sort if necessary
526
- ds = ensure_sorted_by_time(ds)
527
-
528
- # Convert time to pandas.DatetimeIndex for easier manipulation
529
- times = pd.to_datetime(ds["time"].to_numpy())
530
-
531
- # Determine the start and end times
532
- start_time = times[0].floor(f"{sample_interval}s")
533
- end_time = times[-1].ceil(f"{sample_interval}s")
534
-
535
- # Create the expected time grid
536
- expected_times = pd.date_range(start=start_time, end=end_time, freq=f"{sample_interval}s")
537
-
538
- # Convert to numpy arrays
539
- times = times.to_numpy(dtype="M8[s]")
540
- expected_times = expected_times.to_numpy(dtype="M8[s]")
541
-
542
- # Map original times to the nearest expected times
543
- # Calculate the difference between original times and expected times
544
- time_deltas = np.abs(times - expected_times[:, None]).astype(int)
545
-
546
- # Find the index of the closest expected time for each original time
547
- nearest_indices = np.argmin(time_deltas, axis=0)
548
- adjusted_times = expected_times[nearest_indices]
549
-
550
- # Check for duplicates in adjusted times
551
- unique_times, counts = np.unique(adjusted_times, return_counts=True)
552
- duplicates = unique_times[counts > 1]
553
-
554
- # Initialize time quality flag
555
- # - 0 when ok or just rounded to closest 00
556
- # - 1 if previous timestep is missing
557
- # - 2 if next timestep is missing
558
- # - 3 if previous and next timestep is missing
559
- # - 4 if solved duplicated timesteps
560
- # - 5 if needed to drop duplicated timesteps and select the last
561
- flag_previous_missing = 1
562
- flag_next_missing = 2
563
- flag_isolated_timestep = 3
564
- flag_solved_duplicated_timestep = 4
565
- flag_dropped_duplicated_timestep = 5
566
- qc_flag = np.zeros(adjusted_times.shape)
567
-
568
- # Initialize list with the duplicated timesteps index to drop
569
- # - We drop the first occurrence because is likely the shortest interval
570
- idx_to_drop = []
571
-
572
- # Attempt to resolve for duplicates
573
- if duplicates.size > 0:
574
- # Handle duplicates
575
- for dup_time in duplicates:
576
- # Indices of duplicates
577
- dup_indices = np.where(adjusted_times == dup_time)[0]
578
- n_duplicates = len(dup_indices)
579
- # Define previous and following timestep
580
- prev_time = dup_time - pd.Timedelta(seconds=sample_interval)
581
- next_time = dup_time + pd.Timedelta(seconds=sample_interval)
582
- # Try to find missing slots before and after
583
- # - If more than 3 duplicates, impossible to solve !
584
- count_solved = 0
585
- # If the previous timestep is available, set that one
586
- if n_duplicates == 2:
587
- if prev_time not in adjusted_times:
588
- adjusted_times[dup_indices[0]] = prev_time
589
- qc_flag[dup_indices[0]] = flag_solved_duplicated_timestep
590
- count_solved += 1
591
- elif next_time not in adjusted_times:
592
- adjusted_times[dup_indices[-1]] = next_time
593
- qc_flag[dup_indices[-1]] = flag_solved_duplicated_timestep
594
- count_solved += 1
595
- else:
596
- pass
597
- elif n_duplicates == 3:
598
- if prev_time not in adjusted_times:
599
- adjusted_times[dup_indices[0]] = prev_time
600
- qc_flag[dup_indices[0]] = flag_solved_duplicated_timestep
601
- count_solved += 1
602
- if next_time not in adjusted_times:
603
- adjusted_times[dup_indices[-1]] = next_time
604
- qc_flag[dup_indices[-1]] = flag_solved_duplicated_timestep
605
- count_solved += 1
606
- if count_solved != n_duplicates - 1:
607
- idx_to_drop = np.append(idx_to_drop, dup_indices[0:-1])
608
- qc_flag[dup_indices[-1]] = flag_dropped_duplicated_timestep
609
- msg = (
610
- f"Cannot resolve {n_duplicates} duplicated timesteps "
611
- f"(after trailing seconds correction) around {dup_time}."
612
- )
613
- log_warning(logger=logger, msg=msg, verbose=verbose)
614
- if robust:
615
- raise ValueError(msg)
616
-
617
- # Update the time coordinate (Convert to ns for xarray compatibility)
618
- ds = ds.assign_coords({"time": adjusted_times.astype("datetime64[ns]")})
619
-
620
- # Update quality flag values for next and previous timestep is missing
621
- if add_quality_flag:
622
- idx_previous_missing, idx_next_missing, idx_isolated_missing = get_problematic_timestep_indices(
623
- adjusted_times,
624
- sample_interval,
625
- )
626
- qc_flag[idx_previous_missing] = np.maximum(qc_flag[idx_previous_missing], flag_previous_missing)
627
- qc_flag[idx_next_missing] = np.maximum(qc_flag[idx_next_missing], flag_next_missing)
628
- qc_flag[idx_isolated_missing] = np.maximum(qc_flag[idx_isolated_missing], flag_isolated_timestep)
629
-
630
- # If the first timestep is at 00:00 and currently flagged as previous missing (1), reset to 0
631
- # first_time = pd.to_datetime(adjusted_times[0]).time()
632
- # first_expected_time = pd.Timestamp("00:00:00").time()
633
- # if first_time == first_expected_time and qc_flag[0] == flag_previous_missing:
634
- # qc_flag[0] = 0
635
-
636
- # # If the last timestep is flagged and currently flagged as next missing (2), reset it to 0
637
- # last_time = pd.to_datetime(adjusted_times[-1]).time()
638
- # last_time_expected = (pd.Timestamp("00:00:00") - pd.Timedelta(30, unit="seconds")).time()
639
- # # Check if adding one interval would go beyond the end_time
640
- # if last_time == last_time_expected and qc_flag[-1] == flag_next_missing:
641
- # qc_flag[-1] = 0
642
-
643
- # Assign time quality flag coordinate
644
- ds["time_qc"] = xr.DataArray(qc_flag, dims="time")
645
- ds = ds.set_coords("time_qc")
646
-
647
- # Drop duplicated timesteps
648
- # - Using ds = ds.drop_isel({"time": idx_to_drop.astype(int)}) raise:
649
- # --> pandas.errors.InvalidIndexError: Reindexing only valid with uniquely valued Index objects
650
- # --> https://github.com/pydata/xarray/issues/6605
651
- if len(idx_to_drop) > 0:
652
- idx_to_drop = idx_to_drop.astype(int)
653
- idx_valid_timesteps = np.arange(0, ds["time"].size)
654
- idx_valid_timesteps = np.delete(idx_valid_timesteps, idx_to_drop)
655
- ds = ds.isel(time=idx_valid_timesteps)
656
- # Return dataset
657
- return ds
disdrodb/utils/writer.py CHANGED
@@ -22,11 +22,29 @@ import os
22
22
 
23
23
  import xarray as xr
24
24
 
25
- from disdrodb.utils.attrs import set_disdrodb_attrs
25
+ from disdrodb.utils.attrs import get_attrs_dict, set_attrs, set_disdrodb_attrs
26
26
  from disdrodb.utils.directories import create_directory, remove_if_exists
27
+ from disdrodb.utils.encoding import get_encodings_dict, set_encodings
27
28
 
28
29
 
29
- def write_product(ds: xr.Dataset, filepath: str, product: str, force: bool = False) -> None:
30
+ def finalize_product(ds, product=None) -> xr.Dataset:
31
+ """Finalize DISDRODB product."""
32
+ # Add variables attributes
33
+ attrs_dict = get_attrs_dict()
34
+ ds = set_attrs(ds, attrs_dict=attrs_dict)
35
+
36
+ # Add variables encoding
37
+ encodings_dict = get_encodings_dict()
38
+ ds = set_encodings(ds, encodings_dict=encodings_dict)
39
+
40
+ # Add DISDRODB global attributes
41
+ # - e.g. in generate_l2_radar it inherit from input dataset !
42
+ if product is not None:
43
+ ds = set_disdrodb_attrs(ds, product=product)
44
+ return ds
45
+
46
+
47
+ def write_product(ds: xr.Dataset, filepath: str, force: bool = False) -> None:
30
48
  """Save the xarray dataset into a NetCDF file.
31
49
 
32
50
  Parameters
@@ -35,8 +53,6 @@ def write_product(ds: xr.Dataset, filepath: str, product: str, force: bool = Fal
35
53
  Input xarray dataset.
36
54
  filepath : str
37
55
  Output file path.
38
- product: str
39
- DISDRODB product name.
40
56
  force : bool, optional
41
57
  Whether to overwrite existing data.
42
58
  If ``True``, overwrite existing data into destination directories.
@@ -50,8 +66,5 @@ def write_product(ds: xr.Dataset, filepath: str, product: str, force: bool = Fal
50
66
  # - If force=False --> Raise error
51
67
  remove_if_exists(filepath, force=force)
52
68
 
53
- # Update attributes
54
- ds = set_disdrodb_attrs(ds, product=product)
55
-
56
69
  # Write netcdf
57
70
  ds.to_netcdf(filepath, engine="netcdf4")
disdrodb/utils/xarray.py CHANGED
@@ -21,6 +21,8 @@ import numpy as np
21
21
  import xarray as xr
22
22
  from xarray.core import dtypes
23
23
 
24
+ from disdrodb.constants import DIAMETER_COORDS, VELOCITY_COORDS
25
+
24
26
 
25
27
  def xr_get_last_valid_idx(da_condition, dim, fill_value=None):
26
28
  """
@@ -104,6 +106,7 @@ def xr_get_last_valid_idx(da_condition, dim, fill_value=None):
104
106
  def _check_coord_handling(coord_handling):
105
107
  if coord_handling not in {"keep", "drop", "unstack"}:
106
108
  raise ValueError("coord_handling must be one of 'keep', 'drop', or 'unstack'.")
109
+ return coord_handling
107
110
 
108
111
 
109
112
  def _unstack_coordinates(xr_obj, dim, prefix, suffix):
@@ -161,6 +164,8 @@ def unstack_datarray_dimension(da, dim, coord_handling="keep", prefix="", suffix
161
164
  """
162
165
  # Retrieve DataArray name
163
166
  name = da.name
167
+ coord_handling = _check_coord_handling(coord_handling)
168
+
164
169
  # Unstack variables
165
170
  ds = da.to_dataset(dim=dim)
166
171
  rename_dict = {dim_value: f"{prefix}{name}{suffix}{dim_value}" for dim_value in list(ds.data_vars)}
@@ -246,13 +251,9 @@ def define_fill_value_dictionary(xr_obj):
246
251
 
247
252
  def remove_diameter_coordinates(xr_obj):
248
253
  """Drop diameter coordinates from xarray object."""
249
- from disdrodb import DIAMETER_COORDS
250
-
251
254
  return xr_obj.drop_vars(DIAMETER_COORDS, errors="ignore")
252
255
 
253
256
 
254
257
  def remove_velocity_coordinates(xr_obj):
255
258
  """Drop velocity coordinates from xarray object."""
256
- from disdrodb import VELOCITY_COORDS
257
-
258
259
  return xr_obj.drop_vars(VELOCITY_COORDS, errors="ignore")
disdrodb/viz/__init__.py CHANGED
@@ -15,3 +15,16 @@
15
15
  # along with this program. If not, see <http://www.gnu.org/licenses/>.
16
16
  # -----------------------------------------------------------------------------.
17
17
  """DISDRODB Visualization Module."""
18
+ from disdrodb.viz.plots import (
19
+ compute_dense_lines,
20
+ max_blend_images,
21
+ plot_nd,
22
+ to_rgba,
23
+ )
24
+
25
+ __all__ = [
26
+ "compute_dense_lines",
27
+ "max_blend_images",
28
+ "plot_nd",
29
+ "to_rgba",
30
+ ]