disdrodb 0.1.5__py3-none-any.whl → 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +1 -5
- disdrodb/_version.py +2 -2
- disdrodb/accessor/methods.py +22 -4
- disdrodb/api/checks.py +10 -0
- disdrodb/api/io.py +20 -18
- disdrodb/api/path.py +42 -77
- disdrodb/api/search.py +89 -23
- disdrodb/cli/disdrodb_create_summary.py +1 -1
- disdrodb/cli/disdrodb_run_l0.py +1 -1
- disdrodb/cli/disdrodb_run_l0a.py +1 -1
- disdrodb/cli/disdrodb_run_l0b.py +1 -1
- disdrodb/cli/disdrodb_run_l0c.py +1 -1
- disdrodb/cli/disdrodb_run_l1.py +1 -1
- disdrodb/cli/disdrodb_run_l2e.py +1 -1
- disdrodb/cli/disdrodb_run_l2m.py +1 -1
- disdrodb/configs.py +30 -83
- disdrodb/constants.py +4 -3
- disdrodb/data_transfer/download_data.py +4 -2
- disdrodb/docs.py +2 -2
- disdrodb/etc/products/L1/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/LPM/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/LPM_V0/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/PARSIVEL/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/PARSIVEL2/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/PWS100/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/RD80/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/SWS250/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/global.yaml +6 -0
- disdrodb/etc/products/L2E/10MIN.yaml +1 -12
- disdrodb/etc/products/L2E/global.yaml +1 -1
- disdrodb/etc/products/L2M/MODELS/NGAMMA_GS_R_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/global.yaml +1 -1
- disdrodb/issue/checks.py +2 -2
- disdrodb/l0/check_configs.py +1 -1
- disdrodb/l0/configs/LPM/l0a_encodings.yml +0 -1
- disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +0 -4
- disdrodb/l0/configs/LPM/l0b_encodings.yml +9 -9
- disdrodb/l0/configs/LPM/raw_data_format.yml +11 -11
- disdrodb/l0/configs/LPM_V0/bins_diameter.yml +103 -0
- disdrodb/l0/configs/LPM_V0/bins_velocity.yml +103 -0
- disdrodb/l0/configs/LPM_V0/l0a_encodings.yml +45 -0
- disdrodb/l0/configs/LPM_V0/l0b_cf_attrs.yml +180 -0
- disdrodb/l0/configs/LPM_V0/l0b_encodings.yml +410 -0
- disdrodb/l0/configs/LPM_V0/raw_data_format.yml +474 -0
- disdrodb/l0/configs/PARSIVEL/l0b_encodings.yml +1 -1
- disdrodb/l0/configs/PARSIVEL/raw_data_format.yml +8 -8
- disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml +9 -9
- disdrodb/l0/l0_reader.py +2 -2
- disdrodb/l0/l0a_processing.py +6 -2
- disdrodb/l0/l0b_processing.py +26 -19
- disdrodb/l0/l0c_processing.py +17 -3
- disdrodb/l0/manuals/LPM_V0.pdf +0 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +15 -7
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_PI.py +279 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_T.py +276 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +2 -2
- disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_RWANDA_LPM_NC.py +103 -0
- disdrodb/l0/readers/LPM/NORWAY/HAUKELISETER_LPM.py +216 -0
- disdrodb/l0/readers/LPM/NORWAY/NMBU_LPM.py +208 -0
- disdrodb/l0/readers/LPM/UK/WITHWORTH_LPM.py +219 -0
- disdrodb/l0/readers/LPM/USA/CHARLESTON.py +229 -0
- disdrodb/l0/readers/{LPM → LPM_V0}/BELGIUM/ULIEGE.py +33 -49
- disdrodb/l0/readers/LPM_V0/ITALY/GID_LPM_V0.py +240 -0
- disdrodb/l0/readers/PARSIVEL/BASQUECOUNTRY/EUSKALMET_OTT.py +227 -0
- disdrodb/l0/readers/{PARSIVEL2 → PARSIVEL}/NASA/LPVEX.py +16 -28
- disdrodb/l0/readers/PARSIVEL/{GPM → NASA}/MC3E.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +3 -3
- disdrodb/l0/readers/PARSIVEL2/BASQUECOUNTRY/EUSKALMET_OTT2.py +232 -0
- disdrodb/l0/readers/PARSIVEL2/DENMARK/EROSION_raw.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/JAPAN/PRECIP.py +155 -0
- disdrodb/l0/readers/PARSIVEL2/MPI/BCO_PARSIVEL2.py +14 -7
- disdrodb/l0/readers/PARSIVEL2/MPI/BOWTIE.py +8 -3
- disdrodb/l0/readers/PARSIVEL2/NASA/APU.py +28 -5
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/{GPM/GCPEX.py → NORWAY/UIB.py} +54 -29
- disdrodb/l0/readers/PARSIVEL2/PHILIPPINES/{PANGASA.py → PAGASA.py} +6 -3
- disdrodb/l0/readers/PARSIVEL2/SPAIN/GRANADA.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/SWEDEN/SMHI.py +189 -0
- disdrodb/l0/readers/{PARSIVEL/GPM/PIERS.py → PARSIVEL2/USA/CSU.py} +62 -29
- disdrodb/l0/readers/PARSIVEL2/USA/{C3WE.py → CW3E.py} +51 -24
- disdrodb/l0/readers/{PARSIVEL/GPM/IFLOODS.py → RD80/BRAZIL/ATTO_RD80.py} +50 -34
- disdrodb/l0/readers/{SW250 → SWS250}/BELGIUM/KMI.py +1 -1
- disdrodb/l1/beard_model.py +45 -1
- disdrodb/l1/fall_velocity.py +1 -6
- disdrodb/l1/filters.py +2 -0
- disdrodb/l1/processing.py +6 -5
- disdrodb/l1/resampling.py +101 -38
- disdrodb/l2/empirical_dsd.py +12 -8
- disdrodb/l2/processing.py +4 -3
- disdrodb/metadata/search.py +3 -4
- disdrodb/routines/l0.py +4 -4
- disdrodb/routines/l1.py +173 -60
- disdrodb/routines/l2.py +121 -269
- disdrodb/routines/options.py +347 -0
- disdrodb/routines/wrappers.py +9 -1
- disdrodb/scattering/axis_ratio.py +3 -0
- disdrodb/scattering/routines.py +1 -1
- disdrodb/summary/routines.py +765 -724
- disdrodb/utils/archiving.py +51 -44
- disdrodb/utils/attrs.py +1 -1
- disdrodb/utils/compression.py +4 -2
- disdrodb/utils/dask.py +35 -15
- disdrodb/utils/dict.py +33 -0
- disdrodb/utils/encoding.py +1 -1
- disdrodb/utils/manipulations.py +7 -1
- disdrodb/utils/routines.py +9 -8
- disdrodb/utils/time.py +9 -1
- disdrodb/viz/__init__.py +0 -13
- disdrodb/viz/plots.py +209 -0
- {disdrodb-0.1.5.dist-info → disdrodb-0.2.1.dist-info}/METADATA +1 -1
- {disdrodb-0.1.5.dist-info → disdrodb-0.2.1.dist-info}/RECORD +124 -95
- disdrodb/l0/readers/PARSIVEL/GPM/LPVEX.py +0 -85
- /disdrodb/etc/products/L2M/{GAMMA_GS_ND_MAE.yaml → MODELS/GAMMA_GS_ND_MAE.yaml} +0 -0
- /disdrodb/etc/products/L2M/{GAMMA_ML.yaml → MODELS/GAMMA_ML.yaml} +0 -0
- /disdrodb/etc/products/L2M/{LOGNORMAL_GS_LOG_ND_MAE.yaml → MODELS/LOGNORMAL_GS_LOG_ND_MAE.yaml} +0 -0
- /disdrodb/etc/products/L2M/{LOGNORMAL_GS_ND_MAE.yaml → MODELS/LOGNORMAL_GS_ND_MAE.yaml} +0 -0
- /disdrodb/etc/products/L2M/{LOGNORMAL_ML.yaml → MODELS/LOGNORMAL_ML.yaml} +0 -0
- /disdrodb/etc/products/L2M/{NGAMMA_GS_LOG_ND_MAE.yaml → MODELS/NGAMMA_GS_LOG_ND_MAE.yaml} +0 -0
- /disdrodb/etc/products/L2M/{NGAMMA_GS_ND_MAE.yaml → MODELS/NGAMMA_GS_ND_MAE.yaml} +0 -0
- /disdrodb/etc/products/L2M/{NGAMMA_GS_Z_MAE.yaml → MODELS/NGAMMA_GS_Z_MAE.yaml} +0 -0
- /disdrodb/l0/readers/PARSIVEL2/{GPM → NASA}/NSSTC.py +0 -0
- {disdrodb-0.1.5.dist-info → disdrodb-0.2.1.dist-info}/WHEEL +0 -0
- {disdrodb-0.1.5.dist-info → disdrodb-0.2.1.dist-info}/entry_points.txt +0 -0
- {disdrodb-0.1.5.dist-info → disdrodb-0.2.1.dist-info}/licenses/LICENSE +0 -0
- {disdrodb-0.1.5.dist-info → disdrodb-0.2.1.dist-info}/top_level.txt +0 -0
disdrodb/api/search.py
CHANGED
|
@@ -17,6 +17,7 @@ from disdrodb.api.path import (
|
|
|
17
17
|
)
|
|
18
18
|
from disdrodb.configs import get_data_archive_dir, get_metadata_archive_dir
|
|
19
19
|
from disdrodb.constants import PRODUCTS_REQUIREMENTS
|
|
20
|
+
from disdrodb.utils.dict import extract_product_kwargs
|
|
20
21
|
from disdrodb.utils.directories import contains_files, contains_netcdf_or_parquet_files, list_directories, list_files
|
|
21
22
|
from disdrodb.utils.yaml import read_yaml
|
|
22
23
|
|
|
@@ -173,7 +174,15 @@ def list_station_names(
|
|
|
173
174
|
#### Filtering utilities for available_stations
|
|
174
175
|
|
|
175
176
|
|
|
176
|
-
def _finalize_output(list_info, return_tuple):
|
|
177
|
+
def _finalize_output(list_info, return_tuple, metadata_archive_dir, filter_kwargs):
|
|
178
|
+
# Filter stations if metadata filtering values are specified
|
|
179
|
+
if len(filter_kwargs) != 0:
|
|
180
|
+
list_info = select_stations_matching_metadata_values(
|
|
181
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
182
|
+
list_info=list_info,
|
|
183
|
+
filter_kwargs=filter_kwargs,
|
|
184
|
+
)
|
|
185
|
+
|
|
177
186
|
# - Return the (data_source, campaign_name, station_name) tuple
|
|
178
187
|
if return_tuple:
|
|
179
188
|
return list_info
|
|
@@ -192,7 +201,7 @@ def is_disdrodb_data_url_specified(metadata_filepath):
|
|
|
192
201
|
return isinstance(disdrodb_data_url, str) and len(disdrodb_data_url) > 1
|
|
193
202
|
|
|
194
203
|
|
|
195
|
-
def
|
|
204
|
+
def select_stations_with_disdrodb_data_url(metadata_archive_dir, list_info):
|
|
196
205
|
"""Keep only the stations with disdrodb_data_url specified in the metadata file."""
|
|
197
206
|
list_info_with_data = []
|
|
198
207
|
for data_source, campaign_name, station_name in list_info:
|
|
@@ -209,7 +218,41 @@ def keep_list_info_with_disdrodb_data_url(metadata_archive_dir, list_info):
|
|
|
209
218
|
return list_info_with_data
|
|
210
219
|
|
|
211
220
|
|
|
212
|
-
def
|
|
221
|
+
def _matches(metadata_value, expected_value):
|
|
222
|
+
"""Return True if metadata_value matches expected_value."""
|
|
223
|
+
# Case 1: both lists → check any intersection
|
|
224
|
+
if isinstance(metadata_value, list) and isinstance(expected_value, list):
|
|
225
|
+
return any(v in metadata_value for v in expected_value)
|
|
226
|
+
# Case 2: metadata is list → check membership
|
|
227
|
+
if isinstance(metadata_value, list):
|
|
228
|
+
return expected_value in metadata_value
|
|
229
|
+
# Case 3: expected is list → check membership
|
|
230
|
+
if isinstance(expected_value, list):
|
|
231
|
+
return metadata_value in expected_value
|
|
232
|
+
# Case 4: both scalars → direct equality
|
|
233
|
+
return metadata_value == expected_value
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def select_stations_matching_metadata_values(metadata_archive_dir, list_info, filter_kwargs):
|
|
237
|
+
"""Keep only the stations with the specified metadata key matching the specified value."""
|
|
238
|
+
list_info_valid = []
|
|
239
|
+
for data_source, campaign_name, station_name in list_info:
|
|
240
|
+
# Define metadata filepath
|
|
241
|
+
metadata_filepath = define_metadata_filepath(
|
|
242
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
243
|
+
data_source=data_source,
|
|
244
|
+
campaign_name=campaign_name,
|
|
245
|
+
station_name=station_name,
|
|
246
|
+
)
|
|
247
|
+
# Read metadata
|
|
248
|
+
metadata = read_yaml(metadata_filepath)
|
|
249
|
+
if np.all([_matches(metadata.get(k), v) for k, v in filter_kwargs.items()]):
|
|
250
|
+
list_info_valid.append((data_source, campaign_name, station_name))
|
|
251
|
+
|
|
252
|
+
return list_info_valid
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def select_stations_with_product_directory(data_archive_dir, product, list_info):
|
|
213
256
|
"""Keep only the stations with the product directory."""
|
|
214
257
|
list_info_with_product_directory = []
|
|
215
258
|
for data_source, campaign_name, station_name in list_info:
|
|
@@ -228,7 +271,7 @@ def keep_list_info_elements_with_product_directory(data_archive_dir, product, li
|
|
|
228
271
|
return list_info_with_product_directory
|
|
229
272
|
|
|
230
273
|
|
|
231
|
-
def
|
|
274
|
+
def select_stations_with_product_data(data_archive_dir, product, list_info, **product_kwargs):
|
|
232
275
|
"""Keep only the stations with product data."""
|
|
233
276
|
# Define file checking function
|
|
234
277
|
checking_function = contains_files if product == "RAW" else contains_netcdf_or_parquet_files
|
|
@@ -265,7 +308,7 @@ def available_stations(
|
|
|
265
308
|
invalid_fields_policy="raise",
|
|
266
309
|
data_archive_dir=None,
|
|
267
310
|
metadata_archive_dir=None,
|
|
268
|
-
**
|
|
311
|
+
**filter_kwargs,
|
|
269
312
|
):
|
|
270
313
|
"""
|
|
271
314
|
Return stations information for which metadata or product data are available on disk.
|
|
@@ -289,7 +332,7 @@ def available_stations(
|
|
|
289
332
|
|
|
290
333
|
If the DISDRODB product is specified,
|
|
291
334
|
it lists the stations present in the local DISDRODB Data Archive given the specified filtering criteria.
|
|
292
|
-
The default is
|
|
335
|
+
The default is None.
|
|
293
336
|
|
|
294
337
|
data_sources : str or sequence of str, optional
|
|
295
338
|
One or more data source identifiers to filter stations by.
|
|
@@ -342,15 +385,15 @@ def available_stations(
|
|
|
342
385
|
If None, the default metadata base directory is used. Default is None.
|
|
343
386
|
**product_kwargs : dict, optional
|
|
344
387
|
Additional arguments required for some products.
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
388
|
+
It must be specified only for product L1, L2E and L2M products !
|
|
389
|
+
For L1, L2E and L2M products, ``temporal_resolution`` is required.
|
|
390
|
+
FOr L2M product, ``model_name`` is required.
|
|
348
391
|
|
|
349
392
|
Returns
|
|
350
393
|
-------
|
|
351
394
|
list
|
|
352
395
|
If ``return_tuple=True``, return a list of tuples ``(data_source, campaign_name, station_name)``.
|
|
353
|
-
If ``return_tuple=True
|
|
396
|
+
If ``return_tuple=True``, return a list of station names.
|
|
354
397
|
|
|
355
398
|
Examples
|
|
356
399
|
--------
|
|
@@ -361,15 +404,18 @@ def available_stations(
|
|
|
361
404
|
>>> # List stations with raw data available in the local DISDRODB Data Archive
|
|
362
405
|
>>> raw_stations = available_stations(product="RAW", available_data=True)
|
|
363
406
|
>>> # List stations of specific data sources
|
|
364
|
-
>>> stations = available_stations(data_sources=["
|
|
407
|
+
>>> stations = available_stations(data_sources=["NASA", "EPFL"])
|
|
365
408
|
"""
|
|
366
409
|
# Retrieve DISDRODB Data and Metadata Archive directories
|
|
367
410
|
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
368
411
|
product = check_product(product) if product is not None else None
|
|
369
412
|
invalid_fields_policy = check_invalid_fields_policy(invalid_fields_policy)
|
|
370
413
|
|
|
414
|
+
# Extract product_kwargs from filter_kwargs
|
|
415
|
+
product_kwargs = extract_product_kwargs(filter_kwargs, product=product) if product is not None else {}
|
|
416
|
+
|
|
371
417
|
# Retrieve available stations from the Metadata Archive
|
|
372
|
-
# - Raise error if no stations
|
|
418
|
+
# - Raise error if no stations available !
|
|
373
419
|
list_info = list_station_names(
|
|
374
420
|
metadata_archive_dir=metadata_archive_dir,
|
|
375
421
|
data_sources=data_sources,
|
|
@@ -386,24 +432,34 @@ def available_stations(
|
|
|
386
432
|
raise_error_if_empty=raise_error_if_empty,
|
|
387
433
|
msg="No station available in the DISDRODB Metadata Archive.",
|
|
388
434
|
)
|
|
389
|
-
return _finalize_output(
|
|
435
|
+
return _finalize_output(
|
|
436
|
+
list_info,
|
|
437
|
+
return_tuple=return_tuple,
|
|
438
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
439
|
+
filter_kwargs=filter_kwargs,
|
|
440
|
+
)
|
|
390
441
|
|
|
391
442
|
# Return stations in the Metadata Archive with specified disdrodb_data_url
|
|
392
443
|
if product is None and available_data:
|
|
393
|
-
list_info =
|
|
444
|
+
list_info = select_stations_with_disdrodb_data_url(metadata_archive_dir, list_info)
|
|
394
445
|
_raise_an_error_if_no_stations(
|
|
395
446
|
list_info,
|
|
396
447
|
raise_error_if_empty=raise_error_if_empty,
|
|
397
448
|
msg="No station has the disdrodb_data_url specified in the metadata.",
|
|
398
449
|
)
|
|
399
|
-
return _finalize_output(
|
|
450
|
+
return _finalize_output(
|
|
451
|
+
list_info,
|
|
452
|
+
return_tuple=return_tuple,
|
|
453
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
454
|
+
filter_kwargs=filter_kwargs,
|
|
455
|
+
)
|
|
400
456
|
|
|
401
457
|
# If product is specified, select stations available in the local DISDRODB Data Archive
|
|
402
458
|
# - If available_data=False, search for station with the existing product directory (do not check for data)
|
|
403
459
|
data_archive_dir = get_data_archive_dir(data_archive_dir)
|
|
404
460
|
product = check_product(product)
|
|
405
461
|
if not available_data:
|
|
406
|
-
list_info =
|
|
462
|
+
list_info = select_stations_with_product_directory(
|
|
407
463
|
data_archive_dir=data_archive_dir,
|
|
408
464
|
product=product,
|
|
409
465
|
list_info=list_info,
|
|
@@ -413,11 +469,16 @@ def available_stations(
|
|
|
413
469
|
raise_error_if_empty=raise_error_if_empty,
|
|
414
470
|
msg=f"No station product {product} directory available in the local DISDRODB Data Archive.",
|
|
415
471
|
)
|
|
416
|
-
return _finalize_output(
|
|
472
|
+
return _finalize_output(
|
|
473
|
+
list_info,
|
|
474
|
+
return_tuple=return_tuple,
|
|
475
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
476
|
+
filter_kwargs=filter_kwargs,
|
|
477
|
+
)
|
|
417
478
|
|
|
418
479
|
# - If available_data=True, search for station with product data
|
|
419
480
|
product_kwargs = check_product_kwargs(product, product_kwargs)
|
|
420
|
-
list_info =
|
|
481
|
+
list_info = select_stations_with_product_data(
|
|
421
482
|
data_archive_dir=data_archive_dir,
|
|
422
483
|
product=product,
|
|
423
484
|
list_info=list_info,
|
|
@@ -429,7 +490,12 @@ def available_stations(
|
|
|
429
490
|
raise_error_if_empty=raise_error_if_empty,
|
|
430
491
|
msg=f"No station has {product} {product_kwargs} data available in the local DISDRODB Data Archive.",
|
|
431
492
|
)
|
|
432
|
-
return _finalize_output(
|
|
493
|
+
return _finalize_output(
|
|
494
|
+
list_info,
|
|
495
|
+
return_tuple=return_tuple,
|
|
496
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
497
|
+
filter_kwargs=filter_kwargs,
|
|
498
|
+
)
|
|
433
499
|
|
|
434
500
|
|
|
435
501
|
def available_data_sources(
|
|
@@ -441,7 +507,7 @@ def available_data_sources(
|
|
|
441
507
|
invalid_fields_policy="raise",
|
|
442
508
|
data_archive_dir=None,
|
|
443
509
|
metadata_archive_dir=None,
|
|
444
|
-
**
|
|
510
|
+
**kwargs,
|
|
445
511
|
):
|
|
446
512
|
"""Return data sources for which stations are available."""
|
|
447
513
|
list_info = available_stations(
|
|
@@ -455,7 +521,7 @@ def available_data_sources(
|
|
|
455
521
|
invalid_fields_policy=invalid_fields_policy,
|
|
456
522
|
data_archive_dir=data_archive_dir,
|
|
457
523
|
metadata_archive_dir=metadata_archive_dir,
|
|
458
|
-
**
|
|
524
|
+
**kwargs,
|
|
459
525
|
)
|
|
460
526
|
data_sources = [info[0] for info in list_info]
|
|
461
527
|
data_sources = np.unique(data_sources).tolist()
|
|
@@ -471,7 +537,7 @@ def available_campaigns(
|
|
|
471
537
|
invalid_fields_policy="raise",
|
|
472
538
|
data_archive_dir=None,
|
|
473
539
|
metadata_archive_dir=None,
|
|
474
|
-
**
|
|
540
|
+
**kwargs,
|
|
475
541
|
):
|
|
476
542
|
"""Return campaigns names for which stations are available."""
|
|
477
543
|
list_info = available_stations(
|
|
@@ -485,7 +551,7 @@ def available_campaigns(
|
|
|
485
551
|
invalid_fields_policy=invalid_fields_policy,
|
|
486
552
|
data_archive_dir=data_archive_dir,
|
|
487
553
|
metadata_archive_dir=metadata_archive_dir,
|
|
488
|
-
**
|
|
554
|
+
**kwargs,
|
|
489
555
|
)
|
|
490
556
|
campaign_names = [info[1] for info in list_info]
|
|
491
557
|
campaign_names = np.unique(campaign_names).tolist()
|
|
@@ -67,7 +67,7 @@ def disdrodb_create_summary(
|
|
|
67
67
|
Name of data source(s) to process.
|
|
68
68
|
The name(s) must be UPPER CASE.
|
|
69
69
|
If campaign_names and station are not specified, process all stations.
|
|
70
|
-
To specify multiple data sources, write i.e.: --data_sources '
|
|
70
|
+
To specify multiple data sources, write i.e.: --data_sources 'NASA EPFL NCAR'
|
|
71
71
|
campaign_names : str
|
|
72
72
|
Name of the campaign(s) for which to create stations summaries.
|
|
73
73
|
The name(s) must be UPPER CASE.
|
disdrodb/cli/disdrodb_run_l0.py
CHANGED
|
@@ -75,7 +75,7 @@ def disdrodb_run_l0(
|
|
|
75
75
|
Name of data source(s) to process.
|
|
76
76
|
The name(s) must be UPPER CASE.
|
|
77
77
|
If campaign_names and station are not specified, process all stations.
|
|
78
|
-
To specify multiple data sources, write i.e.: --data_sources '
|
|
78
|
+
To specify multiple data sources, write i.e.: --data_sources 'NASA EPFL NCAR'
|
|
79
79
|
campaign_names : str
|
|
80
80
|
Name of the campaign(s) to process.
|
|
81
81
|
The name(s) must be UPPER CASE.
|
disdrodb/cli/disdrodb_run_l0a.py
CHANGED
|
@@ -65,7 +65,7 @@ def disdrodb_run_l0a(
|
|
|
65
65
|
Name of data source(s) to process.
|
|
66
66
|
The name(s) must be UPPER CASE.
|
|
67
67
|
If campaign_names and station are not specified, process all stations.
|
|
68
|
-
To specify multiple data sources, write i.e.: --data_sources '
|
|
68
|
+
To specify multiple data sources, write i.e.: --data_sources 'NASA EPFL NCAR'
|
|
69
69
|
campaign_names : str
|
|
70
70
|
Name of the campaign(s) to process.
|
|
71
71
|
The name(s) must be UPPER CASE.
|
disdrodb/cli/disdrodb_run_l0b.py
CHANGED
|
@@ -68,7 +68,7 @@ def disdrodb_run_l0b(
|
|
|
68
68
|
Name of data source(s) to process.
|
|
69
69
|
The name(s) must be UPPER CASE.
|
|
70
70
|
If campaign_names and station are not specified, process all stations.
|
|
71
|
-
To specify multiple data sources, write i.e.: --data_sources '
|
|
71
|
+
To specify multiple data sources, write i.e.: --data_sources 'NASA EPFL NCAR'
|
|
72
72
|
campaign_names : str
|
|
73
73
|
Name of the campaign(s) to process.
|
|
74
74
|
The name(s) must be UPPER CASE.
|
disdrodb/cli/disdrodb_run_l0c.py
CHANGED
|
@@ -69,7 +69,7 @@ def disdrodb_run_l0c(
|
|
|
69
69
|
Name of data source(s) to process.
|
|
70
70
|
The name(s) must be UPPER CASE.
|
|
71
71
|
If campaign_names and station are not specified, process all stations.
|
|
72
|
-
To specify multiple data sources, write i.e.: --data_sources '
|
|
72
|
+
To specify multiple data sources, write i.e.: --data_sources 'NASA EPFL NCAR'
|
|
73
73
|
campaign_names : str
|
|
74
74
|
Name of the campaign(s) to process.
|
|
75
75
|
The name(s) must be UPPER CASE.
|
disdrodb/cli/disdrodb_run_l1.py
CHANGED
|
@@ -65,7 +65,7 @@ def disdrodb_run_l1(
|
|
|
65
65
|
Name of data source(s) to process.
|
|
66
66
|
The name(s) must be UPPER CASE.
|
|
67
67
|
If campaign_names and station are not specified, process all stations.
|
|
68
|
-
To specify multiple data sources, write i.e.: --data_sources '
|
|
68
|
+
To specify multiple data sources, write i.e.: --data_sources 'NASA EPFL NCAR'
|
|
69
69
|
campaign_names : str
|
|
70
70
|
Name of the campaign(s) to process.
|
|
71
71
|
The name(s) must be UPPER CASE.
|
disdrodb/cli/disdrodb_run_l2e.py
CHANGED
|
@@ -65,7 +65,7 @@ def disdrodb_run_l2e(
|
|
|
65
65
|
Name of data source(s) to process.
|
|
66
66
|
The name(s) must be UPPER CASE.
|
|
67
67
|
If campaign_names and station are not specified, process all stations.
|
|
68
|
-
To specify multiple data sources, write i.e.: --data_sources '
|
|
68
|
+
To specify multiple data sources, write i.e.: --data_sources 'NASA EPFL NCAR'
|
|
69
69
|
campaign_names : str
|
|
70
70
|
Name of the campaign(s) to process.
|
|
71
71
|
The name(s) must be UPPER CASE.
|
disdrodb/cli/disdrodb_run_l2m.py
CHANGED
|
@@ -65,7 +65,7 @@ def disdrodb_run_l2m(
|
|
|
65
65
|
Name of data source(s) to process.
|
|
66
66
|
The name(s) must be UPPER CASE.
|
|
67
67
|
If campaign_names and station are not specified, process all stations.
|
|
68
|
-
To specify multiple data sources, write i.e.: --data_sources '
|
|
68
|
+
To specify multiple data sources, write i.e.: --data_sources 'NASA EPFL NCAR'
|
|
69
69
|
campaign_names : str
|
|
70
70
|
Name of the campaign(s) to process.
|
|
71
71
|
The name(s) must be UPPER CASE.
|
disdrodb/configs.py
CHANGED
|
@@ -37,7 +37,7 @@ def define_configs(
|
|
|
37
37
|
data_archive_dir: Optional[str] = None,
|
|
38
38
|
metadata_archive_dir: Optional[str] = None,
|
|
39
39
|
scattering_table_dir: Optional[str] = None,
|
|
40
|
-
|
|
40
|
+
products_configs_dir: Optional[str] = None,
|
|
41
41
|
folder_partitioning: Optional[str] = None,
|
|
42
42
|
zenodo_token: Optional[str] = None,
|
|
43
43
|
zenodo_sandbox_token: Optional[str] = None,
|
|
@@ -53,7 +53,7 @@ def define_configs(
|
|
|
53
53
|
The directory path where the DISDRODB Metadata Archive is located.
|
|
54
54
|
scattering_table_dir : str
|
|
55
55
|
The directory path where to store DISDRODB T-Matrix scattering tables.
|
|
56
|
-
|
|
56
|
+
products_configs_dir : str
|
|
57
57
|
The directory path where the custom DISDRODB products configurations files are defined.
|
|
58
58
|
folder_partitioning : str
|
|
59
59
|
The folder partitioning scheme used in the DISDRODB Data Archive.
|
|
@@ -64,7 +64,7 @@ def define_configs(
|
|
|
64
64
|
- "year/month/day": Files are stored under subdirectories by year, month and day (<station_dir>/2025/04/01).
|
|
65
65
|
- "year/month_name": Files are stored under subdirectories by year and month name (<station_dir>/2025/April).
|
|
66
66
|
- "year/quarter": Files are stored under subdirectories by year and quarter (<station_dir>/2025/Q2).
|
|
67
|
-
|
|
67
|
+
zenodo_token: str
|
|
68
68
|
Zenodo Access Token. It is required to upload stations data to Zenodo.
|
|
69
69
|
zenodo_sandbox_token: str
|
|
70
70
|
Zenodo Sandbox Access Token. It is required to upload stations data to Zenodo Sandbox.
|
|
@@ -118,8 +118,8 @@ def define_configs(
|
|
|
118
118
|
if zenodo_sandbox_token is not None:
|
|
119
119
|
config_dict["zenodo_sandbox_token"] = zenodo_sandbox_token
|
|
120
120
|
|
|
121
|
-
if
|
|
122
|
-
config_dict["
|
|
121
|
+
if products_configs_dir is not None:
|
|
122
|
+
config_dict["products_configs_dir"] = products_configs_dir
|
|
123
123
|
|
|
124
124
|
# Write the DISDRODB config file
|
|
125
125
|
write_yaml(config_dict, filepath, sort_keys=False)
|
|
@@ -238,105 +238,52 @@ def get_zenodo_token(sandbox: bool):
|
|
|
238
238
|
return token
|
|
239
239
|
|
|
240
240
|
|
|
241
|
-
def
|
|
242
|
-
"""Return the
|
|
241
|
+
def get_default_products_configs_dir():
|
|
242
|
+
"""Return the directory path where DISDRODB products default configuration files are stored."""
|
|
243
243
|
import disdrodb
|
|
244
244
|
|
|
245
|
-
|
|
246
|
-
return
|
|
245
|
+
products_configs_dir = os.path.join(disdrodb.package_dir, "etc", "products")
|
|
246
|
+
return products_configs_dir
|
|
247
247
|
|
|
248
248
|
|
|
249
|
-
def
|
|
250
|
-
"""
|
|
249
|
+
def get_products_configs_dir():
|
|
250
|
+
"""Return the DISDRODB products configuration directory."""
|
|
251
251
|
import disdrodb
|
|
252
252
|
|
|
253
|
-
if "
|
|
254
|
-
|
|
255
|
-
|
|
253
|
+
if os.environ.get("PYTEST_CURRENT_TEST"):
|
|
254
|
+
products_configs_dir = os.path.join(disdrodb.package_dir, "tests", "products")
|
|
255
|
+
else:
|
|
256
|
+
products_configs_dir = disdrodb.config.get("products_configs_dir", None)
|
|
257
|
+
if products_configs_dir is None:
|
|
258
|
+
products_configs_dir = get_default_products_configs_dir()
|
|
259
|
+
return products_configs_dir
|
|
256
260
|
|
|
257
261
|
|
|
258
|
-
def
|
|
262
|
+
def copy_default_products_configs(products_configs_dir):
|
|
259
263
|
"""Copy the default DISDRODB products configuration directory to a custom location.
|
|
260
264
|
|
|
261
|
-
This function duplicates the entire directory of default
|
|
265
|
+
This function duplicates the entire directory of default products settings
|
|
262
266
|
(located at ``disdrodb/etc/products``) into the user-specified
|
|
263
|
-
``
|
|
267
|
+
``products_configs_dir``. Once copied, you can safely edit these files without
|
|
264
268
|
modifying the library's built-in defaults. To have DISDRODB use your
|
|
265
269
|
custom settings, point the global configuration at this new directory
|
|
266
|
-
(e.g by specifying ``
|
|
270
|
+
(e.g by specifying ``products_configs_dir`` with the ``disdrodb.define_configs`` function).
|
|
267
271
|
|
|
268
272
|
Parameters
|
|
269
273
|
----------
|
|
270
|
-
|
|
271
|
-
|
|
274
|
+
products_configs_dir:
|
|
275
|
+
Directory where the default products configuration files
|
|
272
276
|
will be copied. This directory must not already exist, and later
|
|
273
277
|
needs to be referenced in your DISDRODB global configuration.
|
|
274
278
|
|
|
275
279
|
Returns
|
|
276
280
|
-------
|
|
277
|
-
|
|
281
|
+
products_configs_dir
|
|
278
282
|
The path to the newly created custom product configuration directory.
|
|
279
283
|
|
|
280
284
|
"""
|
|
281
|
-
source_dir_path =
|
|
282
|
-
if os.path.exists(
|
|
283
|
-
raise FileExistsError(f"The {
|
|
284
|
-
|
|
285
|
-
return
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
def get_product_options(product, temporal_resolution=None):
|
|
289
|
-
"""Get options for DISDRODB products."""
|
|
290
|
-
import disdrodb
|
|
291
|
-
from disdrodb.api.checks import check_product
|
|
292
|
-
|
|
293
|
-
# Define configs path
|
|
294
|
-
if os.environ.get("PYTEST_CURRENT_TEST"):
|
|
295
|
-
configs_path = os.path.join(disdrodb.__root_path__, "disdrodb", "tests", "products")
|
|
296
|
-
else:
|
|
297
|
-
configs_path = disdrodb.config.get("configs_path", get_product_default_configs_path())
|
|
298
|
-
|
|
299
|
-
# Validate DISDRODB products configuration
|
|
300
|
-
validate_product_configuration(configs_path)
|
|
301
|
-
|
|
302
|
-
# Check product
|
|
303
|
-
check_product(product)
|
|
304
|
-
|
|
305
|
-
# Retrieve global product options
|
|
306
|
-
global_options = read_yaml(os.path.join(configs_path, product, "global.yaml"))
|
|
307
|
-
if temporal_resolution is None:
|
|
308
|
-
global_options = check_availability_radar_simulations(global_options)
|
|
309
|
-
return global_options
|
|
310
|
-
|
|
311
|
-
# If temporal resolutions are specified, drop 'temporal_resolutions' key
|
|
312
|
-
global_options.pop("temporal_resolutions", None)
|
|
313
|
-
custom_options_path = os.path.join(configs_path, product, f"{temporal_resolution}.yaml")
|
|
314
|
-
if not os.path.exists(custom_options_path):
|
|
315
|
-
return global_options
|
|
316
|
-
custom_options = read_yaml(custom_options_path)
|
|
317
|
-
options = global_options.copy()
|
|
318
|
-
options.update(custom_options)
|
|
319
|
-
options = check_availability_radar_simulations(options)
|
|
320
|
-
return options
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
def get_product_temporal_resolutions(product):
|
|
324
|
-
"""Get DISDRODB L2 product temporal aggregations."""
|
|
325
|
-
# Check only L2E and L2M
|
|
326
|
-
return get_product_options(product)["temporal_resolutions"]
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
def get_model_options(product, model_name):
|
|
330
|
-
"""Get DISDRODB L2M model options."""
|
|
331
|
-
import disdrodb
|
|
332
|
-
|
|
333
|
-
configs_path = disdrodb.config.get("configs_path", get_product_default_configs_path())
|
|
334
|
-
model_options_path = os.path.join(configs_path, product, f"{model_name}.yaml")
|
|
335
|
-
model_options = read_yaml(model_options_path)
|
|
336
|
-
return model_options
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
def validate_product_configuration(configs_path):
|
|
340
|
-
"""Validate the DISDRODB products configuration files."""
|
|
341
|
-
# TODO: Implement validation of DISDRODB products configuration files with pydantic
|
|
342
|
-
pass
|
|
285
|
+
source_dir_path = get_default_products_configs_dir()
|
|
286
|
+
if os.path.exists(products_configs_dir):
|
|
287
|
+
raise FileExistsError(f"The {products_configs_dir} directory already exists!")
|
|
288
|
+
products_configs_dir = shutil.copytree(source_dir_path, products_configs_dir)
|
|
289
|
+
return products_configs_dir
|
disdrodb/constants.py
CHANGED
|
@@ -41,14 +41,15 @@ COORDINATES = [
|
|
|
41
41
|
"time",
|
|
42
42
|
"sample_interval",
|
|
43
43
|
]
|
|
44
|
-
OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM", "PWS100", "SWS250"]
|
|
44
|
+
OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM", "LPM_V0", "PWS100", "SWS250"]
|
|
45
45
|
IMPACT_SENSORS = ["RD80"]
|
|
46
46
|
|
|
47
47
|
PRODUCTS = ["RAW", "L0A", "L0B", "L0C", "L1", "L2E", "L2M"]
|
|
48
48
|
|
|
49
49
|
PRODUCTS_ARGUMENTS = {
|
|
50
|
-
"
|
|
51
|
-
"
|
|
50
|
+
"L1": ["temporal_resolution"],
|
|
51
|
+
"L2E": ["temporal_resolution"],
|
|
52
|
+
"L2M": ["temporal_resolution", "model_name"],
|
|
52
53
|
}
|
|
53
54
|
|
|
54
55
|
PRODUCTS_REQUIREMENTS = {
|
|
@@ -166,8 +166,8 @@ def download_archive(
|
|
|
166
166
|
metadata_archive_dir=metadata_archive_dir,
|
|
167
167
|
force=force,
|
|
168
168
|
)
|
|
169
|
-
except Exception:
|
|
170
|
-
print(" - Download error: {e}")
|
|
169
|
+
except Exception as e:
|
|
170
|
+
print(f" - Download error: {e}")
|
|
171
171
|
print(" ")
|
|
172
172
|
|
|
173
173
|
|
|
@@ -397,6 +397,8 @@ def build_webserver_wget_command(url: str, cut_dirs: int, dst_dir: str, force: b
|
|
|
397
397
|
"-r",
|
|
398
398
|
"-np",
|
|
399
399
|
"-nH",
|
|
400
|
+
"--reject",
|
|
401
|
+
"index.html*", # avoid to download Apache autoindex index.html
|
|
400
402
|
f"--cut-dirs={cut_dirs}",
|
|
401
403
|
]
|
|
402
404
|
if force:
|
disdrodb/docs.py
CHANGED
|
@@ -26,10 +26,10 @@ from disdrodb.api.checks import check_sensor_name
|
|
|
26
26
|
|
|
27
27
|
def open_sensor_documentation(sensor_name):
|
|
28
28
|
"""Open the sensor documentation PDF in the browser."""
|
|
29
|
-
from disdrodb import
|
|
29
|
+
from disdrodb import package_dir
|
|
30
30
|
|
|
31
31
|
check_sensor_name(sensor_name)
|
|
32
|
-
docs_filepath = os.path.join(
|
|
32
|
+
docs_filepath = os.path.join(package_dir, "l0", "manuals", sensor_name + ".pdf")
|
|
33
33
|
webbrowser.open(docs_filepath)
|
|
34
34
|
|
|
35
35
|
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
product_options:
|
|
2
|
+
fall_velocity_model: "Beard1976"
|
|
3
|
+
minimum_diameter: 0
|
|
4
|
+
maximum_diameter: 10
|
|
5
|
+
minimum_velocity: 0
|
|
6
|
+
maximum_velocity: 12
|
|
7
|
+
above_velocity_fraction: 0.5
|
|
8
|
+
above_velocity_tolerance: null
|
|
9
|
+
below_velocity_fraction: 0.5
|
|
10
|
+
below_velocity_tolerance: null
|
|
11
|
+
small_diameter_threshold: 1
|
|
12
|
+
small_velocity_threshold: 2.5
|
|
13
|
+
maintain_smallest_drops: True
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
product_options:
|
|
2
|
+
fall_velocity_model: "Beard1976"
|
|
3
|
+
minimum_diameter: 0
|
|
4
|
+
maximum_diameter: 10
|
|
5
|
+
minimum_velocity: 0
|
|
6
|
+
maximum_velocity: 12
|
|
7
|
+
above_velocity_fraction: 0.5
|
|
8
|
+
above_velocity_tolerance: null
|
|
9
|
+
below_velocity_fraction: 0.5
|
|
10
|
+
below_velocity_tolerance: null
|
|
11
|
+
small_diameter_threshold: 1
|
|
12
|
+
small_velocity_threshold: 2.5
|
|
13
|
+
maintain_smallest_drops: True
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
product_options:
|
|
2
|
+
fall_velocity_model: "Beard1976"
|
|
3
|
+
minimum_diameter: 0
|
|
4
|
+
maximum_diameter: 10
|
|
5
|
+
minimum_velocity: 0
|
|
6
|
+
maximum_velocity: 12
|
|
7
|
+
above_velocity_fraction: 0.5
|
|
8
|
+
above_velocity_tolerance: null
|
|
9
|
+
below_velocity_fraction: 0.5
|
|
10
|
+
below_velocity_tolerance: null
|
|
11
|
+
small_diameter_threshold: 1
|
|
12
|
+
small_velocity_threshold: 2.5
|
|
13
|
+
maintain_smallest_drops: True
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
product_options:
|
|
2
|
+
fall_velocity_model: "Beard1976"
|
|
3
|
+
minimum_diameter: 0
|
|
4
|
+
maximum_diameter: 10
|
|
5
|
+
minimum_velocity: 0
|
|
6
|
+
maximum_velocity: 12
|
|
7
|
+
above_velocity_fraction: 0.6
|
|
8
|
+
above_velocity_tolerance: null
|
|
9
|
+
below_velocity_fraction: 0.6
|
|
10
|
+
below_velocity_tolerance: null
|
|
11
|
+
small_diameter_threshold: 1
|
|
12
|
+
small_velocity_threshold: 2.5
|
|
13
|
+
maintain_smallest_drops: True
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
product_options:
|
|
2
|
+
fall_velocity_model: "Beard1976"
|
|
3
|
+
minimum_diameter: 0
|
|
4
|
+
maximum_diameter: 10
|
|
5
|
+
minimum_velocity: 0
|
|
6
|
+
maximum_velocity: 12
|
|
7
|
+
above_velocity_fraction: 0.5
|
|
8
|
+
above_velocity_tolerance: null
|
|
9
|
+
below_velocity_fraction: 0.5
|
|
10
|
+
below_velocity_tolerance: null
|
|
11
|
+
small_diameter_threshold: 1
|
|
12
|
+
small_velocity_threshold: 2.5
|
|
13
|
+
maintain_smallest_drops: True
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
product_options:
|
|
2
|
+
fall_velocity_model: "Beard1976"
|
|
3
|
+
minimum_diameter: 0
|
|
4
|
+
maximum_diameter: 10
|
|
5
|
+
minimum_velocity: 0
|
|
6
|
+
maximum_velocity: 12
|
|
7
|
+
above_velocity_fraction: 0.5
|
|
8
|
+
above_velocity_tolerance: null
|
|
9
|
+
below_velocity_fraction: 0.5
|
|
10
|
+
below_velocity_tolerance: null
|
|
11
|
+
small_diameter_threshold: 1
|
|
12
|
+
small_velocity_threshold: 2.5
|
|
13
|
+
maintain_smallest_drops: True
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
product_options:
|
|
2
|
+
fall_velocity_model: "Beard1976"
|
|
3
|
+
minimum_diameter: 0
|
|
4
|
+
maximum_diameter: 10
|
|
5
|
+
minimum_velocity: 0
|
|
6
|
+
maximum_velocity: 12
|
|
7
|
+
above_velocity_fraction: 0.5
|
|
8
|
+
above_velocity_tolerance: null
|
|
9
|
+
below_velocity_fraction: 0.5
|
|
10
|
+
below_velocity_tolerance: null
|
|
11
|
+
small_diameter_threshold: 1
|
|
12
|
+
small_velocity_threshold: 2.5
|
|
13
|
+
maintain_smallest_drops: True
|