disdrodb 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +4 -0
- disdrodb/_version.py +2 -2
- disdrodb/api/checks.py +70 -47
- disdrodb/api/configs.py +0 -2
- disdrodb/api/create_directories.py +0 -2
- disdrodb/api/info.py +3 -3
- disdrodb/api/io.py +48 -8
- disdrodb/api/path.py +116 -133
- disdrodb/api/search.py +12 -3
- disdrodb/cli/disdrodb_create_summary.py +113 -0
- disdrodb/cli/disdrodb_create_summary_station.py +11 -1
- disdrodb/cli/disdrodb_run_l0a_station.py +1 -1
- disdrodb/cli/disdrodb_run_l0b_station.py +2 -2
- disdrodb/cli/disdrodb_run_l0c_station.py +2 -2
- disdrodb/cli/disdrodb_run_l1_station.py +2 -2
- disdrodb/cli/disdrodb_run_l2e_station.py +2 -2
- disdrodb/cli/disdrodb_run_l2m_station.py +2 -2
- disdrodb/constants.py +1 -1
- disdrodb/data_transfer/download_data.py +123 -7
- disdrodb/etc/products/L1/global.yaml +1 -1
- disdrodb/etc/products/L2E/5MIN.yaml +1 -0
- disdrodb/etc/products/L2E/global.yaml +1 -1
- disdrodb/etc/products/L2M/GAMMA_GS_ND_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/GAMMA_ML.yaml +1 -1
- disdrodb/etc/products/L2M/LOGNORMAL_GS_LOG_ND_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/LOGNORMAL_GS_ND_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/LOGNORMAL_ML.yaml +8 -0
- disdrodb/etc/products/L2M/global.yaml +11 -3
- disdrodb/issue/writer.py +2 -0
- disdrodb/l0/check_configs.py +49 -16
- disdrodb/l0/configs/LPM/l0a_encodings.yml +2 -2
- disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +2 -2
- disdrodb/l0/configs/LPM/l0b_encodings.yml +2 -2
- disdrodb/l0/configs/LPM/raw_data_format.yml +2 -2
- disdrodb/l0/configs/PWS100/l0b_encodings.yml +1 -0
- disdrodb/l0/configs/SWS250/bins_diameter.yml +108 -0
- disdrodb/l0/configs/SWS250/bins_velocity.yml +83 -0
- disdrodb/l0/configs/SWS250/l0a_encodings.yml +18 -0
- disdrodb/l0/configs/SWS250/l0b_cf_attrs.yml +72 -0
- disdrodb/l0/configs/SWS250/l0b_encodings.yml +155 -0
- disdrodb/l0/configs/SWS250/raw_data_format.yml +148 -0
- disdrodb/l0/l0a_processing.py +10 -5
- disdrodb/l0/l0b_nc_processing.py +10 -6
- disdrodb/l0/l0b_processing.py +92 -72
- disdrodb/l0/l0c_processing.py +369 -251
- disdrodb/l0/readers/LPM/ARM/ARM_LPM.py +8 -1
- disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +2 -2
- disdrodb/l0/readers/LPM/BELGIUM/ULIEGE.py +256 -0
- disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +2 -2
- disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +2 -2
- disdrodb/l0/readers/LPM/GERMANY/DWD.py +491 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +2 -2
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +2 -2
- disdrodb/l0/readers/LPM/KIT/CHWALA.py +2 -2
- disdrodb/l0/readers/LPM/SLOVENIA/ARSO.py +107 -12
- disdrodb/l0/readers/LPM/SLOVENIA/UL.py +3 -3
- disdrodb/l0/readers/LPM/SWITZERLAND/INNERERIZ_LPM.py +2 -2
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +5 -14
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +5 -14
- disdrodb/l0/readers/PARSIVEL/SLOVENIA/UL.py +117 -8
- disdrodb/l0/readers/PARSIVEL2/ARM/ARM_PARSIVEL2.py +4 -0
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +10 -14
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +10 -14
- disdrodb/l0/readers/PARSIVEL2/CANADA/UQAM_NC.py +69 -0
- disdrodb/l0/readers/PARSIVEL2/DENMARK/DTU.py +8 -14
- disdrodb/l0/readers/PARSIVEL2/DENMARK/EROSION_raw.py +382 -0
- disdrodb/l0/readers/PARSIVEL2/FINLAND/FMI_PARSIVEL2.py +4 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/OSUG.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/GREECE/NOA.py +127 -0
- disdrodb/l0/readers/PARSIVEL2/ITALY/HYDROX.py +239 -0
- disdrodb/l0/readers/PARSIVEL2/MPI/BCO_PARSIVEL2.py +136 -0
- disdrodb/l0/readers/PARSIVEL2/MPI/BOWTIE.py +220 -0
- disdrodb/l0/readers/PARSIVEL2/NASA/LPVEX.py +109 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/FARM_PARSIVEL2.py +5 -11
- disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_MIPS.py +4 -17
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +5 -14
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +10 -13
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +10 -13
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT_NC.py +3 -0
- disdrodb/l0/readers/PARSIVEL2/PHILIPPINES/PANGASA.py +232 -0
- disdrodb/l0/readers/PARSIVEL2/SPAIN/CENER.py +6 -18
- disdrodb/l0/readers/PARSIVEL2/SPAIN/GRANADA.py +120 -0
- disdrodb/l0/readers/PARSIVEL2/USA/C3WE.py +7 -25
- disdrodb/l0/readers/PWS100/AUSTRIA/HOAL.py +321 -0
- disdrodb/l0/readers/SW250/BELGIUM/KMI.py +239 -0
- disdrodb/l1/beard_model.py +31 -129
- disdrodb/l1/fall_velocity.py +156 -57
- disdrodb/l1/filters.py +25 -28
- disdrodb/l1/processing.py +12 -14
- disdrodb/l1_env/routines.py +46 -17
- disdrodb/l2/empirical_dsd.py +6 -0
- disdrodb/l2/processing.py +3 -3
- disdrodb/metadata/checks.py +132 -125
- disdrodb/metadata/geolocation.py +0 -2
- disdrodb/psd/fitting.py +180 -210
- disdrodb/psd/models.py +1 -1
- disdrodb/routines/__init__.py +54 -0
- disdrodb/{l0/routines.py → routines/l0.py} +288 -418
- disdrodb/{l1/routines.py → routines/l1.py} +60 -92
- disdrodb/{l2/routines.py → routines/l2.py} +284 -485
- disdrodb/{routines.py → routines/wrappers.py} +100 -7
- disdrodb/scattering/axis_ratio.py +95 -85
- disdrodb/scattering/permittivity.py +24 -0
- disdrodb/scattering/routines.py +56 -36
- disdrodb/summary/routines.py +147 -45
- disdrodb/utils/archiving.py +434 -0
- disdrodb/utils/attrs.py +2 -0
- disdrodb/utils/cli.py +5 -5
- disdrodb/utils/dask.py +62 -1
- disdrodb/utils/decorators.py +31 -0
- disdrodb/utils/encoding.py +10 -1
- disdrodb/{l2 → utils}/event.py +1 -66
- disdrodb/utils/logger.py +1 -1
- disdrodb/utils/manipulations.py +22 -12
- disdrodb/utils/routines.py +166 -0
- disdrodb/utils/time.py +5 -293
- disdrodb/utils/xarray.py +3 -0
- disdrodb/viz/plots.py +109 -15
- {disdrodb-0.1.3.dist-info → disdrodb-0.1.5.dist-info}/METADATA +3 -2
- {disdrodb-0.1.3.dist-info → disdrodb-0.1.5.dist-info}/RECORD +124 -96
- {disdrodb-0.1.3.dist-info → disdrodb-0.1.5.dist-info}/entry_points.txt +1 -0
- {disdrodb-0.1.3.dist-info → disdrodb-0.1.5.dist-info}/WHEEL +0 -0
- {disdrodb-0.1.3.dist-info → disdrodb-0.1.5.dist-info}/licenses/LICENSE +0 -0
- {disdrodb-0.1.3.dist-info → disdrodb-0.1.5.dist-info}/top_level.txt +0 -0
disdrodb/api/path.py
CHANGED
|
@@ -63,6 +63,7 @@ def define_disdrodb_path(
|
|
|
63
63
|
The campaign name.
|
|
64
64
|
check_exists : bool, optional
|
|
65
65
|
Whether to check if the directory exists. The default value is ``True``.
|
|
66
|
+
Raise error if the directory does not exist.
|
|
66
67
|
|
|
67
68
|
Returns
|
|
68
69
|
-------
|
|
@@ -81,7 +82,7 @@ def define_disdrodb_path(
|
|
|
81
82
|
dir_path = os.path.join(archive_dir, ARCHIVE_VERSION, data_source, campaign_name)
|
|
82
83
|
if check_exists:
|
|
83
84
|
check_directory_exists(dir_path)
|
|
84
|
-
return dir_path
|
|
85
|
+
return os.path.normpath(dir_path)
|
|
85
86
|
|
|
86
87
|
|
|
87
88
|
def define_data_source_dir(
|
|
@@ -107,6 +108,7 @@ def define_data_source_dir(
|
|
|
107
108
|
If not specified, the path specified in the DISDRODB active configuration will be used.
|
|
108
109
|
check_exists : bool, optional
|
|
109
110
|
Whether to check if the directory exists. The default value is ``False``.
|
|
111
|
+
Raise error if the directory does not exist.
|
|
110
112
|
|
|
111
113
|
Returns
|
|
112
114
|
-------
|
|
@@ -386,7 +388,7 @@ def define_partitioning_tree(time, folder_partitioning):
|
|
|
386
388
|
return os.path.join(year, month, day)
|
|
387
389
|
if folder_partitioning == "year/month_name":
|
|
388
390
|
year = str(time.year)
|
|
389
|
-
month =
|
|
391
|
+
month = time.strftime("%B")
|
|
390
392
|
return os.path.join(year, month)
|
|
391
393
|
if folder_partitioning == "year/quarter":
|
|
392
394
|
year = str(time.year)
|
|
@@ -397,7 +399,7 @@ def define_partitioning_tree(time, folder_partitioning):
|
|
|
397
399
|
raise NotImplementedError(f"Unrecognized '{folder_partitioning}' folder partitioning scheme.")
|
|
398
400
|
|
|
399
401
|
|
|
400
|
-
def define_file_folder_path(obj,
|
|
402
|
+
def define_file_folder_path(obj, dir_path, folder_partitioning):
|
|
401
403
|
"""
|
|
402
404
|
Define the folder path where saving a file based on the dataset's starting time.
|
|
403
405
|
|
|
@@ -405,12 +407,13 @@ def define_file_folder_path(obj, data_dir, folder_partitioning):
|
|
|
405
407
|
----------
|
|
406
408
|
ds : xarray.Dataset or pandas.DataFrame
|
|
407
409
|
The object containing time information.
|
|
408
|
-
|
|
410
|
+
dir : str
|
|
409
411
|
Directory within the DISDRODB Data Archive where DISDRODB product files are to be saved.
|
|
412
|
+
It can be a product directory or a logs directory.
|
|
410
413
|
folder_partitioning : str or None
|
|
411
414
|
Define the subdirectory structure where saving files.
|
|
412
415
|
Allowed values are:
|
|
413
|
-
- None: Files are saved directly in data_dir.
|
|
416
|
+
- None or "": Files are saved directly in data_dir.
|
|
414
417
|
- "year": Files are saved under a subdirectory for the year.
|
|
415
418
|
- "year/month": Files are saved under subdirectories for year and month.
|
|
416
419
|
- "year/month/day": Files are saved under subdirectories for year, month and day
|
|
@@ -432,7 +435,7 @@ def define_file_folder_path(obj, data_dir, folder_partitioning):
|
|
|
432
435
|
|
|
433
436
|
# Build the folder path based on the chosen partition scheme
|
|
434
437
|
partitioning_tree = define_partitioning_tree(time=starting_time, folder_partitioning=folder_partitioning)
|
|
435
|
-
return os.path.join(
|
|
438
|
+
return os.path.normpath(os.path.join(dir_path, partitioning_tree))
|
|
436
439
|
|
|
437
440
|
|
|
438
441
|
def define_product_dir_tree(
|
|
@@ -475,15 +478,14 @@ def define_product_dir_tree(
|
|
|
475
478
|
check_sample_interval(sample_interval)
|
|
476
479
|
temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
|
|
477
480
|
return os.path.join(temporal_resolution)
|
|
478
|
-
if product == "L2M":
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
raise ValueError(f"The product {product} is not defined.")
|
|
481
|
+
# L2M if product == "L2M":
|
|
482
|
+
rolling = product_kwargs.get("rolling")
|
|
483
|
+
sample_interval = product_kwargs.get("sample_interval")
|
|
484
|
+
model_name = product_kwargs.get("model_name")
|
|
485
|
+
check_rolling(rolling)
|
|
486
|
+
check_sample_interval(sample_interval)
|
|
487
|
+
temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
|
|
488
|
+
return os.path.join(model_name, temporal_resolution)
|
|
487
489
|
|
|
488
490
|
|
|
489
491
|
def define_logs_dir(
|
|
@@ -529,7 +531,7 @@ def define_logs_dir(
|
|
|
529
531
|
product=product,
|
|
530
532
|
**product_kwargs,
|
|
531
533
|
)
|
|
532
|
-
logs_dir = os.path.join(campaign_dir, "logs", "files", product, product_dir_tree, station_name)
|
|
534
|
+
logs_dir = os.path.normpath(os.path.join(campaign_dir, "logs", "files", product, product_dir_tree, station_name))
|
|
533
535
|
if check_exists:
|
|
534
536
|
check_directory_exists(logs_dir)
|
|
535
537
|
return str(logs_dir)
|
|
@@ -643,7 +645,7 @@ def define_data_dir(
|
|
|
643
645
|
**product_kwargs,
|
|
644
646
|
)
|
|
645
647
|
# Define data directory
|
|
646
|
-
data_dir = os.path.join(station_dir, product_dir_tree)
|
|
648
|
+
data_dir = os.path.normpath(os.path.join(station_dir, product_dir_tree))
|
|
647
649
|
# Check if directory exists
|
|
648
650
|
if check_exists:
|
|
649
651
|
check_directory_exists(data_dir)
|
|
@@ -674,7 +676,8 @@ def define_filename(
|
|
|
674
676
|
campaign_name: str,
|
|
675
677
|
station_name: str,
|
|
676
678
|
# Filename options
|
|
677
|
-
|
|
679
|
+
start_time=None,
|
|
680
|
+
end_time=None,
|
|
678
681
|
add_version=True,
|
|
679
682
|
add_time_period=True,
|
|
680
683
|
add_extension=True,
|
|
@@ -688,19 +691,22 @@ def define_filename(
|
|
|
688
691
|
|
|
689
692
|
Parameters
|
|
690
693
|
----------
|
|
691
|
-
obj : xarray.Dataset or pandas.DataFrame
|
|
692
|
-
xarray Dataset or pandas DataFrame.
|
|
693
|
-
Required if add_time_period = True.
|
|
694
694
|
campaign_name : str
|
|
695
695
|
Name of the campaign.
|
|
696
696
|
station_name : str
|
|
697
697
|
Name of the station.
|
|
698
|
+
start_time : datetime.datatime, optional
|
|
699
|
+
Start time.
|
|
700
|
+
Required if add_time_period = True.
|
|
701
|
+
end_time : datetime.datatime, optional
|
|
702
|
+
End time.
|
|
703
|
+
Required if add_time_period = True.
|
|
698
704
|
sample_interval : int, optional
|
|
699
705
|
The sampling interval in seconds of the product.
|
|
700
|
-
It must be specified only for product L2E and L2M !
|
|
706
|
+
It must be specified only for product L0C, L1, L2E and L2M !
|
|
701
707
|
rolling : bool, optional
|
|
702
708
|
Whether the dataset has been resampled by aggregating or rolling.
|
|
703
|
-
It must be specified only for product L2E and L2M !
|
|
709
|
+
It must be specified only for product L1, L2E and L2M !
|
|
704
710
|
model_name : str
|
|
705
711
|
The model name of the fitted statistical distribution for the DSD.
|
|
706
712
|
It must be specified only for product L2M !
|
|
@@ -715,19 +721,25 @@ def define_filename(
|
|
|
715
721
|
product = check_product(product)
|
|
716
722
|
product_kwargs = check_product_kwargs(product, product_kwargs)
|
|
717
723
|
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
# - ADD temporal_resolution also to L0A and L0B
|
|
721
|
-
# - Add temporal_resolution also to L0C and L1
|
|
724
|
+
if add_time_period and (start_time is None or end_time is None):
|
|
725
|
+
raise ValueError("If add_time_period=True, specify start_time and end_time.")
|
|
722
726
|
|
|
723
727
|
# -----------------------------------------.
|
|
724
728
|
# Define product name
|
|
725
729
|
product_name = f"{product}"
|
|
730
|
+
|
|
731
|
+
# L0C ... sample interval known only per-file
|
|
732
|
+
# L1 ... in future known a priori
|
|
733
|
+
# if product in ["L1"]:
|
|
734
|
+
# # TODO: HACK FOR CURRENT L0C and L1 log files in create_product_logs
|
|
735
|
+
# sample_interval = product_kwargs.get("sample_interval", 0)
|
|
736
|
+
# temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=False)
|
|
737
|
+
# product_name = f"{product}.{temporal_resolution}"
|
|
726
738
|
if product in ["L2E", "L2M"]:
|
|
727
739
|
rolling = product_kwargs.get("rolling")
|
|
728
740
|
sample_interval = product_kwargs.get("sample_interval")
|
|
729
741
|
temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
|
|
730
|
-
product_name = f"
|
|
742
|
+
product_name = f"{product}.{temporal_resolution}"
|
|
731
743
|
if product in ["L2M"]:
|
|
732
744
|
model_name = product_kwargs.get("model_name")
|
|
733
745
|
product_name = f"L2M_{model_name}.{temporal_resolution}"
|
|
@@ -744,10 +756,9 @@ def define_filename(
|
|
|
744
756
|
# -----------------------------------------.
|
|
745
757
|
# Add time period information
|
|
746
758
|
if add_time_period:
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
filename = f"{filename}.s{starting_time}.e{ending_time}"
|
|
759
|
+
start_time = start_time.strftime("%Y%m%d%H%M%S")
|
|
760
|
+
end_time = end_time.strftime("%Y%m%d%H%M%S")
|
|
761
|
+
filename = f"{filename}.s{start_time}.e{end_time}"
|
|
751
762
|
|
|
752
763
|
# -----------------------------------------.
|
|
753
764
|
# Add product version
|
|
@@ -784,56 +795,40 @@ def define_l0a_filename(df, campaign_name: str, station_name: str) -> str:
|
|
|
784
795
|
L0A file name.
|
|
785
796
|
"""
|
|
786
797
|
starting_time, ending_time = get_file_start_end_time(df)
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
798
|
+
filename = define_filename(
|
|
799
|
+
product="L0A",
|
|
800
|
+
campaign_name=campaign_name,
|
|
801
|
+
station_name=station_name,
|
|
802
|
+
# Filename options
|
|
803
|
+
start_time=starting_time,
|
|
804
|
+
end_time=ending_time,
|
|
805
|
+
add_version=True,
|
|
806
|
+
add_time_period=True,
|
|
807
|
+
add_extension=True,
|
|
808
|
+
)
|
|
791
809
|
return filename
|
|
792
810
|
|
|
793
811
|
|
|
794
812
|
def define_l0b_filename(ds, campaign_name: str, station_name: str) -> str:
|
|
795
|
-
"""Define L0B file name.
|
|
796
|
-
|
|
797
|
-
Parameters
|
|
798
|
-
----------
|
|
799
|
-
ds : xarray.Dataset
|
|
800
|
-
L0B xarray Dataset.
|
|
801
|
-
campaign_name : str
|
|
802
|
-
Name of the campaign.
|
|
803
|
-
station_name : str
|
|
804
|
-
Name of the station.
|
|
805
|
-
|
|
806
|
-
Returns
|
|
807
|
-
-------
|
|
808
|
-
str
|
|
809
|
-
L0B file name.
|
|
810
|
-
"""
|
|
813
|
+
"""Define L0B file name."""
|
|
811
814
|
starting_time, ending_time = get_file_start_end_time(ds)
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
815
|
+
filename = define_filename(
|
|
816
|
+
product="L0B",
|
|
817
|
+
campaign_name=campaign_name,
|
|
818
|
+
station_name=station_name,
|
|
819
|
+
# Filename options
|
|
820
|
+
start_time=starting_time,
|
|
821
|
+
end_time=ending_time,
|
|
822
|
+
add_version=True,
|
|
823
|
+
add_time_period=True,
|
|
824
|
+
add_extension=True,
|
|
825
|
+
)
|
|
816
826
|
return filename
|
|
817
827
|
|
|
818
828
|
|
|
819
829
|
def define_l0c_filename(ds, campaign_name: str, station_name: str) -> str:
|
|
820
|
-
"""Define L0C file name.
|
|
821
|
-
|
|
822
|
-
Parameters
|
|
823
|
-
----------
|
|
824
|
-
ds : xarray.Dataset
|
|
825
|
-
L0B xarray Dataset.
|
|
826
|
-
campaign_name : str
|
|
827
|
-
Name of the campaign.
|
|
828
|
-
station_name : str
|
|
829
|
-
Name of the station.
|
|
830
|
-
|
|
831
|
-
Returns
|
|
832
|
-
-------
|
|
833
|
-
str
|
|
834
|
-
L0B file name.
|
|
835
|
-
"""
|
|
836
|
-
# TODO: add sample_interval as argument
|
|
830
|
+
"""Define L0C file name."""
|
|
831
|
+
# TODO: add sample_interval as function argument s
|
|
837
832
|
sample_interval = int(ensure_sample_interval_in_seconds(ds["sample_interval"]).data.item())
|
|
838
833
|
temporal_resolution = define_temporal_resolution(sample_interval, rolling=False)
|
|
839
834
|
starting_time, ending_time = get_file_start_end_time(ds)
|
|
@@ -845,23 +840,10 @@ def define_l0c_filename(ds, campaign_name: str, station_name: str) -> str:
|
|
|
845
840
|
|
|
846
841
|
|
|
847
842
|
def define_l1_filename(ds, campaign_name, station_name: str) -> str:
|
|
848
|
-
"""Define L1 file name.
|
|
843
|
+
"""Define L1 file name."""
|
|
844
|
+
# TODO: add sample_interval and rolling as function argument
|
|
849
845
|
|
|
850
|
-
|
|
851
|
-
----------
|
|
852
|
-
ds : xarray.Dataset
|
|
853
|
-
L1 xarray Dataset.
|
|
854
|
-
campaign_name : str
|
|
855
|
-
Name of the campaign.
|
|
856
|
-
station_name : str
|
|
857
|
-
Name of the station.
|
|
858
|
-
|
|
859
|
-
Returns
|
|
860
|
-
-------
|
|
861
|
-
str
|
|
862
|
-
L1 file name.
|
|
863
|
-
"""
|
|
864
|
-
# TODO: add sample_interval as argument
|
|
846
|
+
starting_time, ending_time = get_file_start_end_time(ds)
|
|
865
847
|
sample_interval = int(ensure_sample_interval_in_seconds(ds["sample_interval"]).data.item())
|
|
866
848
|
temporal_resolution = define_temporal_resolution(sample_interval, rolling=False)
|
|
867
849
|
starting_time, ending_time = get_file_start_end_time(ds)
|
|
@@ -869,32 +851,41 @@ def define_l1_filename(ds, campaign_name, station_name: str) -> str:
|
|
|
869
851
|
ending_time = ending_time.strftime("%Y%m%d%H%M%S")
|
|
870
852
|
version = ARCHIVE_VERSION
|
|
871
853
|
filename = f"L1.{temporal_resolution}.{campaign_name}.{station_name}.s{starting_time}.e{ending_time}.{version}.nc"
|
|
854
|
+
|
|
855
|
+
# filename = define_filename(
|
|
856
|
+
# product="L1",
|
|
857
|
+
# campaign_name=campaign_name,
|
|
858
|
+
# station_name=station_name,
|
|
859
|
+
# # Filename options
|
|
860
|
+
# start_time=starting_time,
|
|
861
|
+
# end_time=ending_time,
|
|
862
|
+
# add_version=True,
|
|
863
|
+
# add_time_period=True,
|
|
864
|
+
# add_extension=True,
|
|
865
|
+
# # Product options
|
|
866
|
+
# # sample_interval=sample_interval,
|
|
867
|
+
# # rolling=rolling,
|
|
868
|
+
# )
|
|
872
869
|
return filename
|
|
873
870
|
|
|
874
871
|
|
|
875
872
|
def define_l2e_filename(ds, campaign_name: str, station_name: str, sample_interval: int, rolling: bool) -> str:
|
|
876
|
-
"""Define L2E file name.
|
|
877
|
-
|
|
878
|
-
Parameters
|
|
879
|
-
----------
|
|
880
|
-
ds : xarray.Dataset
|
|
881
|
-
L1 xarray Dataset
|
|
882
|
-
campaign_name : str
|
|
883
|
-
Name of the campaign.
|
|
884
|
-
station_name : str
|
|
885
|
-
Name of the station
|
|
886
|
-
|
|
887
|
-
Returns
|
|
888
|
-
-------
|
|
889
|
-
str
|
|
890
|
-
L0B file name.
|
|
891
|
-
"""
|
|
892
|
-
temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
|
|
873
|
+
"""Define L2E file name."""
|
|
893
874
|
starting_time, ending_time = get_file_start_end_time(ds)
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
875
|
+
filename = define_filename(
|
|
876
|
+
product="L2E",
|
|
877
|
+
campaign_name=campaign_name,
|
|
878
|
+
station_name=station_name,
|
|
879
|
+
# Filename options
|
|
880
|
+
start_time=starting_time,
|
|
881
|
+
end_time=ending_time,
|
|
882
|
+
add_version=True,
|
|
883
|
+
add_time_period=True,
|
|
884
|
+
add_extension=True,
|
|
885
|
+
# Product options
|
|
886
|
+
sample_interval=sample_interval,
|
|
887
|
+
rolling=rolling,
|
|
888
|
+
)
|
|
898
889
|
return filename
|
|
899
890
|
|
|
900
891
|
|
|
@@ -906,29 +897,21 @@ def define_l2m_filename(
|
|
|
906
897
|
rolling: bool,
|
|
907
898
|
model_name: str,
|
|
908
899
|
) -> str:
|
|
909
|
-
"""Define L2M file name.
|
|
910
|
-
|
|
911
|
-
Parameters
|
|
912
|
-
----------
|
|
913
|
-
ds : xarray.Dataset
|
|
914
|
-
L1 xarray Dataset
|
|
915
|
-
campaign_name : str
|
|
916
|
-
Name of the campaign.
|
|
917
|
-
station_name : str
|
|
918
|
-
Name of the station
|
|
919
|
-
|
|
920
|
-
Returns
|
|
921
|
-
-------
|
|
922
|
-
str
|
|
923
|
-
L0B file name.
|
|
924
|
-
"""
|
|
925
|
-
temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
|
|
900
|
+
"""Define L2M file name."""
|
|
926
901
|
starting_time, ending_time = get_file_start_end_time(ds)
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
902
|
+
filename = define_filename(
|
|
903
|
+
product="L2M",
|
|
904
|
+
campaign_name=campaign_name,
|
|
905
|
+
station_name=station_name,
|
|
906
|
+
# Filename options
|
|
907
|
+
start_time=starting_time,
|
|
908
|
+
end_time=ending_time,
|
|
909
|
+
add_version=True,
|
|
910
|
+
add_time_period=True,
|
|
911
|
+
add_extension=True,
|
|
912
|
+
# Product options
|
|
913
|
+
sample_interval=sample_interval,
|
|
914
|
+
rolling=rolling,
|
|
915
|
+
model_name=model_name,
|
|
933
916
|
)
|
|
934
917
|
return filename
|
disdrodb/api/search.py
CHANGED
|
@@ -20,6 +20,8 @@ from disdrodb.constants import PRODUCTS_REQUIREMENTS
|
|
|
20
20
|
from disdrodb.utils.directories import contains_files, contains_netcdf_or_parquet_files, list_directories, list_files
|
|
21
21
|
from disdrodb.utils.yaml import read_yaml
|
|
22
22
|
|
|
23
|
+
####-------------------------------------------------------------------------
|
|
24
|
+
|
|
23
25
|
|
|
24
26
|
def get_required_product(product):
|
|
25
27
|
"""Determine the required product for input product processing."""
|
|
@@ -31,7 +33,7 @@ def get_required_product(product):
|
|
|
31
33
|
|
|
32
34
|
|
|
33
35
|
####-------------------------------------------------------------------------
|
|
34
|
-
#### List DISDRODB
|
|
36
|
+
#### List DISDRODB Metadata directories
|
|
35
37
|
|
|
36
38
|
|
|
37
39
|
def list_data_sources(metadata_archive_dir, data_sources=None, invalid_fields_policy="raise"):
|
|
@@ -167,6 +169,10 @@ def list_station_names(
|
|
|
167
169
|
return station_names
|
|
168
170
|
|
|
169
171
|
|
|
172
|
+
####-------------------------------------------------------------------------
|
|
173
|
+
#### Filtering utilities for available_stations
|
|
174
|
+
|
|
175
|
+
|
|
170
176
|
def _finalize_output(list_info, return_tuple):
|
|
171
177
|
# - Return the (data_source, campaign_name, station_name) tuple
|
|
172
178
|
if return_tuple:
|
|
@@ -228,7 +234,6 @@ def keep_list_info_elements_with_product_data(data_archive_dir, product, list_in
|
|
|
228
234
|
checking_function = contains_files if product == "RAW" else contains_netcdf_or_parquet_files
|
|
229
235
|
|
|
230
236
|
# Check presence of data for each station
|
|
231
|
-
# TODO: - In parallel over stations to speed up ?
|
|
232
237
|
list_info_with_product_data = []
|
|
233
238
|
for data_source, campaign_name, station_name in list_info:
|
|
234
239
|
data_dir = define_data_dir(
|
|
@@ -362,10 +367,11 @@ def available_stations(
|
|
|
362
367
|
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
363
368
|
product = check_product(product) if product is not None else None
|
|
364
369
|
invalid_fields_policy = check_invalid_fields_policy(invalid_fields_policy)
|
|
370
|
+
|
|
365
371
|
# Retrieve available stations from the Metadata Archive
|
|
366
372
|
# - Raise error if no stations availables !
|
|
367
373
|
list_info = list_station_names(
|
|
368
|
-
metadata_archive_dir,
|
|
374
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
369
375
|
data_sources=data_sources,
|
|
370
376
|
campaign_names=campaign_names,
|
|
371
377
|
station_names=station_names,
|
|
@@ -484,3 +490,6 @@ def available_campaigns(
|
|
|
484
490
|
campaign_names = [info[1] for info in list_info]
|
|
485
491
|
campaign_names = np.unique(campaign_names).tolist()
|
|
486
492
|
return campaign_names
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
####-------------------------------------------------------------------------
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# -----------------------------------------------------------------------------.
|
|
2
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
3
|
+
#
|
|
4
|
+
# This program is free software: you can redistribute it and/or modify
|
|
5
|
+
# it under the terms of the GNU General Public License as published by
|
|
6
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
7
|
+
# (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This program is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
12
|
+
# GNU General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU General Public License
|
|
15
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
16
|
+
# -----------------------------------------------------------------------------.
|
|
17
|
+
"""Script to create summary figures and tables for a DISDRODB stationn."""
|
|
18
|
+
import sys
|
|
19
|
+
from typing import Optional
|
|
20
|
+
|
|
21
|
+
import click
|
|
22
|
+
|
|
23
|
+
from disdrodb.utils.cli import (
|
|
24
|
+
click_data_archive_dir_option,
|
|
25
|
+
click_metadata_archive_dir_option,
|
|
26
|
+
click_stations_options,
|
|
27
|
+
parse_archive_dir,
|
|
28
|
+
parse_arg_to_list,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
sys.tracebacklimit = 0 # avoid full traceback error if occur
|
|
32
|
+
|
|
33
|
+
# -------------------------------------------------------------------------.
|
|
34
|
+
# Click Command Line Interface decorator
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@click.command()
|
|
38
|
+
@click_stations_options
|
|
39
|
+
@click_data_archive_dir_option
|
|
40
|
+
@click_metadata_archive_dir_option
|
|
41
|
+
@click.option("-p", "--parallel", type=bool, show_default=True, default=False, help="Read files in parallel")
|
|
42
|
+
@click.option(
|
|
43
|
+
"-t",
|
|
44
|
+
"--temporal_resolution",
|
|
45
|
+
type=str,
|
|
46
|
+
show_default=True,
|
|
47
|
+
default="1MIN",
|
|
48
|
+
help="Temporal resolution of the L2E product to be used for the summary.",
|
|
49
|
+
)
|
|
50
|
+
def disdrodb_create_summary(
|
|
51
|
+
# Stations options
|
|
52
|
+
data_sources: Optional[str] = None,
|
|
53
|
+
campaign_names: Optional[str] = None,
|
|
54
|
+
station_names: Optional[str] = None,
|
|
55
|
+
# Processing options:
|
|
56
|
+
parallel=False,
|
|
57
|
+
temporal_resolution="1MIN",
|
|
58
|
+
# DISDRODB root directories
|
|
59
|
+
data_archive_dir: Optional[str] = None,
|
|
60
|
+
metadata_archive_dir: Optional[str] = None,
|
|
61
|
+
):
|
|
62
|
+
r"""Create summary figures and tables for a specific set of DISDRODB stations.
|
|
63
|
+
|
|
64
|
+
Parameters \n
|
|
65
|
+
---------- \n
|
|
66
|
+
data_sources : str
|
|
67
|
+
Name of data source(s) to process.
|
|
68
|
+
The name(s) must be UPPER CASE.
|
|
69
|
+
If campaign_names and station are not specified, process all stations.
|
|
70
|
+
To specify multiple data sources, write i.e.: --data_sources 'GPM EPFL NCAR'
|
|
71
|
+
campaign_names : str
|
|
72
|
+
Name of the campaign(s) for which to create stations summaries.
|
|
73
|
+
The name(s) must be UPPER CASE.
|
|
74
|
+
To specify multiple campaigns, write i.e.: --campaign_names 'IPEX IMPACTS'
|
|
75
|
+
station_names : str
|
|
76
|
+
Station names.
|
|
77
|
+
To specify multiple stations, write i.e.: --station_names 'station1 station2'
|
|
78
|
+
data_archive_dir : str \n
|
|
79
|
+
DISDRODB Data Archive directory \n
|
|
80
|
+
Format: <...>/DISDRODB \n
|
|
81
|
+
If not specified, uses path specified in the DISDRODB active configuration. \n
|
|
82
|
+
"""
|
|
83
|
+
from disdrodb.routines import create_summary
|
|
84
|
+
from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
|
|
85
|
+
|
|
86
|
+
data_archive_dir = parse_archive_dir(data_archive_dir)
|
|
87
|
+
data_sources = parse_arg_to_list(data_sources)
|
|
88
|
+
campaign_names = parse_arg_to_list(campaign_names)
|
|
89
|
+
station_names = parse_arg_to_list(station_names)
|
|
90
|
+
|
|
91
|
+
# -------------------------------------------------------------------------.
|
|
92
|
+
# If parallel=True, set the dask environment
|
|
93
|
+
if parallel:
|
|
94
|
+
cluster, client = initialize_dask_cluster()
|
|
95
|
+
|
|
96
|
+
# -------------------------------------------------------------------------.
|
|
97
|
+
create_summary(
|
|
98
|
+
# Station arguments
|
|
99
|
+
data_sources=data_sources,
|
|
100
|
+
campaign_names=campaign_names,
|
|
101
|
+
station_names=station_names,
|
|
102
|
+
# Options
|
|
103
|
+
parallel=parallel,
|
|
104
|
+
temporal_resolution=temporal_resolution,
|
|
105
|
+
# DISDRODB root directory
|
|
106
|
+
data_archive_dir=data_archive_dir,
|
|
107
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
# -------------------------------------------------------------------------.
|
|
111
|
+
# Close the cluster
|
|
112
|
+
if parallel:
|
|
113
|
+
close_dask_cluster(cluster, client)
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
# You should have received a copy of the GNU General Public License
|
|
15
15
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
16
16
|
# -----------------------------------------------------------------------------.
|
|
17
|
-
"""Script to
|
|
17
|
+
"""Script to create summary figures and tables for a DISDRODB station."""
|
|
18
18
|
import sys
|
|
19
19
|
from typing import Optional
|
|
20
20
|
|
|
@@ -36,6 +36,14 @@ sys.tracebacklimit = 0 # avoid full traceback error if occur
|
|
|
36
36
|
@click_station_arguments
|
|
37
37
|
@click_data_archive_dir_option
|
|
38
38
|
@click.option("-p", "--parallel", type=bool, show_default=True, default=False, help="Read files in parallel")
|
|
39
|
+
@click.option(
|
|
40
|
+
"-t",
|
|
41
|
+
"--temporal_resolution",
|
|
42
|
+
type=str,
|
|
43
|
+
show_default=True,
|
|
44
|
+
default="1MIN",
|
|
45
|
+
help="Temporal resolution of the L2E product to be used for the summary.",
|
|
46
|
+
)
|
|
39
47
|
def disdrodb_create_summary_station(
|
|
40
48
|
# Station arguments
|
|
41
49
|
data_source: str,
|
|
@@ -43,6 +51,7 @@ def disdrodb_create_summary_station(
|
|
|
43
51
|
station_name: str,
|
|
44
52
|
# Processing options:
|
|
45
53
|
parallel=False,
|
|
54
|
+
temporal_resolution="1MIN",
|
|
46
55
|
# DISDRODB root directories
|
|
47
56
|
data_archive_dir: Optional[str] = None,
|
|
48
57
|
):
|
|
@@ -81,6 +90,7 @@ def disdrodb_create_summary_station(
|
|
|
81
90
|
station_name=station_name,
|
|
82
91
|
# Options
|
|
83
92
|
parallel=parallel,
|
|
93
|
+
temporal_resolution=temporal_resolution,
|
|
84
94
|
# DISDRODB root directory
|
|
85
95
|
data_archive_dir=data_archive_dir,
|
|
86
96
|
)
|
|
@@ -89,7 +89,7 @@ def disdrodb_run_l0a_station(
|
|
|
89
89
|
Format: <...>/DISDRODB
|
|
90
90
|
If not specified, uses path specified in the DISDRODB active configuration.
|
|
91
91
|
"""
|
|
92
|
-
from disdrodb.l0
|
|
92
|
+
from disdrodb.routines.l0 import run_l0a_station
|
|
93
93
|
from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
|
|
94
94
|
|
|
95
95
|
data_archive_dir = parse_archive_dir(data_archive_dir)
|
|
@@ -91,7 +91,7 @@ def disdrodb_run_l0b_station(
|
|
|
91
91
|
Format: <...>/DISDRODB
|
|
92
92
|
If not specified, uses path specified in the DISDRODB active configuration.
|
|
93
93
|
"""
|
|
94
|
-
from disdrodb.l0
|
|
94
|
+
from disdrodb.routines.l0 import run_l0b_station
|
|
95
95
|
from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
|
|
96
96
|
|
|
97
97
|
data_archive_dir = parse_archive_dir(data_archive_dir)
|
|
@@ -100,7 +100,7 @@ def disdrodb_run_l0b_station(
|
|
|
100
100
|
# -------------------------------------------------------------------------.
|
|
101
101
|
# If parallel=True, set the dask environment
|
|
102
102
|
if parallel:
|
|
103
|
-
cluster, client = initialize_dask_cluster()
|
|
103
|
+
cluster, client = initialize_dask_cluster(minimum_memory="4GB")
|
|
104
104
|
|
|
105
105
|
# -------------------------------------------------------------------------.
|
|
106
106
|
run_l0b_station(
|
|
@@ -94,7 +94,7 @@ def disdrodb_run_l0c_station(
|
|
|
94
94
|
Format: <...>/DISDRODB
|
|
95
95
|
If not specified, uses path specified in the DISDRODB active configuration.
|
|
96
96
|
"""
|
|
97
|
-
from disdrodb.l0
|
|
97
|
+
from disdrodb.routines.l0 import run_l0c_station
|
|
98
98
|
from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
|
|
99
99
|
|
|
100
100
|
data_archive_dir = parse_archive_dir(data_archive_dir)
|
|
@@ -103,7 +103,7 @@ def disdrodb_run_l0c_station(
|
|
|
103
103
|
# -------------------------------------------------------------------------.
|
|
104
104
|
# If parallel=True, set the dask environment
|
|
105
105
|
if parallel:
|
|
106
|
-
cluster, client = initialize_dask_cluster()
|
|
106
|
+
cluster, client = initialize_dask_cluster(minimum_memory="4GB")
|
|
107
107
|
|
|
108
108
|
# -------------------------------------------------------------------------.
|
|
109
109
|
run_l0c_station(
|
|
@@ -89,7 +89,7 @@ def disdrodb_run_l1_station(
|
|
|
89
89
|
Format: <...>/DISDRODB
|
|
90
90
|
If not specified, uses path specified in the DISDRODB active configuration.
|
|
91
91
|
"""
|
|
92
|
-
from disdrodb.l1
|
|
92
|
+
from disdrodb.routines.l1 import run_l1_station
|
|
93
93
|
from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
|
|
94
94
|
|
|
95
95
|
data_archive_dir = parse_archive_dir(data_archive_dir)
|
|
@@ -97,7 +97,7 @@ def disdrodb_run_l1_station(
|
|
|
97
97
|
# -------------------------------------------------------------------------.
|
|
98
98
|
# If parallel=True, set the dask environment
|
|
99
99
|
if parallel:
|
|
100
|
-
cluster, client = initialize_dask_cluster()
|
|
100
|
+
cluster, client = initialize_dask_cluster(minimum_memory="4GB")
|
|
101
101
|
|
|
102
102
|
# -------------------------------------------------------------------------.
|
|
103
103
|
run_l1_station(
|