disdrodb 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +64 -34
- disdrodb/_config.py +5 -4
- disdrodb/_version.py +16 -3
- disdrodb/accessor/__init__.py +20 -0
- disdrodb/accessor/methods.py +125 -0
- disdrodb/api/checks.py +139 -9
- disdrodb/api/configs.py +4 -2
- disdrodb/api/info.py +10 -10
- disdrodb/api/io.py +237 -18
- disdrodb/api/path.py +81 -75
- disdrodb/api/search.py +6 -6
- disdrodb/cli/disdrodb_create_summary_station.py +91 -0
- disdrodb/cli/disdrodb_run_l0.py +1 -1
- disdrodb/cli/disdrodb_run_l0_station.py +1 -1
- disdrodb/cli/disdrodb_run_l0b.py +1 -1
- disdrodb/cli/disdrodb_run_l0b_station.py +1 -1
- disdrodb/cli/disdrodb_run_l0c.py +1 -1
- disdrodb/cli/disdrodb_run_l0c_station.py +1 -1
- disdrodb/cli/disdrodb_run_l2e_station.py +1 -1
- disdrodb/configs.py +149 -4
- disdrodb/constants.py +61 -0
- disdrodb/data_transfer/download_data.py +145 -14
- disdrodb/etc/configs/attributes.yaml +339 -0
- disdrodb/etc/configs/encodings.yaml +473 -0
- disdrodb/etc/products/L1/global.yaml +13 -0
- disdrodb/etc/products/L2E/10MIN.yaml +12 -0
- disdrodb/etc/products/L2E/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/global.yaml +22 -0
- disdrodb/etc/products/L2M/10MIN.yaml +12 -0
- disdrodb/etc/products/L2M/GAMMA_ML.yaml +8 -0
- disdrodb/etc/products/L2M/NGAMMA_GS_LOG_ND_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/NGAMMA_GS_ND_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/NGAMMA_GS_Z_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/global.yaml +26 -0
- disdrodb/l0/__init__.py +13 -0
- disdrodb/l0/configs/LPM/bins_diameter.yml +3 -3
- disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +4 -4
- disdrodb/l0/configs/PARSIVEL/l0b_cf_attrs.yml +1 -1
- disdrodb/l0/configs/PARSIVEL/l0b_encodings.yml +3 -3
- disdrodb/l0/configs/PARSIVEL/raw_data_format.yml +1 -1
- disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +4 -0
- disdrodb/l0/configs/PARSIVEL2/l0b_cf_attrs.yml +20 -4
- disdrodb/l0/configs/PARSIVEL2/l0b_encodings.yml +44 -3
- disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml +41 -1
- disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +4 -4
- disdrodb/l0/configs/PWS100/raw_data_format.yml +1 -1
- disdrodb/l0/l0a_processing.py +30 -30
- disdrodb/l0/l0b_nc_processing.py +108 -2
- disdrodb/l0/l0b_processing.py +4 -4
- disdrodb/l0/l0c_processing.py +5 -13
- disdrodb/l0/manuals/SWS250.pdf +0 -0
- disdrodb/l0/manuals/VPF730.pdf +0 -0
- disdrodb/l0/manuals/VPF750.pdf +0 -0
- disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_LPM_NC.py +66 -0
- disdrodb/l0/readers/LPM/SLOVENIA/{CRNI_VRH.py → UL.py} +3 -0
- disdrodb/l0/readers/LPM/SWITZERLAND/INNERERIZ_LPM.py +195 -0
- disdrodb/l0/readers/PARSIVEL/GPM/PIERS.py +105 -0
- disdrodb/l0/readers/PARSIVEL/JAPAN/JMA.py +128 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/BELGIUM/ILVO.py +168 -0
- disdrodb/l0/readers/PARSIVEL2/DENMARK/DTU.py +165 -0
- disdrodb/l0/readers/PARSIVEL2/FINLAND/FMI_PARSIVEL2.py +69 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +255 -134
- disdrodb/l0/readers/PARSIVEL2/FRANCE/OSUG.py +525 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +9 -7
- disdrodb/l0/readers/{PARSIVEL → PARSIVEL2}/KIT/BURKINA_FASO.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/KIT/TEAMX.py +123 -0
- disdrodb/l0/readers/PARSIVEL2/NASA/APU.py +120 -0
- disdrodb/l0/readers/PARSIVEL2/{NETHERLANDS/DELFT.py → NCAR/FARM_PARSIVEL2.py} +43 -70
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_MIPS.py +126 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_PIPS.py +165 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +29 -12
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT_NC.py +69 -0
- disdrodb/l0/readers/PARSIVEL2/SPAIN/CENER.py +144 -0
- disdrodb/l0/readers/PARSIVEL2/SPAIN/CR1000DL.py +201 -0
- disdrodb/l0/readers/PARSIVEL2/SPAIN/LIAISE.py +137 -0
- disdrodb/l0/readers/PARSIVEL2/USA/C3WE.py +146 -0
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +105 -99
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100_SIRTA.py +151 -0
- disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py +31 -14
- disdrodb/l0/routines.py +105 -14
- disdrodb/l1/__init__.py +5 -0
- disdrodb/l1/filters.py +34 -20
- disdrodb/l1/processing.py +45 -44
- disdrodb/l1/resampling.py +77 -66
- disdrodb/l1/routines.py +35 -42
- disdrodb/l1_env/routines.py +18 -3
- disdrodb/l2/__init__.py +7 -0
- disdrodb/l2/empirical_dsd.py +58 -10
- disdrodb/l2/event.py +27 -120
- disdrodb/l2/processing.py +267 -116
- disdrodb/l2/routines.py +618 -254
- disdrodb/metadata/standards.py +3 -1
- disdrodb/psd/fitting.py +463 -144
- disdrodb/psd/models.py +8 -5
- disdrodb/routines.py +3 -3
- disdrodb/scattering/__init__.py +16 -4
- disdrodb/scattering/axis_ratio.py +56 -36
- disdrodb/scattering/permittivity.py +486 -0
- disdrodb/scattering/routines.py +701 -159
- disdrodb/summary/__init__.py +17 -0
- disdrodb/summary/routines.py +4120 -0
- disdrodb/utils/attrs.py +68 -125
- disdrodb/utils/compression.py +30 -1
- disdrodb/utils/dask.py +59 -8
- disdrodb/utils/dataframe.py +63 -9
- disdrodb/utils/directories.py +49 -17
- disdrodb/utils/encoding.py +33 -19
- disdrodb/utils/logger.py +13 -6
- disdrodb/utils/manipulations.py +71 -0
- disdrodb/utils/subsetting.py +214 -0
- disdrodb/utils/time.py +165 -19
- disdrodb/utils/writer.py +20 -7
- disdrodb/utils/xarray.py +85 -4
- disdrodb/viz/__init__.py +13 -0
- disdrodb/viz/plots.py +327 -0
- {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/METADATA +3 -2
- {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/RECORD +127 -87
- {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/entry_points.txt +1 -0
- disdrodb/l1/encoding_attrs.py +0 -635
- disdrodb/l2/processing_options.py +0 -213
- /disdrodb/l0/readers/PARSIVEL/SLOVENIA/{UL_FGG.py → UL.py} +0 -0
- {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/WHEEL +0 -0
- {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/licenses/LICENSE +0 -0
- {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# -----------------------------------------------------------------------------.
|
|
2
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
3
|
+
#
|
|
4
|
+
# This program is free software: you can redistribute it and/or modify
|
|
5
|
+
# it under the terms of the GNU General Public License as published by
|
|
6
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
7
|
+
# (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This program is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
12
|
+
# GNU General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU General Public License
|
|
15
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
16
|
+
# -----------------------------------------------------------------------------.
|
|
17
|
+
"""Script to run the DISDRODB L0 station processing."""
|
|
18
|
+
import sys
|
|
19
|
+
from typing import Optional
|
|
20
|
+
|
|
21
|
+
import click
|
|
22
|
+
|
|
23
|
+
from disdrodb.utils.cli import (
|
|
24
|
+
click_data_archive_dir_option,
|
|
25
|
+
click_station_arguments,
|
|
26
|
+
parse_archive_dir,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
sys.tracebacklimit = 0 # avoid full traceback error if occur
|
|
30
|
+
|
|
31
|
+
# -------------------------------------------------------------------------.
|
|
32
|
+
# Click Command Line Interface decorator
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@click.command()
|
|
36
|
+
@click_station_arguments
|
|
37
|
+
@click_data_archive_dir_option
|
|
38
|
+
@click.option("-p", "--parallel", type=bool, show_default=True, default=False, help="Read files in parallel")
|
|
39
|
+
def disdrodb_create_summary_station(
|
|
40
|
+
# Station arguments
|
|
41
|
+
data_source: str,
|
|
42
|
+
campaign_name: str,
|
|
43
|
+
station_name: str,
|
|
44
|
+
# Processing options:
|
|
45
|
+
parallel=False,
|
|
46
|
+
# DISDRODB root directories
|
|
47
|
+
data_archive_dir: Optional[str] = None,
|
|
48
|
+
):
|
|
49
|
+
r"""Create summary figures and tables for a specific DISDRODB station.
|
|
50
|
+
|
|
51
|
+
Parameters \n
|
|
52
|
+
---------- \n
|
|
53
|
+
data_source : str \n
|
|
54
|
+
Institution name (when campaign data spans more than 1 country),
|
|
55
|
+
or country (when all campaigns (or sensor networks) are inside a given country).\n
|
|
56
|
+
Must be UPPER CASE.\n
|
|
57
|
+
campaign_name : str \n
|
|
58
|
+
Campaign name. Must be UPPER CASE.\n
|
|
59
|
+
station_name : str \n
|
|
60
|
+
Station name \n
|
|
61
|
+
data_archive_dir : str \n
|
|
62
|
+
DISDRODB Data Archive directory \n
|
|
63
|
+
Format: <...>/DISDRODB \n
|
|
64
|
+
If not specified, uses path specified in the DISDRODB active configuration. \n
|
|
65
|
+
"""
|
|
66
|
+
from disdrodb.summary.routines import create_station_summary
|
|
67
|
+
from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
|
|
68
|
+
|
|
69
|
+
data_archive_dir = parse_archive_dir(data_archive_dir)
|
|
70
|
+
|
|
71
|
+
# -------------------------------------------------------------------------.
|
|
72
|
+
# If parallel=True, set the dask environment
|
|
73
|
+
if parallel:
|
|
74
|
+
cluster, client = initialize_dask_cluster()
|
|
75
|
+
|
|
76
|
+
# -------------------------------------------------------------------------.
|
|
77
|
+
create_station_summary(
|
|
78
|
+
# Station arguments
|
|
79
|
+
data_source=data_source,
|
|
80
|
+
campaign_name=campaign_name,
|
|
81
|
+
station_name=station_name,
|
|
82
|
+
# Options
|
|
83
|
+
parallel=parallel,
|
|
84
|
+
# DISDRODB root directory
|
|
85
|
+
data_archive_dir=data_archive_dir,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
# -------------------------------------------------------------------------.
|
|
89
|
+
# Close the cluster
|
|
90
|
+
if parallel:
|
|
91
|
+
close_dask_cluster(cluster, client)
|
disdrodb/cli/disdrodb_run_l0.py
CHANGED
|
@@ -115,7 +115,7 @@ def disdrodb_run_l0(
|
|
|
115
115
|
debugging_mode : bool
|
|
116
116
|
If True, it reduces the amount of data to process.
|
|
117
117
|
For L0A, it processes just the first 3 raw data files.
|
|
118
|
-
For L0B, it processes
|
|
118
|
+
For L0B, it processes 100 rows sampled from 3 L0A files.
|
|
119
119
|
The default is False.
|
|
120
120
|
data_archive_dir : str
|
|
121
121
|
DISDRODB Data Archive directory
|
|
@@ -105,7 +105,7 @@ def disdrodb_run_l0_station(
|
|
|
105
105
|
debugging_mode : bool \n
|
|
106
106
|
If True, it reduces the amount of data to process.\n
|
|
107
107
|
For L0A, it processes just the first 3 raw data files for each station.\n
|
|
108
|
-
For L0B, it processes
|
|
108
|
+
For L0B, it processes 100 rows sampled from 3 L0A files for each station.\n
|
|
109
109
|
The default is False.\n
|
|
110
110
|
data_archive_dir : str \n
|
|
111
111
|
DISDRODB Data Archive directory \n
|
disdrodb/cli/disdrodb_run_l0b.py
CHANGED
|
@@ -92,7 +92,7 @@ def disdrodb_run_l0b(
|
|
|
92
92
|
If False, multi-threading is automatically exploited to speed up I/0 tasks.
|
|
93
93
|
debugging_mode : bool
|
|
94
94
|
If True, it reduces the amount of data to process.
|
|
95
|
-
It processes
|
|
95
|
+
It processes 100 rows sampled from 3 L0A files for each station.
|
|
96
96
|
The default is False.
|
|
97
97
|
data_archive_dir : str
|
|
98
98
|
DISDRODB Data Archive directory
|
|
@@ -84,7 +84,7 @@ def disdrodb_run_l0b_station(
|
|
|
84
84
|
If False, multi-threading is automatically exploited to speed up I/0 tasks.
|
|
85
85
|
debugging_mode : bool
|
|
86
86
|
If True, it reduces the amount of data to process.
|
|
87
|
-
It processes
|
|
87
|
+
It processes 100 rows sampled from 3 L0A files.
|
|
88
88
|
The default is False.
|
|
89
89
|
data_archive_dir : str
|
|
90
90
|
DISDRODB Data Archive directory
|
disdrodb/cli/disdrodb_run_l0c.py
CHANGED
|
@@ -93,7 +93,7 @@ def disdrodb_run_l0c(
|
|
|
93
93
|
If False, multi-threading is automatically exploited to speed up I/0 tasks.
|
|
94
94
|
debugging_mode : bool
|
|
95
95
|
If True, it reduces the amount of data to process.
|
|
96
|
-
It processes
|
|
96
|
+
It processes 100 rows sampled from 3 L0A files for each station.
|
|
97
97
|
The default is False.
|
|
98
98
|
remove_l0b: bool, optional
|
|
99
99
|
Whether to remove the processed L0B files. The default value is ``False``.
|
|
@@ -85,7 +85,7 @@ def disdrodb_run_l0c_station(
|
|
|
85
85
|
If False, multi-threading is automatically exploited to speed up I/0 tasks.
|
|
86
86
|
debugging_mode : bool
|
|
87
87
|
If True, it reduces the amount of data to process.
|
|
88
|
-
It processes
|
|
88
|
+
It processes 100 rows sampled from 3 L0A files.
|
|
89
89
|
The default is False.
|
|
90
90
|
remove_l0b: bool, optional
|
|
91
91
|
Whether to remove the processed L0B files. The default value is ``False``.
|
|
@@ -98,7 +98,7 @@ def disdrodb_run_l2e_station(
|
|
|
98
98
|
# -------------------------------------------------------------------------.
|
|
99
99
|
# If parallel=True, set the dask environment
|
|
100
100
|
if parallel:
|
|
101
|
-
cluster, client = initialize_dask_cluster()
|
|
101
|
+
cluster, client = initialize_dask_cluster(minimum_memory="8GB")
|
|
102
102
|
|
|
103
103
|
# -------------------------------------------------------------------------.
|
|
104
104
|
run_l2e_station(
|
disdrodb/configs.py
CHANGED
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
"""DISDRODB Configuration File functions."""
|
|
20
20
|
|
|
21
21
|
import os
|
|
22
|
+
import shutil
|
|
22
23
|
from typing import Optional
|
|
23
24
|
|
|
24
25
|
from disdrodb.utils.yaml import read_yaml, write_yaml
|
|
@@ -32,9 +33,11 @@ def _define_config_filepath():
|
|
|
32
33
|
return filepath
|
|
33
34
|
|
|
34
35
|
|
|
35
|
-
def
|
|
36
|
+
def define_configs(
|
|
36
37
|
data_archive_dir: Optional[str] = None,
|
|
37
38
|
metadata_archive_dir: Optional[str] = None,
|
|
39
|
+
scattering_table_dir: Optional[str] = None,
|
|
40
|
+
configs_path: Optional[str] = None,
|
|
38
41
|
folder_partitioning: Optional[str] = None,
|
|
39
42
|
zenodo_token: Optional[str] = None,
|
|
40
43
|
zenodo_sandbox_token: Optional[str] = None,
|
|
@@ -48,6 +51,10 @@ def define_disdrodb_configs(
|
|
|
48
51
|
The directory path where the DISDRODB Data Archive is located.
|
|
49
52
|
metadata_archive_dir : str
|
|
50
53
|
The directory path where the DISDRODB Metadata Archive is located.
|
|
54
|
+
scattering_table_dir : str
|
|
55
|
+
The directory path where to store DISDRODB T-Matrix scattering tables.
|
|
56
|
+
configs_path : str
|
|
57
|
+
The directory path where the custom DISDRODB products configurations files are defined.
|
|
51
58
|
folder_partitioning : str
|
|
52
59
|
The folder partitioning scheme used in the DISDRODB Data Archive.
|
|
53
60
|
Allowed values are:
|
|
@@ -69,7 +76,13 @@ def define_disdrodb_configs(
|
|
|
69
76
|
The configuration file is used to run the various DISDRODB operations.
|
|
70
77
|
|
|
71
78
|
"""
|
|
72
|
-
|
|
79
|
+
import disdrodb
|
|
80
|
+
from disdrodb.api.checks import (
|
|
81
|
+
check_data_archive_dir,
|
|
82
|
+
check_folder_partitioning,
|
|
83
|
+
check_metadata_archive_dir,
|
|
84
|
+
check_scattering_table_dir,
|
|
85
|
+
)
|
|
73
86
|
|
|
74
87
|
# Define path to .config_disdrodb.yaml file
|
|
75
88
|
filepath = _define_config_filepath()
|
|
@@ -85,9 +98,16 @@ def define_disdrodb_configs(
|
|
|
85
98
|
# Add DISDRODB Data Archive Directory
|
|
86
99
|
if data_archive_dir is not None:
|
|
87
100
|
config_dict["data_archive_dir"] = check_data_archive_dir(data_archive_dir)
|
|
101
|
+
|
|
88
102
|
# Add DISDRODB Metadata Archive Directory
|
|
89
103
|
if metadata_archive_dir is not None:
|
|
90
104
|
config_dict["metadata_archive_dir"] = check_metadata_archive_dir(metadata_archive_dir)
|
|
105
|
+
|
|
106
|
+
# Add DISDRODB Scattering Table Directory
|
|
107
|
+
if scattering_table_dir is not None:
|
|
108
|
+
os.makedirs(scattering_table_dir, exist_ok=True)
|
|
109
|
+
config_dict["scattering_table_dir"] = check_scattering_table_dir(scattering_table_dir)
|
|
110
|
+
|
|
91
111
|
# Add DISDRODB Folder Partitioning
|
|
92
112
|
if folder_partitioning is not None:
|
|
93
113
|
config_dict["folder_partitioning"] = check_folder_partitioning(folder_partitioning)
|
|
@@ -98,13 +118,21 @@ def define_disdrodb_configs(
|
|
|
98
118
|
if zenodo_sandbox_token is not None:
|
|
99
119
|
config_dict["zenodo_sandbox_token"] = zenodo_sandbox_token
|
|
100
120
|
|
|
121
|
+
if configs_path is not None:
|
|
122
|
+
config_dict["configs_path"] = configs_path
|
|
123
|
+
|
|
101
124
|
# Write the DISDRODB config file
|
|
102
125
|
write_yaml(config_dict, filepath, sort_keys=False)
|
|
103
126
|
|
|
104
127
|
print(f"The DISDRODB config file has been {action_msg} successfully!")
|
|
105
128
|
|
|
129
|
+
# Now read the config file and set it as the active configuration
|
|
130
|
+
# - This avoid the need to restart a python session to take effect !
|
|
131
|
+
config_dict = read_configs()
|
|
132
|
+
disdrodb.config.update(config_dict)
|
|
133
|
+
|
|
106
134
|
|
|
107
|
-
def
|
|
135
|
+
def read_configs() -> dict[str, str]:
|
|
108
136
|
"""
|
|
109
137
|
Reads the DISDRODB configuration file and returns a dictionary with the configuration settings.
|
|
110
138
|
|
|
@@ -158,6 +186,19 @@ def get_metadata_archive_dir(metadata_archive_dir=None):
|
|
|
158
186
|
return metadata_archive_dir
|
|
159
187
|
|
|
160
188
|
|
|
189
|
+
def get_scattering_table_dir(scattering_table_dir=None):
|
|
190
|
+
"""Return the directory where DISDRODB save pyTMatrix scattering tables."""
|
|
191
|
+
import disdrodb
|
|
192
|
+
from disdrodb.api.checks import check_scattering_table_dir
|
|
193
|
+
|
|
194
|
+
if scattering_table_dir is None:
|
|
195
|
+
scattering_table_dir = disdrodb.config.get("scattering_table_dir", None)
|
|
196
|
+
if scattering_table_dir is None:
|
|
197
|
+
raise ValueError("The directory where to save DISDRODB T-Matrix scattering tables is not specified.")
|
|
198
|
+
scattering_table_dir = check_scattering_table_dir(scattering_table_dir) # ensure Path converted to str
|
|
199
|
+
return scattering_table_dir
|
|
200
|
+
|
|
201
|
+
|
|
161
202
|
def get_folder_partitioning():
|
|
162
203
|
"""Return the folder partitioning."""
|
|
163
204
|
import disdrodb
|
|
@@ -182,7 +223,7 @@ def get_zenodo_token(sandbox: bool):
|
|
|
182
223
|
host = "zenodo.org"
|
|
183
224
|
token_name = "zenodo_token"
|
|
184
225
|
|
|
185
|
-
# token =
|
|
226
|
+
# token = read_configs().get(token_name, None)
|
|
186
227
|
token = disdrodb.config.get(token_name, None)
|
|
187
228
|
|
|
188
229
|
if token is None:
|
|
@@ -195,3 +236,107 @@ def get_zenodo_token(sandbox: bool):
|
|
|
195
236
|
raise ValueError(f"Missing {token_name} in the DISDRODB config file !")
|
|
196
237
|
|
|
197
238
|
return token
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def get_product_default_configs_path():
|
|
242
|
+
"""Return the paths where DISDRODB products configuration files are stored."""
|
|
243
|
+
import disdrodb
|
|
244
|
+
|
|
245
|
+
configs_path = os.path.join(disdrodb.__root_path__, "disdrodb", "etc", "products")
|
|
246
|
+
return configs_path
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def check_availability_radar_simulations(options):
|
|
250
|
+
"""Check radar simulations are possible for L2E and L2M products."""
|
|
251
|
+
import disdrodb
|
|
252
|
+
|
|
253
|
+
if "radar_enabled" in options and not disdrodb.is_pytmatrix_available():
|
|
254
|
+
options["radar_enabled"] = False
|
|
255
|
+
return options
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def copy_product_default_configs(configs_path):
|
|
259
|
+
"""Copy the default DISDRODB products configuration directory to a custom location.
|
|
260
|
+
|
|
261
|
+
This function duplicates the entire directory of default product settings
|
|
262
|
+
(located at ``disdrodb/etc/products``) into the user-specified
|
|
263
|
+
``configs_path``. Once copied, you can safely edit these files without
|
|
264
|
+
modifying the library's built-in defaults. To have DISDRODB use your
|
|
265
|
+
custom settings, point the global configuration at this new directory
|
|
266
|
+
(e.g by specifying ``configs_path`` with the ``disdrodb.define_configs`` function).
|
|
267
|
+
|
|
268
|
+
Parameters
|
|
269
|
+
----------
|
|
270
|
+
configs_path:
|
|
271
|
+
Destination directory where the default product configuration files
|
|
272
|
+
will be copied. This directory must not already exist, and later
|
|
273
|
+
needs to be referenced in your DISDRODB global configuration.
|
|
274
|
+
|
|
275
|
+
Returns
|
|
276
|
+
-------
|
|
277
|
+
configs_path
|
|
278
|
+
The path to the newly created custom product configuration directory.
|
|
279
|
+
|
|
280
|
+
"""
|
|
281
|
+
source_dir_path = get_product_default_configs_path()
|
|
282
|
+
if os.path.exists(configs_path):
|
|
283
|
+
raise FileExistsError(f"The {configs_path} directory already exists!")
|
|
284
|
+
configs_path = shutil.copytree(source_dir_path, configs_path)
|
|
285
|
+
return configs_path
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def get_product_options(product, temporal_resolution=None):
|
|
289
|
+
"""Get options for DISDRODB products."""
|
|
290
|
+
import disdrodb
|
|
291
|
+
from disdrodb.api.checks import check_product
|
|
292
|
+
|
|
293
|
+
# Define configs path
|
|
294
|
+
if os.environ.get("PYTEST_CURRENT_TEST"):
|
|
295
|
+
configs_path = os.path.join(disdrodb.__root_path__, "disdrodb", "tests", "products")
|
|
296
|
+
else:
|
|
297
|
+
configs_path = disdrodb.config.get("configs_path", get_product_default_configs_path())
|
|
298
|
+
|
|
299
|
+
# Validate DISDRODB products configuration
|
|
300
|
+
validate_product_configuration(configs_path)
|
|
301
|
+
|
|
302
|
+
# Check product
|
|
303
|
+
check_product(product)
|
|
304
|
+
|
|
305
|
+
# Retrieve global product options
|
|
306
|
+
global_options = read_yaml(os.path.join(configs_path, product, "global.yaml"))
|
|
307
|
+
if temporal_resolution is None:
|
|
308
|
+
global_options = check_availability_radar_simulations(global_options)
|
|
309
|
+
return global_options
|
|
310
|
+
|
|
311
|
+
# If temporal resolutions are specified, drop 'temporal_resolutions' key
|
|
312
|
+
global_options.pop("temporal_resolutions", None)
|
|
313
|
+
custom_options_path = os.path.join(configs_path, product, f"{temporal_resolution}.yaml")
|
|
314
|
+
if not os.path.exists(custom_options_path):
|
|
315
|
+
return global_options
|
|
316
|
+
custom_options = read_yaml(custom_options_path)
|
|
317
|
+
options = global_options.copy()
|
|
318
|
+
options.update(custom_options)
|
|
319
|
+
options = check_availability_radar_simulations(options)
|
|
320
|
+
return options
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def get_product_temporal_resolutions(product):
|
|
324
|
+
"""Get DISDRODB L2 product temporal aggregations."""
|
|
325
|
+
# Check only L2E and L2M
|
|
326
|
+
return get_product_options(product)["temporal_resolutions"]
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def get_model_options(product, model_name):
|
|
330
|
+
"""Get DISDRODB L2M model options."""
|
|
331
|
+
import disdrodb
|
|
332
|
+
|
|
333
|
+
configs_path = disdrodb.config.get("configs_path", get_product_default_configs_path())
|
|
334
|
+
model_options_path = os.path.join(configs_path, product, f"{model_name}.yaml")
|
|
335
|
+
model_options = read_yaml(model_options_path)
|
|
336
|
+
return model_options
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def validate_product_configuration(configs_path):
|
|
340
|
+
"""Validate the DISDRODB products configuration files."""
|
|
341
|
+
# TODO: Implement validation of DISDRODB products configuration files with pydantic
|
|
342
|
+
pass
|
disdrodb/constants.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# -----------------------------------------------------------------------------.
|
|
2
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
3
|
+
#
|
|
4
|
+
# This program is free software: you can redistribute it and/or modify
|
|
5
|
+
# it under the terms of the GNU General Public License as published by
|
|
6
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
7
|
+
# (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This program is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
12
|
+
# GNU General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU General Public License
|
|
15
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
16
|
+
# -----------------------------------------------------------------------------.
|
|
17
|
+
"""DISDRODB constants."""
|
|
18
|
+
import importlib
|
|
19
|
+
|
|
20
|
+
ARCHIVE_VERSION = "V0"
|
|
21
|
+
SOFTWARE_VERSION = "V" + importlib.metadata.version("disdrodb")
|
|
22
|
+
CONVENTIONS = "CF-1.10, ACDD-1.3"
|
|
23
|
+
|
|
24
|
+
# Define coordinates names
|
|
25
|
+
DIAMETER_COORDS = ["diameter_bin_center", "diameter_bin_width", "diameter_bin_lower", "diameter_bin_upper"]
|
|
26
|
+
VELOCITY_COORDS = ["velocity_bin_center", "velocity_bin_width", "velocity_bin_lower", "velocity_bin_upper"]
|
|
27
|
+
GEOLOCATION_COORDS = ["longitude", "latitude", "altitude"]
|
|
28
|
+
VELOCITY_DIMENSION = "velocity_bin_center"
|
|
29
|
+
DIAMETER_DIMENSION = "diameter_bin_center"
|
|
30
|
+
COORDINATES = [
|
|
31
|
+
"diameter_bin_center",
|
|
32
|
+
"diameter_bin_width",
|
|
33
|
+
"diameter_bin_upper",
|
|
34
|
+
"velocity_bin_lower",
|
|
35
|
+
"velocity_bin_center",
|
|
36
|
+
"velocity_bin_width",
|
|
37
|
+
"velocity_bin_upper",
|
|
38
|
+
"latitude",
|
|
39
|
+
"longitude",
|
|
40
|
+
"altitude",
|
|
41
|
+
"time",
|
|
42
|
+
"sample_interval",
|
|
43
|
+
]
|
|
44
|
+
OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM", "PWS100"]
|
|
45
|
+
IMPACT_SENSORS = ["RD80"]
|
|
46
|
+
|
|
47
|
+
PRODUCTS = ["RAW", "L0A", "L0B", "L0C", "L1", "L2E", "L2M"]
|
|
48
|
+
|
|
49
|
+
PRODUCTS_ARGUMENTS = {
|
|
50
|
+
"L2E": ["rolling", "sample_interval"],
|
|
51
|
+
"L2M": ["rolling", "sample_interval", "model_name"],
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
PRODUCTS_REQUIREMENTS = {
|
|
55
|
+
"L0A": "RAW",
|
|
56
|
+
"L0B": "L0A",
|
|
57
|
+
"L0C": "L0B",
|
|
58
|
+
"L1": "L0C",
|
|
59
|
+
"L2E": "L1",
|
|
60
|
+
"L2M": "L2E",
|
|
61
|
+
}
|
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
import logging
|
|
22
22
|
import os
|
|
23
23
|
import shutil
|
|
24
|
+
import subprocess
|
|
25
|
+
import urllib.parse
|
|
24
26
|
from typing import Optional, Union
|
|
25
27
|
|
|
26
28
|
import click
|
|
@@ -213,7 +215,7 @@ def download_station(
|
|
|
213
215
|
check_exists=True,
|
|
214
216
|
)
|
|
215
217
|
# Download data
|
|
216
|
-
|
|
218
|
+
download_station_data(metadata_filepath, data_archive_dir=data_archive_dir, force=force)
|
|
217
219
|
|
|
218
220
|
|
|
219
221
|
def _is_valid_disdrodb_data_url(disdrodb_data_url):
|
|
@@ -228,13 +230,25 @@ def _extract_station_files(zip_filepath, station_dir):
|
|
|
228
230
|
os.remove(zip_filepath)
|
|
229
231
|
|
|
230
232
|
|
|
231
|
-
def
|
|
233
|
+
def check_consistent_station_name(metadata_filepath, station_name):
|
|
234
|
+
"""Check consistent station_name between YAML file name and metadata key."""
|
|
235
|
+
# Check consistent station name
|
|
236
|
+
expected_station_name = os.path.basename(metadata_filepath).replace(".yml", "")
|
|
237
|
+
if station_name and str(station_name) != str(expected_station_name):
|
|
238
|
+
raise ValueError(f"Inconsistent station_name values in the {metadata_filepath} file. Download aborted.")
|
|
239
|
+
return station_name
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def download_station_data(metadata_filepath: str, data_archive_dir: str, force: bool = False) -> None:
|
|
232
243
|
"""Download and unzip the station data .
|
|
233
244
|
|
|
234
245
|
Parameters
|
|
235
246
|
----------
|
|
236
247
|
metadata_filepaths : str
|
|
237
248
|
Metadata file path.
|
|
249
|
+
data_archive_dir : str (optional)
|
|
250
|
+
DISDRODB Data Archive directory. Format: ``<...>/DISDRODB``.
|
|
251
|
+
If ``None`` (the default), the disdrodb config variable ``data_archive_dir`` is used.
|
|
238
252
|
force : bool, optional
|
|
239
253
|
If ``True``, delete existing files and redownload it. The default value is ``False``.
|
|
240
254
|
|
|
@@ -247,7 +261,7 @@ def _download_station_data(metadata_filepath: str, data_archive_dir: str, force:
|
|
|
247
261
|
campaign_name = metadata_dict["campaign_name"]
|
|
248
262
|
station_name = metadata_dict["station_name"]
|
|
249
263
|
station_name = check_consistent_station_name(metadata_filepath, station_name)
|
|
250
|
-
# Define the
|
|
264
|
+
# Define the path to the station RAW data directory
|
|
251
265
|
station_dir = define_station_dir(
|
|
252
266
|
data_archive_dir=data_archive_dir,
|
|
253
267
|
data_source=data_source,
|
|
@@ -259,19 +273,136 @@ def _download_station_data(metadata_filepath: str, data_archive_dir: str, force:
|
|
|
259
273
|
disdrodb_data_url = metadata_dict.get("disdrodb_data_url", None)
|
|
260
274
|
if not _is_valid_disdrodb_data_url(disdrodb_data_url):
|
|
261
275
|
raise ValueError(f"Invalid disdrodb_data_url '{disdrodb_data_url}' for station {station_name}")
|
|
262
|
-
# Download file
|
|
263
|
-
zip_filepath = _download_file_from_url(disdrodb_data_url, dst_dir=station_dir, force=force)
|
|
264
|
-
# Extract the stations files from the downloaded station.zip file
|
|
265
|
-
_extract_station_files(zip_filepath, station_dir=station_dir)
|
|
266
276
|
|
|
277
|
+
# Download files
|
|
278
|
+
# - Option 1: Download Zip file containing all station raw data
|
|
279
|
+
if disdrodb_data_url.startswith("https://zenodo.org/") or disdrodb_data_url.startswith("https://cloudnet.fmi.fi/"):
|
|
280
|
+
download_zip_file(url=disdrodb_data_url, dst_dir=station_dir, force=force)
|
|
281
|
+
# - Option 2: Recursive download from a web server via HTTP or HTTPS.
|
|
282
|
+
elif disdrodb_data_url.startswith("http"):
|
|
283
|
+
download_web_server_data(url=disdrodb_data_url, dst_dir=station_dir, force=force, verbose=True)
|
|
284
|
+
else:
|
|
285
|
+
raise NotImplementedError(f"Open a GitHub Issue to enable the download of data from {disdrodb_data_url}.")
|
|
267
286
|
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
287
|
+
|
|
288
|
+
####-----------------------------------------------------------------------------------------.
|
|
289
|
+
#### Download from Web Server via HTTP or HTTPS
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def download_web_server_data(url: str, dst_dir: str, force=True, verbose=True) -> None:
|
|
293
|
+
"""Download data from a web server via HTTP or HTTPS.
|
|
294
|
+
|
|
295
|
+
Use the system's wget command to recursively download all files and subdirectories
|
|
296
|
+
under the given HTTPS “directory” URL. Works on both Windows and Linux, provided
|
|
297
|
+
that wget is installed and on the PATH.
|
|
298
|
+
|
|
299
|
+
1. Ensure wget is available.
|
|
300
|
+
2. Normalize URL to end with '/'.
|
|
301
|
+
3. Compute cut-dirs so that only the last segment of the path remains locally.
|
|
302
|
+
4. Build and run the wget command.
|
|
303
|
+
|
|
304
|
+
Example:
|
|
305
|
+
download_with_wget("https://ruisdael.citg.tudelft.nl/parsivel/PAR001_Cabauw/2021/202101/")
|
|
306
|
+
# → Creates a local folder "202101/" with all files and subfolders.
|
|
307
|
+
"""
|
|
308
|
+
# 1. Ensure wget exists
|
|
309
|
+
ensure_wget_available()
|
|
310
|
+
|
|
311
|
+
# 2. Normalize URL
|
|
312
|
+
url = ensure_trailing_slash(url)
|
|
313
|
+
|
|
314
|
+
# 3. Compute cut-dirs so that only the last URL segment remains locally
|
|
315
|
+
cut_dirs = compute_cut_dirs(url)
|
|
316
|
+
|
|
317
|
+
# 4. Create destination directory if needed
|
|
318
|
+
os.makedirs(dst_dir, exist_ok=True)
|
|
319
|
+
|
|
320
|
+
# 5. Build wget command
|
|
321
|
+
cmd = build_webserver_wget_command(url, cut_dirs=cut_dirs, dst_dir=dst_dir, force=force, verbose=verbose)
|
|
322
|
+
|
|
323
|
+
# 6. Run wget command
|
|
324
|
+
try:
|
|
325
|
+
subprocess.run(cmd, check=True)
|
|
326
|
+
except subprocess.CalledProcessError as e:
|
|
327
|
+
raise subprocess.CalledProcessError(
|
|
328
|
+
returncode=e.returncode,
|
|
329
|
+
cmd=e.cmd,
|
|
330
|
+
output=e.output,
|
|
331
|
+
stderr=e.stderr,
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def ensure_wget_available() -> None:
|
|
336
|
+
"""Raise FileNotFoundError if 'wget' is not on the system PATH."""
|
|
337
|
+
if shutil.which("wget") is None:
|
|
338
|
+
raise FileNotFoundError("The WGET software was not found. Please install WGET or add it to PATH.")
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def ensure_trailing_slash(url: str) -> str:
|
|
342
|
+
"""Return `url` guaranteed to end with a slash."""
|
|
343
|
+
return url if url.endswith("/") else url.rstrip("/") + "/"
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def compute_cut_dirs(url: str) -> int:
|
|
347
|
+
"""Compute the wget cut_dirs value to download directly in `dst_dir`.
|
|
348
|
+
|
|
349
|
+
Given a URL ending with '/', compute the total number of path segments.
|
|
350
|
+
By returning len(segments), we strip away all of them—so that files
|
|
351
|
+
within that final directory land directly in `dst_dir` without creating
|
|
352
|
+
an extra subfolder.
|
|
353
|
+
"""
|
|
354
|
+
parsed = urllib.parse.urlparse(url)
|
|
355
|
+
path = parsed.path.strip("/") # remove leading/trailing '/'
|
|
356
|
+
segments = path.split("/") if path else []
|
|
357
|
+
return len(segments)
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def build_webserver_wget_command(url: str, cut_dirs: int, dst_dir: str, force: bool, verbose: bool) -> list[str]:
|
|
361
|
+
"""Construct the wget command list for subprocess.run.
|
|
362
|
+
|
|
363
|
+
Notes
|
|
364
|
+
-----
|
|
365
|
+
The following wget arguments are used
|
|
366
|
+
- -q : quiet mode (no detailed progress)
|
|
367
|
+
- -r : recursive
|
|
368
|
+
- -np : no parent
|
|
369
|
+
- -nH : no host directories
|
|
370
|
+
- --timestamping: download missing files or when remote version is newer
|
|
371
|
+
- --cut-dirs : strip all but the last path segment from the remote path
|
|
372
|
+
- -P dst_dir : download into `dst_dir`
|
|
373
|
+
- url
|
|
374
|
+
"""
|
|
375
|
+
cmd = ["wget"]
|
|
376
|
+
if not verbose:
|
|
377
|
+
cmd.append("-q")
|
|
378
|
+
cmd += [
|
|
379
|
+
"-r",
|
|
380
|
+
"-np",
|
|
381
|
+
"-nH",
|
|
382
|
+
f"--cut-dirs={cut_dirs}",
|
|
383
|
+
]
|
|
384
|
+
if force:
|
|
385
|
+
cmd.append("--timestamping") # -N
|
|
386
|
+
|
|
387
|
+
# Define source and destination directory
|
|
388
|
+
cmd += [
|
|
389
|
+
"-P",
|
|
390
|
+
dst_dir,
|
|
391
|
+
url,
|
|
392
|
+
]
|
|
393
|
+
return cmd
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
####--------------------------------------------------------------------.
|
|
397
|
+
#### Download from Zenodo
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def download_zip_file(url, dst_dir, force):
|
|
401
|
+
"""Download zip file from zenodo and extract station raw data."""
|
|
402
|
+
# Download zip file
|
|
403
|
+
zip_filepath = _download_file_from_url(url, dst_dir=dst_dir, force=force)
|
|
404
|
+
# Extract the stations files from the downloaded station.zip file
|
|
405
|
+
_extract_station_files(zip_filepath, station_dir=dst_dir)
|
|
275
406
|
|
|
276
407
|
|
|
277
408
|
def _download_file_from_url(url: str, dst_dir: str, force: bool = False) -> str:
|