disdrodb 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. disdrodb/__init__.py +64 -34
  2. disdrodb/_config.py +5 -4
  3. disdrodb/_version.py +16 -3
  4. disdrodb/accessor/__init__.py +20 -0
  5. disdrodb/accessor/methods.py +125 -0
  6. disdrodb/api/checks.py +139 -9
  7. disdrodb/api/configs.py +4 -2
  8. disdrodb/api/info.py +10 -10
  9. disdrodb/api/io.py +237 -18
  10. disdrodb/api/path.py +81 -75
  11. disdrodb/api/search.py +6 -6
  12. disdrodb/cli/disdrodb_create_summary_station.py +91 -0
  13. disdrodb/cli/disdrodb_run_l0.py +1 -1
  14. disdrodb/cli/disdrodb_run_l0_station.py +1 -1
  15. disdrodb/cli/disdrodb_run_l0b.py +1 -1
  16. disdrodb/cli/disdrodb_run_l0b_station.py +1 -1
  17. disdrodb/cli/disdrodb_run_l0c.py +1 -1
  18. disdrodb/cli/disdrodb_run_l0c_station.py +1 -1
  19. disdrodb/cli/disdrodb_run_l2e_station.py +1 -1
  20. disdrodb/configs.py +149 -4
  21. disdrodb/constants.py +61 -0
  22. disdrodb/data_transfer/download_data.py +145 -14
  23. disdrodb/etc/configs/attributes.yaml +339 -0
  24. disdrodb/etc/configs/encodings.yaml +473 -0
  25. disdrodb/etc/products/L1/global.yaml +13 -0
  26. disdrodb/etc/products/L2E/10MIN.yaml +12 -0
  27. disdrodb/etc/products/L2E/1MIN.yaml +1 -0
  28. disdrodb/etc/products/L2E/global.yaml +22 -0
  29. disdrodb/etc/products/L2M/10MIN.yaml +12 -0
  30. disdrodb/etc/products/L2M/GAMMA_ML.yaml +8 -0
  31. disdrodb/etc/products/L2M/NGAMMA_GS_LOG_ND_MAE.yaml +6 -0
  32. disdrodb/etc/products/L2M/NGAMMA_GS_ND_MAE.yaml +6 -0
  33. disdrodb/etc/products/L2M/NGAMMA_GS_Z_MAE.yaml +6 -0
  34. disdrodb/etc/products/L2M/global.yaml +26 -0
  35. disdrodb/l0/__init__.py +13 -0
  36. disdrodb/l0/configs/LPM/bins_diameter.yml +3 -3
  37. disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +4 -4
  38. disdrodb/l0/configs/PARSIVEL/l0b_cf_attrs.yml +1 -1
  39. disdrodb/l0/configs/PARSIVEL/l0b_encodings.yml +3 -3
  40. disdrodb/l0/configs/PARSIVEL/raw_data_format.yml +1 -1
  41. disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +4 -0
  42. disdrodb/l0/configs/PARSIVEL2/l0b_cf_attrs.yml +20 -4
  43. disdrodb/l0/configs/PARSIVEL2/l0b_encodings.yml +44 -3
  44. disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml +41 -1
  45. disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +4 -4
  46. disdrodb/l0/configs/PWS100/raw_data_format.yml +1 -1
  47. disdrodb/l0/l0a_processing.py +30 -30
  48. disdrodb/l0/l0b_nc_processing.py +108 -2
  49. disdrodb/l0/l0b_processing.py +4 -4
  50. disdrodb/l0/l0c_processing.py +5 -13
  51. disdrodb/l0/manuals/SWS250.pdf +0 -0
  52. disdrodb/l0/manuals/VPF730.pdf +0 -0
  53. disdrodb/l0/manuals/VPF750.pdf +0 -0
  54. disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_LPM_NC.py +66 -0
  55. disdrodb/l0/readers/LPM/SLOVENIA/{CRNI_VRH.py → UL.py} +3 -0
  56. disdrodb/l0/readers/LPM/SWITZERLAND/INNERERIZ_LPM.py +195 -0
  57. disdrodb/l0/readers/PARSIVEL/GPM/PIERS.py +105 -0
  58. disdrodb/l0/readers/PARSIVEL/JAPAN/JMA.py +128 -0
  59. disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +1 -1
  60. disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +1 -1
  61. disdrodb/l0/readers/PARSIVEL2/BELGIUM/ILVO.py +168 -0
  62. disdrodb/l0/readers/PARSIVEL2/DENMARK/DTU.py +165 -0
  63. disdrodb/l0/readers/PARSIVEL2/FINLAND/FMI_PARSIVEL2.py +69 -0
  64. disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +255 -134
  65. disdrodb/l0/readers/PARSIVEL2/FRANCE/OSUG.py +525 -0
  66. disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +1 -1
  67. disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +9 -7
  68. disdrodb/l0/readers/{PARSIVEL → PARSIVEL2}/KIT/BURKINA_FASO.py +1 -1
  69. disdrodb/l0/readers/PARSIVEL2/KIT/TEAMX.py +123 -0
  70. disdrodb/l0/readers/PARSIVEL2/NASA/APU.py +120 -0
  71. disdrodb/l0/readers/PARSIVEL2/{NETHERLANDS/DELFT.py → NCAR/FARM_PARSIVEL2.py} +43 -70
  72. disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +1 -1
  73. disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_MIPS.py +126 -0
  74. disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_PIPS.py +165 -0
  75. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +1 -1
  76. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +29 -12
  77. disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT_NC.py +69 -0
  78. disdrodb/l0/readers/PARSIVEL2/SPAIN/CENER.py +144 -0
  79. disdrodb/l0/readers/PARSIVEL2/SPAIN/CR1000DL.py +201 -0
  80. disdrodb/l0/readers/PARSIVEL2/SPAIN/LIAISE.py +137 -0
  81. disdrodb/l0/readers/PARSIVEL2/USA/C3WE.py +146 -0
  82. disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +105 -99
  83. disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100_SIRTA.py +151 -0
  84. disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py +31 -14
  85. disdrodb/l0/routines.py +105 -14
  86. disdrodb/l1/__init__.py +5 -0
  87. disdrodb/l1/filters.py +34 -20
  88. disdrodb/l1/processing.py +45 -44
  89. disdrodb/l1/resampling.py +77 -66
  90. disdrodb/l1/routines.py +35 -42
  91. disdrodb/l1_env/routines.py +18 -3
  92. disdrodb/l2/__init__.py +7 -0
  93. disdrodb/l2/empirical_dsd.py +58 -10
  94. disdrodb/l2/event.py +27 -120
  95. disdrodb/l2/processing.py +267 -116
  96. disdrodb/l2/routines.py +618 -254
  97. disdrodb/metadata/standards.py +3 -1
  98. disdrodb/psd/fitting.py +463 -144
  99. disdrodb/psd/models.py +8 -5
  100. disdrodb/routines.py +3 -3
  101. disdrodb/scattering/__init__.py +16 -4
  102. disdrodb/scattering/axis_ratio.py +56 -36
  103. disdrodb/scattering/permittivity.py +486 -0
  104. disdrodb/scattering/routines.py +701 -159
  105. disdrodb/summary/__init__.py +17 -0
  106. disdrodb/summary/routines.py +4120 -0
  107. disdrodb/utils/attrs.py +68 -125
  108. disdrodb/utils/compression.py +30 -1
  109. disdrodb/utils/dask.py +59 -8
  110. disdrodb/utils/dataframe.py +63 -9
  111. disdrodb/utils/directories.py +49 -17
  112. disdrodb/utils/encoding.py +33 -19
  113. disdrodb/utils/logger.py +13 -6
  114. disdrodb/utils/manipulations.py +71 -0
  115. disdrodb/utils/subsetting.py +214 -0
  116. disdrodb/utils/time.py +165 -19
  117. disdrodb/utils/writer.py +20 -7
  118. disdrodb/utils/xarray.py +85 -4
  119. disdrodb/viz/__init__.py +13 -0
  120. disdrodb/viz/plots.py +327 -0
  121. {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/METADATA +3 -2
  122. {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/RECORD +127 -87
  123. {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/entry_points.txt +1 -0
  124. disdrodb/l1/encoding_attrs.py +0 -635
  125. disdrodb/l2/processing_options.py +0 -213
  126. /disdrodb/l0/readers/PARSIVEL/SLOVENIA/{UL_FGG.py → UL.py} +0 -0
  127. {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/WHEEL +0 -0
  128. {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/licenses/LICENSE +0 -0
  129. {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,91 @@
1
+ # -----------------------------------------------------------------------------.
2
+ # Copyright (c) 2021-2023 DISDRODB developers
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU General Public License
15
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
16
+ # -----------------------------------------------------------------------------.
17
+ """Script to run the DISDRODB L0 station processing."""
18
+ import sys
19
+ from typing import Optional
20
+
21
+ import click
22
+
23
+ from disdrodb.utils.cli import (
24
+ click_data_archive_dir_option,
25
+ click_station_arguments,
26
+ parse_archive_dir,
27
+ )
28
+
29
+ sys.tracebacklimit = 0 # avoid full traceback error if occur
30
+
31
+ # -------------------------------------------------------------------------.
32
+ # Click Command Line Interface decorator
33
+
34
+
35
+ @click.command()
36
+ @click_station_arguments
37
+ @click_data_archive_dir_option
38
+ @click.option("-p", "--parallel", type=bool, show_default=True, default=False, help="Read files in parallel")
39
+ def disdrodb_create_summary_station(
40
+ # Station arguments
41
+ data_source: str,
42
+ campaign_name: str,
43
+ station_name: str,
44
+ # Processing options:
45
+ parallel=False,
46
+ # DISDRODB root directories
47
+ data_archive_dir: Optional[str] = None,
48
+ ):
49
+ r"""Create summary figures and tables for a specific DISDRODB station.
50
+
51
+ Parameters \n
52
+ ---------- \n
53
+ data_source : str \n
54
+ Institution name (when campaign data spans more than 1 country),
55
+ or country (when all campaigns (or sensor networks) are inside a given country).\n
56
+ Must be UPPER CASE.\n
57
+ campaign_name : str \n
58
+ Campaign name. Must be UPPER CASE.\n
59
+ station_name : str \n
60
+ Station name \n
61
+ data_archive_dir : str \n
62
+ DISDRODB Data Archive directory \n
63
+ Format: <...>/DISDRODB \n
64
+ If not specified, uses path specified in the DISDRODB active configuration. \n
65
+ """
66
+ from disdrodb.summary.routines import create_station_summary
67
+ from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
68
+
69
+ data_archive_dir = parse_archive_dir(data_archive_dir)
70
+
71
+ # -------------------------------------------------------------------------.
72
+ # If parallel=True, set the dask environment
73
+ if parallel:
74
+ cluster, client = initialize_dask_cluster()
75
+
76
+ # -------------------------------------------------------------------------.
77
+ create_station_summary(
78
+ # Station arguments
79
+ data_source=data_source,
80
+ campaign_name=campaign_name,
81
+ station_name=station_name,
82
+ # Options
83
+ parallel=parallel,
84
+ # DISDRODB root directory
85
+ data_archive_dir=data_archive_dir,
86
+ )
87
+
88
+ # -------------------------------------------------------------------------.
89
+ # Close the cluster
90
+ if parallel:
91
+ close_dask_cluster(cluster, client)
@@ -115,7 +115,7 @@ def disdrodb_run_l0(
115
115
  debugging_mode : bool
116
116
  If True, it reduces the amount of data to process.
117
117
  For L0A, it processes just the first 3 raw data files.
118
- For L0B, it processes just the first 100 rows of 3 L0A files.
118
+ For L0B, it processes 100 rows sampled from 3 L0A files.
119
119
  The default is False.
120
120
  data_archive_dir : str
121
121
  DISDRODB Data Archive directory
@@ -105,7 +105,7 @@ def disdrodb_run_l0_station(
105
105
  debugging_mode : bool \n
106
106
  If True, it reduces the amount of data to process.\n
107
107
  For L0A, it processes just the first 3 raw data files for each station.\n
108
- For L0B, it processes just the first 100 rows of 3 L0A files for each station.\n
108
+ For L0B, it processes 100 rows sampled from 3 L0A files for each station.\n
109
109
  The default is False.\n
110
110
  data_archive_dir : str \n
111
111
  DISDRODB Data Archive directory \n
@@ -92,7 +92,7 @@ def disdrodb_run_l0b(
92
92
  If False, multi-threading is automatically exploited to speed up I/0 tasks.
93
93
  debugging_mode : bool
94
94
  If True, it reduces the amount of data to process.
95
- It processes just the first 100 rows of 3 L0A files for each station.
95
+ It processes 100 rows sampled from 3 L0A files for each station.
96
96
  The default is False.
97
97
  data_archive_dir : str
98
98
  DISDRODB Data Archive directory
@@ -84,7 +84,7 @@ def disdrodb_run_l0b_station(
84
84
  If False, multi-threading is automatically exploited to speed up I/0 tasks.
85
85
  debugging_mode : bool
86
86
  If True, it reduces the amount of data to process.
87
- It processes just the first 100 rows of 3 L0A files.
87
+ It processes 100 rows sampled from 3 L0A files.
88
88
  The default is False.
89
89
  data_archive_dir : str
90
90
  DISDRODB Data Archive directory
@@ -93,7 +93,7 @@ def disdrodb_run_l0c(
93
93
  If False, multi-threading is automatically exploited to speed up I/0 tasks.
94
94
  debugging_mode : bool
95
95
  If True, it reduces the amount of data to process.
96
- It processes just the first 100 rows of 3 L0A files for each station.
96
+ It processes 100 rows sampled from 3 L0A files for each station.
97
97
  The default is False.
98
98
  remove_l0b: bool, optional
99
99
  Whether to remove the processed L0B files. The default value is ``False``.
@@ -85,7 +85,7 @@ def disdrodb_run_l0c_station(
85
85
  If False, multi-threading is automatically exploited to speed up I/0 tasks.
86
86
  debugging_mode : bool
87
87
  If True, it reduces the amount of data to process.
88
- It processes just the first 100 rows of 3 L0A files.
88
+ It processes 100 rows sampled from 3 L0A files.
89
89
  The default is False.
90
90
  remove_l0b: bool, optional
91
91
  Whether to remove the processed L0B files. The default value is ``False``.
@@ -98,7 +98,7 @@ def disdrodb_run_l2e_station(
98
98
  # -------------------------------------------------------------------------.
99
99
  # If parallel=True, set the dask environment
100
100
  if parallel:
101
- cluster, client = initialize_dask_cluster()
101
+ cluster, client = initialize_dask_cluster(minimum_memory="8GB")
102
102
 
103
103
  # -------------------------------------------------------------------------.
104
104
  run_l2e_station(
disdrodb/configs.py CHANGED
@@ -19,6 +19,7 @@
19
19
  """DISDRODB Configuration File functions."""
20
20
 
21
21
  import os
22
+ import shutil
22
23
  from typing import Optional
23
24
 
24
25
  from disdrodb.utils.yaml import read_yaml, write_yaml
@@ -32,9 +33,11 @@ def _define_config_filepath():
32
33
  return filepath
33
34
 
34
35
 
35
- def define_disdrodb_configs(
36
+ def define_configs(
36
37
  data_archive_dir: Optional[str] = None,
37
38
  metadata_archive_dir: Optional[str] = None,
39
+ scattering_table_dir: Optional[str] = None,
40
+ configs_path: Optional[str] = None,
38
41
  folder_partitioning: Optional[str] = None,
39
42
  zenodo_token: Optional[str] = None,
40
43
  zenodo_sandbox_token: Optional[str] = None,
@@ -48,6 +51,10 @@ def define_disdrodb_configs(
48
51
  The directory path where the DISDRODB Data Archive is located.
49
52
  metadata_archive_dir : str
50
53
  The directory path where the DISDRODB Metadata Archive is located.
54
+ scattering_table_dir : str
55
+ The directory path where to store DISDRODB T-Matrix scattering tables.
56
+ configs_path : str
57
+ The directory path where the custom DISDRODB products configurations files are defined.
51
58
  folder_partitioning : str
52
59
  The folder partitioning scheme used in the DISDRODB Data Archive.
53
60
  Allowed values are:
@@ -69,7 +76,13 @@ def define_disdrodb_configs(
69
76
  The configuration file is used to run the various DISDRODB operations.
70
77
 
71
78
  """
72
- from disdrodb.api.checks import check_data_archive_dir, check_folder_partitioning, check_metadata_archive_dir
79
+ import disdrodb
80
+ from disdrodb.api.checks import (
81
+ check_data_archive_dir,
82
+ check_folder_partitioning,
83
+ check_metadata_archive_dir,
84
+ check_scattering_table_dir,
85
+ )
73
86
 
74
87
  # Define path to .config_disdrodb.yaml file
75
88
  filepath = _define_config_filepath()
@@ -85,9 +98,16 @@ def define_disdrodb_configs(
85
98
  # Add DISDRODB Data Archive Directory
86
99
  if data_archive_dir is not None:
87
100
  config_dict["data_archive_dir"] = check_data_archive_dir(data_archive_dir)
101
+
88
102
  # Add DISDRODB Metadata Archive Directory
89
103
  if metadata_archive_dir is not None:
90
104
  config_dict["metadata_archive_dir"] = check_metadata_archive_dir(metadata_archive_dir)
105
+
106
+ # Add DISDRODB Scattering Table Directory
107
+ if scattering_table_dir is not None:
108
+ os.makedirs(scattering_table_dir, exist_ok=True)
109
+ config_dict["scattering_table_dir"] = check_scattering_table_dir(scattering_table_dir)
110
+
91
111
  # Add DISDRODB Folder Partitioning
92
112
  if folder_partitioning is not None:
93
113
  config_dict["folder_partitioning"] = check_folder_partitioning(folder_partitioning)
@@ -98,13 +118,21 @@ def define_disdrodb_configs(
98
118
  if zenodo_sandbox_token is not None:
99
119
  config_dict["zenodo_sandbox_token"] = zenodo_sandbox_token
100
120
 
121
+ if configs_path is not None:
122
+ config_dict["configs_path"] = configs_path
123
+
101
124
  # Write the DISDRODB config file
102
125
  write_yaml(config_dict, filepath, sort_keys=False)
103
126
 
104
127
  print(f"The DISDRODB config file has been {action_msg} successfully!")
105
128
 
129
+ # Now read the config file and set it as the active configuration
130
+ # - This avoid the need to restart a python session to take effect !
131
+ config_dict = read_configs()
132
+ disdrodb.config.update(config_dict)
133
+
106
134
 
107
- def read_disdrodb_configs() -> dict[str, str]:
135
+ def read_configs() -> dict[str, str]:
108
136
  """
109
137
  Reads the DISDRODB configuration file and returns a dictionary with the configuration settings.
110
138
 
@@ -158,6 +186,19 @@ def get_metadata_archive_dir(metadata_archive_dir=None):
158
186
  return metadata_archive_dir
159
187
 
160
188
 
189
+ def get_scattering_table_dir(scattering_table_dir=None):
190
+ """Return the directory where DISDRODB save pyTMatrix scattering tables."""
191
+ import disdrodb
192
+ from disdrodb.api.checks import check_scattering_table_dir
193
+
194
+ if scattering_table_dir is None:
195
+ scattering_table_dir = disdrodb.config.get("scattering_table_dir", None)
196
+ if scattering_table_dir is None:
197
+ raise ValueError("The directory where to save DISDRODB T-Matrix scattering tables is not specified.")
198
+ scattering_table_dir = check_scattering_table_dir(scattering_table_dir) # ensure Path converted to str
199
+ return scattering_table_dir
200
+
201
+
161
202
  def get_folder_partitioning():
162
203
  """Return the folder partitioning."""
163
204
  import disdrodb
@@ -182,7 +223,7 @@ def get_zenodo_token(sandbox: bool):
182
223
  host = "zenodo.org"
183
224
  token_name = "zenodo_token"
184
225
 
185
- # token = read_disdrodb_configs().get(token_name, None)
226
+ # token = read_configs().get(token_name, None)
186
227
  token = disdrodb.config.get(token_name, None)
187
228
 
188
229
  if token is None:
@@ -195,3 +236,107 @@ def get_zenodo_token(sandbox: bool):
195
236
  raise ValueError(f"Missing {token_name} in the DISDRODB config file !")
196
237
 
197
238
  return token
239
+
240
+
241
+ def get_product_default_configs_path():
242
+ """Return the paths where DISDRODB products configuration files are stored."""
243
+ import disdrodb
244
+
245
+ configs_path = os.path.join(disdrodb.__root_path__, "disdrodb", "etc", "products")
246
+ return configs_path
247
+
248
+
249
+ def check_availability_radar_simulations(options):
250
+ """Check radar simulations are possible for L2E and L2M products."""
251
+ import disdrodb
252
+
253
+ if "radar_enabled" in options and not disdrodb.is_pytmatrix_available():
254
+ options["radar_enabled"] = False
255
+ return options
256
+
257
+
258
+ def copy_product_default_configs(configs_path):
259
+ """Copy the default DISDRODB products configuration directory to a custom location.
260
+
261
+ This function duplicates the entire directory of default product settings
262
+ (located at ``disdrodb/etc/products``) into the user-specified
263
+ ``configs_path``. Once copied, you can safely edit these files without
264
+ modifying the library's built-in defaults. To have DISDRODB use your
265
+ custom settings, point the global configuration at this new directory
266
+ (e.g by specifying ``configs_path`` with the ``disdrodb.define_configs`` function).
267
+
268
+ Parameters
269
+ ----------
270
+ configs_path:
271
+ Destination directory where the default product configuration files
272
+ will be copied. This directory must not already exist, and later
273
+ needs to be referenced in your DISDRODB global configuration.
274
+
275
+ Returns
276
+ -------
277
+ configs_path
278
+ The path to the newly created custom product configuration directory.
279
+
280
+ """
281
+ source_dir_path = get_product_default_configs_path()
282
+ if os.path.exists(configs_path):
283
+ raise FileExistsError(f"The {configs_path} directory already exists!")
284
+ configs_path = shutil.copytree(source_dir_path, configs_path)
285
+ return configs_path
286
+
287
+
288
+ def get_product_options(product, temporal_resolution=None):
289
+ """Get options for DISDRODB products."""
290
+ import disdrodb
291
+ from disdrodb.api.checks import check_product
292
+
293
+ # Define configs path
294
+ if os.environ.get("PYTEST_CURRENT_TEST"):
295
+ configs_path = os.path.join(disdrodb.__root_path__, "disdrodb", "tests", "products")
296
+ else:
297
+ configs_path = disdrodb.config.get("configs_path", get_product_default_configs_path())
298
+
299
+ # Validate DISDRODB products configuration
300
+ validate_product_configuration(configs_path)
301
+
302
+ # Check product
303
+ check_product(product)
304
+
305
+ # Retrieve global product options
306
+ global_options = read_yaml(os.path.join(configs_path, product, "global.yaml"))
307
+ if temporal_resolution is None:
308
+ global_options = check_availability_radar_simulations(global_options)
309
+ return global_options
310
+
311
+ # If temporal resolutions are specified, drop 'temporal_resolutions' key
312
+ global_options.pop("temporal_resolutions", None)
313
+ custom_options_path = os.path.join(configs_path, product, f"{temporal_resolution}.yaml")
314
+ if not os.path.exists(custom_options_path):
315
+ return global_options
316
+ custom_options = read_yaml(custom_options_path)
317
+ options = global_options.copy()
318
+ options.update(custom_options)
319
+ options = check_availability_radar_simulations(options)
320
+ return options
321
+
322
+
323
+ def get_product_temporal_resolutions(product):
324
+ """Get DISDRODB L2 product temporal aggregations."""
325
+ # Check only L2E and L2M
326
+ return get_product_options(product)["temporal_resolutions"]
327
+
328
+
329
+ def get_model_options(product, model_name):
330
+ """Get DISDRODB L2M model options."""
331
+ import disdrodb
332
+
333
+ configs_path = disdrodb.config.get("configs_path", get_product_default_configs_path())
334
+ model_options_path = os.path.join(configs_path, product, f"{model_name}.yaml")
335
+ model_options = read_yaml(model_options_path)
336
+ return model_options
337
+
338
+
339
+ def validate_product_configuration(configs_path):
340
+ """Validate the DISDRODB products configuration files."""
341
+ # TODO: Implement validation of DISDRODB products configuration files with pydantic
342
+ pass
disdrodb/constants.py ADDED
@@ -0,0 +1,61 @@
1
+ # -----------------------------------------------------------------------------.
2
+ # Copyright (c) 2021-2023 DISDRODB developers
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU General Public License
15
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
16
+ # -----------------------------------------------------------------------------.
17
+ """DISDRODB constants."""
18
+ import importlib
19
+
20
+ ARCHIVE_VERSION = "V0"
21
+ SOFTWARE_VERSION = "V" + importlib.metadata.version("disdrodb")
22
+ CONVENTIONS = "CF-1.10, ACDD-1.3"
23
+
24
+ # Define coordinates names
25
+ DIAMETER_COORDS = ["diameter_bin_center", "diameter_bin_width", "diameter_bin_lower", "diameter_bin_upper"]
26
+ VELOCITY_COORDS = ["velocity_bin_center", "velocity_bin_width", "velocity_bin_lower", "velocity_bin_upper"]
27
+ GEOLOCATION_COORDS = ["longitude", "latitude", "altitude"]
28
+ VELOCITY_DIMENSION = "velocity_bin_center"
29
+ DIAMETER_DIMENSION = "diameter_bin_center"
30
+ COORDINATES = [
31
+ "diameter_bin_center",
32
+ "diameter_bin_width",
33
+ "diameter_bin_upper",
34
+ "velocity_bin_lower",
35
+ "velocity_bin_center",
36
+ "velocity_bin_width",
37
+ "velocity_bin_upper",
38
+ "latitude",
39
+ "longitude",
40
+ "altitude",
41
+ "time",
42
+ "sample_interval",
43
+ ]
44
+ OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM", "PWS100"]
45
+ IMPACT_SENSORS = ["RD80"]
46
+
47
+ PRODUCTS = ["RAW", "L0A", "L0B", "L0C", "L1", "L2E", "L2M"]
48
+
49
+ PRODUCTS_ARGUMENTS = {
50
+ "L2E": ["rolling", "sample_interval"],
51
+ "L2M": ["rolling", "sample_interval", "model_name"],
52
+ }
53
+
54
+ PRODUCTS_REQUIREMENTS = {
55
+ "L0A": "RAW",
56
+ "L0B": "L0A",
57
+ "L0C": "L0B",
58
+ "L1": "L0C",
59
+ "L2E": "L1",
60
+ "L2M": "L2E",
61
+ }
@@ -21,6 +21,8 @@
21
21
  import logging
22
22
  import os
23
23
  import shutil
24
+ import subprocess
25
+ import urllib.parse
24
26
  from typing import Optional, Union
25
27
 
26
28
  import click
@@ -213,7 +215,7 @@ def download_station(
213
215
  check_exists=True,
214
216
  )
215
217
  # Download data
216
- _download_station_data(metadata_filepath, data_archive_dir=data_archive_dir, force=force)
218
+ download_station_data(metadata_filepath, data_archive_dir=data_archive_dir, force=force)
217
219
 
218
220
 
219
221
  def _is_valid_disdrodb_data_url(disdrodb_data_url):
@@ -228,13 +230,25 @@ def _extract_station_files(zip_filepath, station_dir):
228
230
  os.remove(zip_filepath)
229
231
 
230
232
 
231
- def _download_station_data(metadata_filepath: str, data_archive_dir: str, force: bool = False) -> None:
233
+ def check_consistent_station_name(metadata_filepath, station_name):
234
+ """Check consistent station_name between YAML file name and metadata key."""
235
+ # Check consistent station name
236
+ expected_station_name = os.path.basename(metadata_filepath).replace(".yml", "")
237
+ if station_name and str(station_name) != str(expected_station_name):
238
+ raise ValueError(f"Inconsistent station_name values in the {metadata_filepath} file. Download aborted.")
239
+ return station_name
240
+
241
+
242
+ def download_station_data(metadata_filepath: str, data_archive_dir: str, force: bool = False) -> None:
232
243
  """Download and unzip the station data .
233
244
 
234
245
  Parameters
235
246
  ----------
236
247
  metadata_filepaths : str
237
248
  Metadata file path.
249
+ data_archive_dir : str (optional)
250
+ DISDRODB Data Archive directory. Format: ``<...>/DISDRODB``.
251
+ If ``None`` (the default), the disdrodb config variable ``data_archive_dir`` is used.
238
252
  force : bool, optional
239
253
  If ``True``, delete existing files and redownload it. The default value is ``False``.
240
254
 
@@ -247,7 +261,7 @@ def _download_station_data(metadata_filepath: str, data_archive_dir: str, force:
247
261
  campaign_name = metadata_dict["campaign_name"]
248
262
  station_name = metadata_dict["station_name"]
249
263
  station_name = check_consistent_station_name(metadata_filepath, station_name)
250
- # Define the destination local filepath path
264
+ # Define the path to the station RAW data directory
251
265
  station_dir = define_station_dir(
252
266
  data_archive_dir=data_archive_dir,
253
267
  data_source=data_source,
@@ -259,19 +273,136 @@ def _download_station_data(metadata_filepath: str, data_archive_dir: str, force:
259
273
  disdrodb_data_url = metadata_dict.get("disdrodb_data_url", None)
260
274
  if not _is_valid_disdrodb_data_url(disdrodb_data_url):
261
275
  raise ValueError(f"Invalid disdrodb_data_url '{disdrodb_data_url}' for station {station_name}")
262
- # Download file
263
- zip_filepath = _download_file_from_url(disdrodb_data_url, dst_dir=station_dir, force=force)
264
- # Extract the stations files from the downloaded station.zip file
265
- _extract_station_files(zip_filepath, station_dir=station_dir)
266
276
 
277
+ # Download files
278
+ # - Option 1: Download Zip file containing all station raw data
279
+ if disdrodb_data_url.startswith("https://zenodo.org/") or disdrodb_data_url.startswith("https://cloudnet.fmi.fi/"):
280
+ download_zip_file(url=disdrodb_data_url, dst_dir=station_dir, force=force)
281
+ # - Option 2: Recursive download from a web server via HTTP or HTTPS.
282
+ elif disdrodb_data_url.startswith("http"):
283
+ download_web_server_data(url=disdrodb_data_url, dst_dir=station_dir, force=force, verbose=True)
284
+ else:
285
+ raise NotImplementedError(f"Open a GitHub Issue to enable the download of data from {disdrodb_data_url}.")
267
286
 
268
- def check_consistent_station_name(metadata_filepath, station_name):
269
- """Check consistent station_name between YAML file name and metadata key."""
270
- # Check consistent station name
271
- expected_station_name = os.path.basename(metadata_filepath).replace(".yml", "")
272
- if station_name and str(station_name) != str(expected_station_name):
273
- raise ValueError(f"Inconsistent station_name values in the {metadata_filepath} file. Download aborted.")
274
- return station_name
287
+
288
+ ####-----------------------------------------------------------------------------------------.
289
+ #### Download from Web Server via HTTP or HTTPS
290
+
291
+
292
+ def download_web_server_data(url: str, dst_dir: str, force=True, verbose=True) -> None:
293
+ """Download data from a web server via HTTP or HTTPS.
294
+
295
+ Use the system's wget command to recursively download all files and subdirectories
296
+ under the given HTTPS “directory” URL. Works on both Windows and Linux, provided
297
+ that wget is installed and on the PATH.
298
+
299
+ 1. Ensure wget is available.
300
+ 2. Normalize URL to end with '/'.
301
+ 3. Compute cut-dirs so that only the last segment of the path remains locally.
302
+ 4. Build and run the wget command.
303
+
304
+ Example:
305
+ download_with_wget("https://ruisdael.citg.tudelft.nl/parsivel/PAR001_Cabauw/2021/202101/")
306
+ # → Creates a local folder "202101/" with all files and subfolders.
307
+ """
308
+ # 1. Ensure wget exists
309
+ ensure_wget_available()
310
+
311
+ # 2. Normalize URL
312
+ url = ensure_trailing_slash(url)
313
+
314
+ # 3. Compute cut-dirs so that only the last URL segment remains locally
315
+ cut_dirs = compute_cut_dirs(url)
316
+
317
+ # 4. Create destination directory if needed
318
+ os.makedirs(dst_dir, exist_ok=True)
319
+
320
+ # 5. Build wget command
321
+ cmd = build_webserver_wget_command(url, cut_dirs=cut_dirs, dst_dir=dst_dir, force=force, verbose=verbose)
322
+
323
+ # 6. Run wget command
324
+ try:
325
+ subprocess.run(cmd, check=True)
326
+ except subprocess.CalledProcessError as e:
327
+ raise subprocess.CalledProcessError(
328
+ returncode=e.returncode,
329
+ cmd=e.cmd,
330
+ output=e.output,
331
+ stderr=e.stderr,
332
+ )
333
+
334
+
335
+ def ensure_wget_available() -> None:
336
+ """Raise FileNotFoundError if 'wget' is not on the system PATH."""
337
+ if shutil.which("wget") is None:
338
+ raise FileNotFoundError("The WGET software was not found. Please install WGET or add it to PATH.")
339
+
340
+
341
+ def ensure_trailing_slash(url: str) -> str:
342
+ """Return `url` guaranteed to end with a slash."""
343
+ return url if url.endswith("/") else url.rstrip("/") + "/"
344
+
345
+
346
+ def compute_cut_dirs(url: str) -> int:
347
+ """Compute the wget cut_dirs value to download directly in `dst_dir`.
348
+
349
+ Given a URL ending with '/', compute the total number of path segments.
350
+ By returning len(segments), we strip away all of them—so that files
351
+ within that final directory land directly in `dst_dir` without creating
352
+ an extra subfolder.
353
+ """
354
+ parsed = urllib.parse.urlparse(url)
355
+ path = parsed.path.strip("/") # remove leading/trailing '/'
356
+ segments = path.split("/") if path else []
357
+ return len(segments)
358
+
359
+
360
+ def build_webserver_wget_command(url: str, cut_dirs: int, dst_dir: str, force: bool, verbose: bool) -> list[str]:
361
+ """Construct the wget command list for subprocess.run.
362
+
363
+ Notes
364
+ -----
365
+ The following wget arguments are used
366
+ - -q : quiet mode (no detailed progress)
367
+ - -r : recursive
368
+ - -np : no parent
369
+ - -nH : no host directories
370
+ - --timestamping: download missing files or when remote version is newer
371
+ - --cut-dirs : strip all but the last path segment from the remote path
372
+ - -P dst_dir : download into `dst_dir`
373
+ - url
374
+ """
375
+ cmd = ["wget"]
376
+ if not verbose:
377
+ cmd.append("-q")
378
+ cmd += [
379
+ "-r",
380
+ "-np",
381
+ "-nH",
382
+ f"--cut-dirs={cut_dirs}",
383
+ ]
384
+ if force:
385
+ cmd.append("--timestamping") # -N
386
+
387
+ # Define source and destination directory
388
+ cmd += [
389
+ "-P",
390
+ dst_dir,
391
+ url,
392
+ ]
393
+ return cmd
394
+
395
+
396
+ ####--------------------------------------------------------------------.
397
+ #### Download from Zenodo
398
+
399
+
400
+ def download_zip_file(url, dst_dir, force):
401
+ """Download zip file from zenodo and extract station raw data."""
402
+ # Download zip file
403
+ zip_filepath = _download_file_from_url(url, dst_dir=dst_dir, force=force)
404
+ # Extract the stations files from the downloaded station.zip file
405
+ _extract_station_files(zip_filepath, station_dir=dst_dir)
275
406
 
276
407
 
277
408
  def _download_file_from_url(url: str, dst_dir: str, force: bool = False) -> str: