disdrodb 0.1.4__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +1 -5
- disdrodb/_version.py +2 -2
- disdrodb/accessor/methods.py +14 -3
- disdrodb/api/checks.py +10 -0
- disdrodb/api/create_directories.py +0 -2
- disdrodb/api/io.py +14 -17
- disdrodb/api/path.py +42 -77
- disdrodb/api/search.py +89 -23
- disdrodb/cli/disdrodb_create_summary.py +11 -1
- disdrodb/cli/disdrodb_create_summary_station.py +10 -0
- disdrodb/cli/disdrodb_run_l0.py +1 -1
- disdrodb/cli/disdrodb_run_l0a.py +1 -1
- disdrodb/cli/disdrodb_run_l0b.py +1 -1
- disdrodb/cli/disdrodb_run_l0c.py +1 -1
- disdrodb/cli/disdrodb_run_l1.py +1 -1
- disdrodb/cli/disdrodb_run_l2e.py +1 -1
- disdrodb/cli/disdrodb_run_l2m.py +1 -1
- disdrodb/configs.py +30 -83
- disdrodb/constants.py +4 -3
- disdrodb/data_transfer/download_data.py +4 -2
- disdrodb/docs.py +2 -2
- disdrodb/etc/products/L1/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/LPM/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/PARSIVEL/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/PARSIVEL2/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/PWS100/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/RD80/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/SWS250/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/global.yaml +7 -1
- disdrodb/etc/products/L2E/10MIN.yaml +1 -12
- disdrodb/etc/products/L2E/5MIN.yaml +1 -0
- disdrodb/etc/products/L2E/global.yaml +1 -1
- disdrodb/etc/products/L2M/MODELS/GAMMA_GS_ND_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/{GAMMA_ML.yaml → MODELS/GAMMA_ML.yaml} +1 -1
- disdrodb/etc/products/L2M/MODELS/LOGNORMAL_GS_LOG_ND_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/MODELS/LOGNORMAL_GS_ND_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/MODELS/LOGNORMAL_ML.yaml +8 -0
- disdrodb/etc/products/L2M/MODELS/NGAMMA_GS_R_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/global.yaml +11 -3
- disdrodb/l0/check_configs.py +49 -16
- disdrodb/l0/configs/LPM/l0a_encodings.yml +2 -2
- disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +2 -2
- disdrodb/l0/configs/LPM/l0b_encodings.yml +2 -2
- disdrodb/l0/configs/LPM/raw_data_format.yml +2 -2
- disdrodb/l0/configs/PARSIVEL/l0b_encodings.yml +1 -1
- disdrodb/l0/configs/PWS100/l0b_encodings.yml +1 -0
- disdrodb/l0/configs/SWS250/bins_diameter.yml +108 -0
- disdrodb/l0/configs/SWS250/bins_velocity.yml +83 -0
- disdrodb/l0/configs/SWS250/l0a_encodings.yml +18 -0
- disdrodb/l0/configs/SWS250/l0b_cf_attrs.yml +72 -0
- disdrodb/l0/configs/SWS250/l0b_encodings.yml +155 -0
- disdrodb/l0/configs/SWS250/raw_data_format.yml +148 -0
- disdrodb/l0/l0_reader.py +2 -2
- disdrodb/l0/l0b_processing.py +70 -15
- disdrodb/l0/l0c_processing.py +7 -3
- disdrodb/l0/readers/LPM/ARM/ARM_LPM.py +1 -1
- disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +2 -2
- disdrodb/l0/readers/LPM/BELGIUM/ULIEGE.py +256 -0
- disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +2 -2
- disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +2 -2
- disdrodb/l0/readers/LPM/GERMANY/DWD.py +491 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +2 -2
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +2 -2
- disdrodb/l0/readers/LPM/KIT/CHWALA.py +2 -2
- disdrodb/l0/readers/LPM/SLOVENIA/ARSO.py +107 -12
- disdrodb/l0/readers/LPM/SLOVENIA/UL.py +3 -3
- disdrodb/l0/readers/LPM/SWITZERLAND/INNERERIZ_LPM.py +2 -2
- disdrodb/l0/readers/PARSIVEL/BASQUECOUNTRY/EUSKALMET_OTT.py +227 -0
- disdrodb/l0/readers/PARSIVEL/{GPM → NASA}/LPVEX.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +5 -14
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +8 -17
- disdrodb/l0/readers/PARSIVEL/SLOVENIA/UL.py +117 -8
- disdrodb/l0/readers/PARSIVEL2/BASQUECOUNTRY/EUSKALMET_OTT2.py +232 -0
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +10 -14
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +10 -14
- disdrodb/l0/readers/PARSIVEL2/DENMARK/DTU.py +8 -14
- disdrodb/l0/readers/PARSIVEL2/DENMARK/EROSION_raw.py +382 -0
- disdrodb/l0/readers/PARSIVEL2/FINLAND/FMI_PARSIVEL2.py +4 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/OSUG.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/GREECE/NOA.py +127 -0
- disdrodb/l0/readers/PARSIVEL2/ITALY/HYDROX.py +239 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/FARM_PARSIVEL2.py +5 -11
- disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_MIPS.py +4 -17
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +5 -14
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +10 -13
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +10 -13
- disdrodb/l0/readers/PARSIVEL2/PHILIPPINES/PAGASA.py +232 -0
- disdrodb/l0/readers/PARSIVEL2/SPAIN/CENER.py +6 -18
- disdrodb/l0/readers/PARSIVEL2/{NASA/LPVEX.py → SPAIN/GRANADA.py} +46 -35
- disdrodb/l0/readers/PARSIVEL2/SWEDEN/SMHI.py +189 -0
- disdrodb/l0/readers/PARSIVEL2/USA/{C3WE.py → CW3E.py} +10 -28
- disdrodb/l0/readers/PWS100/AUSTRIA/HOAL.py +321 -0
- disdrodb/l0/readers/SW250/BELGIUM/KMI.py +239 -0
- disdrodb/l1/beard_model.py +31 -129
- disdrodb/l1/fall_velocity.py +136 -83
- disdrodb/l1/filters.py +25 -28
- disdrodb/l1/processing.py +16 -17
- disdrodb/l1/resampling.py +101 -38
- disdrodb/l1_env/routines.py +46 -17
- disdrodb/l2/empirical_dsd.py +6 -0
- disdrodb/l2/processing.py +6 -5
- disdrodb/metadata/geolocation.py +0 -2
- disdrodb/metadata/search.py +3 -4
- disdrodb/psd/fitting.py +16 -13
- disdrodb/routines/l0.py +2 -2
- disdrodb/routines/l1.py +173 -60
- disdrodb/routines/l2.py +148 -284
- disdrodb/routines/options.py +345 -0
- disdrodb/routines/wrappers.py +14 -1
- disdrodb/scattering/axis_ratio.py +90 -84
- disdrodb/scattering/permittivity.py +6 -0
- disdrodb/summary/routines.py +735 -670
- disdrodb/utils/archiving.py +51 -44
- disdrodb/utils/attrs.py +3 -1
- disdrodb/utils/dask.py +4 -4
- disdrodb/utils/dict.py +33 -0
- disdrodb/utils/encoding.py +6 -1
- disdrodb/utils/routines.py +9 -8
- disdrodb/utils/time.py +11 -3
- disdrodb/viz/__init__.py +0 -13
- disdrodb/viz/plots.py +231 -1
- {disdrodb-0.1.4.dist-info → disdrodb-0.2.0.dist-info}/METADATA +2 -1
- {disdrodb-0.1.4.dist-info → disdrodb-0.2.0.dist-info}/RECORD +135 -103
- /disdrodb/etc/products/L2M/{NGAMMA_GS_LOG_ND_MAE.yaml → MODELS/NGAMMA_GS_LOG_ND_MAE.yaml} +0 -0
- /disdrodb/etc/products/L2M/{NGAMMA_GS_ND_MAE.yaml → MODELS/NGAMMA_GS_ND_MAE.yaml} +0 -0
- /disdrodb/etc/products/L2M/{NGAMMA_GS_Z_MAE.yaml → MODELS/NGAMMA_GS_Z_MAE.yaml} +0 -0
- /disdrodb/l0/readers/PARSIVEL/{GPM → NASA}/IFLOODS.py +0 -0
- /disdrodb/l0/readers/PARSIVEL/{GPM → NASA}/MC3E.py +0 -0
- /disdrodb/l0/readers/PARSIVEL/{GPM → NASA}/PIERS.py +0 -0
- /disdrodb/l0/readers/PARSIVEL2/{GPM → NASA}/GCPEX.py +0 -0
- /disdrodb/l0/readers/PARSIVEL2/{GPM → NASA}/NSSTC.py +0 -0
- {disdrodb-0.1.4.dist-info → disdrodb-0.2.0.dist-info}/WHEEL +0 -0
- {disdrodb-0.1.4.dist-info → disdrodb-0.2.0.dist-info}/entry_points.txt +0 -0
- {disdrodb-0.1.4.dist-info → disdrodb-0.2.0.dist-info}/licenses/LICENSE +0 -0
- {disdrodb-0.1.4.dist-info → disdrodb-0.2.0.dist-info}/top_level.txt +0 -0
disdrodb/routines/l2.py
CHANGED
|
@@ -18,7 +18,6 @@
|
|
|
18
18
|
|
|
19
19
|
import copy
|
|
20
20
|
import datetime
|
|
21
|
-
import json
|
|
22
21
|
import logging
|
|
23
22
|
import os
|
|
24
23
|
import time
|
|
@@ -31,195 +30,51 @@ from disdrodb.api.create_directories import (
|
|
|
31
30
|
create_logs_directory,
|
|
32
31
|
create_product_directory,
|
|
33
32
|
)
|
|
34
|
-
from disdrodb.api.info import group_filepaths
|
|
35
33
|
from disdrodb.api.io import open_netcdf_files
|
|
36
34
|
from disdrodb.api.path import (
|
|
37
35
|
define_file_folder_path,
|
|
38
36
|
define_l2e_filename,
|
|
39
37
|
define_l2m_filename,
|
|
40
|
-
define_temporal_resolution,
|
|
41
38
|
)
|
|
42
39
|
from disdrodb.api.search import get_required_product
|
|
43
40
|
from disdrodb.configs import (
|
|
44
41
|
get_data_archive_dir,
|
|
45
42
|
get_metadata_archive_dir,
|
|
46
|
-
get_model_options,
|
|
47
|
-
get_product_options,
|
|
48
|
-
get_product_temporal_resolutions,
|
|
49
43
|
)
|
|
50
|
-
from disdrodb.l1.resampling import resample_dataset
|
|
51
44
|
from disdrodb.l2.processing import (
|
|
52
45
|
generate_l2_radar,
|
|
53
46
|
generate_l2e,
|
|
54
47
|
generate_l2m,
|
|
55
48
|
)
|
|
56
49
|
from disdrodb.metadata import read_station_metadata
|
|
50
|
+
from disdrodb.routines.options import (
|
|
51
|
+
L2ProcessingOptions,
|
|
52
|
+
get_model_options,
|
|
53
|
+
get_product_temporal_resolutions,
|
|
54
|
+
is_possible_product,
|
|
55
|
+
)
|
|
57
56
|
from disdrodb.scattering.routines import precompute_scattering_tables
|
|
58
|
-
from disdrodb.utils.archiving import define_temporal_partitions, get_files_partitions
|
|
59
57
|
from disdrodb.utils.dask import execute_tasks_safely
|
|
60
58
|
from disdrodb.utils.decorators import delayed_if_parallel, single_threaded_if_parallel
|
|
61
|
-
from disdrodb.utils.list import flatten_list
|
|
62
|
-
|
|
63
|
-
# Logger
|
|
64
59
|
from disdrodb.utils.logger import (
|
|
65
60
|
create_product_logs,
|
|
66
61
|
log_info,
|
|
67
62
|
)
|
|
68
63
|
from disdrodb.utils.routines import (
|
|
69
|
-
is_possible_product,
|
|
70
64
|
run_product_generation,
|
|
71
65
|
try_get_required_filepaths,
|
|
72
66
|
)
|
|
73
|
-
from disdrodb.utils.time import (
|
|
74
|
-
ensure_sample_interval_in_seconds,
|
|
75
|
-
get_resampling_information,
|
|
76
|
-
)
|
|
77
67
|
from disdrodb.utils.writer import write_product
|
|
78
68
|
|
|
79
69
|
logger = logging.getLogger(__name__)
|
|
80
70
|
|
|
81
71
|
|
|
82
|
-
####----------------------------------------------------------------------------.
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
class ProcessingOptions:
|
|
86
|
-
"""Define L2 products processing options."""
|
|
87
|
-
|
|
88
|
-
# TODO: TO MOVE ELSEWHERE (AFTER L1 REFACTORING !)
|
|
89
|
-
|
|
90
|
-
def __init__(self, product, filepaths, parallel, temporal_resolutions=None):
|
|
91
|
-
"""Define L2 products processing options."""
|
|
92
|
-
import disdrodb
|
|
93
|
-
|
|
94
|
-
# ---------------------------------------------------------------------.
|
|
95
|
-
# Define temporal resolutions for which to retrieve processing options
|
|
96
|
-
if temporal_resolutions is None:
|
|
97
|
-
temporal_resolutions = get_product_temporal_resolutions(product)
|
|
98
|
-
elif isinstance(temporal_resolutions, str):
|
|
99
|
-
temporal_resolutions = [temporal_resolutions]
|
|
100
|
-
|
|
101
|
-
# ---------------------------------------------------------------------.
|
|
102
|
-
# Get product options at various temporal resolutions
|
|
103
|
-
dict_product_options = {
|
|
104
|
-
temporal_resolution: get_product_options(product, temporal_resolution=temporal_resolution)
|
|
105
|
-
for temporal_resolution in temporal_resolutions
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
# ---------------------------------------------------------------------.
|
|
109
|
-
# Group filepaths by source sample intervals
|
|
110
|
-
# - Typically the sample interval is fixed and is just one
|
|
111
|
-
# - Some stations might change the sample interval along the years
|
|
112
|
-
# - For each sample interval, separated processing take place here after !
|
|
113
|
-
dict_filepaths = group_filepaths(filepaths, groups="sample_interval")
|
|
114
|
-
|
|
115
|
-
# ---------------------------------------------------------------------.
|
|
116
|
-
# Retrieve processing information for each temporal resolution
|
|
117
|
-
dict_folder_partitioning = {}
|
|
118
|
-
dict_files_partitions = {}
|
|
119
|
-
_cache_dict_list_partitions: dict[str, dict] = {}
|
|
120
|
-
for temporal_resolution in temporal_resolutions:
|
|
121
|
-
|
|
122
|
-
# -------------------------------------------------------------------------.
|
|
123
|
-
# Retrieve product options
|
|
124
|
-
product_options = dict_product_options[temporal_resolution].copy()
|
|
125
|
-
|
|
126
|
-
# Retrieve accumulation_interval and rolling option
|
|
127
|
-
accumulation_interval, rolling = get_resampling_information(temporal_resolution)
|
|
128
|
-
|
|
129
|
-
# Extract processing options
|
|
130
|
-
archive_options = product_options.pop("archive_options")
|
|
131
|
-
|
|
132
|
-
dict_product_options[temporal_resolution] = product_options
|
|
133
|
-
# -------------------------------------------------------------------------.
|
|
134
|
-
# Define folder partitioning
|
|
135
|
-
if "folder_partitioning" not in archive_options:
|
|
136
|
-
dict_folder_partitioning[temporal_resolution] = disdrodb.config.get("folder_partitioning")
|
|
137
|
-
else:
|
|
138
|
-
dict_folder_partitioning[temporal_resolution] = archive_options.pop("folder_partitioning")
|
|
139
|
-
|
|
140
|
-
# -------------------------------------------------------------------------.
|
|
141
|
-
# Define list of temporal partitions
|
|
142
|
-
# - [{start_time: np.datetime64, end_time: np.datetime64}, ....]
|
|
143
|
-
# - Either strategy: "event" or "time_block" or save_by_time_block"
|
|
144
|
-
# - "event" requires loading data into memory to identify events
|
|
145
|
-
# --> Does some data filtering on what to process !
|
|
146
|
-
# - "time_block" does not require loading data into memory
|
|
147
|
-
# --> Does not do data filtering on what to process !
|
|
148
|
-
# --> Here we cache dict_list_partitions so that we don't need to recompute
|
|
149
|
-
# stuffs if processing options are the same
|
|
150
|
-
key = json.dumps(archive_options, sort_keys=True)
|
|
151
|
-
if key not in _cache_dict_list_partitions:
|
|
152
|
-
_cache_dict_list_partitions[key] = {
|
|
153
|
-
sample_interval: define_temporal_partitions(filepaths, parallel=parallel, **archive_options)
|
|
154
|
-
for sample_interval, filepaths in dict_filepaths.items()
|
|
155
|
-
}
|
|
156
|
-
dict_list_partitions = _cache_dict_list_partitions[key].copy() # To avoid in-place replacement
|
|
157
|
-
|
|
158
|
-
# ------------------------------------------------------------------.
|
|
159
|
-
# Group filepaths by temporal partitions
|
|
160
|
-
# - This is done separately for each possible source sample interval
|
|
161
|
-
# - It groups filepaths by start_time and end_time provided by list_partitions
|
|
162
|
-
# - Here 'events' can also simply be period of times ('day', 'months', ...)
|
|
163
|
-
# - When aggregating/resampling/accumulating data, we need to load also
|
|
164
|
-
# some data after the actual event end_time to ensure that the resampled dataset
|
|
165
|
-
# contains the event_end_time
|
|
166
|
-
# --> get_files_partitions adjust the event end_time to accounts for the required "border" data.
|
|
167
|
-
# - ATTENTION: get_files_partitions returns start_time and end_time as datetime objects !
|
|
168
|
-
files_partitions = [
|
|
169
|
-
get_files_partitions(
|
|
170
|
-
list_partitions=list_partitions,
|
|
171
|
-
filepaths=dict_filepaths[sample_interval],
|
|
172
|
-
sample_interval=sample_interval,
|
|
173
|
-
accumulation_interval=accumulation_interval,
|
|
174
|
-
rolling=rolling,
|
|
175
|
-
)
|
|
176
|
-
for sample_interval, list_partitions in dict_list_partitions.items()
|
|
177
|
-
if product != "L2E"
|
|
178
|
-
or is_possible_product(
|
|
179
|
-
accumulation_interval=accumulation_interval,
|
|
180
|
-
sample_interval=sample_interval,
|
|
181
|
-
rolling=rolling,
|
|
182
|
-
)
|
|
183
|
-
]
|
|
184
|
-
files_partitions = flatten_list(files_partitions)
|
|
185
|
-
dict_files_partitions[temporal_resolution] = files_partitions
|
|
186
|
-
|
|
187
|
-
# ------------------------------------------------------------------.
|
|
188
|
-
# Keep only temporal_resolutions for which events could be defined
|
|
189
|
-
# - Remove e.g when not compatible accumulation_interval with source sample_interval
|
|
190
|
-
temporal_resolutions = [
|
|
191
|
-
temporal_resolution
|
|
192
|
-
for temporal_resolution in temporal_resolutions
|
|
193
|
-
if len(dict_files_partitions[temporal_resolution]) > 0
|
|
194
|
-
]
|
|
195
|
-
# ------------------------------------------------------------------.
|
|
196
|
-
# Add attributes
|
|
197
|
-
self.temporal_resolutions = temporal_resolutions
|
|
198
|
-
self.dict_files_partitions = dict_files_partitions
|
|
199
|
-
self.dict_product_options = dict_product_options
|
|
200
|
-
self.dict_folder_partitioning = dict_folder_partitioning
|
|
201
|
-
|
|
202
|
-
def get_files_partitions(self, temporal_resolution):
|
|
203
|
-
"""Return files partitions dictionary for a specific L2E product."""
|
|
204
|
-
return self.dict_files_partitions[temporal_resolution]
|
|
205
|
-
|
|
206
|
-
def get_product_options(self, temporal_resolution):
|
|
207
|
-
"""Return product options dictionary for a specific L2E product."""
|
|
208
|
-
return self.dict_product_options[temporal_resolution]
|
|
209
|
-
|
|
210
|
-
def get_folder_partitioning(self, temporal_resolution):
|
|
211
|
-
"""Return the folder partitioning for a specific L2E product."""
|
|
212
|
-
# to be used for logs and files !
|
|
213
|
-
return self.dict_folder_partitioning[temporal_resolution]
|
|
214
|
-
|
|
215
|
-
|
|
216
72
|
####----------------------------------------------------------------------------.
|
|
217
73
|
#### L2E
|
|
218
74
|
|
|
219
75
|
|
|
220
|
-
def define_l2e_logs_filename(campaign_name, station_name, start_time, end_time,
|
|
76
|
+
def define_l2e_logs_filename(campaign_name, station_name, start_time, end_time, temporal_resolution):
|
|
221
77
|
"""Define L2E logs filename."""
|
|
222
|
-
temporal_resolution = define_temporal_resolution(seconds=accumulation_interval, rolling=rolling)
|
|
223
78
|
starting_time = pd.to_datetime(start_time).strftime("%Y%m%d%H%M%S")
|
|
224
79
|
ending_time = pd.to_datetime(end_time).strftime("%Y%m%d%H%M%S")
|
|
225
80
|
logs_filename = f"L2E.{temporal_resolution}.{campaign_name}.{station_name}.s{starting_time}.e{ending_time}"
|
|
@@ -239,8 +94,7 @@ def _generate_l2e(
|
|
|
239
94
|
campaign_name,
|
|
240
95
|
station_name,
|
|
241
96
|
# L2E options
|
|
242
|
-
|
|
243
|
-
rolling,
|
|
97
|
+
temporal_resolution,
|
|
244
98
|
product_options,
|
|
245
99
|
# Processing options
|
|
246
100
|
force,
|
|
@@ -254,42 +108,28 @@ def _generate_l2e(
|
|
|
254
108
|
# Define product processing function
|
|
255
109
|
def core(
|
|
256
110
|
filepaths,
|
|
111
|
+
start_time,
|
|
112
|
+
end_time,
|
|
257
113
|
campaign_name,
|
|
258
114
|
station_name,
|
|
259
|
-
product_options,
|
|
260
115
|
# Processing options
|
|
261
116
|
logger,
|
|
262
117
|
parallel,
|
|
263
118
|
verbose,
|
|
264
119
|
force,
|
|
265
|
-
#
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
accumulation_interval,
|
|
269
|
-
rolling,
|
|
120
|
+
# Product options
|
|
121
|
+
temporal_resolution,
|
|
122
|
+
product_options,
|
|
270
123
|
# Archiving arguments
|
|
271
124
|
data_dir,
|
|
272
125
|
folder_partitioning,
|
|
273
126
|
):
|
|
274
|
-
"""Define
|
|
127
|
+
"""Define L2E product processing."""
|
|
275
128
|
# Copy to avoid in-place replacement (outside this function)
|
|
276
129
|
product_options = product_options.copy()
|
|
277
130
|
|
|
278
131
|
# Open the dataset over the period of interest
|
|
279
|
-
ds = open_netcdf_files(filepaths, start_time=start_time, end_time=end_time, parallel=False)
|
|
280
|
-
ds = ds.load()
|
|
281
|
-
ds.close()
|
|
282
|
-
|
|
283
|
-
# Resample dataset # TODO: in future to perform in L1
|
|
284
|
-
# - Define sample interval in seconds
|
|
285
|
-
sample_interval = ensure_sample_interval_in_seconds(ds["sample_interval"]).to_numpy().item()
|
|
286
|
-
# - Resample dataset
|
|
287
|
-
ds = resample_dataset(
|
|
288
|
-
ds=ds,
|
|
289
|
-
sample_interval=sample_interval,
|
|
290
|
-
accumulation_interval=accumulation_interval,
|
|
291
|
-
rolling=rolling,
|
|
292
|
-
)
|
|
132
|
+
ds = open_netcdf_files(filepaths, start_time=start_time, end_time=end_time, parallel=False, compute=True)
|
|
293
133
|
|
|
294
134
|
# Extract L2E processing options
|
|
295
135
|
l2e_options = product_options.get("product_options")
|
|
@@ -320,8 +160,7 @@ def _generate_l2e(
|
|
|
320
160
|
ds,
|
|
321
161
|
campaign_name=campaign_name,
|
|
322
162
|
station_name=station_name,
|
|
323
|
-
|
|
324
|
-
rolling=rolling,
|
|
163
|
+
temporal_resolution=temporal_resolution,
|
|
325
164
|
)
|
|
326
165
|
folder_path = define_file_folder_path(ds, dir_path=data_dir, folder_partitioning=folder_partitioning)
|
|
327
166
|
filepath = os.path.join(folder_path, filename)
|
|
@@ -333,14 +172,14 @@ def _generate_l2e(
|
|
|
333
172
|
# Define product processing function kwargs
|
|
334
173
|
core_func_kwargs = dict( # noqa: C408
|
|
335
174
|
filepaths=filepaths,
|
|
175
|
+
start_time=start_time,
|
|
176
|
+
end_time=end_time,
|
|
177
|
+
# Station info
|
|
336
178
|
campaign_name=campaign_name,
|
|
337
179
|
station_name=station_name,
|
|
180
|
+
# Product options
|
|
181
|
+
temporal_resolution=temporal_resolution,
|
|
338
182
|
product_options=product_options,
|
|
339
|
-
# Resampling arguments
|
|
340
|
-
start_time=start_time,
|
|
341
|
-
end_time=end_time,
|
|
342
|
-
accumulation_interval=accumulation_interval,
|
|
343
|
-
rolling=rolling,
|
|
344
183
|
# Archiving arguments
|
|
345
184
|
data_dir=data_dir,
|
|
346
185
|
folder_partitioning=folder_partitioning,
|
|
@@ -449,57 +288,85 @@ def run_l2e_station(
|
|
|
449
288
|
msg = f"{product} processing of station {station_name} has started."
|
|
450
289
|
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
451
290
|
|
|
452
|
-
#
|
|
453
|
-
#
|
|
454
|
-
# - If
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
data_archive_dir=data_archive_dir,
|
|
291
|
+
# ---------------------------------------------------------------------.
|
|
292
|
+
# Retrieve source sampling interval
|
|
293
|
+
# - If a station has varying measurement interval over time, choose the smallest one !
|
|
294
|
+
metadata = read_station_metadata(
|
|
295
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
458
296
|
data_source=data_source,
|
|
459
297
|
campaign_name=campaign_name,
|
|
460
298
|
station_name=station_name,
|
|
461
|
-
product=required_product,
|
|
462
|
-
# Processing options
|
|
463
|
-
debugging_mode=debugging_mode,
|
|
464
299
|
)
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
# -------------------------------------------------------------------------.
|
|
469
|
-
# Retrieve L2E processing options
|
|
470
|
-
l2e_processing_options = ProcessingOptions(product="L2E", filepaths=filepaths, parallel=parallel)
|
|
300
|
+
sample_interval = metadata["measurement_interval"]
|
|
301
|
+
if isinstance(sample_interval, list):
|
|
302
|
+
sample_interval = min(sample_interval)
|
|
471
303
|
|
|
472
|
-
#
|
|
304
|
+
# ---------------------------------------------------------------------.
|
|
473
305
|
# Generate products for each temporal resolution
|
|
474
|
-
#
|
|
475
|
-
# accumulation_interval = 60
|
|
306
|
+
# temporal_resolution = "1MIN"
|
|
476
307
|
# temporal_resolution = "10MIN"
|
|
477
|
-
|
|
478
|
-
|
|
308
|
+
temporal_resolutions = get_product_temporal_resolutions(product)
|
|
309
|
+
for temporal_resolution in temporal_resolutions:
|
|
479
310
|
|
|
480
|
-
|
|
481
|
-
#
|
|
482
|
-
|
|
483
|
-
|
|
311
|
+
# ------------------------------------------------------------------.
|
|
312
|
+
# Check if the product can be generated
|
|
313
|
+
if not is_possible_product(
|
|
314
|
+
temporal_resolution=temporal_resolution,
|
|
315
|
+
sample_interval=sample_interval,
|
|
316
|
+
):
|
|
317
|
+
continue
|
|
484
318
|
|
|
485
|
-
#
|
|
486
|
-
|
|
319
|
+
# ---------------------------------------------------------------------.
|
|
320
|
+
# List files to process
|
|
321
|
+
# - If no data available, print error message and try with other L2E accumulation intervals
|
|
322
|
+
required_product = get_required_product(product)
|
|
323
|
+
filepaths = try_get_required_filepaths(
|
|
324
|
+
data_archive_dir=data_archive_dir,
|
|
325
|
+
data_source=data_source,
|
|
326
|
+
campaign_name=campaign_name,
|
|
327
|
+
station_name=station_name,
|
|
328
|
+
product=required_product,
|
|
329
|
+
# Processing options
|
|
330
|
+
debugging_mode=debugging_mode,
|
|
331
|
+
# Product options
|
|
332
|
+
temporal_resolution=temporal_resolution,
|
|
333
|
+
)
|
|
334
|
+
if filepaths is None:
|
|
335
|
+
continue
|
|
336
|
+
|
|
337
|
+
# ---------------------------------------------------------------------.
|
|
338
|
+
# Retrieve L2E processing options
|
|
339
|
+
l2e_processing_options = L2ProcessingOptions(
|
|
340
|
+
product=product,
|
|
341
|
+
temporal_resolution=temporal_resolution,
|
|
342
|
+
filepaths=filepaths,
|
|
343
|
+
parallel=parallel,
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
# ---------------------------------------------------------------------.
|
|
347
|
+
# Retrieve files temporal partitions
|
|
348
|
+
files_partitions = l2e_processing_options.files_partitions
|
|
349
|
+
|
|
350
|
+
if len(files_partitions) == 0:
|
|
351
|
+
msg = (
|
|
352
|
+
f"{product} processing of {data_source} {campaign_name} {station_name} "
|
|
353
|
+
+ f"has not been launched because of missing {required_product} {temporal_resolution} data."
|
|
354
|
+
)
|
|
355
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
356
|
+
continue
|
|
487
357
|
|
|
488
358
|
# Retrieve folder partitioning (for files and logs)
|
|
489
|
-
folder_partitioning = l2e_processing_options.
|
|
359
|
+
folder_partitioning = l2e_processing_options.folder_partitioning
|
|
490
360
|
|
|
491
361
|
# Retrieve product options
|
|
492
|
-
product_options = l2e_processing_options.
|
|
493
|
-
|
|
494
|
-
# Retrieve accumulation_interval and rolling option
|
|
495
|
-
accumulation_interval, rolling = get_resampling_information(temporal_resolution)
|
|
362
|
+
product_options = l2e_processing_options.product_options
|
|
496
363
|
|
|
497
364
|
# Precompute required scattering tables
|
|
498
365
|
if product_options["radar_enabled"]:
|
|
499
366
|
radar_options = product_options["radar_options"]
|
|
500
367
|
precompute_scattering_tables(verbose=verbose, **radar_options)
|
|
501
368
|
|
|
502
|
-
#
|
|
369
|
+
# ---------------------------------------------------------------------.
|
|
503
370
|
# Create product directory
|
|
504
371
|
data_dir = create_product_directory(
|
|
505
372
|
data_archive_dir=data_archive_dir,
|
|
@@ -510,8 +377,7 @@ def run_l2e_station(
|
|
|
510
377
|
product=product,
|
|
511
378
|
force=force,
|
|
512
379
|
# Option for L2E
|
|
513
|
-
|
|
514
|
-
rolling=rolling,
|
|
380
|
+
temporal_resolution=temporal_resolution,
|
|
515
381
|
)
|
|
516
382
|
|
|
517
383
|
# Define logs directory
|
|
@@ -522,11 +388,10 @@ def run_l2e_station(
|
|
|
522
388
|
campaign_name=campaign_name,
|
|
523
389
|
station_name=station_name,
|
|
524
390
|
# Option for L2E
|
|
525
|
-
|
|
526
|
-
rolling=rolling,
|
|
391
|
+
temporal_resolution=temporal_resolution,
|
|
527
392
|
)
|
|
528
393
|
|
|
529
|
-
#
|
|
394
|
+
# ---------------------------------------------------------------------.
|
|
530
395
|
# Generate files
|
|
531
396
|
# - L2E product generation is optionally parallelized over events
|
|
532
397
|
# - If parallel=True, it does that in parallel using dask.delayed
|
|
@@ -542,15 +407,13 @@ def run_l2e_station(
|
|
|
542
407
|
station_name=station_name,
|
|
543
408
|
start_time=event_info["start_time"],
|
|
544
409
|
end_time=event_info["end_time"],
|
|
545
|
-
|
|
546
|
-
accumulation_interval=accumulation_interval,
|
|
410
|
+
temporal_resolution=temporal_resolution,
|
|
547
411
|
),
|
|
548
412
|
folder_partitioning=folder_partitioning,
|
|
549
413
|
campaign_name=campaign_name,
|
|
550
414
|
station_name=station_name,
|
|
551
415
|
# L2E options
|
|
552
|
-
|
|
553
|
-
accumulation_interval=accumulation_interval,
|
|
416
|
+
temporal_resolution=temporal_resolution,
|
|
554
417
|
product_options=product_options,
|
|
555
418
|
# Processing options
|
|
556
419
|
force=force,
|
|
@@ -570,8 +433,7 @@ def run_l2e_station(
|
|
|
570
433
|
station_name=station_name,
|
|
571
434
|
data_archive_dir=data_archive_dir,
|
|
572
435
|
# Product options
|
|
573
|
-
|
|
574
|
-
rolling=rolling,
|
|
436
|
+
temporal_resolution=temporal_resolution,
|
|
575
437
|
# Logs list
|
|
576
438
|
list_logs=list_logs,
|
|
577
439
|
)
|
|
@@ -586,9 +448,8 @@ def run_l2e_station(
|
|
|
586
448
|
|
|
587
449
|
####----------------------------------------------------------------------------.
|
|
588
450
|
#### L2M
|
|
589
|
-
def define_l2m_logs_filename(campaign_name, station_name, start_time, end_time, model_name,
|
|
451
|
+
def define_l2m_logs_filename(campaign_name, station_name, start_time, end_time, model_name, temporal_resolution):
|
|
590
452
|
"""Define L2M logs filename."""
|
|
591
|
-
temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
|
|
592
453
|
starting_time = pd.to_datetime(start_time).strftime("%Y%m%d%H%M%S")
|
|
593
454
|
ending_time = pd.to_datetime(end_time).strftime("%Y%m%d%H%M%S")
|
|
594
455
|
logs_filename = (
|
|
@@ -610,8 +471,7 @@ def _generate_l2m(
|
|
|
610
471
|
campaign_name,
|
|
611
472
|
station_name,
|
|
612
473
|
# L2M options
|
|
613
|
-
|
|
614
|
-
rolling,
|
|
474
|
+
temporal_resolution,
|
|
615
475
|
model_name,
|
|
616
476
|
product_options,
|
|
617
477
|
# Processing options
|
|
@@ -636,14 +496,13 @@ def _generate_l2m(
|
|
|
636
496
|
force,
|
|
637
497
|
# Product options
|
|
638
498
|
product_options,
|
|
639
|
-
|
|
640
|
-
rolling,
|
|
499
|
+
temporal_resolution,
|
|
641
500
|
model_name,
|
|
642
501
|
# Archiving arguments
|
|
643
502
|
data_dir,
|
|
644
503
|
folder_partitioning,
|
|
645
504
|
):
|
|
646
|
-
"""Define
|
|
505
|
+
"""Define L2M product processing."""
|
|
647
506
|
# Copy to avoid in-place replacement (outside this function)
|
|
648
507
|
product_options = product_options.copy()
|
|
649
508
|
|
|
@@ -655,7 +514,10 @@ def _generate_l2m(
|
|
|
655
514
|
|
|
656
515
|
# Define variables to load
|
|
657
516
|
optimization_kwargs = l2m_options["optimization_kwargs"]
|
|
658
|
-
if "init_method" in optimization_kwargs:
|
|
517
|
+
if "init_method" in optimization_kwargs and optimization_kwargs["init_method"] is None:
|
|
518
|
+
optimization_kwargs["init_method"] = "None"
|
|
519
|
+
|
|
520
|
+
if optimization_kwargs.get("init_method", "None") != "None":
|
|
659
521
|
init_method = optimization_kwargs["init_method"]
|
|
660
522
|
moments = [f"M{order}" for order in init_method.replace("M", "")] + ["M1"]
|
|
661
523
|
else:
|
|
@@ -673,9 +535,14 @@ def _generate_l2m(
|
|
|
673
535
|
|
|
674
536
|
##------------------------------------------------------------------------.
|
|
675
537
|
# Open the netCDF files
|
|
676
|
-
ds = open_netcdf_files(
|
|
677
|
-
|
|
678
|
-
|
|
538
|
+
ds = open_netcdf_files(
|
|
539
|
+
filepaths,
|
|
540
|
+
start_time=start_time,
|
|
541
|
+
end_time=end_time,
|
|
542
|
+
variables=variables,
|
|
543
|
+
parallel=False,
|
|
544
|
+
compute=True,
|
|
545
|
+
)
|
|
679
546
|
|
|
680
547
|
# Produce L2M dataset
|
|
681
548
|
ds = generate_l2m(
|
|
@@ -699,8 +566,7 @@ def _generate_l2m(
|
|
|
699
566
|
ds,
|
|
700
567
|
campaign_name=campaign_name,
|
|
701
568
|
station_name=station_name,
|
|
702
|
-
|
|
703
|
-
rolling=rolling,
|
|
569
|
+
temporal_resolution=temporal_resolution,
|
|
704
570
|
model_name=model_name,
|
|
705
571
|
)
|
|
706
572
|
folder_path = define_file_folder_path(ds, dir_path=data_dir, folder_partitioning=folder_partitioning)
|
|
@@ -722,8 +588,7 @@ def _generate_l2m(
|
|
|
722
588
|
force=force,
|
|
723
589
|
# Product options
|
|
724
590
|
product_options=product_options,
|
|
725
|
-
|
|
726
|
-
rolling=rolling,
|
|
591
|
+
temporal_resolution=temporal_resolution,
|
|
727
592
|
model_name=model_name,
|
|
728
593
|
# Archiving arguments
|
|
729
594
|
data_dir=data_dir,
|
|
@@ -835,19 +700,15 @@ def run_l2m_station(
|
|
|
835
700
|
# Loop
|
|
836
701
|
# temporal_resolution = "1MIN"
|
|
837
702
|
# temporal_resolution = "10MIN"
|
|
838
|
-
temporal_resolutions = get_product_temporal_resolutions(
|
|
703
|
+
temporal_resolutions = get_product_temporal_resolutions(product)
|
|
839
704
|
for temporal_resolution in temporal_resolutions:
|
|
840
705
|
|
|
841
|
-
# Retrieve accumulation_interval and rolling option
|
|
842
|
-
accumulation_interval, rolling = get_resampling_information(temporal_resolution)
|
|
843
|
-
|
|
844
706
|
# ------------------------------------------------------------------.
|
|
845
|
-
#
|
|
846
|
-
if
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
if accumulation_interval < sample_interval:
|
|
707
|
+
# Check if the product can be generated
|
|
708
|
+
if not is_possible_product(
|
|
709
|
+
temporal_resolution=temporal_resolution,
|
|
710
|
+
sample_interval=sample_interval,
|
|
711
|
+
):
|
|
851
712
|
continue
|
|
852
713
|
|
|
853
714
|
# -----------------------------------------------------------------.
|
|
@@ -863,29 +724,28 @@ def run_l2m_station(
|
|
|
863
724
|
# Processing options
|
|
864
725
|
debugging_mode=debugging_mode,
|
|
865
726
|
# Product options
|
|
866
|
-
|
|
867
|
-
rolling=rolling,
|
|
727
|
+
temporal_resolution=temporal_resolution,
|
|
868
728
|
)
|
|
869
729
|
if filepaths is None:
|
|
870
730
|
continue
|
|
871
731
|
|
|
872
732
|
# -------------------------------------------------------------------------.
|
|
873
733
|
# Retrieve L2M processing options
|
|
874
|
-
l2m_processing_options =
|
|
875
|
-
product=
|
|
876
|
-
|
|
734
|
+
l2m_processing_options = L2ProcessingOptions(
|
|
735
|
+
product=product,
|
|
736
|
+
temporal_resolution=temporal_resolution,
|
|
877
737
|
filepaths=filepaths,
|
|
878
738
|
parallel=parallel,
|
|
879
739
|
)
|
|
880
740
|
|
|
881
741
|
# Retrieve folder partitioning (for files and logs)
|
|
882
|
-
folder_partitioning = l2m_processing_options.
|
|
742
|
+
folder_partitioning = l2m_processing_options.folder_partitioning
|
|
883
743
|
|
|
884
744
|
# Retrieve product options
|
|
885
|
-
global_product_options = l2m_processing_options.
|
|
745
|
+
global_product_options = l2m_processing_options.product_options
|
|
886
746
|
|
|
887
747
|
# Retrieve files temporal partitions
|
|
888
|
-
files_partitions = l2m_processing_options.
|
|
748
|
+
files_partitions = l2m_processing_options.files_partitions
|
|
889
749
|
|
|
890
750
|
if len(files_partitions) == 0:
|
|
891
751
|
msg = (
|
|
@@ -898,6 +758,7 @@ def run_l2m_station(
|
|
|
898
758
|
# -----------------------------------------------------------------.
|
|
899
759
|
# Loop over distributions to fit
|
|
900
760
|
# model_name = "GAMMA_ML"
|
|
761
|
+
# model_name = "LOGNORMAL_GS_ND_MAE"
|
|
901
762
|
# model_options = l2m_options["models"][model_name]
|
|
902
763
|
# Retrieve list of models to fit
|
|
903
764
|
models = global_product_options.pop("models")
|
|
@@ -917,30 +778,37 @@ def run_l2m_station(
|
|
|
917
778
|
precompute_scattering_tables(verbose=verbose, **radar_options)
|
|
918
779
|
|
|
919
780
|
# -----------------------------------------------------------------.
|
|
920
|
-
msg = f"Production of L2M_{model_name} for
|
|
781
|
+
msg = f"Production of L2M_{model_name} for {temporal_resolution} has started."
|
|
921
782
|
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
922
783
|
msg = f"Estimating {psd_model} parameters using {optimization}."
|
|
923
784
|
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
924
785
|
|
|
925
786
|
# -------------------------------------------------------------.
|
|
926
787
|
# Create product directory
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
788
|
+
try:
|
|
789
|
+
data_dir = create_product_directory(
|
|
790
|
+
# DISDRODB root directories
|
|
791
|
+
data_archive_dir=data_archive_dir,
|
|
792
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
793
|
+
# Station arguments
|
|
794
|
+
data_source=data_source,
|
|
795
|
+
campaign_name=campaign_name,
|
|
796
|
+
station_name=station_name,
|
|
797
|
+
# Processing options
|
|
798
|
+
product=product,
|
|
799
|
+
force=force,
|
|
800
|
+
# Option for L2E
|
|
801
|
+
temporal_resolution=temporal_resolution,
|
|
802
|
+
# Option for L2M
|
|
803
|
+
model_name=model_name,
|
|
804
|
+
)
|
|
805
|
+
except Exception:
|
|
806
|
+
msg = (
|
|
807
|
+
f"Production of L2M_{model_name} for {temporal_resolution} data has been "
|
|
808
|
+
+ "skipped because the product already exists and force=False."
|
|
809
|
+
)
|
|
810
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
811
|
+
continue
|
|
944
812
|
|
|
945
813
|
# Define logs directory
|
|
946
814
|
logs_dir = create_logs_directory(
|
|
@@ -951,8 +819,7 @@ def run_l2m_station(
|
|
|
951
819
|
campaign_name=campaign_name,
|
|
952
820
|
station_name=station_name,
|
|
953
821
|
# Option for L2E
|
|
954
|
-
|
|
955
|
-
rolling=rolling,
|
|
822
|
+
temporal_resolution=temporal_resolution,
|
|
956
823
|
# Option for L2M
|
|
957
824
|
model_name=model_name,
|
|
958
825
|
)
|
|
@@ -973,15 +840,13 @@ def run_l2m_station(
|
|
|
973
840
|
start_time=event_info["start_time"],
|
|
974
841
|
end_time=event_info["end_time"],
|
|
975
842
|
model_name=model_name,
|
|
976
|
-
|
|
977
|
-
rolling=rolling,
|
|
843
|
+
temporal_resolution=temporal_resolution,
|
|
978
844
|
),
|
|
979
845
|
folder_partitioning=folder_partitioning,
|
|
980
846
|
campaign_name=campaign_name,
|
|
981
847
|
station_name=station_name,
|
|
982
848
|
# L2M options
|
|
983
|
-
|
|
984
|
-
rolling=rolling,
|
|
849
|
+
temporal_resolution=temporal_resolution,
|
|
985
850
|
model_name=model_name,
|
|
986
851
|
product_options=product_options,
|
|
987
852
|
# Processing options
|
|
@@ -1005,8 +870,7 @@ def run_l2m_station(
|
|
|
1005
870
|
data_archive_dir=data_archive_dir,
|
|
1006
871
|
# Product options
|
|
1007
872
|
model_name=model_name,
|
|
1008
|
-
|
|
1009
|
-
rolling=rolling,
|
|
873
|
+
temporal_resolution=temporal_resolution,
|
|
1010
874
|
# Logs list
|
|
1011
875
|
list_logs=list_logs,
|
|
1012
876
|
)
|