disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +132 -15
- disdrodb/_config.py +4 -2
- disdrodb/_version.py +9 -4
- disdrodb/api/checks.py +264 -237
- disdrodb/api/configs.py +4 -8
- disdrodb/api/create_directories.py +235 -290
- disdrodb/api/info.py +217 -26
- disdrodb/api/io.py +306 -270
- disdrodb/api/path.py +597 -173
- disdrodb/api/search.py +486 -0
- disdrodb/{metadata/scripts → cli}/disdrodb_check_metadata_archive.py +12 -7
- disdrodb/{utils/pandas.py → cli/disdrodb_data_archive_directory.py} +9 -18
- disdrodb/cli/disdrodb_download_archive.py +86 -0
- disdrodb/cli/disdrodb_download_metadata_archive.py +53 -0
- disdrodb/cli/disdrodb_download_station.py +84 -0
- disdrodb/{api/scripts → cli}/disdrodb_initialize_station.py +22 -10
- disdrodb/cli/disdrodb_metadata_archive_directory.py +32 -0
- disdrodb/{data_transfer/scripts/disdrodb_download_station.py → cli/disdrodb_open_data_archive.py} +22 -22
- disdrodb/cli/disdrodb_open_logs_directory.py +69 -0
- disdrodb/{data_transfer/scripts/disdrodb_upload_station.py → cli/disdrodb_open_metadata_archive.py} +22 -24
- disdrodb/cli/disdrodb_open_metadata_directory.py +71 -0
- disdrodb/cli/disdrodb_open_product_directory.py +74 -0
- disdrodb/cli/disdrodb_open_readers_directory.py +32 -0
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0.py +38 -31
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0_station.py +32 -30
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a_station.py +24 -33
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b_station.py +25 -34
- disdrodb/cli/disdrodb_run_l0c.py +130 -0
- disdrodb/cli/disdrodb_run_l0c_station.py +129 -0
- disdrodb/cli/disdrodb_run_l1.py +122 -0
- disdrodb/cli/disdrodb_run_l1_station.py +121 -0
- disdrodb/cli/disdrodb_run_l2e.py +122 -0
- disdrodb/cli/disdrodb_run_l2e_station.py +122 -0
- disdrodb/cli/disdrodb_run_l2m.py +122 -0
- disdrodb/cli/disdrodb_run_l2m_station.py +122 -0
- disdrodb/cli/disdrodb_upload_archive.py +105 -0
- disdrodb/cli/disdrodb_upload_station.py +98 -0
- disdrodb/configs.py +90 -25
- disdrodb/data_transfer/__init__.py +22 -0
- disdrodb/data_transfer/download_data.py +87 -90
- disdrodb/data_transfer/upload_data.py +64 -37
- disdrodb/data_transfer/zenodo.py +15 -18
- disdrodb/docs.py +1 -1
- disdrodb/issue/__init__.py +17 -4
- disdrodb/issue/checks.py +10 -23
- disdrodb/issue/reader.py +9 -12
- disdrodb/issue/writer.py +14 -17
- disdrodb/l0/__init__.py +17 -26
- disdrodb/l0/check_configs.py +35 -23
- disdrodb/l0/check_standards.py +46 -51
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_diameter.yml +44 -44
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_velocity.yml +40 -40
- disdrodb/l0/configs/LPM/l0a_encodings.yml +80 -0
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_cf_attrs.yml +84 -65
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_encodings.yml +50 -9
- disdrodb/l0/configs/{Thies_LPM → LPM}/raw_data_format.yml +285 -245
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_diameter.yml +66 -66
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL/l0a_encodings.yml +32 -0
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_cf_attrs.yml +23 -21
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_encodings.yml +17 -17
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/raw_data_format.yml +77 -77
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_diameter.yml +64 -64
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +39 -0
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_cf_attrs.yml +28 -26
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_encodings.yml +20 -20
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/raw_data_format.yml +107 -107
- disdrodb/l0/configs/PWS100/bins_diameter.yml +173 -0
- disdrodb/l0/configs/PWS100/bins_velocity.yml +173 -0
- disdrodb/l0/configs/PWS100/l0a_encodings.yml +19 -0
- disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +76 -0
- disdrodb/l0/configs/PWS100/l0b_encodings.yml +176 -0
- disdrodb/l0/configs/PWS100/raw_data_format.yml +182 -0
- disdrodb/l0/configs/{RD_80 → RD80}/bins_diameter.yml +40 -40
- disdrodb/l0/configs/RD80/l0a_encodings.yml +16 -0
- disdrodb/l0/configs/{RD_80 → RD80}/l0b_cf_attrs.yml +3 -3
- disdrodb/l0/configs/RD80/l0b_encodings.yml +135 -0
- disdrodb/l0/configs/{RD_80 → RD80}/raw_data_format.yml +46 -50
- disdrodb/l0/l0_reader.py +216 -340
- disdrodb/l0/l0a_processing.py +237 -208
- disdrodb/l0/l0b_nc_processing.py +227 -80
- disdrodb/l0/l0b_processing.py +96 -174
- disdrodb/l0/l0c_processing.py +627 -0
- disdrodb/l0/readers/{ARM → LPM/ARM}/ARM_LPM.py +36 -58
- disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +236 -0
- disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +185 -0
- disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +185 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +195 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +210 -0
- disdrodb/l0/readers/{BRAZIL/GOAMAZON_LPM.py → LPM/KIT/CHWALA.py} +97 -76
- disdrodb/l0/readers/LPM/SLOVENIA/ARSO.py +197 -0
- disdrodb/l0/readers/LPM/SLOVENIA/CRNI_VRH.py +197 -0
- disdrodb/l0/readers/{UK → LPM/UK}/DIVEN.py +14 -35
- disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +157 -0
- disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +113 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/ARCTIC_2021.py +40 -57
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/COMMON_2011.py +37 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/DAVOS_2009_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_2009.py +34 -51
- disdrodb/l0/readers/{EPFL/PARADISO_2014.py → PARSIVEL/EPFL/EPFL_ROOF_2008.py} +38 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2012.py +33 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GENEPI_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007_2.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HPICONET_2010.py +34 -51
- disdrodb/l0/readers/{EPFL/EPFL_ROOF_2010.py → PARSIVEL/EPFL/HYMEX_LTE_SOP2.py} +37 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +111 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HYMEX_LTE_SOP4.py +36 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2018.py +34 -52
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2019.py +38 -56
- disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PARSIVEL_2007.py +27 -45
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PLATO_2019.py +24 -44
- disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +140 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RACLETS_2019_WJF.py +41 -59
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RIETHOLZBACH_2011.py +34 -51
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +117 -0
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +137 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/UNIL_2022.py +42 -55
- disdrodb/l0/readers/PARSIVEL/GPM/IFLOODS.py +104 -0
- disdrodb/l0/readers/{GPM → PARSIVEL/GPM}/LPVEX.py +29 -48
- disdrodb/l0/readers/PARSIVEL/GPM/MC3E.py +184 -0
- disdrodb/l0/readers/PARSIVEL/KIT/BURKINA_FASO.py +133 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +113 -0
- disdrodb/l0/readers/{NCAR/VORTEX_SE_2016_P1.py → PARSIVEL/NCAR/OWLES_MIPS.py} +46 -72
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +125 -0
- disdrodb/l0/readers/{NCAR/OWLES_MIPS.py → PARSIVEL/NCAR/PLOWS_MIPS.py} +45 -64
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +114 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +176 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +183 -0
- disdrodb/l0/readers/PARSIVEL/SLOVENIA/UL_FGG.py +121 -0
- disdrodb/l0/readers/{ARM/ARM_LD.py → PARSIVEL2/ARM/ARM_PARSIVEL2.py} +27 -50
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/{DENMARK → PARSIVEL2/DENMARK}/EROSION_nc.py +14 -35
- disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +189 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +119 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +104 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/NSSTC.py +176 -0
- disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +32 -0
- disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +56 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +120 -0
- disdrodb/l0/readers/{NCAR → PARSIVEL2/NCAR}/PECAN_MIPS.py +45 -64
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +181 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +160 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +160 -0
- disdrodb/l0/readers/{NCAR/PLOWS_MIPS.py → PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py} +49 -66
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +118 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +152 -0
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT.py +166 -0
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +150 -0
- disdrodb/l0/readers/{NCAR/RELAMPAGO_RD80.py → RD80/BRAZIL/CHUVA_RD80.py} +36 -60
- disdrodb/l0/readers/{BRAZIL → RD80/BRAZIL}/GOAMAZON_RD80.py +36 -55
- disdrodb/l0/readers/{NCAR → RD80/NCAR}/CINDY_2011_RD80.py +35 -54
- disdrodb/l0/readers/{BRAZIL/CHUVA_RD80.py → RD80/NCAR/RELAMPAGO_RD80.py} +40 -54
- disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py +274 -0
- disdrodb/l0/readers/template_reader_raw_netcdf_data.py +62 -0
- disdrodb/l0/readers/{reader_template.py → template_reader_raw_text_data.py} +20 -44
- disdrodb/l0/routines.py +885 -581
- disdrodb/l0/standards.py +77 -238
- disdrodb/l0/template_tools.py +105 -110
- disdrodb/l1/__init__.py +17 -0
- disdrodb/l1/beard_model.py +716 -0
- disdrodb/l1/encoding_attrs.py +635 -0
- disdrodb/l1/fall_velocity.py +260 -0
- disdrodb/l1/filters.py +192 -0
- disdrodb/l1/processing.py +202 -0
- disdrodb/l1/resampling.py +236 -0
- disdrodb/l1/routines.py +358 -0
- disdrodb/l1_env/__init__.py +17 -0
- disdrodb/l1_env/routines.py +38 -0
- disdrodb/l2/__init__.py +17 -0
- disdrodb/l2/empirical_dsd.py +1833 -0
- disdrodb/l2/event.py +388 -0
- disdrodb/l2/processing.py +528 -0
- disdrodb/l2/processing_options.py +213 -0
- disdrodb/l2/routines.py +868 -0
- disdrodb/metadata/__init__.py +9 -2
- disdrodb/metadata/checks.py +180 -124
- disdrodb/metadata/download.py +81 -0
- disdrodb/metadata/geolocation.py +146 -0
- disdrodb/metadata/info.py +20 -13
- disdrodb/metadata/manipulation.py +3 -3
- disdrodb/metadata/reader.py +59 -8
- disdrodb/metadata/search.py +77 -144
- disdrodb/metadata/standards.py +83 -80
- disdrodb/metadata/writer.py +10 -16
- disdrodb/psd/__init__.py +38 -0
- disdrodb/psd/fitting.py +2146 -0
- disdrodb/psd/models.py +774 -0
- disdrodb/routines.py +1412 -0
- disdrodb/scattering/__init__.py +28 -0
- disdrodb/scattering/axis_ratio.py +344 -0
- disdrodb/scattering/routines.py +456 -0
- disdrodb/utils/__init__.py +17 -0
- disdrodb/utils/attrs.py +208 -0
- disdrodb/utils/cli.py +269 -0
- disdrodb/utils/compression.py +60 -42
- disdrodb/utils/dask.py +62 -0
- disdrodb/utils/dataframe.py +342 -0
- disdrodb/utils/decorators.py +110 -0
- disdrodb/utils/directories.py +107 -46
- disdrodb/utils/encoding.py +127 -0
- disdrodb/utils/list.py +29 -0
- disdrodb/utils/logger.py +168 -46
- disdrodb/utils/time.py +657 -0
- disdrodb/utils/warnings.py +30 -0
- disdrodb/utils/writer.py +57 -0
- disdrodb/utils/xarray.py +138 -47
- disdrodb/utils/yaml.py +0 -1
- disdrodb/viz/__init__.py +17 -0
- disdrodb/viz/plots.py +17 -0
- disdrodb-0.1.1.dist-info/METADATA +294 -0
- disdrodb-0.1.1.dist-info/RECORD +232 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/WHEEL +1 -1
- disdrodb-0.1.1.dist-info/entry_points.txt +30 -0
- disdrodb/data_transfer/scripts/disdrodb_download_archive.py +0 -53
- disdrodb/data_transfer/scripts/disdrodb_upload_archive.py +0 -57
- disdrodb/l0/configs/OTT_Parsivel/l0a_encodings.yml +0 -32
- disdrodb/l0/configs/OTT_Parsivel2/l0a_encodings.yml +0 -39
- disdrodb/l0/configs/RD_80/l0a_encodings.yml +0 -16
- disdrodb/l0/configs/RD_80/l0b_encodings.yml +0 -135
- disdrodb/l0/configs/Thies_LPM/l0a_encodings.yml +0 -80
- disdrodb/l0/io.py +0 -257
- disdrodb/l0/l0_processing.py +0 -1091
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_OTT.py +0 -178
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_THIES.py +0 -247
- disdrodb/l0/readers/BRAZIL/CHUVA_LPM.py +0 -204
- disdrodb/l0/readers/BRAZIL/CHUVA_OTT.py +0 -183
- disdrodb/l0/readers/BRAZIL/GOAMAZON_OTT.py +0 -183
- disdrodb/l0/readers/CHINA/CHONGQING.py +0 -131
- disdrodb/l0/readers/EPFL/EPFL_ROOF_2008.py +0 -128
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP2.py +0 -127
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP3.py +0 -129
- disdrodb/l0/readers/EPFL/RACLETS_2019.py +0 -158
- disdrodb/l0/readers/EPFL/SAMOYLOV_2017.py +0 -136
- disdrodb/l0/readers/EPFL/SAMOYLOV_2019.py +0 -158
- disdrodb/l0/readers/FRANCE/SIRTA_OTT2.py +0 -138
- disdrodb/l0/readers/GPM/GCPEX.py +0 -123
- disdrodb/l0/readers/GPM/IFLOODS.py +0 -123
- disdrodb/l0/readers/GPM/MC3E.py +0 -123
- disdrodb/l0/readers/GPM/NSSTC.py +0 -164
- disdrodb/l0/readers/ITALY/GID.py +0 -199
- disdrodb/l0/readers/MEXICO/OH_IIUNAM_nc.py +0 -92
- disdrodb/l0/readers/NCAR/CCOPE_2015.py +0 -133
- disdrodb/l0/readers/NCAR/PECAN_FP3.py +0 -137
- disdrodb/l0/readers/NCAR/PECAN_MOBILE.py +0 -144
- disdrodb/l0/readers/NCAR/RELAMPAGO_OTT.py +0 -195
- disdrodb/l0/readers/NCAR/SNOWIE_PJ.py +0 -172
- disdrodb/l0/readers/NCAR/SNOWIE_SB.py +0 -179
- disdrodb/l0/readers/NCAR/VORTEX2_2009.py +0 -133
- disdrodb/l0/readers/NCAR/VORTEX2_2010.py +0 -188
- disdrodb/l0/readers/NCAR/VORTEX2_2010_UF.py +0 -191
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_P2.py +0 -135
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_PIPS.py +0 -170
- disdrodb/l0/readers/NETHERLANDS/DELFT.py +0 -187
- disdrodb/l0/readers/SPAIN/SBEGUERIA.py +0 -179
- disdrodb/l0/scripts/disdrodb_run_l0b_concat.py +0 -93
- disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py +0 -85
- disdrodb/utils/netcdf.py +0 -452
- disdrodb/utils/scripts.py +0 -102
- disdrodb-0.0.21.dist-info/AUTHORS.md +0 -18
- disdrodb-0.0.21.dist-info/METADATA +0 -186
- disdrodb-0.0.21.dist-info/RECORD +0 -168
- disdrodb-0.0.21.dist-info/entry_points.txt +0 -15
- /disdrodb/l0/configs/{RD_80 → RD80}/bins_velocity.yml +0 -0
- /disdrodb/l0/manuals/{Thies_LPM.pdf → LPM.pdf} +0 -0
- /disdrodb/l0/manuals/{ODM_470.pdf → ODM470.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel.pdf → PARSIVEL.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel2.pdf → PARSIVEL2.pdf} +0 -0
- /disdrodb/l0/manuals/{PWS_100.pdf → PWS100.pdf} +0 -0
- /disdrodb/l0/manuals/{RD_80.pdf → RD80.pdf} +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info/licenses}/LICENSE +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/top_level.txt +0 -0
disdrodb/l0/routines.py
CHANGED
|
@@ -16,702 +16,1006 @@
|
|
|
16
16
|
# You should have received a copy of the GNU General Public License
|
|
17
17
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
18
18
|
# -----------------------------------------------------------------------------.
|
|
19
|
-
"""Implement DISDRODB
|
|
19
|
+
"""Implement DISDRODB L0 processing."""
|
|
20
20
|
|
|
21
21
|
import datetime
|
|
22
22
|
import logging
|
|
23
|
+
import os
|
|
23
24
|
import time
|
|
25
|
+
from typing import Optional
|
|
24
26
|
|
|
25
|
-
import
|
|
27
|
+
import dask
|
|
26
28
|
|
|
29
|
+
from disdrodb.api.checks import check_sensor_name
|
|
30
|
+
|
|
31
|
+
# Directory
|
|
32
|
+
from disdrodb.api.create_directories import (
|
|
33
|
+
create_l0_directory_structure,
|
|
34
|
+
create_logs_directory,
|
|
35
|
+
create_product_directory,
|
|
36
|
+
)
|
|
37
|
+
from disdrodb.api.io import find_files, remove_product
|
|
38
|
+
from disdrodb.api.path import (
|
|
39
|
+
define_file_folder_path,
|
|
40
|
+
define_l0a_filename,
|
|
41
|
+
define_l0b_filename,
|
|
42
|
+
define_l0c_filename,
|
|
43
|
+
define_metadata_filepath,
|
|
44
|
+
)
|
|
45
|
+
from disdrodb.api.search import get_required_product
|
|
46
|
+
from disdrodb.configs import get_data_archive_dir, get_folder_partitioning, get_metadata_archive_dir
|
|
47
|
+
from disdrodb.issue import read_station_issue
|
|
48
|
+
from disdrodb.l0.l0_reader import get_reader
|
|
49
|
+
from disdrodb.l0.l0a_processing import (
|
|
50
|
+
read_l0a_dataframe,
|
|
51
|
+
sanitize_df,
|
|
52
|
+
write_l0a,
|
|
53
|
+
)
|
|
54
|
+
from disdrodb.l0.l0b_nc_processing import sanitize_ds
|
|
55
|
+
from disdrodb.l0.l0b_processing import (
|
|
56
|
+
create_l0b_from_l0a,
|
|
57
|
+
set_l0b_encodings,
|
|
58
|
+
write_l0b,
|
|
59
|
+
)
|
|
60
|
+
from disdrodb.l0.l0c_processing import (
|
|
61
|
+
create_daily_file,
|
|
62
|
+
get_files_per_days,
|
|
63
|
+
retrieve_possible_measurement_intervals,
|
|
64
|
+
)
|
|
65
|
+
from disdrodb.metadata import read_station_metadata
|
|
66
|
+
from disdrodb.utils.decorators import delayed_if_parallel, single_threaded_if_parallel
|
|
67
|
+
|
|
68
|
+
# Logger
|
|
27
69
|
from disdrodb.utils.logger import (
|
|
28
|
-
|
|
29
|
-
|
|
70
|
+
close_logger,
|
|
71
|
+
create_logger_file,
|
|
72
|
+
create_product_logs,
|
|
73
|
+
log_error,
|
|
30
74
|
log_info,
|
|
31
75
|
)
|
|
32
|
-
|
|
76
|
+
|
|
77
|
+
# log_warning,
|
|
78
|
+
from disdrodb.utils.writer import write_product
|
|
79
|
+
from disdrodb.utils.yaml import read_yaml
|
|
33
80
|
|
|
34
81
|
logger = logging.getLogger(__name__)
|
|
35
82
|
|
|
36
|
-
|
|
37
|
-
####
|
|
83
|
+
# -----------------------------------------------------------------------------.
|
|
84
|
+
#### Creation of L0A and L0B Single Station File
|
|
38
85
|
|
|
39
86
|
|
|
40
|
-
|
|
41
|
-
|
|
87
|
+
@delayed_if_parallel
|
|
88
|
+
@single_threaded_if_parallel
|
|
89
|
+
def _generate_l0a(
|
|
90
|
+
filepath,
|
|
91
|
+
data_dir,
|
|
92
|
+
logs_dir,
|
|
93
|
+
campaign_name,
|
|
94
|
+
station_name,
|
|
95
|
+
# Processing info
|
|
96
|
+
reader,
|
|
97
|
+
metadata,
|
|
98
|
+
issue_dict,
|
|
99
|
+
# Processing options
|
|
100
|
+
force,
|
|
101
|
+
verbose,
|
|
102
|
+
parallel,
|
|
103
|
+
):
|
|
104
|
+
"""Generate L0A file from raw file."""
|
|
105
|
+
# Define product
|
|
106
|
+
product = "L0A"
|
|
107
|
+
|
|
108
|
+
# Define folder partitioning
|
|
109
|
+
folder_partitioning = get_folder_partitioning()
|
|
110
|
+
|
|
111
|
+
# Retrieve sensor name
|
|
112
|
+
sensor_name = metadata["sensor_name"]
|
|
113
|
+
|
|
114
|
+
##------------------------------------------------------------------------.
|
|
115
|
+
# Create file logger
|
|
116
|
+
filename = os.path.basename(filepath)
|
|
117
|
+
logger, logger_filepath = create_logger_file(
|
|
118
|
+
logs_dir=logs_dir,
|
|
119
|
+
filename=filename,
|
|
120
|
+
parallel=parallel,
|
|
121
|
+
)
|
|
42
122
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
"""
|
|
48
|
-
function = click.option(
|
|
49
|
-
"--data_sources",
|
|
50
|
-
type=str,
|
|
51
|
-
show_default=True,
|
|
52
|
-
default="",
|
|
53
|
-
help="DISDRODB data sources to process",
|
|
54
|
-
)(function)
|
|
55
|
-
function = click.option(
|
|
56
|
-
"--campaign_names",
|
|
57
|
-
type=str,
|
|
58
|
-
show_default=True,
|
|
59
|
-
default="",
|
|
60
|
-
help="DISDRODB campaign names to process",
|
|
61
|
-
)(function)
|
|
62
|
-
function = click.option(
|
|
63
|
-
"--station_names",
|
|
64
|
-
type=str,
|
|
65
|
-
show_default=True,
|
|
66
|
-
default="",
|
|
67
|
-
help="DISDRODB station names to process",
|
|
68
|
-
)(function)
|
|
69
|
-
return function
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
def click_l0_processing_options(function: object):
|
|
73
|
-
"""Click command line default parameters for L0 processing options.
|
|
123
|
+
##------------------------------------------------------------------------.
|
|
124
|
+
# Log start processing
|
|
125
|
+
msg = f"{product} processing of {filename} has started."
|
|
126
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
74
127
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
help="Process files in parallel",
|
|
87
|
-
)(function)
|
|
88
|
-
function = click.option(
|
|
89
|
-
"-d",
|
|
90
|
-
"--debugging_mode",
|
|
91
|
-
type=bool,
|
|
92
|
-
show_default=True,
|
|
93
|
-
default=False,
|
|
94
|
-
help="Switch to debugging mode",
|
|
95
|
-
)(function)
|
|
96
|
-
function = click.option("-v", "--verbose", type=bool, show_default=True, default=True, help="Verbose")(function)
|
|
97
|
-
function = click.option(
|
|
98
|
-
"-f",
|
|
99
|
-
"--force",
|
|
100
|
-
type=bool,
|
|
101
|
-
show_default=True,
|
|
102
|
-
default=False,
|
|
103
|
-
help="Force overwriting",
|
|
104
|
-
)(function)
|
|
105
|
-
return function
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
def click_remove_l0a_option(function: object):
|
|
109
|
-
function = click.option(
|
|
110
|
-
"--remove_l0a",
|
|
111
|
-
type=bool,
|
|
112
|
-
show_default=True,
|
|
113
|
-
default=False,
|
|
114
|
-
help="If true, remove the L0A files once the L0B processing is terminated.",
|
|
115
|
-
)(function)
|
|
116
|
-
return function
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
def click_l0_archive_options(function: object):
|
|
120
|
-
"""Click command line arguments for L0 processing archiving of a station.
|
|
128
|
+
##------------------------------------------------------------------------.
|
|
129
|
+
### - Read raw file into a dataframe and sanitize for L0A format
|
|
130
|
+
try:
|
|
131
|
+
df = reader(filepath, logger=logger)
|
|
132
|
+
df = sanitize_df(
|
|
133
|
+
df=df,
|
|
134
|
+
sensor_name=sensor_name,
|
|
135
|
+
verbose=verbose,
|
|
136
|
+
issue_dict=issue_dict,
|
|
137
|
+
logger=logger,
|
|
138
|
+
)
|
|
121
139
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
"--l0b_concat",
|
|
129
|
-
type=bool,
|
|
130
|
-
show_default=True,
|
|
131
|
-
default=False,
|
|
132
|
-
help="Produce single L0B netCDF file.",
|
|
133
|
-
)(function)
|
|
134
|
-
function = click.option(
|
|
135
|
-
"--remove_l0b",
|
|
136
|
-
type=bool,
|
|
137
|
-
show_default=True,
|
|
138
|
-
default=False,
|
|
139
|
-
help="If true, remove all source L0B files once L0B concatenation is terminated.",
|
|
140
|
-
)(function)
|
|
141
|
-
function = click.option(
|
|
142
|
-
"--remove_l0a",
|
|
143
|
-
type=bool,
|
|
144
|
-
show_default=True,
|
|
145
|
-
default=False,
|
|
146
|
-
help="If true, remove the L0A files once the L0B processing is terminated.",
|
|
147
|
-
)(function)
|
|
148
|
-
function = click.option(
|
|
149
|
-
"-l0b",
|
|
150
|
-
"--l0b_processing",
|
|
151
|
-
type=bool,
|
|
152
|
-
show_default=True,
|
|
153
|
-
default=True,
|
|
154
|
-
help="Perform L0B processing.",
|
|
155
|
-
)(function)
|
|
156
|
-
function = click.option(
|
|
157
|
-
"-l0a",
|
|
158
|
-
"--l0a_processing",
|
|
159
|
-
type=bool,
|
|
160
|
-
show_default=True,
|
|
161
|
-
default=True,
|
|
162
|
-
help="Perform L0A processing.",
|
|
163
|
-
)(function)
|
|
164
|
-
return function
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
def click_l0b_concat_options(function: object):
|
|
168
|
-
"""Click command line default parameters for L0B concatenation.
|
|
140
|
+
##--------------------------------------------------------------------.
|
|
141
|
+
#### - Write to Parquet
|
|
142
|
+
filename = define_l0a_filename(df=df, campaign_name=campaign_name, station_name=station_name)
|
|
143
|
+
folder_path = define_file_folder_path(df, data_dir=data_dir, folder_partitioning=folder_partitioning)
|
|
144
|
+
filepath = os.path.join(folder_path, filename)
|
|
145
|
+
write_l0a(df=df, filepath=filepath, force=force, logger=logger, verbose=verbose)
|
|
169
146
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
Function.
|
|
174
|
-
"""
|
|
175
|
-
function = click.option(
|
|
176
|
-
"--remove_l0b",
|
|
177
|
-
type=bool,
|
|
178
|
-
show_default=True,
|
|
179
|
-
default=False,
|
|
180
|
-
help="If true, remove all source L0B files once L0B concatenation is terminated.",
|
|
181
|
-
)(function)
|
|
182
|
-
function = click.option("-v", "--verbose", type=bool, show_default=True, default=False, help="Verbose")(function)
|
|
183
|
-
return function
|
|
147
|
+
##--------------------------------------------------------------------.
|
|
148
|
+
# Clean environment
|
|
149
|
+
del df
|
|
184
150
|
|
|
151
|
+
# Log end processing
|
|
152
|
+
msg = f"{product} processing of {filename} has ended."
|
|
153
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
185
154
|
|
|
186
|
-
|
|
187
|
-
|
|
155
|
+
# Otherwise log the error
|
|
156
|
+
except Exception as e:
|
|
157
|
+
error_type = str(type(e).__name__)
|
|
158
|
+
msg = f"{error_type}: {e}"
|
|
159
|
+
log_error(logger=logger, msg=msg, verbose=verbose)
|
|
188
160
|
|
|
161
|
+
# Close the file logger
|
|
162
|
+
close_logger(logger)
|
|
189
163
|
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
164
|
+
# Return the logger file path
|
|
165
|
+
return logger_filepath
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@delayed_if_parallel
|
|
169
|
+
@single_threaded_if_parallel
|
|
170
|
+
def _generate_l0b(
|
|
171
|
+
filepath,
|
|
172
|
+
data_dir,
|
|
173
|
+
logs_dir,
|
|
193
174
|
campaign_name,
|
|
194
175
|
station_name,
|
|
176
|
+
# Processing info
|
|
177
|
+
metadata,
|
|
195
178
|
# Processing options
|
|
196
|
-
force
|
|
197
|
-
verbose
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
base_dir: str = None,
|
|
179
|
+
force,
|
|
180
|
+
verbose,
|
|
181
|
+
parallel,
|
|
182
|
+
debugging_mode,
|
|
201
183
|
):
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
184
|
+
# Define product
|
|
185
|
+
product = "L0B"
|
|
186
|
+
|
|
187
|
+
# Define folder partitioning
|
|
188
|
+
folder_partitioning = get_folder_partitioning()
|
|
189
|
+
|
|
190
|
+
# -----------------------------------------------------------------.
|
|
191
|
+
# Create file logger
|
|
192
|
+
filename = os.path.basename(filepath)
|
|
193
|
+
logger, logger_filepath = create_logger_file(
|
|
194
|
+
logs_dir=logs_dir,
|
|
195
|
+
filename=filename,
|
|
196
|
+
parallel=parallel,
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
##------------------------------------------------------------------------.
|
|
200
|
+
# Log start processing
|
|
201
|
+
msg = f"{product} processing of {filename} has started."
|
|
202
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
203
|
+
|
|
204
|
+
##------------------------------------------------------------------------.
|
|
205
|
+
# Retrieve sensor name
|
|
206
|
+
sensor_name = metadata["sensor_name"]
|
|
207
|
+
check_sensor_name(sensor_name)
|
|
208
|
+
|
|
209
|
+
##------------------------------------------------------------------------.
|
|
210
|
+
try:
|
|
211
|
+
# Read L0A Apache Parquet file
|
|
212
|
+
df = read_l0a_dataframe(filepath, logger=logger, verbose=verbose, debugging_mode=debugging_mode)
|
|
213
|
+
|
|
214
|
+
# -----------------------------------------------------------------.
|
|
215
|
+
# Create xarray Dataset
|
|
216
|
+
ds = create_l0b_from_l0a(df=df, metadata=metadata, logger=logger, verbose=verbose)
|
|
217
|
+
|
|
218
|
+
# -----------------------------------------------------------------.
|
|
219
|
+
# Write L0B netCDF4 dataset
|
|
220
|
+
filename = define_l0b_filename(ds=ds, campaign_name=campaign_name, station_name=station_name)
|
|
221
|
+
folder_path = define_file_folder_path(ds, data_dir=data_dir, folder_partitioning=folder_partitioning)
|
|
222
|
+
filepath = os.path.join(folder_path, filename)
|
|
223
|
+
write_l0b(ds, filepath=filepath, force=force)
|
|
224
|
+
|
|
225
|
+
##--------------------------------------------------------------------.
|
|
226
|
+
# Clean environment
|
|
227
|
+
del ds, df
|
|
228
|
+
|
|
229
|
+
# Log end processing
|
|
230
|
+
msg = f"{product} processing of {filename} has ended."
|
|
231
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
232
|
+
|
|
233
|
+
# Otherwise log the error
|
|
234
|
+
except Exception as e:
|
|
235
|
+
error_type = str(type(e).__name__)
|
|
236
|
+
msg = f"{error_type}: {e}"
|
|
237
|
+
log_error(logger, msg, verbose=verbose)
|
|
238
|
+
|
|
239
|
+
# Close the file logger
|
|
240
|
+
close_logger(logger)
|
|
241
|
+
|
|
242
|
+
# Return the logger file path
|
|
243
|
+
return logger_filepath
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _generate_l0b_from_nc(
|
|
247
|
+
filepath,
|
|
248
|
+
data_dir,
|
|
249
|
+
logs_dir,
|
|
230
250
|
campaign_name,
|
|
231
251
|
station_name,
|
|
252
|
+
# Processing info
|
|
253
|
+
reader,
|
|
254
|
+
metadata,
|
|
255
|
+
issue_dict,
|
|
232
256
|
# Processing options
|
|
233
|
-
force
|
|
234
|
-
verbose
|
|
235
|
-
|
|
236
|
-
parallel: bool = True,
|
|
237
|
-
base_dir: str = None,
|
|
238
|
-
remove_l0a: bool = False,
|
|
257
|
+
force,
|
|
258
|
+
verbose,
|
|
259
|
+
parallel,
|
|
239
260
|
):
|
|
240
|
-
|
|
241
|
-
#
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
261
|
+
|
|
262
|
+
# -----------------------------------------------------------------.
|
|
263
|
+
# Define product name
|
|
264
|
+
product = "L0B"
|
|
265
|
+
|
|
266
|
+
# Define folder partitioning
|
|
267
|
+
folder_partitioning = get_folder_partitioning()
|
|
268
|
+
|
|
269
|
+
# Retrieve sensor name
|
|
270
|
+
sensor_name = metadata["sensor_name"]
|
|
271
|
+
|
|
272
|
+
# -----------------------------------------------------------------.
|
|
273
|
+
# Create file logger
|
|
274
|
+
filename = os.path.basename(filepath)
|
|
275
|
+
logger, logger_filepath = create_logger_file(
|
|
276
|
+
logs_dir=logs_dir,
|
|
277
|
+
filename=filename,
|
|
278
|
+
parallel=parallel,
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
##------------------------------------------------------------------------.
|
|
282
|
+
# Log start processing
|
|
283
|
+
msg = f"{product} processing of {filename} has started."
|
|
284
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
285
|
+
|
|
286
|
+
##------------------------------------------------------------------------.
|
|
287
|
+
### - Read raw netCDF and sanitize for L0B format
|
|
288
|
+
try:
|
|
289
|
+
ds = reader(filepath, logger=logger)
|
|
290
|
+
ds = sanitize_ds(
|
|
291
|
+
ds=ds,
|
|
292
|
+
sensor_name=sensor_name,
|
|
293
|
+
metadata=metadata,
|
|
294
|
+
issue_dict=issue_dict,
|
|
295
|
+
verbose=verbose,
|
|
296
|
+
logger=logger,
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
# -----------------------------------------------------------------.
|
|
300
|
+
# Write L0B netCDF4 dataset
|
|
301
|
+
filename = define_l0b_filename(ds=ds, campaign_name=campaign_name, station_name=station_name)
|
|
302
|
+
folder_path = define_file_folder_path(ds, data_dir=data_dir, folder_partitioning=folder_partitioning)
|
|
303
|
+
filepath = os.path.join(folder_path, filename)
|
|
304
|
+
write_l0b(ds, filepath=filepath, force=force)
|
|
305
|
+
|
|
306
|
+
##--------------------------------------------------------------------.
|
|
307
|
+
# Clean environment
|
|
308
|
+
del ds
|
|
309
|
+
|
|
310
|
+
# Log end processing
|
|
311
|
+
msg = f"L0B processing of {filename} has ended."
|
|
312
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
313
|
+
|
|
314
|
+
# Otherwise log the error
|
|
315
|
+
except Exception as e:
|
|
316
|
+
error_type = str(type(e).__name__)
|
|
317
|
+
msg = f"{error_type}: {e}"
|
|
318
|
+
log_error(logger, msg, verbose=verbose)
|
|
319
|
+
|
|
320
|
+
# Close the file logger
|
|
321
|
+
close_logger(logger)
|
|
322
|
+
|
|
323
|
+
# Return the logger file path
|
|
324
|
+
return logger_filepath
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
@delayed_if_parallel
|
|
328
|
+
@single_threaded_if_parallel
|
|
329
|
+
def _generate_l0c(
|
|
330
|
+
day,
|
|
331
|
+
filepaths,
|
|
332
|
+
data_dir,
|
|
333
|
+
logs_dir,
|
|
334
|
+
metadata_filepath,
|
|
269
335
|
campaign_name,
|
|
270
336
|
station_name,
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
337
|
+
# Processing options
|
|
338
|
+
force,
|
|
339
|
+
verbose,
|
|
340
|
+
parallel, # this is used only to initialize the correct logger !
|
|
274
341
|
):
|
|
275
|
-
|
|
342
|
+
# -----------------------------------------------------------------.
|
|
343
|
+
# Define product name
|
|
344
|
+
product = "L0C"
|
|
345
|
+
|
|
346
|
+
# Define folder partitioning
|
|
347
|
+
folder_partitioning = get_folder_partitioning()
|
|
348
|
+
|
|
349
|
+
# -----------------------------------------------------------------.
|
|
350
|
+
# Create file logger
|
|
351
|
+
logger, logger_filepath = create_logger_file(
|
|
352
|
+
logs_dir=logs_dir,
|
|
353
|
+
filename=day,
|
|
354
|
+
parallel=parallel,
|
|
355
|
+
)
|
|
276
356
|
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
357
|
+
##------------------------------------------------------------------------.
|
|
358
|
+
# Log start processing
|
|
359
|
+
msg = f"{product} processing for {day} has started."
|
|
360
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
361
|
+
|
|
362
|
+
##------------------------------------------------------------------------.
|
|
363
|
+
### Core computation
|
|
364
|
+
try:
|
|
365
|
+
# Retrieve measurement_intervals
|
|
366
|
+
# - TODO: in future available from dataset
|
|
367
|
+
metadata = read_yaml(metadata_filepath)
|
|
368
|
+
measurement_intervals = retrieve_possible_measurement_intervals(metadata)
|
|
369
|
+
|
|
370
|
+
# Produce L0C datasets
|
|
371
|
+
dict_ds = create_daily_file(
|
|
372
|
+
day=day,
|
|
373
|
+
filepaths=filepaths,
|
|
374
|
+
measurement_intervals=measurement_intervals,
|
|
375
|
+
ensure_variables_equality=True,
|
|
376
|
+
logger=logger,
|
|
377
|
+
verbose=verbose,
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
# Write a dataset for each sample interval
|
|
381
|
+
for ds in dict_ds.values(): # (sample_interval, ds)
|
|
382
|
+
# Write L0C netCDF4 dataset
|
|
383
|
+
if ds["time"].size > 1:
|
|
384
|
+
# Get sensor name from dataset
|
|
385
|
+
sensor_name = ds.attrs.get("sensor_name")
|
|
386
|
+
campaign_name = ds.attrs.get("campaign_name")
|
|
387
|
+
station_name = ds.attrs.get("station_name")
|
|
388
|
+
|
|
389
|
+
# Set encodings
|
|
390
|
+
ds = set_l0b_encodings(ds=ds, sensor_name=sensor_name)
|
|
391
|
+
|
|
392
|
+
# Define filepath
|
|
393
|
+
filename = define_l0c_filename(ds, campaign_name=campaign_name, station_name=station_name)
|
|
394
|
+
folder_path = define_file_folder_path(ds, data_dir=data_dir, folder_partitioning=folder_partitioning)
|
|
395
|
+
filepath = os.path.join(folder_path, filename)
|
|
396
|
+
|
|
397
|
+
# Write to disk
|
|
398
|
+
write_product(ds, product=product, filepath=filepath, force=force)
|
|
399
|
+
|
|
400
|
+
# Clean environment
|
|
401
|
+
del ds
|
|
402
|
+
|
|
403
|
+
# Log end processing
|
|
404
|
+
msg = f"{product} processing for {day} has ended."
|
|
405
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
406
|
+
|
|
407
|
+
##--------------------------------------------------------------------.
|
|
408
|
+
# Otherwise log the error
|
|
409
|
+
except Exception as e:
|
|
410
|
+
error_type = str(type(e).__name__)
|
|
411
|
+
msg = f"{error_type}: {e}"
|
|
412
|
+
log_error(logger, msg, verbose=verbose)
|
|
413
|
+
|
|
414
|
+
# Close the file logger
|
|
415
|
+
close_logger(logger)
|
|
416
|
+
|
|
417
|
+
# Return the logger file path
|
|
418
|
+
return logger_filepath
|
|
292
419
|
|
|
293
420
|
|
|
294
421
|
####--------------------------------------------------------------------------.
|
|
295
|
-
####
|
|
422
|
+
#### DISDRODB Station Functions
|
|
296
423
|
|
|
297
424
|
|
|
298
|
-
def
|
|
425
|
+
def run_l0a_station(
|
|
426
|
+
# Station arguments
|
|
299
427
|
data_source,
|
|
300
428
|
campaign_name,
|
|
301
429
|
station_name,
|
|
302
|
-
# L0 archive options
|
|
303
|
-
l0a_processing: bool = True,
|
|
304
|
-
l0b_processing: bool = True,
|
|
305
|
-
l0b_concat: bool = True,
|
|
306
|
-
remove_l0a: bool = False,
|
|
307
|
-
remove_l0b: bool = False,
|
|
308
430
|
# Processing options
|
|
309
431
|
force: bool = False,
|
|
310
432
|
verbose: bool = False,
|
|
311
433
|
debugging_mode: bool = False,
|
|
312
434
|
parallel: bool = True,
|
|
313
|
-
|
|
435
|
+
# DISDRODB root directories
|
|
436
|
+
data_archive_dir: Optional[str] = None,
|
|
437
|
+
metadata_archive_dir: Optional[str] = None,
|
|
314
438
|
):
|
|
315
|
-
"""
|
|
439
|
+
"""
|
|
440
|
+
Run the L0A processing of a specific DISDRODB station when invoked from the terminal.
|
|
441
|
+
|
|
442
|
+
This function is intended to be called through the ``disdrodb_run_l0a_station``
|
|
443
|
+
command-line interface.
|
|
316
444
|
|
|
317
445
|
Parameters
|
|
318
446
|
----------
|
|
319
447
|
data_source : str
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
Must be UPPER CASE.
|
|
448
|
+
The name of the institution (for campaigns spanning multiple countries) or
|
|
449
|
+
the name of the country (for campaigns or sensor networks within a single country).
|
|
450
|
+
Must be provided in UPPER CASE.
|
|
323
451
|
campaign_name : str
|
|
324
|
-
|
|
452
|
+
The name of the campaign. Must be provided in UPPER CASE.
|
|
325
453
|
station_name : str
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
If
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
The default is False
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
The default is False.
|
|
345
|
-
force : bool
|
|
346
|
-
If True, overwrite existing data into destination directories.
|
|
347
|
-
If False, raise an error if there are already data into destination directories.
|
|
348
|
-
The default is False.
|
|
349
|
-
verbose : bool
|
|
350
|
-
Whether to print detailed processing information into terminal.
|
|
351
|
-
The default is True.
|
|
352
|
-
parallel : bool
|
|
353
|
-
If True, the files are processed simultaneously in multiple processes.
|
|
354
|
-
Each process will use a single thread to avoid issues with the HDF/netCDF library.
|
|
355
|
-
By default, the number of process is defined with os.cpu_count().
|
|
356
|
-
If False, the files are processed sequentially in a single process.
|
|
357
|
-
If False, multi-threading is automatically exploited to speed up I/0 tasks.
|
|
358
|
-
debugging_mode : bool
|
|
359
|
-
If True, it reduces the amount of data to process.
|
|
360
|
-
For L0A, it processes just the first 3 raw data files for each station.
|
|
361
|
-
For L0B, it processes just the first 100 rows of 3 L0A files for each station.
|
|
362
|
-
The default is False.
|
|
363
|
-
base_dir : str (optional)
|
|
364
|
-
Base directory of DISDRODB. Format: <...>/DISDRODB
|
|
365
|
-
If None (the default), the disdrodb config variable 'dir' is used.
|
|
454
|
+
The name of the station.
|
|
455
|
+
force : bool, optional
|
|
456
|
+
If ``True``, existing data in the destination directories will be overwritten.
|
|
457
|
+
If ``False`` (default), an error will be raised if data already exists in the destination directories.
|
|
458
|
+
verbose : bool, optional
|
|
459
|
+
If ``True`` (default), detailed processing information will be printed to the terminal.
|
|
460
|
+
If ``False``, less information will be displayed.
|
|
461
|
+
parallel : bool, optional
|
|
462
|
+
If ``True``, files will be processed in multiple processes simultaneously
|
|
463
|
+
with each process using a single thread.
|
|
464
|
+
If ``False`` (default), files will be processed sequentially in a single process,
|
|
465
|
+
and multi-threading will be automatically exploited to speed up I/O tasks.
|
|
466
|
+
debugging_mode : bool, optional
|
|
467
|
+
If ``True``, the amount of data processed will be reduced.
|
|
468
|
+
Only the first 3 raw data files will be processed. The default value is ``False``.
|
|
469
|
+
data_archive_dir : str, optional
|
|
470
|
+
The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``.
|
|
471
|
+
If not specified, the path specified in the DISDRODB active configuration will be used.
|
|
366
472
|
"""
|
|
473
|
+
# Retrieve DISDRODB Metadata Archive and Data Archive root directories
|
|
474
|
+
data_archive_dir = get_data_archive_dir(data_archive_dir)
|
|
475
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
476
|
+
|
|
477
|
+
# Read metadata
|
|
478
|
+
metadata = read_station_metadata(
|
|
479
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
480
|
+
data_source=data_source,
|
|
481
|
+
campaign_name=campaign_name,
|
|
482
|
+
station_name=station_name,
|
|
483
|
+
)
|
|
367
484
|
|
|
368
|
-
#
|
|
485
|
+
# ------------------------------------------------------------------------.
|
|
486
|
+
# Define raw data ingestion chain
|
|
487
|
+
# --> If raw data are netCDF files, this routine produces directly L0B files
|
|
488
|
+
# --> Otherwise, it produces L0A files.
|
|
489
|
+
if metadata["raw_data_format"] == "netcdf":
|
|
490
|
+
generate_standardized_files = _generate_l0b_from_nc
|
|
491
|
+
product = "L0B"
|
|
492
|
+
else:
|
|
493
|
+
generate_standardized_files = _generate_l0a
|
|
494
|
+
product = "L0A"
|
|
495
|
+
|
|
496
|
+
# ------------------------------------------------------------------------.
|
|
497
|
+
# Start product processing
|
|
369
498
|
t_i = time.time()
|
|
370
|
-
msg = f"
|
|
499
|
+
msg = f"{product} processing of station {station_name} has started."
|
|
371
500
|
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
372
501
|
|
|
373
|
-
# ------------------------------------------------------------------.
|
|
374
|
-
# L0A processing
|
|
375
|
-
if l0a_processing:
|
|
376
|
-
run_disdrodb_l0a_station(
|
|
377
|
-
# Station arguments
|
|
378
|
-
base_dir=base_dir,
|
|
379
|
-
data_source=data_source,
|
|
380
|
-
campaign_name=campaign_name,
|
|
381
|
-
station_name=station_name,
|
|
382
|
-
# Processing options
|
|
383
|
-
force=force,
|
|
384
|
-
verbose=verbose,
|
|
385
|
-
debugging_mode=debugging_mode,
|
|
386
|
-
parallel=parallel,
|
|
387
|
-
)
|
|
388
|
-
# ------------------------------------------------------------------.
|
|
389
|
-
# L0B processing
|
|
390
|
-
if l0b_processing:
|
|
391
|
-
run_disdrodb_l0b_station(
|
|
392
|
-
# Station arguments
|
|
393
|
-
base_dir=base_dir,
|
|
394
|
-
data_source=data_source,
|
|
395
|
-
campaign_name=campaign_name,
|
|
396
|
-
station_name=station_name,
|
|
397
|
-
# Processing options
|
|
398
|
-
force=force,
|
|
399
|
-
verbose=verbose,
|
|
400
|
-
debugging_mode=debugging_mode,
|
|
401
|
-
parallel=parallel,
|
|
402
|
-
remove_l0a=remove_l0a,
|
|
403
|
-
)
|
|
404
|
-
|
|
405
502
|
# ------------------------------------------------------------------------.
|
|
406
|
-
#
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
503
|
+
# Create directory structure
|
|
504
|
+
data_dir = create_l0_directory_structure(
|
|
505
|
+
data_archive_dir=data_archive_dir,
|
|
506
|
+
data_source=data_source,
|
|
507
|
+
campaign_name=campaign_name,
|
|
508
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
509
|
+
product=product, # L0A or L0B
|
|
510
|
+
station_name=station_name,
|
|
511
|
+
force=force,
|
|
512
|
+
)
|
|
416
513
|
|
|
417
514
|
# -------------------------------------------------------------------------.
|
|
418
|
-
#
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
515
|
+
# Define logs directory
|
|
516
|
+
logs_dir = create_logs_directory(
|
|
517
|
+
product=product,
|
|
518
|
+
data_archive_dir=data_archive_dir,
|
|
519
|
+
data_source=data_source,
|
|
520
|
+
campaign_name=campaign_name,
|
|
521
|
+
station_name=station_name,
|
|
522
|
+
)
|
|
423
523
|
|
|
524
|
+
# -----------------------------------------------------------------.
|
|
525
|
+
# Read issue YAML file
|
|
526
|
+
issue_dict = read_station_issue(
|
|
527
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
528
|
+
data_source=data_source,
|
|
529
|
+
campaign_name=campaign_name,
|
|
530
|
+
station_name=station_name,
|
|
531
|
+
)
|
|
424
532
|
|
|
425
|
-
|
|
426
|
-
|
|
533
|
+
##------------------------------------------------------------------------.
|
|
534
|
+
# Retrieve sensor name
|
|
535
|
+
sensor_name = metadata["sensor_name"]
|
|
536
|
+
check_sensor_name(sensor_name)
|
|
427
537
|
|
|
538
|
+
# Retrieve sensor name
|
|
539
|
+
reader_reference = metadata["reader"]
|
|
428
540
|
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
if len(list_info) == 0:
|
|
432
|
-
msg = "No stations available given the provided `data_sources` and `campaign_names` arguments !"
|
|
433
|
-
raise ValueError(msg)
|
|
541
|
+
# Retrieve glob patterns
|
|
542
|
+
glob_pattern = metadata["raw_data_glob_pattern"]
|
|
434
543
|
|
|
544
|
+
##------------------------------------------------------------------------.
|
|
545
|
+
# Retrieve reader
|
|
546
|
+
reader = get_reader(reader_reference, sensor_name=sensor_name)
|
|
435
547
|
|
|
436
|
-
|
|
437
|
-
#
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
548
|
+
# -------------------------------------------------------------------------.
|
|
549
|
+
# List files to process
|
|
550
|
+
filepaths = find_files(
|
|
551
|
+
data_source=data_source,
|
|
552
|
+
campaign_name=campaign_name,
|
|
553
|
+
station_name=station_name,
|
|
554
|
+
product="RAW",
|
|
555
|
+
debugging_mode=debugging_mode,
|
|
556
|
+
data_archive_dir=data_archive_dir,
|
|
557
|
+
glob_pattern=glob_pattern,
|
|
558
|
+
)
|
|
444
559
|
|
|
560
|
+
# Print the number of files to be processed
|
|
561
|
+
n_files = len(filepaths)
|
|
562
|
+
msg = f"{n_files} raw files are ready to be processed."
|
|
563
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
445
564
|
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
565
|
+
# -----------------------------------------------------------------.
|
|
566
|
+
# Generate L0A/L0B files
|
|
567
|
+
# - Loop over the files and save the L0A Apache Parquet files.
|
|
568
|
+
# - If parallel=True, it does that in parallel using dask.delayed
|
|
569
|
+
list_tasks = [
|
|
570
|
+
generate_standardized_files(
|
|
571
|
+
filepath=filepath,
|
|
572
|
+
data_dir=data_dir,
|
|
573
|
+
logs_dir=logs_dir,
|
|
574
|
+
campaign_name=campaign_name,
|
|
575
|
+
station_name=station_name,
|
|
576
|
+
# Reader argument
|
|
577
|
+
reader=reader,
|
|
578
|
+
# Processing info
|
|
579
|
+
metadata=metadata,
|
|
580
|
+
issue_dict=issue_dict,
|
|
581
|
+
# Processing options
|
|
582
|
+
force=force,
|
|
583
|
+
verbose=verbose,
|
|
584
|
+
parallel=parallel,
|
|
585
|
+
)
|
|
586
|
+
for filepath in filepaths
|
|
587
|
+
]
|
|
588
|
+
list_logs = dask.compute(*list_tasks) if parallel else list_tasks
|
|
589
|
+
|
|
590
|
+
# -----------------------------------------------------------------.
|
|
591
|
+
# Define product summary logs
|
|
592
|
+
create_product_logs(
|
|
593
|
+
product=product,
|
|
594
|
+
data_source=data_source,
|
|
595
|
+
campaign_name=campaign_name,
|
|
596
|
+
station_name=station_name,
|
|
597
|
+
data_archive_dir=data_archive_dir,
|
|
598
|
+
# Logs list
|
|
599
|
+
list_logs=list_logs,
|
|
600
|
+
)
|
|
601
|
+
|
|
602
|
+
# ---------------------------------------------------------------------.
|
|
603
|
+
# End product processing
|
|
604
|
+
timedelta_str = str(datetime.timedelta(seconds=round(time.time() - t_i)))
|
|
605
|
+
msg = f"{product} processing of station {station_name} completed in {timedelta_str}"
|
|
606
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
607
|
+
# ---------------------------------------------------------------------.
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def run_l0b_station(
|
|
611
|
+
# Station arguments
|
|
612
|
+
data_source,
|
|
613
|
+
campaign_name,
|
|
614
|
+
station_name,
|
|
615
|
+
# L0B processing options
|
|
464
616
|
remove_l0a: bool = False,
|
|
465
|
-
remove_l0b: bool = False,
|
|
466
617
|
# Processing options
|
|
467
618
|
force: bool = False,
|
|
468
|
-
verbose: bool =
|
|
469
|
-
debugging_mode: bool = False,
|
|
619
|
+
verbose: bool = True,
|
|
470
620
|
parallel: bool = True,
|
|
471
|
-
|
|
621
|
+
debugging_mode: bool = False,
|
|
622
|
+
# DISDRODB root directories
|
|
623
|
+
data_archive_dir: Optional[str] = None,
|
|
624
|
+
metadata_archive_dir: Optional[str] = None,
|
|
472
625
|
):
|
|
473
|
-
"""
|
|
626
|
+
"""
|
|
627
|
+
Run the L0B processing of a specific DISDRODB station when invoked from the terminal.
|
|
474
628
|
|
|
475
|
-
This function
|
|
476
|
-
|
|
477
|
-
stations matching the provided data_sources, campaign_names and station_names.
|
|
629
|
+
This function is intended to be called through the ``disdrodb_run_l0b_station``
|
|
630
|
+
command-line interface.
|
|
478
631
|
|
|
479
632
|
Parameters
|
|
480
633
|
----------
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
The name
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
If
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
The
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
It takes places only if l0b_concat = True
|
|
510
|
-
The default is False.
|
|
511
|
-
force : bool
|
|
512
|
-
If True, overwrite existing data into destination directories.
|
|
513
|
-
If False, raise an error if there are already data into destination directories.
|
|
514
|
-
The default is False.
|
|
515
|
-
verbose : bool
|
|
516
|
-
Whether to print detailed processing information into terminal.
|
|
517
|
-
The default is True.
|
|
518
|
-
parallel : bool
|
|
519
|
-
If True, the files are processed simultaneously in multiple processes.
|
|
520
|
-
Each process will use a single thread to avoid issues with the HDF/netCDF library.
|
|
521
|
-
By default, the number of process is defined with os.cpu_count().
|
|
522
|
-
If False, the files are processed sequentially in a single process.
|
|
523
|
-
If False, multi-threading is automatically exploited to speed up I/0 tasks.
|
|
524
|
-
debugging_mode : bool
|
|
525
|
-
If True, it reduces the amount of data to process.
|
|
526
|
-
For L0A, it processes just the first 3 raw data files.
|
|
527
|
-
For L0B, it processes just the first 100 rows of 3 L0A files.
|
|
528
|
-
The default is False.
|
|
529
|
-
base_dir : str (optional)
|
|
530
|
-
Base directory of DISDRODB. Format: <...>/DISDRODB
|
|
531
|
-
If None (the default), the disdrodb config variable 'dir' is used.
|
|
634
|
+
data_source : str
|
|
635
|
+
The name of the institution (for campaigns spanning multiple countries) or
|
|
636
|
+
the name of the country (for campaigns or sensor networks within a single country).
|
|
637
|
+
Must be provided in UPPER CASE.
|
|
638
|
+
campaign_name : str
|
|
639
|
+
The name of the campaign. Must be provided in UPPER CASE.
|
|
640
|
+
station_name : str
|
|
641
|
+
The name of the station.
|
|
642
|
+
force : bool, optional
|
|
643
|
+
If ``True``, existing data in the destination directories will be overwritten.
|
|
644
|
+
If ``False`` (default), an error will be raised if data already exists in the destination directories.
|
|
645
|
+
verbose : bool, optional
|
|
646
|
+
If ``True`` (default), detailed processing information will be printed to the terminal.
|
|
647
|
+
If ``False``, less information will be displayed.
|
|
648
|
+
parallel : bool, optional
|
|
649
|
+
If ``True``, files will be processed in multiple processes simultaneously,
|
|
650
|
+
with each process using a single thread to avoid issues with the HDF/netCDF library.
|
|
651
|
+
If ``False`` (default), files will be processed sequentially in a single process,
|
|
652
|
+
and multi-threading will be automatically exploited to speed up I/O tasks.
|
|
653
|
+
debugging_mode : bool, optional
|
|
654
|
+
If ``True``, the amount of data processed will be reduced.
|
|
655
|
+
Only the first 100 rows of 3 L0A files will be processed. The default value is ``False``.
|
|
656
|
+
remove_l0a: bool, optional
|
|
657
|
+
Whether to remove the processed L0A files. The default value is ``False``.
|
|
658
|
+
data_archive_dir : str, optional
|
|
659
|
+
The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``.
|
|
660
|
+
If not specified, the path specified in the DISDRODB active configuration will be used.
|
|
661
|
+
|
|
532
662
|
"""
|
|
533
|
-
|
|
663
|
+
# Define product
|
|
664
|
+
product = "L0B"
|
|
665
|
+
|
|
666
|
+
# Retrieve DISDRODB base directory
|
|
667
|
+
data_archive_dir = get_data_archive_dir(data_archive_dir)
|
|
668
|
+
|
|
669
|
+
# Retrieve DISDRODB Metadata Archive directory
|
|
670
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
671
|
+
|
|
672
|
+
# -----------------------------------------------------------------.
|
|
673
|
+
# Retrieve metadata
|
|
674
|
+
metadata = read_station_metadata(
|
|
675
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
676
|
+
data_source=data_source,
|
|
677
|
+
campaign_name=campaign_name,
|
|
678
|
+
station_name=station_name,
|
|
679
|
+
)
|
|
680
|
+
|
|
681
|
+
# Skip run_l0b processing if the raw data are netCDFs
|
|
682
|
+
# - L0B produced when running L0A ...
|
|
683
|
+
if metadata["raw_data_format"] == "netcdf":
|
|
684
|
+
return
|
|
685
|
+
|
|
686
|
+
# -----------------------------------------------------------------.
|
|
687
|
+
# Start L0B processing
|
|
688
|
+
t_i = time.time()
|
|
689
|
+
msg = f"{product} processing of station_name {station_name} has started."
|
|
690
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
534
691
|
|
|
535
|
-
#
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
base_dir=base_dir,
|
|
692
|
+
# -----------------------------------------------------------------.
|
|
693
|
+
# Define logs directory
|
|
694
|
+
logs_dir = create_logs_directory(
|
|
539
695
|
product=product,
|
|
540
|
-
|
|
541
|
-
|
|
696
|
+
data_archive_dir=data_archive_dir,
|
|
697
|
+
data_source=data_source,
|
|
698
|
+
campaign_name=campaign_name,
|
|
699
|
+
station_name=station_name,
|
|
542
700
|
)
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
701
|
+
|
|
702
|
+
# Create product directory
|
|
703
|
+
data_dir = create_product_directory(
|
|
704
|
+
data_archive_dir=data_archive_dir,
|
|
705
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
706
|
+
data_source=data_source,
|
|
707
|
+
campaign_name=campaign_name,
|
|
708
|
+
station_name=station_name,
|
|
709
|
+
product=product,
|
|
710
|
+
force=force,
|
|
711
|
+
)
|
|
712
|
+
|
|
713
|
+
##----------------------------------------------------------------.
|
|
714
|
+
# Get L0A files for the station
|
|
715
|
+
required_product = get_required_product(product)
|
|
716
|
+
flag_not_available_data = False
|
|
717
|
+
try:
|
|
718
|
+
filepaths = find_files(
|
|
719
|
+
data_archive_dir=data_archive_dir,
|
|
556
720
|
data_source=data_source,
|
|
557
721
|
campaign_name=campaign_name,
|
|
558
722
|
station_name=station_name,
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
723
|
+
product=required_product,
|
|
724
|
+
debugging_mode=debugging_mode,
|
|
725
|
+
)
|
|
726
|
+
except Exception as e:
|
|
727
|
+
print(str(e)) # Case where no file paths available
|
|
728
|
+
flag_not_available_data = True
|
|
729
|
+
|
|
730
|
+
# -------------------------------------------------------------------------.
|
|
731
|
+
# If no data available, print error message and return None
|
|
732
|
+
if flag_not_available_data:
|
|
733
|
+
msg = (
|
|
734
|
+
f"{product} processing of {data_source} {campaign_name} {station_name}"
|
|
735
|
+
+ f"has not been launched because of missing {required_product} data."
|
|
736
|
+
)
|
|
737
|
+
print(msg)
|
|
738
|
+
return
|
|
739
|
+
|
|
740
|
+
##----------------------------------------------------------------.
|
|
741
|
+
# Generate L0B files
|
|
742
|
+
# Loop over the L0A files and save the L0B netCDF files.
|
|
743
|
+
# - If parallel=True, it does that in parallel using dask.bag
|
|
744
|
+
# Settings npartitions=len(filepaths) enable to wait prior task on a core
|
|
745
|
+
# finish before starting a new one.
|
|
746
|
+
# BUG: If debugging_mode=True and parallel=True a subtle bug can currently occur when
|
|
747
|
+
# two processes with a subsetted L0A files want to create the same L0B files !
|
|
748
|
+
list_tasks = [
|
|
749
|
+
_generate_l0b(
|
|
750
|
+
filepath=filepath,
|
|
751
|
+
data_dir=data_dir,
|
|
752
|
+
logs_dir=logs_dir,
|
|
753
|
+
metadata=metadata,
|
|
754
|
+
campaign_name=campaign_name,
|
|
755
|
+
station_name=station_name,
|
|
566
756
|
force=force,
|
|
567
757
|
verbose=verbose,
|
|
568
758
|
debugging_mode=debugging_mode,
|
|
569
759
|
parallel=parallel,
|
|
570
760
|
)
|
|
571
|
-
|
|
761
|
+
for filepath in filepaths
|
|
762
|
+
]
|
|
763
|
+
list_logs = dask.compute(*list_tasks) if parallel else list_tasks
|
|
764
|
+
# if not parallel:
|
|
765
|
+
# list_logs = [
|
|
766
|
+
# _generate_l0b(
|
|
767
|
+
# filepath=filepath,
|
|
768
|
+
# data_dir=data_dir,
|
|
769
|
+
# logs_dir=logs_dir,
|
|
770
|
+
# metadata=metadata,
|
|
771
|
+
# campaign_name=campaign_name,
|
|
772
|
+
# station_name=station_name,
|
|
773
|
+
# force=force,
|
|
774
|
+
# verbose=verbose,
|
|
775
|
+
# debugging_mode=debugging_mode,
|
|
776
|
+
# parallel=parallel,
|
|
777
|
+
# )
|
|
778
|
+
# for filepath in filepaths
|
|
779
|
+
# ]
|
|
780
|
+
|
|
781
|
+
# else:
|
|
782
|
+
# bag = db.from_sequence(filepaths, npartitions=len(filepaths))
|
|
783
|
+
# list_logs = bag.map(
|
|
784
|
+
# _generate_l0b,
|
|
785
|
+
# data_dir=data_dir,
|
|
786
|
+
# logs_dir=logs_dir,
|
|
787
|
+
# metadata=metadata,
|
|
788
|
+
# campaign_name=campaign_name,
|
|
789
|
+
# station_name=station_name,
|
|
790
|
+
# force=force,
|
|
791
|
+
# verbose=verbose,
|
|
792
|
+
# debugging_mode=debugging_mode,
|
|
793
|
+
# parallel=parallel,
|
|
794
|
+
# ).compute()
|
|
795
|
+
|
|
796
|
+
# -----------------------------------------------------------------.
|
|
797
|
+
# Define L0B summary logs
|
|
798
|
+
create_product_logs(
|
|
799
|
+
product=product,
|
|
800
|
+
data_source=data_source,
|
|
801
|
+
campaign_name=campaign_name,
|
|
802
|
+
station_name=station_name,
|
|
803
|
+
data_archive_dir=data_archive_dir,
|
|
804
|
+
# Logs list
|
|
805
|
+
list_logs=list_logs,
|
|
806
|
+
)
|
|
807
|
+
|
|
808
|
+
# -----------------------------------------------------------------.
|
|
809
|
+
# End L0B processing
|
|
810
|
+
timedelta_str = str(datetime.timedelta(seconds=round(time.time() - t_i)))
|
|
811
|
+
msg = f"{product} processing of station_name {station_name} completed in {timedelta_str}"
|
|
812
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
813
|
+
|
|
814
|
+
# -----------------------------------------------------------------.
|
|
815
|
+
# Option to remove L0A
|
|
816
|
+
if remove_l0a:
|
|
817
|
+
remove_product(
|
|
818
|
+
data_archive_dir=data_archive_dir,
|
|
819
|
+
product="L0A",
|
|
820
|
+
data_source=data_source,
|
|
821
|
+
campaign_name=campaign_name,
|
|
822
|
+
station_name=station_name,
|
|
823
|
+
logger=logger,
|
|
824
|
+
verbose=verbose,
|
|
825
|
+
)
|
|
572
826
|
|
|
573
827
|
|
|
574
|
-
def
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
828
|
+
def run_l0c_station(
|
|
829
|
+
# Station arguments
|
|
830
|
+
data_source,
|
|
831
|
+
campaign_name,
|
|
832
|
+
station_name,
|
|
833
|
+
# L0C processing options
|
|
834
|
+
remove_l0b: bool = False,
|
|
578
835
|
# Processing options
|
|
579
836
|
force: bool = False,
|
|
580
|
-
verbose: bool =
|
|
581
|
-
debugging_mode: bool = False,
|
|
837
|
+
verbose: bool = True,
|
|
582
838
|
parallel: bool = True,
|
|
583
|
-
|
|
839
|
+
debugging_mode: bool = False,
|
|
840
|
+
# DISDRODB root directories
|
|
841
|
+
data_archive_dir: Optional[str] = None,
|
|
842
|
+
metadata_archive_dir: Optional[str] = None,
|
|
584
843
|
):
|
|
585
|
-
"""
|
|
844
|
+
"""
|
|
845
|
+
Run the L0C processing of a specific DISDRODB station when invoked from the terminal.
|
|
846
|
+
|
|
847
|
+
The DISDRODB L0A and L0B routines just convert source raw data into netCDF format.
|
|
848
|
+
The DISDRODB L0C routine ingests L0B files and performs data homogenization.
|
|
849
|
+
The DISDRODB L0C routine takes care of:
|
|
850
|
+
|
|
851
|
+
- removing duplicated timesteps across files,
|
|
852
|
+
- merging/splitting files into daily files,
|
|
853
|
+
- regularizing timesteps for potentially trailing seconds,
|
|
854
|
+
- ensuring L0C files with unique sample intervals.
|
|
586
855
|
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
856
|
+
Duplicated timesteps are automatically dropped if their variable values coincides,
|
|
857
|
+
otherwise an error is raised.
|
|
858
|
+
|
|
859
|
+
This function is intended to be called through the ``disdrodb_run_l0c_station``
|
|
860
|
+
command-line interface.
|
|
590
861
|
|
|
591
862
|
Parameters
|
|
592
863
|
----------
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
The name
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
If
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
If True
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
Base directory of DISDRODB. Format: <...>/DISDRODB
|
|
622
|
-
If None (the default), the disdrodb config variable 'dir' is used.
|
|
864
|
+
data_source : str
|
|
865
|
+
The name of the institution (for campaigns spanning multiple countries) or
|
|
866
|
+
the name of the country (for campaigns or sensor networks within a single country).
|
|
867
|
+
Must be provided in UPPER CASE.
|
|
868
|
+
campaign_name : str
|
|
869
|
+
The name of the campaign. Must be provided in UPPER CASE.
|
|
870
|
+
station_name : str
|
|
871
|
+
The name of the station.
|
|
872
|
+
force : bool, optional
|
|
873
|
+
If ``True``, existing data in the destination directories will be overwritten.
|
|
874
|
+
If ``False`` (default), an error will be raised if data already exists in the destination directories.
|
|
875
|
+
verbose : bool, optional
|
|
876
|
+
If ``True`` (default), detailed processing information will be printed to the terminal.
|
|
877
|
+
If ``False``, less information will be displayed.
|
|
878
|
+
parallel : bool, optional
|
|
879
|
+
If ``True``, files will be processed in multiple processes simultaneously,
|
|
880
|
+
with each process using a single thread to avoid issues with the HDF/netCDF library.
|
|
881
|
+
If ``False`` (default), files will be processed sequentially in a single process,
|
|
882
|
+
and multi-threading will be automatically exploited to speed up I/O tasks.
|
|
883
|
+
debugging_mode : bool, optional
|
|
884
|
+
If ``True``, the amount of data processed will be reduced.
|
|
885
|
+
Only the first 3 files will be processed. The default value is ``False``.
|
|
886
|
+
remove_l0b: bool, optional
|
|
887
|
+
Whether to remove the processed L0B files. The default value is ``False``.
|
|
888
|
+
data_archive_dir : str, optional
|
|
889
|
+
The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``.
|
|
890
|
+
If not specified, the path specified in the DISDRODB active configuration will be used.
|
|
891
|
+
|
|
623
892
|
"""
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
data_sources=data_sources,
|
|
627
|
-
campaign_names=campaign_names,
|
|
628
|
-
station_names=station_names,
|
|
629
|
-
# L0 archive options
|
|
630
|
-
l0a_processing=True,
|
|
631
|
-
l0b_processing=False,
|
|
632
|
-
l0b_concat=False,
|
|
633
|
-
remove_l0a=False,
|
|
634
|
-
remove_l0b=False,
|
|
635
|
-
# Processing options
|
|
636
|
-
force=force,
|
|
637
|
-
verbose=verbose,
|
|
638
|
-
debugging_mode=debugging_mode,
|
|
639
|
-
parallel=parallel,
|
|
640
|
-
)
|
|
893
|
+
# Define product
|
|
894
|
+
product = "L0C"
|
|
641
895
|
|
|
896
|
+
# Define base directory
|
|
897
|
+
data_archive_dir = get_data_archive_dir(data_archive_dir)
|
|
642
898
|
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
campaign_names=None,
|
|
646
|
-
station_names=None,
|
|
647
|
-
# Processing options
|
|
648
|
-
force: bool = False,
|
|
649
|
-
verbose: bool = False,
|
|
650
|
-
debugging_mode: bool = False,
|
|
651
|
-
parallel: bool = True,
|
|
652
|
-
base_dir: str = None,
|
|
653
|
-
remove_l0a: bool = False,
|
|
654
|
-
):
|
|
655
|
-
run_disdrodb_l0(
|
|
656
|
-
base_dir=base_dir,
|
|
657
|
-
data_sources=data_sources,
|
|
658
|
-
campaign_names=campaign_names,
|
|
659
|
-
station_names=station_names,
|
|
660
|
-
# L0 archive options
|
|
661
|
-
l0a_processing=False,
|
|
662
|
-
l0b_processing=True,
|
|
663
|
-
l0b_concat=False,
|
|
664
|
-
remove_l0a=remove_l0a,
|
|
665
|
-
remove_l0b=False,
|
|
666
|
-
# Processing options
|
|
667
|
-
force=force,
|
|
668
|
-
verbose=verbose,
|
|
669
|
-
debugging_mode=debugging_mode,
|
|
670
|
-
parallel=parallel,
|
|
671
|
-
)
|
|
899
|
+
# Retrieve DISDRODB Metadata Archive directory
|
|
900
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
672
901
|
|
|
902
|
+
# ------------------------------------------------------------------------.
|
|
903
|
+
# Start processing
|
|
904
|
+
t_i = time.time()
|
|
905
|
+
msg = f"{product} processing of station {station_name} has started."
|
|
906
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
673
907
|
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
)
|
|
683
|
-
"""Concatenate the L0B files of the DISDRODB archive.
|
|
908
|
+
# ------------------------------------------------------------------------.
|
|
909
|
+
# Define logs directory
|
|
910
|
+
logs_dir = create_logs_directory(
|
|
911
|
+
product=product,
|
|
912
|
+
data_archive_dir=data_archive_dir,
|
|
913
|
+
data_source=data_source,
|
|
914
|
+
campaign_name=campaign_name,
|
|
915
|
+
station_name=station_name,
|
|
916
|
+
)
|
|
684
917
|
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
918
|
+
# Create product directory
|
|
919
|
+
data_dir = create_product_directory(
|
|
920
|
+
data_archive_dir=data_archive_dir,
|
|
921
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
922
|
+
data_source=data_source,
|
|
923
|
+
campaign_name=campaign_name,
|
|
924
|
+
station_name=station_name,
|
|
925
|
+
product=product,
|
|
926
|
+
force=force,
|
|
927
|
+
)
|
|
688
928
|
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
929
|
+
# ------------------------------------------------------------------------.
|
|
930
|
+
# Define metadata filepath
|
|
931
|
+
metadata_filepath = define_metadata_filepath(
|
|
932
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
933
|
+
data_source=data_source,
|
|
934
|
+
campaign_name=campaign_name,
|
|
935
|
+
station_name=station_name,
|
|
694
936
|
)
|
|
695
937
|
|
|
696
|
-
|
|
697
|
-
|
|
938
|
+
# -------------------------------------------------------------------------.
|
|
939
|
+
# List files to process
|
|
940
|
+
required_product = get_required_product(product)
|
|
941
|
+
flag_not_available_data = False
|
|
942
|
+
try:
|
|
943
|
+
filepaths = find_files(
|
|
944
|
+
data_archive_dir=data_archive_dir,
|
|
945
|
+
data_source=data_source,
|
|
946
|
+
campaign_name=campaign_name,
|
|
947
|
+
station_name=station_name,
|
|
948
|
+
product=required_product,
|
|
949
|
+
# Processing options
|
|
950
|
+
debugging_mode=debugging_mode,
|
|
951
|
+
)
|
|
952
|
+
except Exception as e:
|
|
953
|
+
print(str(e)) # Case where no file paths available
|
|
954
|
+
flag_not_available_data = True
|
|
698
955
|
|
|
699
|
-
#
|
|
700
|
-
|
|
701
|
-
|
|
956
|
+
# -------------------------------------------------------------------------.
|
|
957
|
+
# If no data available, print error message and return None
|
|
958
|
+
if flag_not_available_data:
|
|
959
|
+
msg = (
|
|
960
|
+
f"{product} processing of {data_source} {campaign_name} {station_name}"
|
|
961
|
+
+ f"has not been launched because of missing {required_product} data."
|
|
962
|
+
)
|
|
963
|
+
print(msg)
|
|
964
|
+
return
|
|
702
965
|
|
|
703
|
-
#
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
966
|
+
# -------------------------------------------------------------------------.
|
|
967
|
+
# Retrieve dictionary with the required files for each day.
|
|
968
|
+
dict_days_files = get_files_per_days(filepaths)
|
|
969
|
+
|
|
970
|
+
# -----------------------------------------------------------------.
|
|
971
|
+
# Generate L0C files
|
|
972
|
+
# - Loop over the L0 netCDF files and generate L1 files.
|
|
973
|
+
# - If parallel=True, it does that in parallel using dask.delayed
|
|
974
|
+
list_tasks = [
|
|
975
|
+
_generate_l0c(
|
|
976
|
+
day=day,
|
|
977
|
+
filepaths=filepaths,
|
|
978
|
+
data_dir=data_dir,
|
|
979
|
+
logs_dir=logs_dir,
|
|
980
|
+
metadata_filepath=metadata_filepath,
|
|
709
981
|
campaign_name=campaign_name,
|
|
710
982
|
station_name=station_name,
|
|
711
|
-
|
|
983
|
+
# Processing options
|
|
984
|
+
force=force,
|
|
712
985
|
verbose=verbose,
|
|
986
|
+
parallel=parallel,
|
|
713
987
|
)
|
|
714
|
-
|
|
988
|
+
for day, filepaths in dict_days_files.items()
|
|
989
|
+
]
|
|
990
|
+
list_logs = dask.compute(*list_tasks) if parallel else list_tasks
|
|
991
|
+
|
|
992
|
+
# -----------------------------------------------------------------.
|
|
993
|
+
# Define summary logs
|
|
994
|
+
create_product_logs(
|
|
995
|
+
product=product,
|
|
996
|
+
data_source=data_source,
|
|
997
|
+
campaign_name=campaign_name,
|
|
998
|
+
station_name=station_name,
|
|
999
|
+
data_archive_dir=data_archive_dir,
|
|
1000
|
+
# Logs list
|
|
1001
|
+
list_logs=list_logs,
|
|
1002
|
+
)
|
|
715
1003
|
|
|
1004
|
+
# ---------------------------------------------------------------------.
|
|
1005
|
+
# End processing
|
|
1006
|
+
timedelta_str = str(datetime.timedelta(seconds=round(time.time() - t_i)))
|
|
1007
|
+
msg = f"{product} processing of station {station_name} completed in {timedelta_str}"
|
|
1008
|
+
log_info(logger=logger, msg=msg, verbose=verbose)
|
|
716
1009
|
|
|
717
|
-
|
|
1010
|
+
# -----------------------------------------------------------------.
|
|
1011
|
+
# Option to remove L0B
|
|
1012
|
+
if remove_l0b:
|
|
1013
|
+
remove_product(
|
|
1014
|
+
data_archive_dir=data_archive_dir,
|
|
1015
|
+
product="L0B",
|
|
1016
|
+
data_source=data_source,
|
|
1017
|
+
campaign_name=campaign_name,
|
|
1018
|
+
station_name=station_name,
|
|
1019
|
+
logger=logger,
|
|
1020
|
+
verbose=verbose,
|
|
1021
|
+
)
|