disdrodb 0.2.1__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +3 -1
- disdrodb/_config.py +2 -3
- disdrodb/_version.py +2 -2
- disdrodb/accessor/__init__.py +2 -1
- disdrodb/accessor/methods.py +10 -9
- disdrodb/api/checks.py +3 -7
- disdrodb/api/configs.py +1 -3
- disdrodb/api/create_directories.py +4 -6
- disdrodb/api/info.py +1 -3
- disdrodb/api/io.py +233 -32
- disdrodb/api/path.py +3 -7
- disdrodb/cli/disdrodb_check_metadata_archive.py +3 -2
- disdrodb/cli/disdrodb_check_products_options.py +45 -0
- disdrodb/cli/disdrodb_create_summary.py +54 -28
- disdrodb/cli/disdrodb_create_summary_station.py +41 -20
- disdrodb/cli/disdrodb_data_archive_directory.py +2 -3
- disdrodb/cli/disdrodb_download_archive.py +50 -30
- disdrodb/cli/disdrodb_download_metadata_archive.py +28 -16
- disdrodb/cli/disdrodb_download_station.py +58 -29
- disdrodb/cli/disdrodb_initialize_station.py +43 -23
- disdrodb/cli/disdrodb_metadata_archive_directory.py +2 -3
- disdrodb/cli/disdrodb_open_data_archive.py +17 -13
- disdrodb/cli/disdrodb_open_logs_directory.py +31 -21
- disdrodb/cli/disdrodb_open_metadata_archive.py +26 -13
- disdrodb/cli/disdrodb_open_metadata_directory.py +34 -23
- disdrodb/cli/disdrodb_open_product_directory.py +39 -23
- disdrodb/cli/disdrodb_open_readers_directory.py +2 -3
- disdrodb/cli/disdrodb_run.py +189 -0
- disdrodb/cli/disdrodb_run_l0.py +61 -70
- disdrodb/cli/disdrodb_run_l0_station.py +50 -55
- disdrodb/cli/disdrodb_run_l0a.py +53 -51
- disdrodb/cli/disdrodb_run_l0a_station.py +41 -40
- disdrodb/cli/disdrodb_run_l0b.py +51 -51
- disdrodb/cli/disdrodb_run_l0b_station.py +40 -39
- disdrodb/cli/disdrodb_run_l0c.py +56 -53
- disdrodb/cli/disdrodb_run_l0c_station.py +44 -41
- disdrodb/cli/disdrodb_run_l1.py +55 -51
- disdrodb/cli/disdrodb_run_l1_station.py +43 -40
- disdrodb/cli/disdrodb_run_l2e.py +56 -51
- disdrodb/cli/disdrodb_run_l2e_station.py +44 -40
- disdrodb/cli/disdrodb_run_l2m.py +55 -51
- disdrodb/cli/disdrodb_run_l2m_station.py +43 -40
- disdrodb/cli/disdrodb_run_station.py +184 -0
- disdrodb/cli/disdrodb_upload_archive.py +51 -42
- disdrodb/cli/disdrodb_upload_station.py +42 -36
- disdrodb/configs.py +20 -16
- disdrodb/constants.py +5 -2
- disdrodb/data_transfer/__init__.py +1 -3
- disdrodb/data_transfer/download_data.py +45 -61
- disdrodb/data_transfer/upload_data.py +7 -11
- disdrodb/data_transfer/zenodo.py +2 -4
- disdrodb/docs.py +1 -3
- disdrodb/etc/configs/attributes.yaml +52 -2
- disdrodb/etc/configs/encodings.yaml +45 -1
- disdrodb/etc/products/L0C/ODM470/global.yaml +5 -0
- disdrodb/etc/products/L0C/global.yaml +5 -0
- disdrodb/etc/products/L1/ODM470/global.yaml +6 -0
- disdrodb/etc/products/L1/global.yaml +0 -13
- disdrodb/etc/products/L2E/LPM/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/LPM/global.yaml +36 -0
- disdrodb/etc/products/L2E/LPM_V0/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/LPM_V0/global.yaml +36 -0
- disdrodb/etc/products/L2E/ODM470/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/ODM470/global.yaml +36 -0
- disdrodb/etc/products/L2E/PARSIVEL/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/PARSIVEL/global.yaml +36 -0
- disdrodb/etc/products/L2E/PARSIVEL2/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/PARSIVEL2/global.yaml +36 -0
- disdrodb/etc/products/L2E/PWS100/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/PWS100/global.yaml +36 -0
- disdrodb/etc/products/L2E/RD80/1MIN.yaml +19 -0
- disdrodb/etc/products/L2E/SWS250/1MIN.yaml +19 -0
- disdrodb/etc/products/L2E/global.yaml +16 -2
- disdrodb/fall_velocity/__init__.py +47 -0
- disdrodb/fall_velocity/graupel.py +484 -0
- disdrodb/fall_velocity/hail.py +288 -0
- disdrodb/{l1/fall_velocity.py → fall_velocity/rain.py} +265 -44
- disdrodb/issue/__init__.py +1 -3
- disdrodb/issue/checks.py +2 -3
- disdrodb/issue/reader.py +2 -3
- disdrodb/issue/writer.py +2 -5
- disdrodb/l0/__init__.py +2 -1
- disdrodb/l0/check_configs.py +36 -29
- disdrodb/l0/check_standards.py +1 -4
- disdrodb/l0/configs/LPM/l0a_encodings.yml +17 -17
- disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +55 -55
- disdrodb/l0/configs/LPM/l0b_encodings.yml +17 -17
- disdrodb/l0/configs/LPM/raw_data_format.yml +17 -17
- disdrodb/l0/configs/LPM_V0/l0a_encodings.yml +2 -2
- disdrodb/l0/configs/LPM_V0/l0b_cf_attrs.yml +2 -2
- disdrodb/l0/configs/LPM_V0/l0b_encodings.yml +2 -2
- disdrodb/l0/configs/LPM_V0/raw_data_format.yml +2 -2
- disdrodb/l0/configs/ODM470/bins_diameter.yml +643 -0
- disdrodb/l0/configs/ODM470/bins_velocity.yml +0 -0
- disdrodb/l0/configs/ODM470/l0a_encodings.yml +11 -0
- disdrodb/l0/configs/ODM470/l0b_cf_attrs.yml +46 -0
- disdrodb/l0/configs/ODM470/l0b_encodings.yml +106 -0
- disdrodb/l0/configs/ODM470/raw_data_format.yml +111 -0
- disdrodb/l0/configs/PARSIVEL/l0b_cf_attrs.yml +1 -1
- disdrodb/l0/l0_reader.py +2 -3
- disdrodb/l0/l0a_processing.py +6 -8
- disdrodb/l0/l0b_nc_processing.py +3 -6
- disdrodb/l0/l0b_processing.py +2 -16
- disdrodb/l0/l0c_processing.py +29 -12
- disdrodb/l0/readers/LPM/ARM/ARM_LPM.py +2 -1
- disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +18 -18
- disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +18 -18
- disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +18 -18
- disdrodb/l0/readers/LPM/GERMANY/DWD.py +244 -63
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +65 -23
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_AQ.py +277 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_PI.py +19 -18
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_T.py +23 -19
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +19 -21
- disdrodb/l0/readers/LPM/KIT/CHWALA.py +19 -20
- disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_LPM_NC.py +1 -1
- disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_RWANDA_LPM_NC.py +18 -18
- disdrodb/l0/readers/LPM/NORWAY/HAUKELISETER_LPM.py +19 -20
- disdrodb/l0/readers/LPM/NORWAY/NMBU_LPM.py +19 -20
- disdrodb/l0/readers/LPM/SLOVENIA/ARSO.py +19 -20
- disdrodb/l0/readers/LPM/SLOVENIA/UL.py +19 -20
- disdrodb/l0/readers/LPM/SWITZERLAND/INNERERIZ_LPM.py +19 -20
- disdrodb/l0/readers/LPM/UK/DIVEN.py +1 -1
- disdrodb/l0/readers/LPM/UK/WITHWORTH_LPM.py +19 -20
- disdrodb/l0/readers/LPM/USA/CHARLESTON.py +19 -20
- disdrodb/l0/readers/LPM/USA/DEVEX.py +255 -0
- disdrodb/l0/readers/LPM_V0/BELGIUM/ULIEGE.py +3 -5
- disdrodb/l0/readers/LPM_V0/ITALY/GID_LPM_V0.py +4 -3
- disdrodb/l0/readers/ODM470/OCEAN/OCEANRAIN.py +124 -0
- disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +1 -1
- disdrodb/l0/readers/PARSIVEL/BASQUECOUNTRY/EUSKALMET_OTT.py +2 -1
- disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +2 -3
- disdrodb/l0/readers/PARSIVEL/EPFL/ARCTIC_2021.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/COMMON_2011.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/DAVOS_2009_2011.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_2009.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2008.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2011.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2012.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/GENEPI_2007.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/GRAND_ST_BERNARD_2007.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/GRAND_ST_BERNARD_2007_2.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/HPICONET_2010.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP2.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP4.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/LOCARNO_2018.py +1 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/LOCARNO_2019.py +1 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/PARSIVEL_2007.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/PLATO_2019.py +1 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019_WJF.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/RIETHOLZBACH_2011.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/UNIL_2022.py +2 -1
- disdrodb/l0/readers/PARSIVEL/JAPAN/JMA.py +1 -1
- disdrodb/l0/readers/PARSIVEL/KOREA/ICEPOP_MSC.py +159 -0
- disdrodb/l0/readers/PARSIVEL/NASA/LPVEX.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NASA/MC3E.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/OWLES_MIPS.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/PLOWS_MIPS.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +1 -3
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +1 -3
- disdrodb/l0/readers/PARSIVEL/SLOVENIA/UL.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/ARM/ARM_PARSIVEL2.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/BASQUECOUNTRY/EUSKALMET_OTT2.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/BELGIUM/ILVO.py +2 -3
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/CANADA/UQAM_NC.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/DENMARK/DTU.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/DENMARK/EROSION_nc.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/DENMARK/EROSION_raw.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/FINLAND/FMI_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +2 -3
- disdrodb/l0/readers/PARSIVEL2/FRANCE/OSUG.py +2 -2
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +1 -3
- disdrodb/l0/readers/PARSIVEL2/GREECE/NOA.py +4 -3
- disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +1 -3
- disdrodb/l0/readers/PARSIVEL2/ITALY/HYDROX.py +6 -3
- disdrodb/l0/readers/PARSIVEL2/JAPAN/PRECIP.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/KIT/BURKINA_FASO.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/KIT/TEAMX.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/KOREA/ICEPOP_MSC.py +161 -0
- disdrodb/l0/readers/PARSIVEL2/KOREA/ICEPOP_UCLM.py +126 -0
- disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/MPI/BCO_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/MPI/BOWTIE.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NASA/APU.py +3 -1
- disdrodb/l0/readers/PARSIVEL2/NASA/NSSTC.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/FARM_PARSIVEL2.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_MIPS.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_MIPS.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_PIPS.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py +2 -3
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT_NC.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NORWAY/UIB.py +10 -2
- disdrodb/l0/readers/PARSIVEL2/PHILIPPINES/PAGASA.py +2 -3
- disdrodb/l0/readers/PARSIVEL2/SPAIN/CENER.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/SPAIN/CR1000DL.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/SPAIN/GRANADA.py +2 -3
- disdrodb/l0/readers/PARSIVEL2/SPAIN/LIAISE.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/SWEDEN/SMHI.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/USA/CSU.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/USA/CW3E.py +2 -1
- disdrodb/l0/readers/PWS100/AUSTRIA/HOAL.py +2 -3
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +2 -3
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100_SIRTA.py +2 -1
- disdrodb/l0/readers/RD80/BRAZIL/ATTO_RD80.py +1 -3
- disdrodb/l0/readers/RD80/BRAZIL/CHUVA_RD80.py +1 -3
- disdrodb/l0/readers/RD80/BRAZIL/GOAMAZON_RD80.py +1 -3
- disdrodb/l0/readers/RD80/NCAR/CINDY_2011_RD80.py +1 -3
- disdrodb/l0/readers/RD80/NCAR/RELAMPAGO_RD80.py +1 -3
- disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py +2 -3
- disdrodb/l0/readers/SWS250/BELGIUM/KMI.py +2 -3
- disdrodb/l0/readers/template_reader_raw_netcdf_data.py +2 -3
- disdrodb/l0/readers/template_reader_raw_text_data.py +2 -3
- disdrodb/l0/standards.py +4 -5
- disdrodb/l0/template_tools.py +7 -11
- disdrodb/l1/__init__.py +2 -1
- disdrodb/l1/classification.py +914 -0
- disdrodb/l1/processing.py +36 -106
- disdrodb/l1/resampling.py +13 -3
- disdrodb/l1_env/__init__.py +1 -1
- disdrodb/l1_env/routines.py +7 -6
- disdrodb/l2/__init__.py +2 -1
- disdrodb/l2/empirical_dsd.py +58 -31
- disdrodb/l2/processing.py +327 -61
- disdrodb/metadata/checks.py +10 -13
- disdrodb/metadata/download.py +5 -4
- disdrodb/metadata/geolocation.py +3 -4
- disdrodb/metadata/info.py +3 -5
- disdrodb/metadata/manipulation.py +1 -3
- disdrodb/metadata/reader.py +1 -3
- disdrodb/metadata/search.py +1 -4
- disdrodb/metadata/standards.py +1 -3
- disdrodb/metadata/writer.py +1 -3
- disdrodb/physics/__init__.py +17 -0
- disdrodb/physics/atmosphere.py +273 -0
- disdrodb/physics/water.py +131 -0
- disdrodb/physics/wrappers.py +63 -0
- disdrodb/psd/__init__.py +1 -2
- disdrodb/psd/fitting.py +23 -9
- disdrodb/psd/models.py +2 -1
- disdrodb/routines/__init__.py +6 -1
- disdrodb/routines/l0.py +39 -25
- disdrodb/routines/l1.py +23 -16
- disdrodb/routines/l2.py +12 -9
- disdrodb/routines/options.py +117 -73
- disdrodb/routines/options_validation.py +728 -0
- disdrodb/routines/wrappers.py +460 -40
- disdrodb/scattering/__init__.py +1 -2
- disdrodb/scattering/axis_ratio.py +6 -6
- disdrodb/scattering/permittivity.py +9 -8
- disdrodb/scattering/routines.py +33 -15
- disdrodb/summary/__init__.py +1 -1
- disdrodb/summary/routines.py +95 -30
- disdrodb/utils/__init__.py +1 -1
- disdrodb/utils/archiving.py +18 -10
- disdrodb/utils/attrs.py +7 -5
- disdrodb/utils/cli.py +8 -10
- disdrodb/utils/compression.py +10 -13
- disdrodb/utils/coords.py +45 -0
- disdrodb/utils/dask.py +7 -5
- disdrodb/utils/dataframe.py +5 -6
- disdrodb/utils/decorators.py +3 -4
- disdrodb/utils/dict.py +1 -1
- disdrodb/utils/directories.py +5 -7
- disdrodb/utils/encoding.py +4 -5
- disdrodb/utils/event.py +1 -1
- disdrodb/utils/list.py +1 -3
- disdrodb/utils/logger.py +1 -3
- disdrodb/utils/manipulations.py +175 -4
- disdrodb/utils/pydantic.py +81 -0
- disdrodb/utils/routines.py +2 -3
- disdrodb/utils/subsetting.py +1 -1
- disdrodb/utils/time.py +6 -4
- disdrodb/utils/warnings.py +2 -3
- disdrodb/utils/writer.py +5 -3
- disdrodb/utils/xarray.py +31 -3
- disdrodb/utils/yaml.py +1 -3
- disdrodb/viz/__init__.py +1 -1
- disdrodb/viz/plots.py +193 -18
- {disdrodb-0.2.1.dist-info → disdrodb-0.4.0.dist-info}/METADATA +5 -4
- disdrodb-0.4.0.dist-info/RECORD +361 -0
- {disdrodb-0.2.1.dist-info → disdrodb-0.4.0.dist-info}/entry_points.txt +3 -0
- disdrodb/etc/products/L1/1MIN.yaml +0 -13
- disdrodb/etc/products/L1/LPM/1MIN.yaml +0 -13
- disdrodb/etc/products/L1/LPM_V0/1MIN.yaml +0 -13
- disdrodb/etc/products/L1/PARSIVEL/1MIN.yaml +0 -13
- disdrodb/etc/products/L1/PARSIVEL2/1MIN.yaml +0 -13
- disdrodb/etc/products/L1/PWS100/1MIN.yaml +0 -13
- disdrodb/etc/products/L1/RD80/1MIN.yaml +0 -13
- disdrodb/etc/products/L1/SWS250/1MIN.yaml +0 -13
- disdrodb/etc/products/L2M/10MIN.yaml +0 -12
- disdrodb/l1/beard_model.py +0 -662
- disdrodb/l1/filters.py +0 -205
- disdrodb-0.2.1.dist-info/RECORD +0 -329
- {disdrodb-0.2.1.dist-info → disdrodb-0.4.0.dist-info}/WHEEL +0 -0
- {disdrodb-0.2.1.dist-info → disdrodb-0.4.0.dist-info}/licenses/LICENSE +0 -0
- {disdrodb-0.2.1.dist-info → disdrodb-0.4.0.dist-info}/top_level.txt +0 -0
disdrodb/routines/wrappers.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# -----------------------------------------------------------------------------.
|
|
2
|
-
# Copyright (c) 2021-
|
|
2
|
+
# Copyright (c) 2021-2026 DISDRODB developers
|
|
3
3
|
#
|
|
4
4
|
# This program is free software: you can redistribute it and/or modify
|
|
5
5
|
# it under the terms of the GNU General Public License as published by
|
|
@@ -15,13 +15,82 @@
|
|
|
15
15
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
16
16
|
# -----------------------------------------------------------------------------.
|
|
17
17
|
"""DISDRODB CLI routine wrappers."""
|
|
18
|
+
|
|
18
19
|
import datetime
|
|
19
20
|
import time
|
|
20
|
-
from typing import Optional
|
|
21
21
|
|
|
22
22
|
from disdrodb.api.search import available_stations, get_required_product
|
|
23
23
|
from disdrodb.utils.cli import execute_cmd
|
|
24
24
|
|
|
25
|
+
|
|
26
|
+
def validate_processing_flags_and_get_required_product(**processing_flags):
|
|
27
|
+
"""Validate processing flags and determine the required input product.
|
|
28
|
+
|
|
29
|
+
Validates that the processing chain is logically consistent and returns
|
|
30
|
+
the product needed as input for the earliest requested processing step.
|
|
31
|
+
|
|
32
|
+
The processing chain must be contiguous - you cannot skip intermediate levels.
|
|
33
|
+
For example: l1_processing=True, l2e_processing=False, l2m_processing=True is invalid.
|
|
34
|
+
|
|
35
|
+
Raises
|
|
36
|
+
------
|
|
37
|
+
ValueError
|
|
38
|
+
If no processing levels are enabled or if the processing chain has gaps.
|
|
39
|
+
|
|
40
|
+
Returns
|
|
41
|
+
-------
|
|
42
|
+
str
|
|
43
|
+
The required input product for the processing chain.
|
|
44
|
+
"""
|
|
45
|
+
# Define processing chain order and their required inputs
|
|
46
|
+
PROCESSING_CHAIN = [
|
|
47
|
+
("l0a_processing", "L0A"),
|
|
48
|
+
("l0b_processing", "L0B"),
|
|
49
|
+
("l0c_processing", "L0C"),
|
|
50
|
+
("l1_processing", "L1"),
|
|
51
|
+
("l2e_processing", "L2E"),
|
|
52
|
+
("l2m_processing", "L2M"),
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
# Filter only the processing flags we care about and get enabled levels
|
|
56
|
+
enabled_levels = []
|
|
57
|
+
for level, product in PROCESSING_CHAIN:
|
|
58
|
+
if processing_flags.get(level, False):
|
|
59
|
+
enabled_levels.append((level, product))
|
|
60
|
+
|
|
61
|
+
# Check if at least one processing level is enabled
|
|
62
|
+
if not enabled_levels:
|
|
63
|
+
raise ValueError("At least one processing level must be enabled.")
|
|
64
|
+
|
|
65
|
+
# Get indices of enabled levels in the original chain
|
|
66
|
+
enabled_indices = []
|
|
67
|
+
for level, _ in enabled_levels:
|
|
68
|
+
for i, (chain_level, _) in enumerate(PROCESSING_CHAIN):
|
|
69
|
+
if level == chain_level:
|
|
70
|
+
enabled_indices.append(i)
|
|
71
|
+
break
|
|
72
|
+
|
|
73
|
+
# Check for gaps: all indices between first and last enabled should be present
|
|
74
|
+
first_idx = min(enabled_indices)
|
|
75
|
+
last_idx = max(enabled_indices)
|
|
76
|
+
|
|
77
|
+
expected_indices = set(range(first_idx, last_idx + 1))
|
|
78
|
+
actual_indices = set(enabled_indices)
|
|
79
|
+
|
|
80
|
+
missing_indices = expected_indices - actual_indices
|
|
81
|
+
if missing_indices:
|
|
82
|
+
missing_levels = [PROCESSING_CHAIN[i][0] for i in missing_indices]
|
|
83
|
+
enabled_level_names = [level for level, _ in enabled_levels]
|
|
84
|
+
raise ValueError(
|
|
85
|
+
f"Processing chain has gaps. Enabled levels: {enabled_level_names}. "
|
|
86
|
+
f"Missing intermediate levels: {missing_levels}. "
|
|
87
|
+
f"All levels between the first and last enabled level must be enabled.",
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
# Return the required input product for the earliest enabled level
|
|
91
|
+
return get_required_product(enabled_levels[0][1])
|
|
92
|
+
|
|
93
|
+
|
|
25
94
|
####--------------------------------------------------------------------------.
|
|
26
95
|
#### Run DISDRODB Station Processing
|
|
27
96
|
|
|
@@ -42,8 +111,8 @@ def run_l0_station(
|
|
|
42
111
|
debugging_mode: bool = False,
|
|
43
112
|
parallel: bool = True,
|
|
44
113
|
# DISDRODB root directories
|
|
45
|
-
data_archive_dir:
|
|
46
|
-
metadata_archive_dir:
|
|
114
|
+
data_archive_dir: str | None = None,
|
|
115
|
+
metadata_archive_dir: str | None = None,
|
|
47
116
|
):
|
|
48
117
|
"""Run the L0 processing of a specific DISDRODB station from the terminal.
|
|
49
118
|
|
|
@@ -100,6 +169,13 @@ def run_l0_station(
|
|
|
100
169
|
in the DISDRODB active configuration.
|
|
101
170
|
"""
|
|
102
171
|
# ---------------------------------------------------------------------.
|
|
172
|
+
# Validate processing flags
|
|
173
|
+
_ = validate_processing_flags_and_get_required_product(
|
|
174
|
+
l0a_processing=l0a_processing,
|
|
175
|
+
l0b_processing=l0b_processing,
|
|
176
|
+
l0c_processing=l0c_processing,
|
|
177
|
+
)
|
|
178
|
+
# ---------------------------------------------------------------------.
|
|
103
179
|
t_i = time.time()
|
|
104
180
|
print(f"L0 processing of station {station_name} has started.")
|
|
105
181
|
|
|
@@ -177,8 +253,8 @@ def run_l0a_station(
|
|
|
177
253
|
debugging_mode: bool = False,
|
|
178
254
|
parallel: bool = True,
|
|
179
255
|
# DISDRODB root directories
|
|
180
|
-
data_archive_dir:
|
|
181
|
-
metadata_archive_dir:
|
|
256
|
+
data_archive_dir: str | None = None,
|
|
257
|
+
metadata_archive_dir: str | None = None,
|
|
182
258
|
):
|
|
183
259
|
"""
|
|
184
260
|
Run the L0A processing of a station by invoking the disdrodb_run_l0a_station command in the terminal.
|
|
@@ -255,8 +331,8 @@ def run_l0b_station(
|
|
|
255
331
|
debugging_mode: bool = False,
|
|
256
332
|
parallel: bool = True,
|
|
257
333
|
# DISDRODB root directories
|
|
258
|
-
data_archive_dir:
|
|
259
|
-
metadata_archive_dir:
|
|
334
|
+
data_archive_dir: str | None = None,
|
|
335
|
+
metadata_archive_dir: str | None = None,
|
|
260
336
|
):
|
|
261
337
|
"""
|
|
262
338
|
Run the L0B processing of a station by invoking the disdrodb_run_l0b_station command in the terminal.
|
|
@@ -339,8 +415,8 @@ def run_l0c_station(
|
|
|
339
415
|
debugging_mode: bool = False,
|
|
340
416
|
parallel: bool = True,
|
|
341
417
|
# DISDRODB root directories
|
|
342
|
-
data_archive_dir:
|
|
343
|
-
metadata_archive_dir:
|
|
418
|
+
data_archive_dir: str | None = None,
|
|
419
|
+
metadata_archive_dir: str | None = None,
|
|
344
420
|
):
|
|
345
421
|
"""
|
|
346
422
|
Run the L0C processing of a station by invoking the disdrodb_run_l0c_station command in the terminal.
|
|
@@ -421,8 +497,8 @@ def run_l1_station(
|
|
|
421
497
|
debugging_mode: bool = False,
|
|
422
498
|
parallel: bool = True,
|
|
423
499
|
# DISDRODB root directories
|
|
424
|
-
data_archive_dir:
|
|
425
|
-
metadata_archive_dir:
|
|
500
|
+
data_archive_dir: str | None = None,
|
|
501
|
+
metadata_archive_dir: str | None = None,
|
|
426
502
|
):
|
|
427
503
|
"""
|
|
428
504
|
Run the L1 processing of a station by invoking the disdrodb_run_l1_station command in the terminal.
|
|
@@ -497,8 +573,8 @@ def run_l2e_station(
|
|
|
497
573
|
debugging_mode: bool = False,
|
|
498
574
|
parallel: bool = True,
|
|
499
575
|
# DISDRODB root directories
|
|
500
|
-
data_archive_dir:
|
|
501
|
-
metadata_archive_dir:
|
|
576
|
+
data_archive_dir: str | None = None,
|
|
577
|
+
metadata_archive_dir: str | None = None,
|
|
502
578
|
):
|
|
503
579
|
"""
|
|
504
580
|
Run the L2E processing of a station by invoking the disdrodb_run_l2e_station command in the terminal.
|
|
@@ -573,8 +649,8 @@ def run_l2m_station(
|
|
|
573
649
|
debugging_mode: bool = False,
|
|
574
650
|
parallel: bool = True,
|
|
575
651
|
# DISDRODB root directories
|
|
576
|
-
data_archive_dir:
|
|
577
|
-
metadata_archive_dir:
|
|
652
|
+
data_archive_dir: str | None = None,
|
|
653
|
+
metadata_archive_dir: str | None = None,
|
|
578
654
|
):
|
|
579
655
|
"""
|
|
580
656
|
Run the L2M processing of a station by invoking the disdrodb_run_l2m_station command in the terminal.
|
|
@@ -681,8 +757,8 @@ def run_l0a(
|
|
|
681
757
|
debugging_mode: bool = False,
|
|
682
758
|
parallel: bool = True,
|
|
683
759
|
# DISDRODB root directories
|
|
684
|
-
data_archive_dir:
|
|
685
|
-
metadata_archive_dir:
|
|
760
|
+
data_archive_dir: str | None = None,
|
|
761
|
+
metadata_archive_dir: str | None = None,
|
|
686
762
|
):
|
|
687
763
|
"""Run the L0A processing of DISDRODB stations.
|
|
688
764
|
|
|
@@ -784,8 +860,8 @@ def run_l0b(
|
|
|
784
860
|
debugging_mode: bool = False,
|
|
785
861
|
parallel: bool = True,
|
|
786
862
|
# DISDRODB root directories
|
|
787
|
-
data_archive_dir:
|
|
788
|
-
metadata_archive_dir:
|
|
863
|
+
data_archive_dir: str | None = None,
|
|
864
|
+
metadata_archive_dir: str | None = None,
|
|
789
865
|
):
|
|
790
866
|
"""Run the L0B processing of DISDRODB stations.
|
|
791
867
|
|
|
@@ -893,8 +969,8 @@ def run_l0c(
|
|
|
893
969
|
debugging_mode: bool = False,
|
|
894
970
|
parallel: bool = True,
|
|
895
971
|
# DISDRODB root directories
|
|
896
|
-
data_archive_dir:
|
|
897
|
-
metadata_archive_dir:
|
|
972
|
+
data_archive_dir: str | None = None,
|
|
973
|
+
metadata_archive_dir: str | None = None,
|
|
898
974
|
):
|
|
899
975
|
"""Run the L0C processing of DISDRODB stations.
|
|
900
976
|
|
|
@@ -1008,8 +1084,8 @@ def run_l0(
|
|
|
1008
1084
|
debugging_mode: bool = False,
|
|
1009
1085
|
parallel: bool = True,
|
|
1010
1086
|
# DISDRODB root directories
|
|
1011
|
-
data_archive_dir:
|
|
1012
|
-
metadata_archive_dir:
|
|
1087
|
+
data_archive_dir: str | None = None,
|
|
1088
|
+
metadata_archive_dir: str | None = None,
|
|
1013
1089
|
):
|
|
1014
1090
|
"""Run the L0 processing of DISDRODB stations.
|
|
1015
1091
|
|
|
@@ -1075,13 +1151,12 @@ def run_l0(
|
|
|
1075
1151
|
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
1076
1152
|
in the DISDRODB active configuration.
|
|
1077
1153
|
"""
|
|
1078
|
-
#
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
required_product = get_required_product("L0A")
|
|
1154
|
+
# Determine required product based on the lowest level processing requested
|
|
1155
|
+
required_product = validate_processing_flags_and_get_required_product(
|
|
1156
|
+
l0a_processing=l0a_processing,
|
|
1157
|
+
l0b_processing=l0b_processing,
|
|
1158
|
+
l0c_processing=l0c_processing,
|
|
1159
|
+
)
|
|
1085
1160
|
|
|
1086
1161
|
# Get list of available stations
|
|
1087
1162
|
list_info = available_stations(
|
|
@@ -1138,8 +1213,8 @@ def run_l1(
|
|
|
1138
1213
|
debugging_mode: bool = False,
|
|
1139
1214
|
parallel: bool = True,
|
|
1140
1215
|
# DISDRODB root directories
|
|
1141
|
-
data_archive_dir:
|
|
1142
|
-
metadata_archive_dir:
|
|
1216
|
+
data_archive_dir: str | None = None,
|
|
1217
|
+
metadata_archive_dir: str | None = None,
|
|
1143
1218
|
):
|
|
1144
1219
|
"""Run the L1 processing of DISDRODB stations.
|
|
1145
1220
|
|
|
@@ -1242,8 +1317,8 @@ def run_l2e(
|
|
|
1242
1317
|
debugging_mode: bool = False,
|
|
1243
1318
|
parallel: bool = True,
|
|
1244
1319
|
# DISDRODB root directories
|
|
1245
|
-
data_archive_dir:
|
|
1246
|
-
metadata_archive_dir:
|
|
1320
|
+
data_archive_dir: str | None = None,
|
|
1321
|
+
metadata_archive_dir: str | None = None,
|
|
1247
1322
|
):
|
|
1248
1323
|
"""Run the L2E processing of DISDRODB stations.
|
|
1249
1324
|
|
|
@@ -1345,8 +1420,8 @@ def run_l2m(
|
|
|
1345
1420
|
debugging_mode: bool = False,
|
|
1346
1421
|
parallel: bool = True,
|
|
1347
1422
|
# DISDRODB root directories
|
|
1348
|
-
data_archive_dir:
|
|
1349
|
-
metadata_archive_dir:
|
|
1423
|
+
data_archive_dir: str | None = None,
|
|
1424
|
+
metadata_archive_dir: str | None = None,
|
|
1350
1425
|
):
|
|
1351
1426
|
"""Run the L2M processing of DISDRODB stations.
|
|
1352
1427
|
|
|
@@ -1438,6 +1513,354 @@ def run_l2m(
|
|
|
1438
1513
|
print(f"{product} processing of {data_source} {campaign_name} {station_name} station ended.")
|
|
1439
1514
|
|
|
1440
1515
|
|
|
1516
|
+
####--------------------------------------------------------------------------.
|
|
1517
|
+
#### Full Processing Wrappers
|
|
1518
|
+
def run_station(
|
|
1519
|
+
data_source,
|
|
1520
|
+
campaign_name,
|
|
1521
|
+
station_name,
|
|
1522
|
+
# L0 archive options
|
|
1523
|
+
l0a_processing: bool = True,
|
|
1524
|
+
l0b_processing: bool = True,
|
|
1525
|
+
l0c_processing: bool = True,
|
|
1526
|
+
remove_l0a: bool = False,
|
|
1527
|
+
remove_l0b: bool = False,
|
|
1528
|
+
# Higher level processing options
|
|
1529
|
+
l1_processing: bool = True,
|
|
1530
|
+
l2e_processing: bool = True,
|
|
1531
|
+
l2m_processing: bool = True,
|
|
1532
|
+
# Processing options
|
|
1533
|
+
force: bool = False,
|
|
1534
|
+
verbose: bool = False,
|
|
1535
|
+
debugging_mode: bool = False,
|
|
1536
|
+
parallel: bool = True,
|
|
1537
|
+
# DISDRODB root directories
|
|
1538
|
+
data_archive_dir: str | None = None,
|
|
1539
|
+
metadata_archive_dir: str | None = None,
|
|
1540
|
+
):
|
|
1541
|
+
"""Run the complete processing chain of a specific DISDRODB station from the terminal.
|
|
1542
|
+
|
|
1543
|
+
Parameters
|
|
1544
|
+
----------
|
|
1545
|
+
data_source : str
|
|
1546
|
+
Institution name (when campaign data spans more than 1 country),
|
|
1547
|
+
or country (when all campaigns (or sensor networks) are inside a given country).
|
|
1548
|
+
Must be UPPER CASE.
|
|
1549
|
+
campaign_name : str
|
|
1550
|
+
Campaign name. Must be UPPER CASE.
|
|
1551
|
+
station_name : str
|
|
1552
|
+
Station name
|
|
1553
|
+
l0a_processing : bool
|
|
1554
|
+
Whether to launch processing to generate L0A Apache Parquet file(s) from raw data.
|
|
1555
|
+
The default value is ``True``.
|
|
1556
|
+
l0b_processing : bool
|
|
1557
|
+
Whether to launch processing to generate L0B netCDF4 file(s) from L0A data.
|
|
1558
|
+
The default value is ``True``.
|
|
1559
|
+
l0c_processing : bool
|
|
1560
|
+
Whether to launch processing to generate L0C netCDF4 file(s) from L0B data.
|
|
1561
|
+
The default value is ``True``.
|
|
1562
|
+
remove_l0a : bool
|
|
1563
|
+
Whether to keep the L0A files after having generated the L0B netCDF products.
|
|
1564
|
+
The default value is ``False``.
|
|
1565
|
+
remove_l0b : bool
|
|
1566
|
+
Whether to remove the L0B files after having produced L0C netCDF files.
|
|
1567
|
+
The default is False.
|
|
1568
|
+
l1_processing : bool
|
|
1569
|
+
Whether to launch processing to generate L1 netCDF4 file(s) from L0C data.
|
|
1570
|
+
The default value is ``True``.
|
|
1571
|
+
l2e_processing : bool
|
|
1572
|
+
Whether to launch processing to generate L2E netCDF4 file(s) from L1 data.
|
|
1573
|
+
The default value is ``True``.
|
|
1574
|
+
l2m_processing : bool
|
|
1575
|
+
Whether to launch processing to generate L2M netCDF4 file(s) from L1 data.
|
|
1576
|
+
The default value is ``True``.
|
|
1577
|
+
force : bool
|
|
1578
|
+
If ``True``, overwrite existing data into destination directories.
|
|
1579
|
+
If ``False``, raise an error if there are already data into destination directories.
|
|
1580
|
+
The default value is ``False``.
|
|
1581
|
+
verbose : bool
|
|
1582
|
+
Whether to print detailed processing information into terminal.
|
|
1583
|
+
The default value is ``True``.
|
|
1584
|
+
parallel : bool
|
|
1585
|
+
If ``True``, the files are processed simultaneously in multiple processes.
|
|
1586
|
+
Each process will use a single thread to avoid issues with the HDF/netCDF library.
|
|
1587
|
+
By default, the number of process is defined with ``os.cpu_count()``.
|
|
1588
|
+
If ``False``, the files are processed sequentially in a single process.
|
|
1589
|
+
If ``False``, multi-threading is automatically exploited to speed up I/0 tasks.
|
|
1590
|
+
debugging_mode : bool
|
|
1591
|
+
If ``True``, it reduces the amount of data to process.
|
|
1592
|
+
For L0A, it processes just the first 3 raw data files for each station.
|
|
1593
|
+
For L0B, it processes 100 rows sampled from 3 L0A files for each station.
|
|
1594
|
+
The default value is ``False``.
|
|
1595
|
+
data_archive_dir : str (optional)
|
|
1596
|
+
The directory path where the DISDRODB Data Archive is located.
|
|
1597
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
1598
|
+
If ``None``, it uses the ``data_archive_dir`` path specified
|
|
1599
|
+
in the DISDRODB active configuration.
|
|
1600
|
+
metadata_archive_dir : str (optional)
|
|
1601
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
1602
|
+
The directory path must end with ``<...>/DISDRODB-METADATA/DISDRODB``.
|
|
1603
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
1604
|
+
in the DISDRODB active configuration.
|
|
1605
|
+
"""
|
|
1606
|
+
# ---------------------------------------------------------------------.
|
|
1607
|
+
# Validate processing flags
|
|
1608
|
+
_ = validate_processing_flags_and_get_required_product(
|
|
1609
|
+
l0a_processing=l0a_processing,
|
|
1610
|
+
l0b_processing=l0b_processing,
|
|
1611
|
+
l0c_processing=l0c_processing,
|
|
1612
|
+
l1_processing=l1_processing,
|
|
1613
|
+
l2e_processing=l2e_processing,
|
|
1614
|
+
l2m_processing=l2m_processing,
|
|
1615
|
+
)
|
|
1616
|
+
|
|
1617
|
+
# ---------------------------------------------------------------------.
|
|
1618
|
+
t_i = time.time()
|
|
1619
|
+
print(f"Complete processing of station {station_name} has started.")
|
|
1620
|
+
|
|
1621
|
+
# ------------------------------------------------------------------.
|
|
1622
|
+
# L0 processing (L0A, L0B, L0C)
|
|
1623
|
+
if l0a_processing or l0b_processing or l0c_processing:
|
|
1624
|
+
run_l0_station(
|
|
1625
|
+
# DISDRODB root directories
|
|
1626
|
+
data_archive_dir=data_archive_dir,
|
|
1627
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
1628
|
+
# Station arguments
|
|
1629
|
+
data_source=data_source,
|
|
1630
|
+
campaign_name=campaign_name,
|
|
1631
|
+
station_name=station_name,
|
|
1632
|
+
# L0 archive options
|
|
1633
|
+
l0a_processing=l0a_processing,
|
|
1634
|
+
l0b_processing=l0b_processing,
|
|
1635
|
+
l0c_processing=l0c_processing,
|
|
1636
|
+
remove_l0a=remove_l0a,
|
|
1637
|
+
remove_l0b=remove_l0b,
|
|
1638
|
+
# Processing options
|
|
1639
|
+
force=force,
|
|
1640
|
+
verbose=verbose,
|
|
1641
|
+
debugging_mode=debugging_mode,
|
|
1642
|
+
parallel=parallel,
|
|
1643
|
+
)
|
|
1644
|
+
|
|
1645
|
+
# ------------------------------------------------------------------.
|
|
1646
|
+
# L1 processing
|
|
1647
|
+
if l1_processing:
|
|
1648
|
+
run_l1_station(
|
|
1649
|
+
# DISDRODB root directories
|
|
1650
|
+
data_archive_dir=data_archive_dir,
|
|
1651
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
1652
|
+
# Station arguments
|
|
1653
|
+
data_source=data_source,
|
|
1654
|
+
campaign_name=campaign_name,
|
|
1655
|
+
station_name=station_name,
|
|
1656
|
+
# Processing options
|
|
1657
|
+
force=force,
|
|
1658
|
+
verbose=verbose,
|
|
1659
|
+
debugging_mode=debugging_mode,
|
|
1660
|
+
parallel=parallel,
|
|
1661
|
+
)
|
|
1662
|
+
|
|
1663
|
+
# ------------------------------------------------------------------.
|
|
1664
|
+
# L2E processing
|
|
1665
|
+
if l2e_processing:
|
|
1666
|
+
run_l2e_station(
|
|
1667
|
+
# DISDRODB root directories
|
|
1668
|
+
data_archive_dir=data_archive_dir,
|
|
1669
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
1670
|
+
# Station arguments
|
|
1671
|
+
data_source=data_source,
|
|
1672
|
+
campaign_name=campaign_name,
|
|
1673
|
+
station_name=station_name,
|
|
1674
|
+
# Processing options
|
|
1675
|
+
force=force,
|
|
1676
|
+
verbose=verbose,
|
|
1677
|
+
debugging_mode=debugging_mode,
|
|
1678
|
+
parallel=parallel,
|
|
1679
|
+
)
|
|
1680
|
+
|
|
1681
|
+
# ------------------------------------------------------------------.
|
|
1682
|
+
# L2M processing
|
|
1683
|
+
if l2m_processing:
|
|
1684
|
+
run_l2m_station(
|
|
1685
|
+
# DISDRODB root directories
|
|
1686
|
+
data_archive_dir=data_archive_dir,
|
|
1687
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
1688
|
+
# Station arguments
|
|
1689
|
+
data_source=data_source,
|
|
1690
|
+
campaign_name=campaign_name,
|
|
1691
|
+
station_name=station_name,
|
|
1692
|
+
# Processing options
|
|
1693
|
+
force=force,
|
|
1694
|
+
verbose=verbose,
|
|
1695
|
+
debugging_mode=debugging_mode,
|
|
1696
|
+
parallel=parallel,
|
|
1697
|
+
)
|
|
1698
|
+
|
|
1699
|
+
# -------------------------------------------------------------------------.
|
|
1700
|
+
# End of complete processing for station
|
|
1701
|
+
timedelta_str = str(datetime.timedelta(seconds=round(time.time() - t_i)))
|
|
1702
|
+
print(f"Complete processing of station {station_name} completed in {timedelta_str}")
|
|
1703
|
+
|
|
1704
|
+
|
|
1705
|
+
def run(
|
|
1706
|
+
data_sources=None,
|
|
1707
|
+
campaign_names=None,
|
|
1708
|
+
station_names=None,
|
|
1709
|
+
# L0 archive options
|
|
1710
|
+
l0a_processing: bool = True,
|
|
1711
|
+
l0b_processing: bool = True,
|
|
1712
|
+
l0c_processing: bool = True,
|
|
1713
|
+
remove_l0a: bool = False,
|
|
1714
|
+
remove_l0b: bool = False,
|
|
1715
|
+
# Higher level processing options
|
|
1716
|
+
l1_processing: bool = True,
|
|
1717
|
+
l2e_processing: bool = True,
|
|
1718
|
+
l2m_processing: bool = True,
|
|
1719
|
+
# Processing options
|
|
1720
|
+
force: bool = False,
|
|
1721
|
+
verbose: bool = False,
|
|
1722
|
+
debugging_mode: bool = False,
|
|
1723
|
+
parallel: bool = True,
|
|
1724
|
+
# DISDRODB root directories
|
|
1725
|
+
data_archive_dir: str | None = None,
|
|
1726
|
+
metadata_archive_dir: str | None = None,
|
|
1727
|
+
):
|
|
1728
|
+
"""Run the complete processing chain of DISDRODB stations.
|
|
1729
|
+
|
|
1730
|
+
This function allows to launch the complete processing of many DISDRODB stations with a single command.
|
|
1731
|
+
From the list of all available DISDRODB stations, it runs the processing of the
|
|
1732
|
+
stations matching the provided data_sources, campaign_names and station_names.
|
|
1733
|
+
|
|
1734
|
+
Parameters
|
|
1735
|
+
----------
|
|
1736
|
+
data_sources : list
|
|
1737
|
+
Name of data source(s) to process.
|
|
1738
|
+
The name(s) must be UPPER CASE.
|
|
1739
|
+
If campaign_names and station are not specified, process all stations.
|
|
1740
|
+
The default value is ``None``.
|
|
1741
|
+
campaign_names : list
|
|
1742
|
+
Name of the campaign(s) to process.
|
|
1743
|
+
The name(s) must be UPPER CASE.
|
|
1744
|
+
The default value is ``None``.
|
|
1745
|
+
station_names : list
|
|
1746
|
+
Station names to process.
|
|
1747
|
+
The default value is ``None``.
|
|
1748
|
+
l0a_processing : bool
|
|
1749
|
+
Whether to launch processing to generate L0A Apache Parquet file(s) from raw data.
|
|
1750
|
+
The default value is ``True``.
|
|
1751
|
+
l0b_processing : bool
|
|
1752
|
+
Whether to launch processing to generate L0B netCDF4 file(s) from L0A data.
|
|
1753
|
+
The default value is ``True``.
|
|
1754
|
+
l0c_processing : bool
|
|
1755
|
+
Whether to launch processing to generate L0C netCDF4 file(s) from L0B data.
|
|
1756
|
+
The default value is ``True``.
|
|
1757
|
+
remove_l0a : bool
|
|
1758
|
+
Whether to keep the L0A files after having generated the L0B netCDF products.
|
|
1759
|
+
The default value is ``False``.
|
|
1760
|
+
remove_l0b : bool
|
|
1761
|
+
Whether to remove the L0B files after having produced L0C netCDF files.
|
|
1762
|
+
The default value is ``False``.
|
|
1763
|
+
l1_processing : bool
|
|
1764
|
+
Whether to launch processing to generate L1 netCDF4 file(s) from L0C data.
|
|
1765
|
+
The default value is ``True``.
|
|
1766
|
+
l2e_processing : bool
|
|
1767
|
+
Whether to launch processing to generate L2E netCDF4 file(s) from L1 data.
|
|
1768
|
+
The default value is ``True``.
|
|
1769
|
+
l2m_processing : bool
|
|
1770
|
+
Whether to launch processing to generate L2M netCDF4 file(s) from L1 data.
|
|
1771
|
+
The default value is ``True``.
|
|
1772
|
+
force : bool
|
|
1773
|
+
If ``True``, overwrite existing data into destination directories.
|
|
1774
|
+
If ``False``, raise an error if there are already data into destination directories.
|
|
1775
|
+
The default value is ``False``.
|
|
1776
|
+
verbose : bool
|
|
1777
|
+
Whether to print detailed processing information into terminal.
|
|
1778
|
+
The default value is ``False``.
|
|
1779
|
+
parallel : bool
|
|
1780
|
+
If ``True``, the files are processed simultaneously in multiple processes.
|
|
1781
|
+
Each process will use a single thread to avoid issues with the HDF/netCDF library.
|
|
1782
|
+
By default, the number of process is defined with ``os.cpu_count()``.
|
|
1783
|
+
If ``False``, the files are processed sequentially in a single process.
|
|
1784
|
+
If ``False``, multi-threading is automatically exploited to speed up I/0 tasks.
|
|
1785
|
+
debugging_mode : bool
|
|
1786
|
+
If ``True``, it reduces the amount of data to process.
|
|
1787
|
+
For L0A, it processes just the first 3 raw data files.
|
|
1788
|
+
For L0B, it processes 100 rows sampled from 3 L0A files.
|
|
1789
|
+
The default value is ``False``.
|
|
1790
|
+
data_archive_dir : str (optional)
|
|
1791
|
+
The directory path where the DISDRODB Data Archive is located.
|
|
1792
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
1793
|
+
If ``None``, it uses the ``data_archive_dir`` path specified
|
|
1794
|
+
in the DISDRODB active configuration.
|
|
1795
|
+
metadata_archive_dir : str (optional)
|
|
1796
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
1797
|
+
The directory path must end with ``<...>/DISDRODB-METADATA/DISDRODB``.
|
|
1798
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
1799
|
+
in the DISDRODB active configuration.
|
|
1800
|
+
"""
|
|
1801
|
+
# Determine required product based on the lowest level processing requested
|
|
1802
|
+
required_product = validate_processing_flags_and_get_required_product(
|
|
1803
|
+
l0a_processing=l0a_processing,
|
|
1804
|
+
l0b_processing=l0b_processing,
|
|
1805
|
+
l0c_processing=l0c_processing,
|
|
1806
|
+
l1_processing=l1_processing,
|
|
1807
|
+
l2e_processing=l2e_processing,
|
|
1808
|
+
l2m_processing=l2m_processing,
|
|
1809
|
+
)
|
|
1810
|
+
|
|
1811
|
+
# Get list of available stations
|
|
1812
|
+
list_info = available_stations(
|
|
1813
|
+
# DISDRODB root directories
|
|
1814
|
+
data_archive_dir=data_archive_dir,
|
|
1815
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
1816
|
+
# Stations arguments
|
|
1817
|
+
data_sources=data_sources,
|
|
1818
|
+
campaign_names=campaign_names,
|
|
1819
|
+
station_names=station_names,
|
|
1820
|
+
# Search options
|
|
1821
|
+
product=required_product,
|
|
1822
|
+
raise_error_if_empty=True,
|
|
1823
|
+
)
|
|
1824
|
+
|
|
1825
|
+
# Print message
|
|
1826
|
+
n_stations = len(list_info)
|
|
1827
|
+
print(f"Complete processing of {n_stations} stations started.")
|
|
1828
|
+
|
|
1829
|
+
# Loop over stations
|
|
1830
|
+
for data_source, campaign_name, station_name in list_info:
|
|
1831
|
+
print(f"Complete processing of {data_source} {campaign_name} {station_name} station started.")
|
|
1832
|
+
# Run processing
|
|
1833
|
+
run_station(
|
|
1834
|
+
# DISDRODB root directories
|
|
1835
|
+
data_archive_dir=data_archive_dir,
|
|
1836
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
1837
|
+
# Station arguments
|
|
1838
|
+
data_source=data_source,
|
|
1839
|
+
campaign_name=campaign_name,
|
|
1840
|
+
station_name=station_name,
|
|
1841
|
+
# L0 archive options
|
|
1842
|
+
l0a_processing=l0a_processing,
|
|
1843
|
+
l0b_processing=l0b_processing,
|
|
1844
|
+
l0c_processing=l0c_processing,
|
|
1845
|
+
remove_l0a=remove_l0a,
|
|
1846
|
+
remove_l0b=remove_l0b,
|
|
1847
|
+
# Higher level processing options
|
|
1848
|
+
l1_processing=l1_processing,
|
|
1849
|
+
l2e_processing=l2e_processing,
|
|
1850
|
+
l2m_processing=l2m_processing,
|
|
1851
|
+
# Process options
|
|
1852
|
+
force=force,
|
|
1853
|
+
verbose=verbose,
|
|
1854
|
+
debugging_mode=debugging_mode,
|
|
1855
|
+
parallel=parallel,
|
|
1856
|
+
)
|
|
1857
|
+
print(f"Complete processing of {data_source} {campaign_name} {station_name} station ended.")
|
|
1858
|
+
|
|
1859
|
+
print(f"Complete processing of {n_stations} stations completed.")
|
|
1860
|
+
|
|
1861
|
+
|
|
1862
|
+
####--------------------------------------------------------------------------.
|
|
1863
|
+
#### Summary Wrappers
|
|
1441
1864
|
def create_summary(
|
|
1442
1865
|
data_sources=None,
|
|
1443
1866
|
campaign_names=None,
|
|
@@ -1508,6 +1931,3 @@ def create_summary(
|
|
|
1508
1931
|
temporal_resolution=temporal_resolution,
|
|
1509
1932
|
)
|
|
1510
1933
|
print("Creation of station summaries has terminated.")
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
####--------------------------------------------------------------------------.
|
disdrodb/scattering/__init__.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# -----------------------------------------------------------------------------.
|
|
2
|
-
# Copyright (c) 2021-
|
|
2
|
+
# Copyright (c) 2021-2026 DISDRODB developers
|
|
3
3
|
#
|
|
4
4
|
# This program is free software: you can redistribute it and/or modify
|
|
5
5
|
# it under the terms of the GNU General Public License as published by
|
|
@@ -16,7 +16,6 @@
|
|
|
16
16
|
# -----------------------------------------------------------------------------.
|
|
17
17
|
"""Implement PSD scattering routines."""
|
|
18
18
|
|
|
19
|
-
|
|
20
19
|
from disdrodb.scattering.axis_ratio import available_axis_ratio_models, get_axis_ratio_model
|
|
21
20
|
from disdrodb.scattering.permittivity import available_permittivity_models, get_refractive_index
|
|
22
21
|
from disdrodb.scattering.routines import (
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# -----------------------------------------------------------------------------.
|
|
2
|
-
# Copyright (c) 2021-
|
|
2
|
+
# Copyright (c) 2021-2026 DISDRODB developers
|
|
3
3
|
#
|
|
4
4
|
# This program is free software: you can redistribute it and/or modify
|
|
5
5
|
# it under the terms of the GNU General Public License as published by
|
|
@@ -79,9 +79,9 @@ def get_axis_ratio_andsager_1999(diameter):
|
|
|
79
79
|
return axis_ratio
|
|
80
80
|
|
|
81
81
|
|
|
82
|
-
def
|
|
82
|
+
def get_axis_ratio_parsivel(diameter):
|
|
83
83
|
"""
|
|
84
|
-
Compute the axis ratio of raindrops
|
|
84
|
+
Compute the axis ratio of raindrops as used by OTT Parsivel sensors.
|
|
85
85
|
|
|
86
86
|
This axis ratio is assumed by OTT Parsivel sensors internally to compute the
|
|
87
87
|
reported particle size (Deq).
|
|
@@ -279,7 +279,7 @@ def get_axis_ratio_thurai_2007(diameter):
|
|
|
279
279
|
AXIS_RATIO_MODELS = {
|
|
280
280
|
"Thurai2005": get_axis_ratio_thurai_2005,
|
|
281
281
|
"Thurai2007": get_axis_ratio_thurai_2007,
|
|
282
|
-
"
|
|
282
|
+
"Parsivel": get_axis_ratio_parsivel,
|
|
283
283
|
"Brandes2002": get_axis_ratio_brandes_2002,
|
|
284
284
|
"Pruppacher1970": get_axis_ratio_pruppacher_1970,
|
|
285
285
|
"Beard1987": get_axis_ratio_beard_1987,
|
|
@@ -307,7 +307,7 @@ def get_axis_ratio_model(model):
|
|
|
307
307
|
----------
|
|
308
308
|
model : str
|
|
309
309
|
The model to use for calculating the axis ratio. Available models are:
|
|
310
|
-
'Thurai2005', 'Thurai2007', '
|
|
310
|
+
'Thurai2005', 'Thurai2007', 'Parsivel', 'Brandes2002',
|
|
311
311
|
'Pruppacher1970', 'Beard1987', 'Andsager1999'.
|
|
312
312
|
|
|
313
313
|
Returns
|
|
@@ -339,7 +339,7 @@ def get_axis_ratio(diameter, model):
|
|
|
339
339
|
Raindrops diameter in mm.
|
|
340
340
|
model : str
|
|
341
341
|
The axis ratio model to use for calculating the axis ratio. Available models are:
|
|
342
|
-
'Thurai2005', 'Thurai2007', '
|
|
342
|
+
'Thurai2005', 'Thurai2007', 'Parsivel', 'Brandes2002',
|
|
343
343
|
'Pruppacher1970', 'Beard1987', 'Andsager1999'.
|
|
344
344
|
|
|
345
345
|
Returns
|