disdrodb 0.2.1__py3-none-any.whl ā 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +3 -1
- disdrodb/_config.py +2 -3
- disdrodb/_version.py +2 -2
- disdrodb/accessor/__init__.py +2 -1
- disdrodb/accessor/methods.py +10 -9
- disdrodb/api/checks.py +3 -7
- disdrodb/api/configs.py +1 -3
- disdrodb/api/create_directories.py +4 -6
- disdrodb/api/info.py +1 -3
- disdrodb/api/io.py +233 -32
- disdrodb/api/path.py +3 -7
- disdrodb/cli/disdrodb_check_metadata_archive.py +3 -2
- disdrodb/cli/disdrodb_check_products_options.py +45 -0
- disdrodb/cli/disdrodb_create_summary.py +54 -28
- disdrodb/cli/disdrodb_create_summary_station.py +41 -20
- disdrodb/cli/disdrodb_data_archive_directory.py +2 -3
- disdrodb/cli/disdrodb_download_archive.py +50 -30
- disdrodb/cli/disdrodb_download_metadata_archive.py +28 -16
- disdrodb/cli/disdrodb_download_station.py +58 -29
- disdrodb/cli/disdrodb_initialize_station.py +43 -23
- disdrodb/cli/disdrodb_metadata_archive_directory.py +2 -3
- disdrodb/cli/disdrodb_open_data_archive.py +17 -13
- disdrodb/cli/disdrodb_open_logs_directory.py +31 -21
- disdrodb/cli/disdrodb_open_metadata_archive.py +26 -13
- disdrodb/cli/disdrodb_open_metadata_directory.py +34 -23
- disdrodb/cli/disdrodb_open_product_directory.py +39 -23
- disdrodb/cli/disdrodb_open_readers_directory.py +2 -3
- disdrodb/cli/disdrodb_run.py +189 -0
- disdrodb/cli/disdrodb_run_l0.py +61 -70
- disdrodb/cli/disdrodb_run_l0_station.py +50 -55
- disdrodb/cli/disdrodb_run_l0a.py +53 -51
- disdrodb/cli/disdrodb_run_l0a_station.py +41 -40
- disdrodb/cli/disdrodb_run_l0b.py +51 -51
- disdrodb/cli/disdrodb_run_l0b_station.py +40 -39
- disdrodb/cli/disdrodb_run_l0c.py +56 -53
- disdrodb/cli/disdrodb_run_l0c_station.py +44 -41
- disdrodb/cli/disdrodb_run_l1.py +55 -51
- disdrodb/cli/disdrodb_run_l1_station.py +43 -40
- disdrodb/cli/disdrodb_run_l2e.py +56 -51
- disdrodb/cli/disdrodb_run_l2e_station.py +44 -40
- disdrodb/cli/disdrodb_run_l2m.py +55 -51
- disdrodb/cli/disdrodb_run_l2m_station.py +43 -40
- disdrodb/cli/disdrodb_run_station.py +184 -0
- disdrodb/cli/disdrodb_upload_archive.py +51 -42
- disdrodb/cli/disdrodb_upload_station.py +42 -36
- disdrodb/configs.py +20 -16
- disdrodb/constants.py +5 -2
- disdrodb/data_transfer/__init__.py +1 -3
- disdrodb/data_transfer/download_data.py +45 -61
- disdrodb/data_transfer/upload_data.py +7 -11
- disdrodb/data_transfer/zenodo.py +2 -4
- disdrodb/docs.py +1 -3
- disdrodb/etc/configs/attributes.yaml +52 -2
- disdrodb/etc/configs/encodings.yaml +45 -1
- disdrodb/etc/products/L0C/ODM470/global.yaml +5 -0
- disdrodb/etc/products/L0C/global.yaml +5 -0
- disdrodb/etc/products/L1/ODM470/global.yaml +6 -0
- disdrodb/etc/products/L1/global.yaml +0 -13
- disdrodb/etc/products/L2E/LPM/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/LPM/global.yaml +36 -0
- disdrodb/etc/products/L2E/LPM_V0/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/LPM_V0/global.yaml +36 -0
- disdrodb/etc/products/L2E/ODM470/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/ODM470/global.yaml +36 -0
- disdrodb/etc/products/L2E/PARSIVEL/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/PARSIVEL/global.yaml +36 -0
- disdrodb/etc/products/L2E/PARSIVEL2/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/PARSIVEL2/global.yaml +36 -0
- disdrodb/etc/products/L2E/PWS100/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/PWS100/global.yaml +36 -0
- disdrodb/etc/products/L2E/RD80/1MIN.yaml +19 -0
- disdrodb/etc/products/L2E/SWS250/1MIN.yaml +19 -0
- disdrodb/etc/products/L2E/global.yaml +16 -2
- disdrodb/fall_velocity/__init__.py +47 -0
- disdrodb/fall_velocity/graupel.py +484 -0
- disdrodb/fall_velocity/hail.py +288 -0
- disdrodb/{l1/fall_velocity.py ā fall_velocity/rain.py} +265 -44
- disdrodb/issue/__init__.py +1 -3
- disdrodb/issue/checks.py +2 -3
- disdrodb/issue/reader.py +2 -3
- disdrodb/issue/writer.py +2 -5
- disdrodb/l0/__init__.py +2 -1
- disdrodb/l0/check_configs.py +36 -29
- disdrodb/l0/check_standards.py +1 -4
- disdrodb/l0/configs/LPM/l0a_encodings.yml +17 -17
- disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +55 -55
- disdrodb/l0/configs/LPM/l0b_encodings.yml +17 -17
- disdrodb/l0/configs/LPM/raw_data_format.yml +17 -17
- disdrodb/l0/configs/LPM_V0/l0a_encodings.yml +2 -2
- disdrodb/l0/configs/LPM_V0/l0b_cf_attrs.yml +2 -2
- disdrodb/l0/configs/LPM_V0/l0b_encodings.yml +2 -2
- disdrodb/l0/configs/LPM_V0/raw_data_format.yml +2 -2
- disdrodb/l0/configs/ODM470/bins_diameter.yml +643 -0
- disdrodb/l0/configs/ODM470/bins_velocity.yml +0 -0
- disdrodb/l0/configs/ODM470/l0a_encodings.yml +11 -0
- disdrodb/l0/configs/ODM470/l0b_cf_attrs.yml +46 -0
- disdrodb/l0/configs/ODM470/l0b_encodings.yml +106 -0
- disdrodb/l0/configs/ODM470/raw_data_format.yml +111 -0
- disdrodb/l0/configs/PARSIVEL/l0b_cf_attrs.yml +1 -1
- disdrodb/l0/l0_reader.py +2 -3
- disdrodb/l0/l0a_processing.py +6 -8
- disdrodb/l0/l0b_nc_processing.py +3 -6
- disdrodb/l0/l0b_processing.py +2 -16
- disdrodb/l0/l0c_processing.py +29 -12
- disdrodb/l0/readers/LPM/ARM/ARM_LPM.py +2 -1
- disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +18 -18
- disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +18 -18
- disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +18 -18
- disdrodb/l0/readers/LPM/GERMANY/DWD.py +244 -63
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +65 -23
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_AQ.py +277 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_PI.py +19 -18
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_T.py +23 -19
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +19 -21
- disdrodb/l0/readers/LPM/KIT/CHWALA.py +19 -20
- disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_LPM_NC.py +1 -1
- disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_RWANDA_LPM_NC.py +18 -18
- disdrodb/l0/readers/LPM/NORWAY/HAUKELISETER_LPM.py +19 -20
- disdrodb/l0/readers/LPM/NORWAY/NMBU_LPM.py +19 -20
- disdrodb/l0/readers/LPM/SLOVENIA/ARSO.py +19 -20
- disdrodb/l0/readers/LPM/SLOVENIA/UL.py +19 -20
- disdrodb/l0/readers/LPM/SWITZERLAND/INNERERIZ_LPM.py +19 -20
- disdrodb/l0/readers/LPM/UK/DIVEN.py +1 -1
- disdrodb/l0/readers/LPM/UK/WITHWORTH_LPM.py +19 -20
- disdrodb/l0/readers/LPM/USA/CHARLESTON.py +19 -20
- disdrodb/l0/readers/LPM/USA/DEVEX.py +255 -0
- disdrodb/l0/readers/LPM_V0/BELGIUM/ULIEGE.py +3 -5
- disdrodb/l0/readers/LPM_V0/ITALY/GID_LPM_V0.py +4 -3
- disdrodb/l0/readers/ODM470/OCEAN/OCEANRAIN.py +124 -0
- disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +1 -1
- disdrodb/l0/readers/PARSIVEL/BASQUECOUNTRY/EUSKALMET_OTT.py +2 -1
- disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +2 -3
- disdrodb/l0/readers/PARSIVEL/EPFL/ARCTIC_2021.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/COMMON_2011.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/DAVOS_2009_2011.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_2009.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2008.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2011.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2012.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/GENEPI_2007.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/GRAND_ST_BERNARD_2007.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/GRAND_ST_BERNARD_2007_2.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/HPICONET_2010.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP2.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP4.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/LOCARNO_2018.py +1 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/LOCARNO_2019.py +1 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/PARSIVEL_2007.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/PLATO_2019.py +1 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019_WJF.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/RIETHOLZBACH_2011.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +2 -1
- disdrodb/l0/readers/PARSIVEL/EPFL/UNIL_2022.py +2 -1
- disdrodb/l0/readers/PARSIVEL/JAPAN/JMA.py +1 -1
- disdrodb/l0/readers/PARSIVEL/KOREA/ICEPOP_MSC.py +159 -0
- disdrodb/l0/readers/PARSIVEL/NASA/LPVEX.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NASA/MC3E.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/OWLES_MIPS.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/PLOWS_MIPS.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +1 -3
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +1 -3
- disdrodb/l0/readers/PARSIVEL/SLOVENIA/UL.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/ARM/ARM_PARSIVEL2.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/BASQUECOUNTRY/EUSKALMET_OTT2.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/BELGIUM/ILVO.py +2 -3
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/CANADA/UQAM_NC.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/DENMARK/DTU.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/DENMARK/EROSION_nc.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/DENMARK/EROSION_raw.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/FINLAND/FMI_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +2 -3
- disdrodb/l0/readers/PARSIVEL2/FRANCE/OSUG.py +2 -2
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +1 -3
- disdrodb/l0/readers/PARSIVEL2/GREECE/NOA.py +4 -3
- disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +1 -3
- disdrodb/l0/readers/PARSIVEL2/ITALY/HYDROX.py +6 -3
- disdrodb/l0/readers/PARSIVEL2/JAPAN/PRECIP.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/KIT/BURKINA_FASO.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/KIT/TEAMX.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/KOREA/ICEPOP_MSC.py +161 -0
- disdrodb/l0/readers/PARSIVEL2/KOREA/ICEPOP_UCLM.py +126 -0
- disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/MPI/BCO_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/MPI/BOWTIE.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NASA/APU.py +3 -1
- disdrodb/l0/readers/PARSIVEL2/NASA/NSSTC.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/FARM_PARSIVEL2.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_MIPS.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_MIPS.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_PIPS.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py +2 -3
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT_NC.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NORWAY/UIB.py +10 -2
- disdrodb/l0/readers/PARSIVEL2/PHILIPPINES/PAGASA.py +2 -3
- disdrodb/l0/readers/PARSIVEL2/SPAIN/CENER.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/SPAIN/CR1000DL.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/SPAIN/GRANADA.py +2 -3
- disdrodb/l0/readers/PARSIVEL2/SPAIN/LIAISE.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/SWEDEN/SMHI.py +2 -1
- disdrodb/l0/readers/PARSIVEL2/USA/CSU.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/USA/CW3E.py +2 -1
- disdrodb/l0/readers/PWS100/AUSTRIA/HOAL.py +2 -3
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +2 -3
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100_SIRTA.py +2 -1
- disdrodb/l0/readers/RD80/BRAZIL/ATTO_RD80.py +1 -3
- disdrodb/l0/readers/RD80/BRAZIL/CHUVA_RD80.py +1 -3
- disdrodb/l0/readers/RD80/BRAZIL/GOAMAZON_RD80.py +1 -3
- disdrodb/l0/readers/RD80/NCAR/CINDY_2011_RD80.py +1 -3
- disdrodb/l0/readers/RD80/NCAR/RELAMPAGO_RD80.py +1 -3
- disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py +2 -3
- disdrodb/l0/readers/SWS250/BELGIUM/KMI.py +2 -3
- disdrodb/l0/readers/template_reader_raw_netcdf_data.py +2 -3
- disdrodb/l0/readers/template_reader_raw_text_data.py +2 -3
- disdrodb/l0/standards.py +4 -5
- disdrodb/l0/template_tools.py +7 -11
- disdrodb/l1/__init__.py +2 -1
- disdrodb/l1/classification.py +914 -0
- disdrodb/l1/processing.py +36 -106
- disdrodb/l1/resampling.py +13 -3
- disdrodb/l1_env/__init__.py +1 -1
- disdrodb/l1_env/routines.py +7 -6
- disdrodb/l2/__init__.py +2 -1
- disdrodb/l2/empirical_dsd.py +58 -31
- disdrodb/l2/processing.py +327 -61
- disdrodb/metadata/checks.py +10 -13
- disdrodb/metadata/download.py +5 -4
- disdrodb/metadata/geolocation.py +3 -4
- disdrodb/metadata/info.py +3 -5
- disdrodb/metadata/manipulation.py +1 -3
- disdrodb/metadata/reader.py +1 -3
- disdrodb/metadata/search.py +1 -4
- disdrodb/metadata/standards.py +1 -3
- disdrodb/metadata/writer.py +1 -3
- disdrodb/physics/__init__.py +17 -0
- disdrodb/physics/atmosphere.py +273 -0
- disdrodb/physics/water.py +131 -0
- disdrodb/physics/wrappers.py +63 -0
- disdrodb/psd/__init__.py +1 -2
- disdrodb/psd/fitting.py +23 -9
- disdrodb/psd/models.py +2 -1
- disdrodb/routines/__init__.py +6 -1
- disdrodb/routines/l0.py +39 -25
- disdrodb/routines/l1.py +23 -16
- disdrodb/routines/l2.py +12 -9
- disdrodb/routines/options.py +117 -73
- disdrodb/routines/options_validation.py +728 -0
- disdrodb/routines/wrappers.py +460 -40
- disdrodb/scattering/__init__.py +1 -2
- disdrodb/scattering/axis_ratio.py +6 -6
- disdrodb/scattering/permittivity.py +9 -8
- disdrodb/scattering/routines.py +33 -15
- disdrodb/summary/__init__.py +1 -1
- disdrodb/summary/routines.py +95 -30
- disdrodb/utils/__init__.py +1 -1
- disdrodb/utils/archiving.py +18 -10
- disdrodb/utils/attrs.py +7 -5
- disdrodb/utils/cli.py +8 -10
- disdrodb/utils/compression.py +10 -13
- disdrodb/utils/coords.py +45 -0
- disdrodb/utils/dask.py +7 -5
- disdrodb/utils/dataframe.py +5 -6
- disdrodb/utils/decorators.py +3 -4
- disdrodb/utils/dict.py +1 -1
- disdrodb/utils/directories.py +5 -7
- disdrodb/utils/encoding.py +4 -5
- disdrodb/utils/event.py +1 -1
- disdrodb/utils/list.py +1 -3
- disdrodb/utils/logger.py +1 -3
- disdrodb/utils/manipulations.py +175 -4
- disdrodb/utils/pydantic.py +81 -0
- disdrodb/utils/routines.py +2 -3
- disdrodb/utils/subsetting.py +1 -1
- disdrodb/utils/time.py +6 -4
- disdrodb/utils/warnings.py +2 -3
- disdrodb/utils/writer.py +5 -3
- disdrodb/utils/xarray.py +31 -3
- disdrodb/utils/yaml.py +1 -3
- disdrodb/viz/__init__.py +1 -1
- disdrodb/viz/plots.py +193 -18
- {disdrodb-0.2.1.dist-info ā disdrodb-0.4.0.dist-info}/METADATA +5 -4
- disdrodb-0.4.0.dist-info/RECORD +361 -0
- {disdrodb-0.2.1.dist-info ā disdrodb-0.4.0.dist-info}/entry_points.txt +3 -0
- disdrodb/etc/products/L1/1MIN.yaml +0 -13
- disdrodb/etc/products/L1/LPM/1MIN.yaml +0 -13
- disdrodb/etc/products/L1/LPM_V0/1MIN.yaml +0 -13
- disdrodb/etc/products/L1/PARSIVEL/1MIN.yaml +0 -13
- disdrodb/etc/products/L1/PARSIVEL2/1MIN.yaml +0 -13
- disdrodb/etc/products/L1/PWS100/1MIN.yaml +0 -13
- disdrodb/etc/products/L1/RD80/1MIN.yaml +0 -13
- disdrodb/etc/products/L1/SWS250/1MIN.yaml +0 -13
- disdrodb/etc/products/L2M/10MIN.yaml +0 -12
- disdrodb/l1/beard_model.py +0 -662
- disdrodb/l1/filters.py +0 -205
- disdrodb-0.2.1.dist-info/RECORD +0 -329
- {disdrodb-0.2.1.dist-info ā disdrodb-0.4.0.dist-info}/WHEEL +0 -0
- {disdrodb-0.2.1.dist-info ā disdrodb-0.4.0.dist-info}/licenses/LICENSE +0 -0
- {disdrodb-0.2.1.dist-info ā disdrodb-0.4.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,728 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Any, Literal
|
|
4
|
+
|
|
5
|
+
from pydantic import Field, field_validator, model_validator
|
|
6
|
+
|
|
7
|
+
from disdrodb.api.checks import check_folder_partitioning, check_temporal_resolution
|
|
8
|
+
from disdrodb.api.configs import available_sensor_names
|
|
9
|
+
from disdrodb.configs import get_products_configs_dir
|
|
10
|
+
from disdrodb.fall_velocity.rain import check_rain_fall_velocity_model
|
|
11
|
+
from disdrodb.psd.fitting import PSD_MODELS, check_optimization, check_optimization_kwargs
|
|
12
|
+
from disdrodb.routines.options import get_l2m_model_settings_files, get_model_options, get_product_options
|
|
13
|
+
from disdrodb.scattering.axis_ratio import check_axis_ratio_model
|
|
14
|
+
from disdrodb.scattering.permittivity import check_permittivity_model
|
|
15
|
+
from disdrodb.scattering.routines import ensure_numerical_frequency
|
|
16
|
+
from disdrodb.utils.archiving import check_freq
|
|
17
|
+
from disdrodb.utils.pydantic import CustomBaseModel
|
|
18
|
+
|
|
19
|
+
# -------------------------------------------------------------------------------------------------------.
|
|
20
|
+
## Check product options structure (products_configs_dir)
|
|
21
|
+
# products_configs_dir : check exists
|
|
22
|
+
# Check presence of L0C, L1, L2E and L2M directories
|
|
23
|
+
# If inside product directory (L0C-L2E) there is another directory must correspond to a sensor_name
|
|
24
|
+
# --> available_sensor_names(), otherwise raise error
|
|
25
|
+
# --> Presence of sensor directory is not mandatory
|
|
26
|
+
# If sensor directory is specified, there must be yaml file inside, otherwise raise error
|
|
27
|
+
# Check there is a global.yaml file inside each product directory
|
|
28
|
+
# Check there is the MODEL directory in L2M directory and there are YAML files inside.
|
|
29
|
+
|
|
30
|
+
# Check global product options YAML files
|
|
31
|
+
# Check global product options YAML files per sensor
|
|
32
|
+
# Check custom temporal resolution product options YAML files per sensor
|
|
33
|
+
# -------------------------------------------------------------------------------------------------------.
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def validate_product_configuration_structure(products_configs_dir):
|
|
37
|
+
"""
|
|
38
|
+
Validate the DISDRODB products configuration directory structure.
|
|
39
|
+
|
|
40
|
+
Parameters
|
|
41
|
+
----------
|
|
42
|
+
products_configs_dir : str
|
|
43
|
+
Path to the products configuration directory.
|
|
44
|
+
|
|
45
|
+
Raises
|
|
46
|
+
------
|
|
47
|
+
FileNotFoundError
|
|
48
|
+
If required directories or files are missing.
|
|
49
|
+
ValueError
|
|
50
|
+
If directory structure is invalid.
|
|
51
|
+
"""
|
|
52
|
+
products_configs_path = Path(products_configs_dir)
|
|
53
|
+
|
|
54
|
+
# Check that products_configs_dir exists
|
|
55
|
+
if not products_configs_path.exists():
|
|
56
|
+
raise FileNotFoundError(f"Products configuration directory not found: {products_configs_dir}")
|
|
57
|
+
|
|
58
|
+
# Define required product directories
|
|
59
|
+
required_products = ["L0C", "L1", "L2E", "L2M"]
|
|
60
|
+
available_sensors = available_sensor_names()
|
|
61
|
+
|
|
62
|
+
# Check presence of required product directories
|
|
63
|
+
for product in required_products:
|
|
64
|
+
product_dir = products_configs_path / product
|
|
65
|
+
if not product_dir.exists():
|
|
66
|
+
raise FileNotFoundError(f"Required product directory not found: {product_dir}")
|
|
67
|
+
|
|
68
|
+
# Check for global.yaml in each product directory
|
|
69
|
+
for product in required_products:
|
|
70
|
+
product_dir = products_configs_path / product
|
|
71
|
+
global_yaml = product_dir / "global.yaml"
|
|
72
|
+
if not global_yaml.exists():
|
|
73
|
+
raise FileNotFoundError(f"Required global.yaml file not found in: {product_dir}")
|
|
74
|
+
|
|
75
|
+
# Check subdirectories within product directories (L0C-L2E)
|
|
76
|
+
if product in ["L0C", "L1", "L2E", "L2M"]:
|
|
77
|
+
_validate_sensor_subdirectories(product_dir, available_sensors, product)
|
|
78
|
+
|
|
79
|
+
# Special validation for L2M directory
|
|
80
|
+
_validate_l2m_structure(products_configs_path / "L2M")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _validate_sensor_subdirectories(product_dir, available_sensors, product_name):
|
|
84
|
+
"""
|
|
85
|
+
Validate sensor subdirectories within a product directory.
|
|
86
|
+
|
|
87
|
+
Parameters
|
|
88
|
+
----------
|
|
89
|
+
product_dir : Path
|
|
90
|
+
Path to the product directory.
|
|
91
|
+
available_sensors : list
|
|
92
|
+
list of available sensor names.
|
|
93
|
+
product_name : str
|
|
94
|
+
Name of the product (for error messages).
|
|
95
|
+
"""
|
|
96
|
+
# Get all subdirectories (excluding files)
|
|
97
|
+
subdirs = [d for d in product_dir.iterdir() if d.is_dir()]
|
|
98
|
+
|
|
99
|
+
for subdir in subdirs:
|
|
100
|
+
sensor_name = subdir.name
|
|
101
|
+
|
|
102
|
+
if product_name == "L2M" and subdir.name == "MODELS":
|
|
103
|
+
continue
|
|
104
|
+
|
|
105
|
+
# Check if subdirectory name corresponds to a valid sensor name
|
|
106
|
+
if sensor_name not in available_sensors:
|
|
107
|
+
raise ValueError(
|
|
108
|
+
f"Invalid sensor directory '{sensor_name}' in {product_name}. " f"Must be one of: {available_sensors}",
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
# Check that sensor directory contains at least one YAML file
|
|
112
|
+
yaml_files = list(subdir.glob("*.yaml")) + list(subdir.glob("*.yml"))
|
|
113
|
+
if not yaml_files:
|
|
114
|
+
raise FileNotFoundError(
|
|
115
|
+
f"No YAML files found in sensor directory: {subdir}. "
|
|
116
|
+
f"Sensor directories must contain at least one YAML configuration file.",
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _validate_l2m_structure(l2m_dir):
|
|
121
|
+
"""
|
|
122
|
+
Validate L2M directory structure including MODELS subdirectory.
|
|
123
|
+
|
|
124
|
+
Parameters
|
|
125
|
+
----------
|
|
126
|
+
l2m_dir : Path
|
|
127
|
+
Path to the L2M directory.
|
|
128
|
+
"""
|
|
129
|
+
models_dir = l2m_dir / "MODELS"
|
|
130
|
+
|
|
131
|
+
# Check for MODELS directory in L2M
|
|
132
|
+
if not models_dir.exists():
|
|
133
|
+
raise FileNotFoundError(f"Required MODELS directory not found in: {l2m_dir}")
|
|
134
|
+
|
|
135
|
+
# Check that MODELS directory contains YAML files
|
|
136
|
+
yaml_files = list(models_dir.glob("*.yaml")) + list(models_dir.glob("*.yml"))
|
|
137
|
+
if not yaml_files:
|
|
138
|
+
raise FileNotFoundError(
|
|
139
|
+
f"No YAML model configuration files found in: {models_dir}. "
|
|
140
|
+
f"MODELS directory must contain at least one model YAML file.",
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def validate_temporal_resolution_consistency(products_configs_dir):
|
|
145
|
+
"""
|
|
146
|
+
Validate temporal resolution consistency across products for each sensor.
|
|
147
|
+
|
|
148
|
+
Raises warnings if:
|
|
149
|
+
- L1 temporal_resolutions doesn't include all L2E and L2M temporal resolutions
|
|
150
|
+
- L2E temporal_resolutions doesn't include all L2M temporal resolutions
|
|
151
|
+
"""
|
|
152
|
+
sensor_names = available_sensor_names()
|
|
153
|
+
for sensor_name in sensor_names:
|
|
154
|
+
# Get temporal resolutions for each product
|
|
155
|
+
l1_options = get_product_options(
|
|
156
|
+
product="L1",
|
|
157
|
+
sensor_name=sensor_name,
|
|
158
|
+
products_configs_dir=products_configs_dir,
|
|
159
|
+
)
|
|
160
|
+
l2e_options = get_product_options(
|
|
161
|
+
product="L2E",
|
|
162
|
+
sensor_name=sensor_name,
|
|
163
|
+
products_configs_dir=products_configs_dir,
|
|
164
|
+
)
|
|
165
|
+
l2m_options = get_product_options(
|
|
166
|
+
product="L2M",
|
|
167
|
+
sensor_name=sensor_name,
|
|
168
|
+
products_configs_dir=products_configs_dir,
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
l1_temporal_resolutions = set(l1_options.get("temporal_resolutions", []))
|
|
172
|
+
l2e_temporal_resolutions = set(l2e_options.get("temporal_resolutions", []))
|
|
173
|
+
l2m_temporal_resolutions = set(l2m_options.get("temporal_resolutions", []))
|
|
174
|
+
|
|
175
|
+
# Check L1 includes all L2E temporal resolutions
|
|
176
|
+
missing_l2e_in_l1 = l2e_temporal_resolutions - l1_temporal_resolutions
|
|
177
|
+
if missing_l2e_in_l1:
|
|
178
|
+
print(
|
|
179
|
+
f"WARNING. Sensor '{sensor_name}': L1 temporal_resolutions {sorted(l1_temporal_resolutions)} "
|
|
180
|
+
f"does not include all L2E temporal resolutions. Missing: {sorted(missing_l2e_in_l1)}. "
|
|
181
|
+
f"L2E temporal_resolutions: {sorted(l2e_temporal_resolutions)}",
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
# Check L1 includes all L2M temporal resolutions
|
|
185
|
+
missing_l2m_in_l1 = l2m_temporal_resolutions - l1_temporal_resolutions
|
|
186
|
+
if missing_l2m_in_l1:
|
|
187
|
+
print(
|
|
188
|
+
f"WARNING. Sensor '{sensor_name}': L1 temporal_resolutions {sorted(l1_temporal_resolutions)} "
|
|
189
|
+
f"does not include all L2M temporal resolutions. Missing: {sorted(missing_l2m_in_l1)}. "
|
|
190
|
+
f"L2M temporal_resolutions: {sorted(l2m_temporal_resolutions)}",
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
# Check L2E includes all L2M temporal resolutions
|
|
194
|
+
missing_l2m_in_l2e = l2m_temporal_resolutions - l2e_temporal_resolutions
|
|
195
|
+
if missing_l2m_in_l2e:
|
|
196
|
+
print(
|
|
197
|
+
f"WARNING. Sensor '{sensor_name}': L2E temporal_resolutions {sorted(l2e_temporal_resolutions)} "
|
|
198
|
+
f"does not include all L2M temporal resolutions. Missing: {sorted(missing_l2m_in_l2e)}. "
|
|
199
|
+
f"L2M temporal_resolutions: {sorted(l2m_temporal_resolutions)}",
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
####------------------------------------------------------------------------------------------------
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
class TimeBlockStrategyOptions(CustomBaseModel):
|
|
207
|
+
"""Strategy options for time_block strategy."""
|
|
208
|
+
|
|
209
|
+
freq: str = Field(..., description="Frequency for time block partitioning")
|
|
210
|
+
|
|
211
|
+
@field_validator("freq")
|
|
212
|
+
@classmethod
|
|
213
|
+
def validate_freq(cls, v):
|
|
214
|
+
"""Validate frequency using check_freq function."""
|
|
215
|
+
check_freq(v)
|
|
216
|
+
return v
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
class EventStrategyOptions(CustomBaseModel):
|
|
220
|
+
"""Strategy options for event strategy."""
|
|
221
|
+
|
|
222
|
+
variable: str = Field(..., description="Variable to define events")
|
|
223
|
+
detection_threshold: int = Field(..., ge=0, description="Minimum number of drops")
|
|
224
|
+
neighbor_min_size: int = Field(..., ge=0, description="Minimum neighbor size")
|
|
225
|
+
neighbor_time_interval: str = Field(..., description="Neighbor time interval")
|
|
226
|
+
event_max_time_gap: str = Field(..., description="Maximum time gap for events")
|
|
227
|
+
event_min_duration: str = Field(..., description="Minimum event duration")
|
|
228
|
+
event_min_size: int = Field(..., ge=0, description="Minimum event size")
|
|
229
|
+
|
|
230
|
+
@field_validator("neighbor_time_interval", "event_max_time_gap", "event_min_duration")
|
|
231
|
+
@classmethod
|
|
232
|
+
def validate_time_intervals(cls, v):
|
|
233
|
+
"""Validate time interval strings using check_temporal_resolution."""
|
|
234
|
+
check_temporal_resolution(v)
|
|
235
|
+
return v
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
class ArchiveOptions(CustomBaseModel):
|
|
239
|
+
"""Archive options configuration."""
|
|
240
|
+
|
|
241
|
+
strategy: Literal["time_block", "event"] = Field(..., description="Archiving strategy")
|
|
242
|
+
strategy_options: TimeBlockStrategyOptions | EventStrategyOptions = Field(
|
|
243
|
+
...,
|
|
244
|
+
description="Strategy-specific options",
|
|
245
|
+
)
|
|
246
|
+
folder_partitioning: str = Field(..., description="Folder partitioning scheme")
|
|
247
|
+
|
|
248
|
+
@field_validator("folder_partitioning")
|
|
249
|
+
@classmethod
|
|
250
|
+
def validate_folder_partitioning(cls, v):
|
|
251
|
+
"""Validate folder partitioning."""
|
|
252
|
+
check_folder_partitioning(v)
|
|
253
|
+
return v
|
|
254
|
+
|
|
255
|
+
@model_validator(mode="after")
|
|
256
|
+
def validate_strategy_options(self):
|
|
257
|
+
"""Ensure strategy_options match the selected strategy."""
|
|
258
|
+
expected_type = {
|
|
259
|
+
"time_block": TimeBlockStrategyOptions,
|
|
260
|
+
"event": EventStrategyOptions,
|
|
261
|
+
}[self.strategy]
|
|
262
|
+
|
|
263
|
+
if not isinstance(self.strategy_options, expected_type):
|
|
264
|
+
raise ValueError(
|
|
265
|
+
f"{self.strategy} strategy requires {expected_type.__name__}, "
|
|
266
|
+
f"got {type(self.strategy_options).__name__}",
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
return self
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
class ArchiveOptionsTimeBlock(ArchiveOptions):
|
|
273
|
+
"""Archive options configuration for L0C and L1 products (time_block only)."""
|
|
274
|
+
|
|
275
|
+
@field_validator("strategy", mode="after")
|
|
276
|
+
@classmethod
|
|
277
|
+
def validate_strategy_early(cls, v):
|
|
278
|
+
"""Validate that strategy is 'time_block' and fail early if not."""
|
|
279
|
+
if v != "time_block":
|
|
280
|
+
raise ValueError("L0C and L1 products require strategy 'time_block'.")
|
|
281
|
+
return v
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
class RadarOptions(CustomBaseModel):
|
|
285
|
+
"""Radar simulation options."""
|
|
286
|
+
|
|
287
|
+
frequency: str | int | float | list[str | int | float] = Field(
|
|
288
|
+
...,
|
|
289
|
+
description="Radar frequency bands or numeric frequency values (in GHz)",
|
|
290
|
+
)
|
|
291
|
+
num_points: int | float | list[int | float] = Field(
|
|
292
|
+
...,
|
|
293
|
+
description="Number of points for T-matrix simulation",
|
|
294
|
+
)
|
|
295
|
+
diameter_max: int | float | list[int | float] = Field(
|
|
296
|
+
...,
|
|
297
|
+
description="Maximum diameter for T-matrix simulation",
|
|
298
|
+
)
|
|
299
|
+
canting_angle_std: int | float | list[int | float] = Field(
|
|
300
|
+
...,
|
|
301
|
+
description="Canting angle standard deviation",
|
|
302
|
+
)
|
|
303
|
+
axis_ratio_model: str | list[str] = Field(
|
|
304
|
+
...,
|
|
305
|
+
description="Axis ratio model",
|
|
306
|
+
)
|
|
307
|
+
permittivity_model: str | list[str] = Field(
|
|
308
|
+
...,
|
|
309
|
+
description="Permittivity model",
|
|
310
|
+
)
|
|
311
|
+
water_temperature: int | float | list[int | float] = Field(
|
|
312
|
+
...,
|
|
313
|
+
description="Water temperature in Celsius",
|
|
314
|
+
)
|
|
315
|
+
elevation_angle: int | float | list[int | float] = Field(
|
|
316
|
+
...,
|
|
317
|
+
description="Elevation angle in degrees",
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
# Normalization: make sure all are lists
|
|
321
|
+
@field_validator(
|
|
322
|
+
"frequency",
|
|
323
|
+
"axis_ratio_model",
|
|
324
|
+
"permittivity_model",
|
|
325
|
+
"num_points",
|
|
326
|
+
"diameter_max",
|
|
327
|
+
"canting_angle_std",
|
|
328
|
+
"water_temperature",
|
|
329
|
+
"elevation_angle",
|
|
330
|
+
mode="before",
|
|
331
|
+
)
|
|
332
|
+
@classmethod
|
|
333
|
+
def ensure_list(cls, v):
|
|
334
|
+
"""Normalize single values to lists."""
|
|
335
|
+
if not isinstance(v, list):
|
|
336
|
+
return [v]
|
|
337
|
+
return v
|
|
338
|
+
|
|
339
|
+
@field_validator("frequency")
|
|
340
|
+
@classmethod
|
|
341
|
+
def validate_frequency_bands(cls, frequencies):
|
|
342
|
+
"""Validate radar frequency bands."""
|
|
343
|
+
return ensure_numerical_frequency(frequencies)
|
|
344
|
+
|
|
345
|
+
@field_validator("axis_ratio_model")
|
|
346
|
+
@classmethod
|
|
347
|
+
def validate_axis_ratio_model(cls, axis_ratio_models):
|
|
348
|
+
"""Validate axis ratio models."""
|
|
349
|
+
return [check_axis_ratio_model(axis_ratio_model) for axis_ratio_model in axis_ratio_models]
|
|
350
|
+
|
|
351
|
+
@field_validator("permittivity_model")
|
|
352
|
+
@classmethod
|
|
353
|
+
def validate_permittivity_model(cls, permittivity_models):
|
|
354
|
+
"""Validate permittivity models."""
|
|
355
|
+
return [check_permittivity_model(permittivity_model) for permittivity_model in permittivity_models]
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
class L2EProductOptions(CustomBaseModel):
|
|
359
|
+
"""L2E product-specific options."""
|
|
360
|
+
|
|
361
|
+
compute_spectra: bool = Field(..., description="Whether to compute spectra")
|
|
362
|
+
compute_percentage_contribution: bool = Field(
|
|
363
|
+
...,
|
|
364
|
+
description="Whether to compute percentage contribution",
|
|
365
|
+
)
|
|
366
|
+
minimum_ndrops: int = Field(..., ge=0, description="Minimum number of drops")
|
|
367
|
+
minimum_nbins: int = Field(..., ge=0, description="Minimum number of bins")
|
|
368
|
+
minimum_rain_rate: float = Field(..., ge=0, description="Minimum rain rate threshold")
|
|
369
|
+
fall_velocity_model: str = Field(..., description="Fall velocity model to use")
|
|
370
|
+
minimum_diameter: float = Field(..., ge=0, description="Minimum diameter threshold")
|
|
371
|
+
maximum_diameter: float = Field(..., gt=0, description="Maximum diameter threshold")
|
|
372
|
+
minimum_velocity: float = Field(..., ge=0, description="Minimum velocity threshold")
|
|
373
|
+
maximum_velocity: float = Field(..., gt=0, description="Maximum velocity threshold")
|
|
374
|
+
keep_mixed_precipitation: bool = Field(..., description="Whether to keep mixed precipitation")
|
|
375
|
+
above_velocity_fraction: float | None = Field(..., ge=0, le=1, description="Above velocity fraction")
|
|
376
|
+
above_velocity_tolerance: float = Field(..., ge=0, description="Above velocity tolerance")
|
|
377
|
+
below_velocity_fraction: float | None = Field(..., ge=0, le=1, description="Below velocity fraction")
|
|
378
|
+
below_velocity_tolerance: float = Field(..., ge=0, description="Below velocity tolerance")
|
|
379
|
+
maintain_drops_smaller_than: float = Field(..., ge=0, description="Maintain drops smaller than threshold")
|
|
380
|
+
maintain_drops_slower_than: float = Field(..., ge=0, description="Maintain drops slower than threshold")
|
|
381
|
+
maintain_smallest_drops: bool = Field(..., description="Whether to maintain smallest drops")
|
|
382
|
+
remove_splashing_drops: bool = Field(..., description="Whether to remove splashing drops")
|
|
383
|
+
|
|
384
|
+
@model_validator(mode="after")
|
|
385
|
+
def validate_diameter_range(self):
|
|
386
|
+
"""Validate that maximum_diameter > minimum_diameter."""
|
|
387
|
+
if self.maximum_diameter <= self.minimum_diameter:
|
|
388
|
+
raise ValueError("maximum_diameter must be greater than minimum_diameter")
|
|
389
|
+
return self
|
|
390
|
+
|
|
391
|
+
@model_validator(mode="after")
|
|
392
|
+
def validate_velocity_range(self):
|
|
393
|
+
"""Validate that maximum_velocity > minimum_velocity."""
|
|
394
|
+
if self.maximum_velocity <= self.minimum_velocity:
|
|
395
|
+
raise ValueError("maximum_velocity must be greater than minimum_velocity")
|
|
396
|
+
return self
|
|
397
|
+
|
|
398
|
+
@field_validator("fall_velocity_model")
|
|
399
|
+
@classmethod
|
|
400
|
+
def validate_fall_velocity_model(cls, fall_velocity_model):
|
|
401
|
+
"""Validate fall velocity model."""
|
|
402
|
+
return check_rain_fall_velocity_model(fall_velocity_model)
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
class L2MProductOptions(CustomBaseModel):
|
|
406
|
+
"""L2M product-specific options."""
|
|
407
|
+
|
|
408
|
+
fall_velocity_model: str = Field(..., description="Fall velocity model to use")
|
|
409
|
+
diameter_min: float = Field(..., ge=0, description="Minimum diameter threshold")
|
|
410
|
+
diameter_max: float = Field(..., gt=0, description="Maximum diameter threshold")
|
|
411
|
+
diameter_spacing: float = Field(..., gt=0, description="Diameter spacing for grid")
|
|
412
|
+
gof_metrics: bool = Field(..., description="Whether to compute goodness-of-fit metrics")
|
|
413
|
+
minimum_ndrops: int = Field(..., ge=0, description="Minimum number of drops")
|
|
414
|
+
minimum_nbins: int = Field(..., ge=0, description="Minimum number of bins with drops")
|
|
415
|
+
minimum_rain_rate: float = Field(..., ge=0, description="Minimum rain rate threshold")
|
|
416
|
+
|
|
417
|
+
@model_validator(mode="after")
|
|
418
|
+
def validate_diameter_range(self):
|
|
419
|
+
"""Validate that diameter_max > diameter_min."""
|
|
420
|
+
if self.diameter_max <= self.diameter_min:
|
|
421
|
+
raise ValueError("diameter_max must be greater than diameter_min")
|
|
422
|
+
return self
|
|
423
|
+
|
|
424
|
+
@field_validator("fall_velocity_model")
|
|
425
|
+
@classmethod
|
|
426
|
+
def validate_fall_velocity_model(cls, fall_velocity_model):
|
|
427
|
+
"""Validate fall velocity model."""
|
|
428
|
+
return check_rain_fall_velocity_model(fall_velocity_model)
|
|
429
|
+
|
|
430
|
+
@model_validator(mode="after")
|
|
431
|
+
def validate_diameter_grid(self):
|
|
432
|
+
"""Validate diameter grid configuration."""
|
|
433
|
+
# Check that diameter_spacing is reasonable relative to the range
|
|
434
|
+
diameter_range = self.diameter_max - self.diameter_min
|
|
435
|
+
if self.diameter_spacing > diameter_range:
|
|
436
|
+
raise ValueError("diameter_spacing cannot be larger than the diameter range.")
|
|
437
|
+
return self
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
class TemporalResolutionsValidationMixin:
|
|
441
|
+
"""Mixin for temporal resolutions validation."""
|
|
442
|
+
|
|
443
|
+
@field_validator("temporal_resolutions")
|
|
444
|
+
@classmethod
|
|
445
|
+
def validate_temporal_resolutions(cls, temporal_resolutions: list[str]):
|
|
446
|
+
"""Validate temporal resolutions list."""
|
|
447
|
+
if not temporal_resolutions:
|
|
448
|
+
raise ValueError("temporal_resolutions cannot be empty.")
|
|
449
|
+
|
|
450
|
+
for temporal_resolution in temporal_resolutions:
|
|
451
|
+
check_temporal_resolution(temporal_resolution)
|
|
452
|
+
|
|
453
|
+
if len(temporal_resolutions) != len(set(temporal_resolutions)):
|
|
454
|
+
raise ValueError("temporal_resolutions contains duplicates.")
|
|
455
|
+
return temporal_resolutions
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
####------------------------------------------------------------------------------
|
|
459
|
+
#### Validation of L2M models settings
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
class L2MModelConfig(CustomBaseModel):
|
|
463
|
+
"""L2M model configuration validation."""
|
|
464
|
+
|
|
465
|
+
psd_model: str = Field(..., description="PSD model name")
|
|
466
|
+
optimization: str = Field(..., description="Optimization method")
|
|
467
|
+
optimization_kwargs: dict[str, Any] = Field(..., description="Optimization-specific parameters")
|
|
468
|
+
|
|
469
|
+
@field_validator("psd_model")
|
|
470
|
+
@classmethod
|
|
471
|
+
def validate_psd_model(cls, psd_model):
|
|
472
|
+
"""Validate psd_model."""
|
|
473
|
+
valid_psd_models = PSD_MODELS
|
|
474
|
+
if psd_model not in valid_psd_models:
|
|
475
|
+
raise ValueError(f"Invalid psd_model '{psd_model}'. Must be one of {valid_psd_models}")
|
|
476
|
+
return psd_model
|
|
477
|
+
|
|
478
|
+
@field_validator("optimization")
|
|
479
|
+
@classmethod
|
|
480
|
+
def validate_optimization(cls, optimization):
|
|
481
|
+
"""Validate optimization method."""
|
|
482
|
+
return check_optimization(optimization)
|
|
483
|
+
|
|
484
|
+
@model_validator(mode="after")
|
|
485
|
+
def validate_optimization_kwargs(self):
|
|
486
|
+
"""Validate that optimization_kwargs matches the optimization method."""
|
|
487
|
+
# Use the existing validation function
|
|
488
|
+
check_optimization_kwargs(
|
|
489
|
+
optimization_kwargs=self.optimization_kwargs,
|
|
490
|
+
optimization=self.optimization,
|
|
491
|
+
psd_model=self.psd_model,
|
|
492
|
+
)
|
|
493
|
+
return self
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def validate_l2m_model_configs(products_configs_dir: str):
|
|
497
|
+
"""
|
|
498
|
+
Validate all L2M model configuration files.
|
|
499
|
+
|
|
500
|
+
Parameters
|
|
501
|
+
----------
|
|
502
|
+
products_configs_dir : str
|
|
503
|
+
Path to products configuration directory.
|
|
504
|
+
|
|
505
|
+
Raises
|
|
506
|
+
------
|
|
507
|
+
ValidationError
|
|
508
|
+
If any L2M model configuration is invalid.
|
|
509
|
+
"""
|
|
510
|
+
# Get all L2M model configuration files
|
|
511
|
+
model_settings_files = get_l2m_model_settings_files(products_configs_dir)
|
|
512
|
+
|
|
513
|
+
validation_errors = []
|
|
514
|
+
|
|
515
|
+
for model_file in model_settings_files:
|
|
516
|
+
model_name = os.path.basename(model_file).replace(".yaml", "")
|
|
517
|
+
try:
|
|
518
|
+
# Load model configuration
|
|
519
|
+
model_config = get_model_options(model_name, products_configs_dir=products_configs_dir)
|
|
520
|
+
# Validate configuration
|
|
521
|
+
L2MModelConfig(**model_config)
|
|
522
|
+
except Exception as e:
|
|
523
|
+
error_msg = f"L2M model '{model_name}' configuration validation failed:\n{e}"
|
|
524
|
+
validation_errors.append(error_msg)
|
|
525
|
+
|
|
526
|
+
# Report all validation errors at once
|
|
527
|
+
if validation_errors:
|
|
528
|
+
error_summary = f"\n{'='*80}\n".join(validation_errors)
|
|
529
|
+
raise ValueError(
|
|
530
|
+
f"L2M model configuration validation failed for {len(validation_errors)} model(s):\n\n"
|
|
531
|
+
f"{'='*80}\n{error_summary}\n{'='*80}",
|
|
532
|
+
)
|
|
533
|
+
|
|
534
|
+
print("š All L2M models configurations validated successfully!")
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
####------------------------------------------------------------------------------
|
|
538
|
+
#### Validation of DISDRODB products settings
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
class L1ProductConfig(CustomBaseModel):
|
|
542
|
+
"""L0C product configuration model."""
|
|
543
|
+
|
|
544
|
+
archive_options: ArchiveOptionsTimeBlock = Field(..., description="Archive configuration options")
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
class L2EProductConfig(CustomBaseModel):
|
|
548
|
+
"""L2E product configuration model."""
|
|
549
|
+
|
|
550
|
+
archive_options: ArchiveOptions = Field(..., description="Archive configuration options")
|
|
551
|
+
product_options: L2EProductOptions = Field(..., description="L2E product-specific options")
|
|
552
|
+
radar_enabled: bool = Field(..., description="Whether radar simulation is enabled")
|
|
553
|
+
radar_options: RadarOptions = Field(..., description="Radar simulation options")
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
class L2MProductConfig(CustomBaseModel):
|
|
557
|
+
"""L2M product configuration model."""
|
|
558
|
+
|
|
559
|
+
models: list[str] = Field(..., description="list of L2M models to use")
|
|
560
|
+
archive_options: ArchiveOptions = Field(..., description="Archive configuration options")
|
|
561
|
+
product_options: L2MProductOptions = Field(..., description="L2M product-specific options")
|
|
562
|
+
radar_enabled: bool = Field(..., description="Whether radar simulation is enabled")
|
|
563
|
+
radar_options: RadarOptions = Field(..., description="Radar simulation options")
|
|
564
|
+
|
|
565
|
+
@field_validator("models", mode="after")
|
|
566
|
+
@classmethod
|
|
567
|
+
def validate_models(cls, models):
|
|
568
|
+
"""Validate L2M models list."""
|
|
569
|
+
if not models:
|
|
570
|
+
raise ValueError("'models' list cannot be empty")
|
|
571
|
+
|
|
572
|
+
# Check for duplicates
|
|
573
|
+
if len(models) != len(set(models)):
|
|
574
|
+
raise ValueError("'models' list contains duplicates")
|
|
575
|
+
|
|
576
|
+
# Retrieve products configuration directory
|
|
577
|
+
products_configs_dir = get_products_configs_dir()
|
|
578
|
+
|
|
579
|
+
# Get available model YAML files
|
|
580
|
+
model_settings_paths = get_l2m_model_settings_files(products_configs_dir)
|
|
581
|
+
|
|
582
|
+
# Get available model names
|
|
583
|
+
available_models = [os.path.basename(path).replace(".yaml", "") for path in model_settings_paths]
|
|
584
|
+
|
|
585
|
+
# Check each requested model has a corresponding YAML file
|
|
586
|
+
missing_models = [model for model in models if model not in available_models]
|
|
587
|
+
if missing_models:
|
|
588
|
+
raise ValueError(
|
|
589
|
+
f"L2M model configuration files not found for: {missing_models}. "
|
|
590
|
+
f"Available models: {sorted(available_models)}",
|
|
591
|
+
)
|
|
592
|
+
return models
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
class L0CProductGlobalConfig(CustomBaseModel):
|
|
596
|
+
"""L0C product configuration model."""
|
|
597
|
+
|
|
598
|
+
archive_options: ArchiveOptionsTimeBlock = Field(..., description="Archive configuration options")
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
class L1ProductGlobalConfig(L1ProductConfig, TemporalResolutionsValidationMixin):
|
|
602
|
+
"""L1 product configuration model."""
|
|
603
|
+
|
|
604
|
+
temporal_resolutions: list[str] = Field(..., description="list of temporal resolution")
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
class L2EProductGlobalConfig(L2EProductConfig, TemporalResolutionsValidationMixin):
|
|
608
|
+
"""L1 product configuration model."""
|
|
609
|
+
|
|
610
|
+
temporal_resolutions: list[str] = Field(..., description="list of temporal resolution")
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
class L2MProductGlobalConfig(L2MProductConfig, TemporalResolutionsValidationMixin):
|
|
614
|
+
"""L2M product configuration model."""
|
|
615
|
+
|
|
616
|
+
temporal_resolutions: list[str] = Field(..., description="list of temporal resolutions")
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
def validate_all_product_yaml_files(products_configs_dir):
|
|
620
|
+
"""
|
|
621
|
+
Validate all DISDRODB product YAML configuration files.
|
|
622
|
+
|
|
623
|
+
Raises
|
|
624
|
+
------
|
|
625
|
+
ValidationError
|
|
626
|
+
If any YAML file validation fails with detailed information.
|
|
627
|
+
"""
|
|
628
|
+
# Define product validators mapping
|
|
629
|
+
product_global_validators = {
|
|
630
|
+
"L0C": L0CProductGlobalConfig,
|
|
631
|
+
"L1": L1ProductGlobalConfig,
|
|
632
|
+
"L2E": L2EProductGlobalConfig,
|
|
633
|
+
"L2M": L2MProductGlobalConfig,
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
# Define custom temporal resolution validators (without temporal_resolutions field)
|
|
637
|
+
custom_temporal_validators = {
|
|
638
|
+
"L1": L1ProductConfig,
|
|
639
|
+
"L2E": L2EProductConfig,
|
|
640
|
+
"L2M": L2MProductConfig,
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
products = ["L0C", "L1", "L2E", "L2M"]
|
|
644
|
+
sensor_names = available_sensor_names()
|
|
645
|
+
|
|
646
|
+
validation_errors = []
|
|
647
|
+
|
|
648
|
+
for product in products:
|
|
649
|
+
# 1. Test global YAML (product-level)
|
|
650
|
+
product_options = get_product_options(product=product, products_configs_dir=products_configs_dir)
|
|
651
|
+
try:
|
|
652
|
+
validator_class = product_global_validators[product]
|
|
653
|
+
validator_class(**product_options)
|
|
654
|
+
except Exception as e:
|
|
655
|
+
error_msg = f"Global {product} configuration validation failed:\n{e}"
|
|
656
|
+
validation_errors.append(error_msg)
|
|
657
|
+
|
|
658
|
+
# 2. Test YAML per sensor
|
|
659
|
+
for sensor_name in sensor_names:
|
|
660
|
+
# Test sensor-level global YAML
|
|
661
|
+
product_options = get_product_options(
|
|
662
|
+
product=product,
|
|
663
|
+
sensor_name=sensor_name,
|
|
664
|
+
products_configs_dir=products_configs_dir,
|
|
665
|
+
)
|
|
666
|
+
try:
|
|
667
|
+
validator_class = product_global_validators[product]
|
|
668
|
+
validator_class(**product_options)
|
|
669
|
+
except Exception as e:
|
|
670
|
+
error_msg = f"{product}/{sensor_name} configuration validation failed:\n" f"{e}"
|
|
671
|
+
validation_errors.append(error_msg)
|
|
672
|
+
continue
|
|
673
|
+
|
|
674
|
+
# 3. Test custom temporal resolution YAML files for given sensor (not for L0C)
|
|
675
|
+
if "temporal_resolutions" in product_options:
|
|
676
|
+
# Retrieve product validator class
|
|
677
|
+
for temporal_resolution in product_options["temporal_resolutions"]:
|
|
678
|
+
try:
|
|
679
|
+
validator_class = custom_temporal_validators[product]
|
|
680
|
+
custom_product_options = get_product_options(
|
|
681
|
+
product=product,
|
|
682
|
+
temporal_resolution=temporal_resolution,
|
|
683
|
+
sensor_name=sensor_name,
|
|
684
|
+
products_configs_dir=products_configs_dir,
|
|
685
|
+
)
|
|
686
|
+
validator_class(**custom_product_options)
|
|
687
|
+
except Exception as e:
|
|
688
|
+
error_msg = (
|
|
689
|
+
f"{product}/{sensor_name}/{temporal_resolution} configuration validation failed:\n" f"{e}"
|
|
690
|
+
)
|
|
691
|
+
validation_errors.append(error_msg)
|
|
692
|
+
|
|
693
|
+
# Report all validation errors at once
|
|
694
|
+
if validation_errors:
|
|
695
|
+
error_summary = f"\n{'='*80}\n".join(validation_errors)
|
|
696
|
+
raise ValueError(
|
|
697
|
+
f"YAML configuration validation failed for {len(validation_errors)} file(s):\n\n"
|
|
698
|
+
f"{'='*80}\n{error_summary}\n{'='*80}",
|
|
699
|
+
)
|
|
700
|
+
|
|
701
|
+
print("\nš All products configurations validated successfully!")
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
####-----------------------------------------------------------------------------------------------.
|
|
705
|
+
#### Wrapper
|
|
706
|
+
|
|
707
|
+
|
|
708
|
+
def validate_products_configurations(products_configs_dir=None):
|
|
709
|
+
"""Validate the DISDRODB products configuration files."""
|
|
710
|
+
import disdrodb
|
|
711
|
+
|
|
712
|
+
products_configs_dir = get_products_configs_dir(products_configs_dir=products_configs_dir)
|
|
713
|
+
|
|
714
|
+
with disdrodb.config.set({"products_configs_dir": products_configs_dir}):
|
|
715
|
+
# Validate directory structure first
|
|
716
|
+
validate_product_configuration_structure(products_configs_dir)
|
|
717
|
+
|
|
718
|
+
# Validate all DISDRODB products global configuration files with pydantic
|
|
719
|
+
validate_all_product_yaml_files(products_configs_dir)
|
|
720
|
+
|
|
721
|
+
# Check temporal resolution consistency
|
|
722
|
+
validate_temporal_resolution_consistency(products_configs_dir)
|
|
723
|
+
|
|
724
|
+
# Validate L2M model options
|
|
725
|
+
validate_l2m_model_configs(products_configs_dir)
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
####-----------------------------------------------------------------------------------------------.
|