disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +132 -15
- disdrodb/_config.py +4 -2
- disdrodb/_version.py +9 -4
- disdrodb/api/checks.py +264 -237
- disdrodb/api/configs.py +4 -8
- disdrodb/api/create_directories.py +235 -290
- disdrodb/api/info.py +217 -26
- disdrodb/api/io.py +306 -270
- disdrodb/api/path.py +597 -173
- disdrodb/api/search.py +486 -0
- disdrodb/{metadata/scripts → cli}/disdrodb_check_metadata_archive.py +12 -7
- disdrodb/{utils/pandas.py → cli/disdrodb_data_archive_directory.py} +9 -18
- disdrodb/cli/disdrodb_download_archive.py +86 -0
- disdrodb/cli/disdrodb_download_metadata_archive.py +53 -0
- disdrodb/cli/disdrodb_download_station.py +84 -0
- disdrodb/{api/scripts → cli}/disdrodb_initialize_station.py +22 -10
- disdrodb/cli/disdrodb_metadata_archive_directory.py +32 -0
- disdrodb/{data_transfer/scripts/disdrodb_download_station.py → cli/disdrodb_open_data_archive.py} +22 -22
- disdrodb/cli/disdrodb_open_logs_directory.py +69 -0
- disdrodb/{data_transfer/scripts/disdrodb_upload_station.py → cli/disdrodb_open_metadata_archive.py} +22 -24
- disdrodb/cli/disdrodb_open_metadata_directory.py +71 -0
- disdrodb/cli/disdrodb_open_product_directory.py +74 -0
- disdrodb/cli/disdrodb_open_readers_directory.py +32 -0
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0.py +38 -31
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0_station.py +32 -30
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a_station.py +24 -33
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b_station.py +25 -34
- disdrodb/cli/disdrodb_run_l0c.py +130 -0
- disdrodb/cli/disdrodb_run_l0c_station.py +129 -0
- disdrodb/cli/disdrodb_run_l1.py +122 -0
- disdrodb/cli/disdrodb_run_l1_station.py +121 -0
- disdrodb/cli/disdrodb_run_l2e.py +122 -0
- disdrodb/cli/disdrodb_run_l2e_station.py +122 -0
- disdrodb/cli/disdrodb_run_l2m.py +122 -0
- disdrodb/cli/disdrodb_run_l2m_station.py +122 -0
- disdrodb/cli/disdrodb_upload_archive.py +105 -0
- disdrodb/cli/disdrodb_upload_station.py +98 -0
- disdrodb/configs.py +90 -25
- disdrodb/data_transfer/__init__.py +22 -0
- disdrodb/data_transfer/download_data.py +87 -90
- disdrodb/data_transfer/upload_data.py +64 -37
- disdrodb/data_transfer/zenodo.py +15 -18
- disdrodb/docs.py +1 -1
- disdrodb/issue/__init__.py +17 -4
- disdrodb/issue/checks.py +10 -23
- disdrodb/issue/reader.py +9 -12
- disdrodb/issue/writer.py +14 -17
- disdrodb/l0/__init__.py +17 -26
- disdrodb/l0/check_configs.py +35 -23
- disdrodb/l0/check_standards.py +46 -51
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_diameter.yml +44 -44
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_velocity.yml +40 -40
- disdrodb/l0/configs/LPM/l0a_encodings.yml +80 -0
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_cf_attrs.yml +84 -65
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_encodings.yml +50 -9
- disdrodb/l0/configs/{Thies_LPM → LPM}/raw_data_format.yml +285 -245
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_diameter.yml +66 -66
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL/l0a_encodings.yml +32 -0
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_cf_attrs.yml +23 -21
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_encodings.yml +17 -17
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/raw_data_format.yml +77 -77
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_diameter.yml +64 -64
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +39 -0
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_cf_attrs.yml +28 -26
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_encodings.yml +20 -20
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/raw_data_format.yml +107 -107
- disdrodb/l0/configs/PWS100/bins_diameter.yml +173 -0
- disdrodb/l0/configs/PWS100/bins_velocity.yml +173 -0
- disdrodb/l0/configs/PWS100/l0a_encodings.yml +19 -0
- disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +76 -0
- disdrodb/l0/configs/PWS100/l0b_encodings.yml +176 -0
- disdrodb/l0/configs/PWS100/raw_data_format.yml +182 -0
- disdrodb/l0/configs/{RD_80 → RD80}/bins_diameter.yml +40 -40
- disdrodb/l0/configs/RD80/l0a_encodings.yml +16 -0
- disdrodb/l0/configs/{RD_80 → RD80}/l0b_cf_attrs.yml +3 -3
- disdrodb/l0/configs/RD80/l0b_encodings.yml +135 -0
- disdrodb/l0/configs/{RD_80 → RD80}/raw_data_format.yml +46 -50
- disdrodb/l0/l0_reader.py +216 -340
- disdrodb/l0/l0a_processing.py +237 -208
- disdrodb/l0/l0b_nc_processing.py +227 -80
- disdrodb/l0/l0b_processing.py +96 -174
- disdrodb/l0/l0c_processing.py +627 -0
- disdrodb/l0/readers/{ARM → LPM/ARM}/ARM_LPM.py +36 -58
- disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +236 -0
- disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +185 -0
- disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +185 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +195 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +210 -0
- disdrodb/l0/readers/{BRAZIL/GOAMAZON_LPM.py → LPM/KIT/CHWALA.py} +97 -76
- disdrodb/l0/readers/LPM/SLOVENIA/ARSO.py +197 -0
- disdrodb/l0/readers/LPM/SLOVENIA/CRNI_VRH.py +197 -0
- disdrodb/l0/readers/{UK → LPM/UK}/DIVEN.py +14 -35
- disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +157 -0
- disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +113 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/ARCTIC_2021.py +40 -57
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/COMMON_2011.py +37 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/DAVOS_2009_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_2009.py +34 -51
- disdrodb/l0/readers/{EPFL/PARADISO_2014.py → PARSIVEL/EPFL/EPFL_ROOF_2008.py} +38 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2012.py +33 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GENEPI_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007_2.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HPICONET_2010.py +34 -51
- disdrodb/l0/readers/{EPFL/EPFL_ROOF_2010.py → PARSIVEL/EPFL/HYMEX_LTE_SOP2.py} +37 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +111 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HYMEX_LTE_SOP4.py +36 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2018.py +34 -52
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2019.py +38 -56
- disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PARSIVEL_2007.py +27 -45
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PLATO_2019.py +24 -44
- disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +140 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RACLETS_2019_WJF.py +41 -59
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RIETHOLZBACH_2011.py +34 -51
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +117 -0
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +137 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/UNIL_2022.py +42 -55
- disdrodb/l0/readers/PARSIVEL/GPM/IFLOODS.py +104 -0
- disdrodb/l0/readers/{GPM → PARSIVEL/GPM}/LPVEX.py +29 -48
- disdrodb/l0/readers/PARSIVEL/GPM/MC3E.py +184 -0
- disdrodb/l0/readers/PARSIVEL/KIT/BURKINA_FASO.py +133 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +113 -0
- disdrodb/l0/readers/{NCAR/VORTEX_SE_2016_P1.py → PARSIVEL/NCAR/OWLES_MIPS.py} +46 -72
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +125 -0
- disdrodb/l0/readers/{NCAR/OWLES_MIPS.py → PARSIVEL/NCAR/PLOWS_MIPS.py} +45 -64
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +114 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +176 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +183 -0
- disdrodb/l0/readers/PARSIVEL/SLOVENIA/UL_FGG.py +121 -0
- disdrodb/l0/readers/{ARM/ARM_LD.py → PARSIVEL2/ARM/ARM_PARSIVEL2.py} +27 -50
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/{DENMARK → PARSIVEL2/DENMARK}/EROSION_nc.py +14 -35
- disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +189 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +119 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +104 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/NSSTC.py +176 -0
- disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +32 -0
- disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +56 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +120 -0
- disdrodb/l0/readers/{NCAR → PARSIVEL2/NCAR}/PECAN_MIPS.py +45 -64
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +181 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +160 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +160 -0
- disdrodb/l0/readers/{NCAR/PLOWS_MIPS.py → PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py} +49 -66
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +118 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +152 -0
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT.py +166 -0
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +150 -0
- disdrodb/l0/readers/{NCAR/RELAMPAGO_RD80.py → RD80/BRAZIL/CHUVA_RD80.py} +36 -60
- disdrodb/l0/readers/{BRAZIL → RD80/BRAZIL}/GOAMAZON_RD80.py +36 -55
- disdrodb/l0/readers/{NCAR → RD80/NCAR}/CINDY_2011_RD80.py +35 -54
- disdrodb/l0/readers/{BRAZIL/CHUVA_RD80.py → RD80/NCAR/RELAMPAGO_RD80.py} +40 -54
- disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py +274 -0
- disdrodb/l0/readers/template_reader_raw_netcdf_data.py +62 -0
- disdrodb/l0/readers/{reader_template.py → template_reader_raw_text_data.py} +20 -44
- disdrodb/l0/routines.py +885 -581
- disdrodb/l0/standards.py +77 -238
- disdrodb/l0/template_tools.py +105 -110
- disdrodb/l1/__init__.py +17 -0
- disdrodb/l1/beard_model.py +716 -0
- disdrodb/l1/encoding_attrs.py +635 -0
- disdrodb/l1/fall_velocity.py +260 -0
- disdrodb/l1/filters.py +192 -0
- disdrodb/l1/processing.py +202 -0
- disdrodb/l1/resampling.py +236 -0
- disdrodb/l1/routines.py +358 -0
- disdrodb/l1_env/__init__.py +17 -0
- disdrodb/l1_env/routines.py +38 -0
- disdrodb/l2/__init__.py +17 -0
- disdrodb/l2/empirical_dsd.py +1833 -0
- disdrodb/l2/event.py +388 -0
- disdrodb/l2/processing.py +528 -0
- disdrodb/l2/processing_options.py +213 -0
- disdrodb/l2/routines.py +868 -0
- disdrodb/metadata/__init__.py +9 -2
- disdrodb/metadata/checks.py +180 -124
- disdrodb/metadata/download.py +81 -0
- disdrodb/metadata/geolocation.py +146 -0
- disdrodb/metadata/info.py +20 -13
- disdrodb/metadata/manipulation.py +3 -3
- disdrodb/metadata/reader.py +59 -8
- disdrodb/metadata/search.py +77 -144
- disdrodb/metadata/standards.py +83 -80
- disdrodb/metadata/writer.py +10 -16
- disdrodb/psd/__init__.py +38 -0
- disdrodb/psd/fitting.py +2146 -0
- disdrodb/psd/models.py +774 -0
- disdrodb/routines.py +1412 -0
- disdrodb/scattering/__init__.py +28 -0
- disdrodb/scattering/axis_ratio.py +344 -0
- disdrodb/scattering/routines.py +456 -0
- disdrodb/utils/__init__.py +17 -0
- disdrodb/utils/attrs.py +208 -0
- disdrodb/utils/cli.py +269 -0
- disdrodb/utils/compression.py +60 -42
- disdrodb/utils/dask.py +62 -0
- disdrodb/utils/dataframe.py +342 -0
- disdrodb/utils/decorators.py +110 -0
- disdrodb/utils/directories.py +107 -46
- disdrodb/utils/encoding.py +127 -0
- disdrodb/utils/list.py +29 -0
- disdrodb/utils/logger.py +168 -46
- disdrodb/utils/time.py +657 -0
- disdrodb/utils/warnings.py +30 -0
- disdrodb/utils/writer.py +57 -0
- disdrodb/utils/xarray.py +138 -47
- disdrodb/utils/yaml.py +0 -1
- disdrodb/viz/__init__.py +17 -0
- disdrodb/viz/plots.py +17 -0
- disdrodb-0.1.1.dist-info/METADATA +294 -0
- disdrodb-0.1.1.dist-info/RECORD +232 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/WHEEL +1 -1
- disdrodb-0.1.1.dist-info/entry_points.txt +30 -0
- disdrodb/data_transfer/scripts/disdrodb_download_archive.py +0 -53
- disdrodb/data_transfer/scripts/disdrodb_upload_archive.py +0 -57
- disdrodb/l0/configs/OTT_Parsivel/l0a_encodings.yml +0 -32
- disdrodb/l0/configs/OTT_Parsivel2/l0a_encodings.yml +0 -39
- disdrodb/l0/configs/RD_80/l0a_encodings.yml +0 -16
- disdrodb/l0/configs/RD_80/l0b_encodings.yml +0 -135
- disdrodb/l0/configs/Thies_LPM/l0a_encodings.yml +0 -80
- disdrodb/l0/io.py +0 -257
- disdrodb/l0/l0_processing.py +0 -1091
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_OTT.py +0 -178
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_THIES.py +0 -247
- disdrodb/l0/readers/BRAZIL/CHUVA_LPM.py +0 -204
- disdrodb/l0/readers/BRAZIL/CHUVA_OTT.py +0 -183
- disdrodb/l0/readers/BRAZIL/GOAMAZON_OTT.py +0 -183
- disdrodb/l0/readers/CHINA/CHONGQING.py +0 -131
- disdrodb/l0/readers/EPFL/EPFL_ROOF_2008.py +0 -128
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP2.py +0 -127
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP3.py +0 -129
- disdrodb/l0/readers/EPFL/RACLETS_2019.py +0 -158
- disdrodb/l0/readers/EPFL/SAMOYLOV_2017.py +0 -136
- disdrodb/l0/readers/EPFL/SAMOYLOV_2019.py +0 -158
- disdrodb/l0/readers/FRANCE/SIRTA_OTT2.py +0 -138
- disdrodb/l0/readers/GPM/GCPEX.py +0 -123
- disdrodb/l0/readers/GPM/IFLOODS.py +0 -123
- disdrodb/l0/readers/GPM/MC3E.py +0 -123
- disdrodb/l0/readers/GPM/NSSTC.py +0 -164
- disdrodb/l0/readers/ITALY/GID.py +0 -199
- disdrodb/l0/readers/MEXICO/OH_IIUNAM_nc.py +0 -92
- disdrodb/l0/readers/NCAR/CCOPE_2015.py +0 -133
- disdrodb/l0/readers/NCAR/PECAN_FP3.py +0 -137
- disdrodb/l0/readers/NCAR/PECAN_MOBILE.py +0 -144
- disdrodb/l0/readers/NCAR/RELAMPAGO_OTT.py +0 -195
- disdrodb/l0/readers/NCAR/SNOWIE_PJ.py +0 -172
- disdrodb/l0/readers/NCAR/SNOWIE_SB.py +0 -179
- disdrodb/l0/readers/NCAR/VORTEX2_2009.py +0 -133
- disdrodb/l0/readers/NCAR/VORTEX2_2010.py +0 -188
- disdrodb/l0/readers/NCAR/VORTEX2_2010_UF.py +0 -191
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_P2.py +0 -135
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_PIPS.py +0 -170
- disdrodb/l0/readers/NETHERLANDS/DELFT.py +0 -187
- disdrodb/l0/readers/SPAIN/SBEGUERIA.py +0 -179
- disdrodb/l0/scripts/disdrodb_run_l0b_concat.py +0 -93
- disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py +0 -85
- disdrodb/utils/netcdf.py +0 -452
- disdrodb/utils/scripts.py +0 -102
- disdrodb-0.0.21.dist-info/AUTHORS.md +0 -18
- disdrodb-0.0.21.dist-info/METADATA +0 -186
- disdrodb-0.0.21.dist-info/RECORD +0 -168
- disdrodb-0.0.21.dist-info/entry_points.txt +0 -15
- /disdrodb/l0/configs/{RD_80 → RD80}/bins_velocity.yml +0 -0
- /disdrodb/l0/manuals/{Thies_LPM.pdf → LPM.pdf} +0 -0
- /disdrodb/l0/manuals/{ODM_470.pdf → ODM470.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel.pdf → PARSIVEL.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel2.pdf → PARSIVEL2.pdf} +0 -0
- /disdrodb/l0/manuals/{PWS_100.pdf → PWS100.pdf} +0 -0
- /disdrodb/l0/manuals/{RD_80.pdf → RD80.pdf} +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info/licenses}/LICENSE +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/top_level.txt +0 -0
disdrodb/metadata/__init__.py
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
|
+
from disdrodb.metadata.download import download_metadata_archive
|
|
1
2
|
from disdrodb.metadata.info import get_archive_metadata_key_value
|
|
2
|
-
from disdrodb.metadata.reader import read_station_metadata
|
|
3
|
+
from disdrodb.metadata.reader import read_metadata_archive, read_station_metadata
|
|
3
4
|
from disdrodb.metadata.search import get_list_metadata
|
|
4
5
|
|
|
5
|
-
__all__ = [
|
|
6
|
+
__all__ = [
|
|
7
|
+
"download_metadata_archive",
|
|
8
|
+
"get_archive_metadata_key_value",
|
|
9
|
+
"get_list_metadata",
|
|
10
|
+
"read_metadata_archive",
|
|
11
|
+
"read_station_metadata",
|
|
12
|
+
]
|
disdrodb/metadata/checks.py
CHANGED
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
"""Check metadata."""
|
|
20
20
|
|
|
21
21
|
import os
|
|
22
|
-
from typing import Union
|
|
22
|
+
from typing import Optional, Union
|
|
23
23
|
|
|
24
24
|
import numpy as np
|
|
25
25
|
|
|
@@ -27,10 +27,10 @@ from disdrodb.api.info import (
|
|
|
27
27
|
infer_campaign_name_from_path,
|
|
28
28
|
infer_data_source_from_path,
|
|
29
29
|
)
|
|
30
|
-
from disdrodb.configs import
|
|
30
|
+
from disdrodb.configs import get_metadata_archive_dir
|
|
31
31
|
from disdrodb.metadata.reader import read_station_metadata
|
|
32
32
|
from disdrodb.metadata.search import get_list_metadata
|
|
33
|
-
from disdrodb.metadata.standards import
|
|
33
|
+
from disdrodb.metadata.standards import METADATA_KEYS, METADATA_VALUES
|
|
34
34
|
from disdrodb.utils.yaml import read_yaml
|
|
35
35
|
|
|
36
36
|
#### --------------------------------------------------------------------------.
|
|
@@ -40,19 +40,17 @@ from disdrodb.utils.yaml import read_yaml
|
|
|
40
40
|
def get_metadata_missing_keys(metadata):
|
|
41
41
|
"""Return the DISDRODB metadata keys which are missing."""
|
|
42
42
|
keys = list(metadata.keys())
|
|
43
|
-
valid_keys = get_valid_metadata_keys()
|
|
44
43
|
# Identify missing keys
|
|
45
|
-
idx_missing_keys = np.where(np.isin(
|
|
46
|
-
missing_keys = np.array(
|
|
44
|
+
idx_missing_keys = np.where(np.isin(METADATA_KEYS, keys, invert=True))[0]
|
|
45
|
+
missing_keys = np.array(METADATA_KEYS)[idx_missing_keys].tolist()
|
|
47
46
|
return missing_keys
|
|
48
47
|
|
|
49
48
|
|
|
50
49
|
def get_metadata_invalid_keys(metadata):
|
|
51
50
|
"""Return the DISDRODB metadata keys which are not valid."""
|
|
52
51
|
keys = list(metadata.keys())
|
|
53
|
-
valid_keys = get_valid_metadata_keys()
|
|
54
52
|
# Identify invalid keys
|
|
55
|
-
idx_invalid_keys = np.where(np.isin(keys,
|
|
53
|
+
idx_invalid_keys = np.where(np.isin(keys, METADATA_KEYS, invert=True))[0]
|
|
56
54
|
invalid_keys = np.array(keys)[idx_invalid_keys].tolist()
|
|
57
55
|
return invalid_keys
|
|
58
56
|
|
|
@@ -67,22 +65,31 @@ def _check_metadata_keys(metadata):
|
|
|
67
65
|
missing_keys = get_metadata_missing_keys(metadata)
|
|
68
66
|
if len(missing_keys) > 0:
|
|
69
67
|
raise ValueError(f"Missing metadata keys: {missing_keys}")
|
|
70
|
-
return None
|
|
71
68
|
|
|
72
69
|
|
|
73
70
|
def _check_metadata_values(metadata):
|
|
74
|
-
"""Check validity of metadata values
|
|
71
|
+
"""Check validity of metadata values.
|
|
75
72
|
|
|
76
73
|
If null is specified in the YAML files (or None in the dict) raise error.
|
|
74
|
+
For specific keys, check that values match one of the allowed options in METADATA_VALUES.
|
|
77
75
|
"""
|
|
78
76
|
for key, value in metadata.items():
|
|
77
|
+
# Check for None/null values
|
|
79
78
|
if isinstance(value, type(None)):
|
|
80
79
|
raise ValueError(f"The metadata key {key} has None or null value. Use '' instead.")
|
|
81
|
-
|
|
80
|
+
|
|
81
|
+
# Check that values match allowed options for specific keys
|
|
82
|
+
if key in METADATA_VALUES:
|
|
83
|
+
allowed_values = METADATA_VALUES[key]
|
|
84
|
+
if value not in allowed_values:
|
|
85
|
+
allowed_str = ", ".join([f"'{v}'" for v in allowed_values])
|
|
86
|
+
raise ValueError(
|
|
87
|
+
f"Invalid value '{value}' for metadata key '{key}'. " f"Allowed values are: {allowed_str}.",
|
|
88
|
+
)
|
|
82
89
|
|
|
83
90
|
|
|
84
91
|
def _check_metadata_campaign_name(metadata, expected_name):
|
|
85
|
-
"""Check metadata campaign_name
|
|
92
|
+
"""Check metadata ``campaign_name``."""
|
|
86
93
|
if "campaign_name" not in metadata:
|
|
87
94
|
raise ValueError("The metadata file does not contain the 'campaign_name' key.")
|
|
88
95
|
campaign_name = metadata["campaign_name"]
|
|
@@ -90,13 +97,12 @@ def _check_metadata_campaign_name(metadata, expected_name):
|
|
|
90
97
|
raise ValueError("The 'campaign_name' key in the metadata is empty.")
|
|
91
98
|
if campaign_name != expected_name:
|
|
92
99
|
raise ValueError(
|
|
93
|
-
f"The campaign_name in the metadata is '{campaign_name}' but the campaign directory is '{expected_name}'"
|
|
100
|
+
f"The campaign_name in the metadata is '{campaign_name}' but the campaign directory is '{expected_name}'",
|
|
94
101
|
)
|
|
95
|
-
return None
|
|
96
102
|
|
|
97
103
|
|
|
98
104
|
def _check_metadata_data_source(metadata, expected_name):
|
|
99
|
-
"""Check metadata data_source
|
|
105
|
+
"""Check metadata ``data_source``."""
|
|
100
106
|
if "data_source" not in metadata:
|
|
101
107
|
raise ValueError("The metadata file does not contain the 'data_source' key.")
|
|
102
108
|
data_source = metadata["data_source"]
|
|
@@ -104,15 +110,15 @@ def _check_metadata_data_source(metadata, expected_name):
|
|
|
104
110
|
raise ValueError("The 'data_source' key in the metadata is empty.")
|
|
105
111
|
if data_source != expected_name:
|
|
106
112
|
raise ValueError(
|
|
107
|
-
f"The data_source in the metadata is '{data_source}' but the data_source directory is '{expected_name}'"
|
|
113
|
+
f"The data_source in the metadata is '{data_source}' but the data_source directory is '{expected_name}'",
|
|
108
114
|
)
|
|
109
|
-
return None
|
|
110
115
|
|
|
111
116
|
|
|
112
117
|
def _check_metadata_station_name(metadata, expected_name):
|
|
113
|
-
"""Check metadata
|
|
118
|
+
"""Check metadata ``station_name``.
|
|
114
119
|
|
|
115
|
-
This function does not check that data are available for the station!
|
|
120
|
+
This function does not check that data are available for the station!
|
|
121
|
+
"""
|
|
116
122
|
if "station_name" not in metadata:
|
|
117
123
|
raise ValueError("The metadata file does not contain the 'station_name' key.")
|
|
118
124
|
station_name = metadata["station_name"]
|
|
@@ -122,29 +128,37 @@ def _check_metadata_station_name(metadata, expected_name):
|
|
|
122
128
|
raise ValueError("The 'station_name' key in the metadata is empty.")
|
|
123
129
|
if station_name != expected_name:
|
|
124
130
|
raise ValueError(
|
|
125
|
-
f"The station_name in the metadata is '{station_name}' but the metadata file is named
|
|
131
|
+
f"The station_name in the metadata is '{station_name}' but the metadata file is named"
|
|
132
|
+
f" '{expected_name}.yml'",
|
|
126
133
|
)
|
|
127
|
-
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _check_metadata_measurement_interval(metadata):
|
|
137
|
+
"""Check metadata ``measurement_interval``."""
|
|
138
|
+
from disdrodb.api.checks import check_measurement_intervals
|
|
139
|
+
|
|
140
|
+
if "measurement_interval" not in metadata:
|
|
141
|
+
raise ValueError("The metadata file does not contain the 'measurement_interval' key.")
|
|
142
|
+
measurement_intervals = metadata["measurement_interval"]
|
|
143
|
+
_ = check_measurement_intervals(measurement_intervals)
|
|
128
144
|
|
|
129
145
|
|
|
130
146
|
def _check_metadata_sensor_name(metadata):
|
|
131
147
|
from disdrodb.api.checks import check_sensor_name
|
|
132
148
|
|
|
133
149
|
sensor_name = metadata["sensor_name"]
|
|
134
|
-
check_sensor_name(sensor_name
|
|
135
|
-
return None
|
|
150
|
+
check_sensor_name(sensor_name)
|
|
136
151
|
|
|
137
152
|
|
|
138
|
-
def
|
|
153
|
+
def check_station_metadata(data_source, campaign_name, station_name, metadata_archive_dir=None):
|
|
139
154
|
"""Check DISDRODB metadata compliance."""
|
|
140
|
-
from disdrodb.l0.l0_reader import
|
|
155
|
+
from disdrodb.l0.l0_reader import check_metadata_reader
|
|
141
156
|
|
|
142
157
|
metadata = read_station_metadata(
|
|
143
|
-
base_dir=base_dir,
|
|
144
|
-
product=product,
|
|
145
158
|
data_source=data_source,
|
|
146
159
|
campaign_name=campaign_name,
|
|
147
160
|
station_name=station_name,
|
|
161
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
148
162
|
)
|
|
149
163
|
_check_metadata_keys(metadata)
|
|
150
164
|
_check_metadata_values(metadata)
|
|
@@ -152,8 +166,8 @@ def check_metadata_compliance(data_source, campaign_name, station_name, base_dir
|
|
|
152
166
|
_check_metadata_data_source(metadata, expected_name=data_source)
|
|
153
167
|
_check_metadata_station_name(metadata, expected_name=station_name)
|
|
154
168
|
_check_metadata_sensor_name(metadata)
|
|
155
|
-
|
|
156
|
-
|
|
169
|
+
_check_metadata_measurement_interval(metadata)
|
|
170
|
+
check_metadata_reader(metadata)
|
|
157
171
|
|
|
158
172
|
|
|
159
173
|
#### --------------------------------------------------------------------------.
|
|
@@ -174,15 +188,13 @@ def _check_lonlat_type(longitude, latitude):
|
|
|
174
188
|
def _check_lonlat_validity(longitude, latitude):
|
|
175
189
|
if longitude == -9999 or latitude == -9999:
|
|
176
190
|
raise ValueError("Missing lat lon coordinates (-9999).")
|
|
177
|
-
|
|
191
|
+
if longitude > 180 or longitude < -180:
|
|
178
192
|
raise ValueError("Invalid longitude (outside [-180, 180])")
|
|
179
|
-
|
|
193
|
+
if latitude > 90 or latitude < -90:
|
|
180
194
|
raise ValueError("Invalid latitude (outside [-90, 90])")
|
|
181
|
-
else:
|
|
182
|
-
pass
|
|
183
195
|
|
|
184
196
|
|
|
185
|
-
def
|
|
197
|
+
def check_station_metadata_geolocation(metadata) -> None:
|
|
186
198
|
"""Identify metadata with missing or wrong geolocation."""
|
|
187
199
|
# Get longitude, latitude and platform type
|
|
188
200
|
longitude = metadata.get("longitude")
|
|
@@ -198,7 +210,6 @@ def check_metadata_geolocation(metadata) -> None:
|
|
|
198
210
|
# - If fixed platform
|
|
199
211
|
else:
|
|
200
212
|
_check_lonlat_validity(longitude=longitude, latitude=latitude)
|
|
201
|
-
return None
|
|
202
213
|
|
|
203
214
|
|
|
204
215
|
def identify_missing_metadata_coords(metadata_filepaths: str) -> None:
|
|
@@ -212,13 +223,12 @@ def identify_missing_metadata_coords(metadata_filepaths: str) -> None:
|
|
|
212
223
|
Raises
|
|
213
224
|
------
|
|
214
225
|
TypeError
|
|
215
|
-
Error if latitude or longitude coordinates are not present or are wrongly formatted.
|
|
226
|
+
Error if ``latitude`` or ``longitude`` coordinates are not present or are wrongly formatted.
|
|
216
227
|
|
|
217
228
|
"""
|
|
218
229
|
for filepath in metadata_filepaths:
|
|
219
230
|
metadata = read_yaml(filepath)
|
|
220
|
-
|
|
221
|
-
return None
|
|
231
|
+
check_station_metadata_geolocation(metadata)
|
|
222
232
|
|
|
223
233
|
|
|
224
234
|
def identify_empty_metadata_keys(metadata_filepaths: list, keys: Union[str, list]) -> None:
|
|
@@ -231,7 +241,6 @@ def identify_empty_metadata_keys(metadata_filepaths: list, keys: Union[str, list
|
|
|
231
241
|
keys : Union[str,list]
|
|
232
242
|
Attributes to verify the presence.
|
|
233
243
|
"""
|
|
234
|
-
|
|
235
244
|
if isinstance(keys, str):
|
|
236
245
|
keys = [keys]
|
|
237
246
|
|
|
@@ -240,31 +249,37 @@ def identify_empty_metadata_keys(metadata_filepaths: list, keys: Union[str, list
|
|
|
240
249
|
metadata = read_yaml(filepath)
|
|
241
250
|
if len(str(metadata.get(key, ""))) == 0: # ensure is string to avoid error
|
|
242
251
|
print(f"Empty {key} at: ", filepath)
|
|
243
|
-
return None
|
|
244
252
|
|
|
245
253
|
|
|
246
254
|
#### --------------------------------------------------------------------------.
|
|
247
255
|
#### Check Metadata Archive
|
|
248
256
|
|
|
249
257
|
|
|
250
|
-
def
|
|
251
|
-
"""Check that all metadata files have valid keys
|
|
258
|
+
def check_metadata_archive_keys(metadata_archive_dir: Optional[str] = None) -> bool:
|
|
259
|
+
"""Check that all metadata files have valid keys.
|
|
252
260
|
|
|
253
261
|
Parameters
|
|
254
262
|
----------
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
263
|
+
metadata_archive_dir : str (optional)
|
|
264
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
265
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
266
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
267
|
+
in the DISDRODB active configuration.
|
|
258
268
|
|
|
259
269
|
Returns
|
|
260
270
|
-------
|
|
261
271
|
bool
|
|
262
|
-
If the check succeeds, the result is True
|
|
272
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
263
273
|
"""
|
|
264
274
|
is_valid = True
|
|
265
|
-
|
|
275
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
266
276
|
list_metadata_paths = get_list_metadata(
|
|
267
|
-
|
|
277
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
278
|
+
data_sources=None,
|
|
279
|
+
campaign_names=None,
|
|
280
|
+
station_names=None,
|
|
281
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
282
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
268
283
|
)
|
|
269
284
|
for filepath in list_metadata_paths:
|
|
270
285
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -272,8 +287,7 @@ def check_archive_metadata_keys(base_dir: str = None) -> bool:
|
|
|
272
287
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
273
288
|
|
|
274
289
|
metadata = read_station_metadata(
|
|
275
|
-
|
|
276
|
-
product="RAW",
|
|
290
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
277
291
|
data_source=data_source,
|
|
278
292
|
campaign_name=campaign_name,
|
|
279
293
|
station_name=station_name,
|
|
@@ -288,24 +302,31 @@ def check_archive_metadata_keys(base_dir: str = None) -> bool:
|
|
|
288
302
|
return is_valid
|
|
289
303
|
|
|
290
304
|
|
|
291
|
-
def
|
|
292
|
-
"""Check metadata campaign_name
|
|
305
|
+
def check_metadata_archive_campaign_name(metadata_archive_dir: Optional[str] = None) -> bool:
|
|
306
|
+
"""Check metadata ``campaign_name``.
|
|
293
307
|
|
|
294
308
|
Parameters
|
|
295
309
|
----------
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
310
|
+
metadata_archive_dir : str (optional)
|
|
311
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
312
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
313
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
314
|
+
in the DISDRODB active configuration.
|
|
299
315
|
|
|
300
316
|
Returns
|
|
301
317
|
-------
|
|
302
318
|
bool
|
|
303
|
-
If the check succeeds, the result is True
|
|
319
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
304
320
|
"""
|
|
305
321
|
is_valid = True
|
|
306
|
-
|
|
322
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
307
323
|
list_metadata_paths = get_list_metadata(
|
|
308
|
-
|
|
324
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
325
|
+
data_sources=None,
|
|
326
|
+
campaign_names=None,
|
|
327
|
+
station_names=None,
|
|
328
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
329
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
309
330
|
)
|
|
310
331
|
for filepath in list_metadata_paths:
|
|
311
332
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -313,8 +334,7 @@ def check_archive_metadata_campaign_name(base_dir: str = None) -> bool:
|
|
|
313
334
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
314
335
|
|
|
315
336
|
metadata = read_station_metadata(
|
|
316
|
-
|
|
317
|
-
product="RAW",
|
|
337
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
318
338
|
data_source=data_source,
|
|
319
339
|
campaign_name=campaign_name,
|
|
320
340
|
station_name=station_name,
|
|
@@ -328,24 +348,31 @@ def check_archive_metadata_campaign_name(base_dir: str = None) -> bool:
|
|
|
328
348
|
return is_valid
|
|
329
349
|
|
|
330
350
|
|
|
331
|
-
def
|
|
332
|
-
"""Check metadata data_source
|
|
351
|
+
def check_metadata_archive_data_source(metadata_archive_dir: Optional[str] = None) -> bool:
|
|
352
|
+
"""Check metadata ``data_source``.
|
|
333
353
|
|
|
334
354
|
Parameters
|
|
335
355
|
----------
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
356
|
+
metadata_archive_dir : str (optional)
|
|
357
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
358
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
359
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
360
|
+
in the DISDRODB active configuration.
|
|
339
361
|
|
|
340
362
|
Returns
|
|
341
363
|
-------
|
|
342
364
|
bool
|
|
343
|
-
If the check succeeds, the result is True
|
|
365
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
344
366
|
"""
|
|
345
367
|
is_valid = True
|
|
346
|
-
|
|
368
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
347
369
|
list_metadata_paths = get_list_metadata(
|
|
348
|
-
|
|
370
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
371
|
+
data_sources=None,
|
|
372
|
+
campaign_names=None,
|
|
373
|
+
station_names=None,
|
|
374
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
375
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
349
376
|
)
|
|
350
377
|
for filepath in list_metadata_paths:
|
|
351
378
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -353,8 +380,7 @@ def check_archive_metadata_data_source(base_dir: str = None) -> bool:
|
|
|
353
380
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
354
381
|
|
|
355
382
|
metadata = read_station_metadata(
|
|
356
|
-
|
|
357
|
-
product="RAW",
|
|
383
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
358
384
|
data_source=data_source,
|
|
359
385
|
campaign_name=campaign_name,
|
|
360
386
|
station_name=station_name,
|
|
@@ -368,24 +394,31 @@ def check_archive_metadata_data_source(base_dir: str = None) -> bool:
|
|
|
368
394
|
return is_valid
|
|
369
395
|
|
|
370
396
|
|
|
371
|
-
def
|
|
372
|
-
"""Check metadata
|
|
397
|
+
def check_metadata_archive_sensor_name(metadata_archive_dir: Optional[str] = None) -> bool:
|
|
398
|
+
"""Check metadata ``sensor_name``.
|
|
373
399
|
|
|
374
400
|
Parameters
|
|
375
401
|
----------
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
402
|
+
metadata_archive_dir : str (optional)
|
|
403
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
404
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
405
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
406
|
+
in the DISDRODB active configuration.
|
|
379
407
|
|
|
380
408
|
Returns
|
|
381
409
|
-------
|
|
382
410
|
bool
|
|
383
|
-
If the check succeeds, the result is True
|
|
411
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
384
412
|
"""
|
|
385
413
|
is_valid = True
|
|
386
|
-
|
|
414
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
387
415
|
list_metadata_paths = get_list_metadata(
|
|
388
|
-
|
|
416
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
417
|
+
data_sources=None,
|
|
418
|
+
campaign_names=None,
|
|
419
|
+
station_names=None,
|
|
420
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
421
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
389
422
|
)
|
|
390
423
|
for filepath in list_metadata_paths:
|
|
391
424
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -393,8 +426,7 @@ def check_archive_metadata_sensor_name(base_dir: str = None) -> bool:
|
|
|
393
426
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
394
427
|
|
|
395
428
|
metadata = read_station_metadata(
|
|
396
|
-
|
|
397
|
-
product="RAW",
|
|
429
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
398
430
|
data_source=data_source,
|
|
399
431
|
campaign_name=campaign_name,
|
|
400
432
|
station_name=station_name,
|
|
@@ -408,24 +440,31 @@ def check_archive_metadata_sensor_name(base_dir: str = None) -> bool:
|
|
|
408
440
|
return is_valid
|
|
409
441
|
|
|
410
442
|
|
|
411
|
-
def
|
|
412
|
-
"""Check metadata
|
|
443
|
+
def check_metadata_archive_station_name(metadata_archive_dir: Optional[str] = None) -> bool:
|
|
444
|
+
"""Check metadata ``station_name``.
|
|
413
445
|
|
|
414
446
|
Parameters
|
|
415
447
|
----------
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
448
|
+
metadata_archive_dir : str (optional)
|
|
449
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
450
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
451
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
452
|
+
in the DISDRODB active configuration.
|
|
419
453
|
|
|
420
454
|
Returns
|
|
421
455
|
-------
|
|
422
456
|
bool
|
|
423
|
-
If the check succeeds, the result is True
|
|
457
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
424
458
|
"""
|
|
425
459
|
is_valid = True
|
|
426
|
-
|
|
460
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
427
461
|
list_metadata_paths = get_list_metadata(
|
|
428
|
-
|
|
462
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
463
|
+
data_sources=None,
|
|
464
|
+
campaign_names=None,
|
|
465
|
+
station_names=None,
|
|
466
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
467
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
429
468
|
)
|
|
430
469
|
for filepath in list_metadata_paths:
|
|
431
470
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -433,8 +472,7 @@ def check_archive_metadata_station_name(base_dir: str = None) -> bool:
|
|
|
433
472
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
434
473
|
|
|
435
474
|
metadata = read_station_metadata(
|
|
436
|
-
|
|
437
|
-
product="RAW",
|
|
475
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
438
476
|
data_source=data_source,
|
|
439
477
|
campaign_name=campaign_name,
|
|
440
478
|
station_name=station_name,
|
|
@@ -448,26 +486,33 @@ def check_archive_metadata_station_name(base_dir: str = None) -> bool:
|
|
|
448
486
|
return is_valid
|
|
449
487
|
|
|
450
488
|
|
|
451
|
-
def
|
|
452
|
-
"""Check if the reader key is available and there is the associated reader.
|
|
489
|
+
def check_metadata_archive_reader(metadata_archive_dir: Optional[str] = None) -> bool:
|
|
490
|
+
"""Check if the ``reader`` key is available and there is the associated reader.
|
|
453
491
|
|
|
454
492
|
Parameters
|
|
455
493
|
----------
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
494
|
+
metadata_archive_dir : str (optional)
|
|
495
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
496
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
497
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
498
|
+
in the DISDRODB active configuration.
|
|
459
499
|
|
|
460
500
|
Returns
|
|
461
501
|
-------
|
|
462
502
|
bool
|
|
463
|
-
If the check succeeds, the result is True
|
|
503
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
464
504
|
"""
|
|
465
|
-
from disdrodb.l0.l0_reader import
|
|
505
|
+
from disdrodb.l0.l0_reader import check_metadata_reader
|
|
466
506
|
|
|
467
507
|
is_valid = True
|
|
468
|
-
|
|
508
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
469
509
|
list_metadata_paths = get_list_metadata(
|
|
470
|
-
|
|
510
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
511
|
+
data_sources=None,
|
|
512
|
+
campaign_names=None,
|
|
513
|
+
station_names=None,
|
|
514
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
515
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
471
516
|
)
|
|
472
517
|
for filepath in list_metadata_paths:
|
|
473
518
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -475,14 +520,13 @@ def check_archive_metadata_reader(base_dir: str = None) -> bool:
|
|
|
475
520
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
476
521
|
|
|
477
522
|
metadata = read_station_metadata(
|
|
478
|
-
|
|
479
|
-
product="RAW",
|
|
523
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
480
524
|
data_source=data_source,
|
|
481
525
|
campaign_name=campaign_name,
|
|
482
526
|
station_name=station_name,
|
|
483
527
|
)
|
|
484
528
|
try:
|
|
485
|
-
|
|
529
|
+
check_metadata_reader(metadata)
|
|
486
530
|
except Exception as e:
|
|
487
531
|
is_valid = False
|
|
488
532
|
print(f"Error for {data_source} {campaign_name} {station_name}.")
|
|
@@ -490,27 +534,34 @@ def check_archive_metadata_reader(base_dir: str = None) -> bool:
|
|
|
490
534
|
return is_valid
|
|
491
535
|
|
|
492
536
|
|
|
493
|
-
def
|
|
537
|
+
def check_metadata_archive(metadata_archive_dir: Optional[str] = None, raise_error=False):
|
|
494
538
|
"""Check the archive metadata compliance.
|
|
495
539
|
|
|
496
540
|
Parameters
|
|
497
541
|
----------
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
542
|
+
metadata_archive_dir : str (optional)
|
|
543
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
544
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
545
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
546
|
+
in the DISDRODB active configuration.
|
|
501
547
|
raise_error: bool (optional)
|
|
502
548
|
Whether to raise an error and interrupt the archive check if a
|
|
503
|
-
metadata is not compliant. The default is False
|
|
549
|
+
metadata is not compliant. The default value is ``False``.
|
|
504
550
|
|
|
505
551
|
Returns
|
|
506
552
|
-------
|
|
507
553
|
bool
|
|
508
|
-
If the check succeeds, the result is True
|
|
554
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
509
555
|
"""
|
|
510
556
|
is_valid = True
|
|
511
|
-
|
|
557
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
512
558
|
list_metadata_paths = get_list_metadata(
|
|
513
|
-
|
|
559
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
560
|
+
data_sources=None,
|
|
561
|
+
campaign_names=None,
|
|
562
|
+
station_names=None,
|
|
563
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
564
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
514
565
|
)
|
|
515
566
|
for filepath in list_metadata_paths:
|
|
516
567
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -518,8 +569,8 @@ def check_archive_metadata_compliance(base_dir: str = None, raise_error=False):
|
|
|
518
569
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
519
570
|
# Check compliance
|
|
520
571
|
try:
|
|
521
|
-
|
|
522
|
-
|
|
572
|
+
check_station_metadata(
|
|
573
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
523
574
|
data_source=data_source,
|
|
524
575
|
campaign_name=campaign_name,
|
|
525
576
|
station_name=station_name,
|
|
@@ -530,30 +581,36 @@ def check_archive_metadata_compliance(base_dir: str = None, raise_error=False):
|
|
|
530
581
|
msg = msg + f"The error is: {e}."
|
|
531
582
|
if raise_error:
|
|
532
583
|
raise ValueError(msg)
|
|
533
|
-
|
|
534
|
-
print(msg)
|
|
584
|
+
print(msg)
|
|
535
585
|
|
|
536
586
|
return is_valid
|
|
537
587
|
|
|
538
588
|
|
|
539
|
-
def
|
|
589
|
+
def check_metadata_archive_geolocation(metadata_archive_dir: Optional[str] = None):
|
|
540
590
|
"""Check the metadata files have missing or wrong geolocation..
|
|
541
591
|
|
|
542
592
|
Parameters
|
|
543
593
|
----------
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
594
|
+
metadata_archive_dir : str (optional)
|
|
595
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
596
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
597
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
598
|
+
in the DISDRODB active configuration.
|
|
547
599
|
|
|
548
600
|
Returns
|
|
549
601
|
-------
|
|
550
602
|
bool
|
|
551
|
-
If the check succeeds, the result is True
|
|
603
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
552
604
|
"""
|
|
553
605
|
is_valid = True
|
|
554
|
-
|
|
606
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
555
607
|
list_metadata_paths = get_list_metadata(
|
|
556
|
-
|
|
608
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
609
|
+
data_sources=None,
|
|
610
|
+
campaign_names=None,
|
|
611
|
+
station_names=None,
|
|
612
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
613
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
557
614
|
)
|
|
558
615
|
for filepath in list_metadata_paths:
|
|
559
616
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -561,14 +618,13 @@ def check_archive_metadata_geolocation(base_dir: str = None):
|
|
|
561
618
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
562
619
|
|
|
563
620
|
metadata = read_station_metadata(
|
|
564
|
-
|
|
565
|
-
product="RAW",
|
|
621
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
566
622
|
data_source=data_source,
|
|
567
623
|
campaign_name=campaign_name,
|
|
568
624
|
station_name=station_name,
|
|
569
625
|
)
|
|
570
626
|
try:
|
|
571
|
-
|
|
627
|
+
check_station_metadata_geolocation(metadata)
|
|
572
628
|
except Exception as e:
|
|
573
629
|
is_valid = False
|
|
574
630
|
print(f"Missing information for {data_source} {campaign_name} {station_name}.")
|