disdrodb 0.0.21__py3-none-any.whl → 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +132 -15
- disdrodb/_config.py +4 -2
- disdrodb/_version.py +9 -4
- disdrodb/api/checks.py +264 -237
- disdrodb/api/configs.py +4 -8
- disdrodb/api/create_directories.py +235 -290
- disdrodb/api/info.py +217 -26
- disdrodb/api/io.py +295 -269
- disdrodb/api/path.py +597 -173
- disdrodb/api/search.py +486 -0
- disdrodb/{metadata/scripts → cli}/disdrodb_check_metadata_archive.py +12 -7
- disdrodb/{utils/pandas.py → cli/disdrodb_data_archive_directory.py} +9 -18
- disdrodb/cli/disdrodb_download_archive.py +86 -0
- disdrodb/cli/disdrodb_download_metadata_archive.py +53 -0
- disdrodb/cli/disdrodb_download_station.py +84 -0
- disdrodb/{api/scripts → cli}/disdrodb_initialize_station.py +22 -10
- disdrodb/cli/disdrodb_metadata_archive_directory.py +32 -0
- disdrodb/{data_transfer/scripts/disdrodb_download_station.py → cli/disdrodb_open_data_archive.py} +22 -22
- disdrodb/cli/disdrodb_open_logs_directory.py +69 -0
- disdrodb/{data_transfer/scripts/disdrodb_upload_station.py → cli/disdrodb_open_metadata_archive.py} +22 -24
- disdrodb/cli/disdrodb_open_metadata_directory.py +71 -0
- disdrodb/cli/disdrodb_open_product_directory.py +74 -0
- disdrodb/cli/disdrodb_open_readers_directory.py +32 -0
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0.py +38 -31
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0_station.py +32 -30
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a_station.py +24 -33
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b_station.py +25 -34
- disdrodb/cli/disdrodb_run_l0c.py +130 -0
- disdrodb/cli/disdrodb_run_l0c_station.py +129 -0
- disdrodb/cli/disdrodb_run_l1.py +122 -0
- disdrodb/cli/disdrodb_run_l1_station.py +121 -0
- disdrodb/cli/disdrodb_run_l2e.py +122 -0
- disdrodb/cli/disdrodb_run_l2e_station.py +122 -0
- disdrodb/cli/disdrodb_run_l2m.py +122 -0
- disdrodb/cli/disdrodb_run_l2m_station.py +122 -0
- disdrodb/cli/disdrodb_upload_archive.py +105 -0
- disdrodb/cli/disdrodb_upload_station.py +98 -0
- disdrodb/configs.py +90 -25
- disdrodb/data_transfer/__init__.py +22 -0
- disdrodb/data_transfer/download_data.py +87 -90
- disdrodb/data_transfer/upload_data.py +64 -37
- disdrodb/data_transfer/zenodo.py +15 -18
- disdrodb/docs.py +1 -1
- disdrodb/issue/__init__.py +17 -4
- disdrodb/issue/checks.py +10 -23
- disdrodb/issue/reader.py +9 -12
- disdrodb/issue/writer.py +14 -17
- disdrodb/l0/__init__.py +17 -26
- disdrodb/l0/check_configs.py +35 -23
- disdrodb/l0/check_standards.py +32 -42
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_diameter.yml +44 -44
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_velocity.yml +40 -40
- disdrodb/l0/configs/LPM/l0a_encodings.yml +80 -0
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_cf_attrs.yml +62 -59
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_encodings.yml +9 -9
- disdrodb/l0/configs/{Thies_LPM → LPM}/raw_data_format.yml +245 -245
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_diameter.yml +66 -66
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL/l0a_encodings.yml +32 -0
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_cf_attrs.yml +22 -20
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_encodings.yml +17 -17
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/raw_data_format.yml +77 -77
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_diameter.yml +64 -64
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +39 -0
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_cf_attrs.yml +24 -22
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_encodings.yml +20 -20
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/raw_data_format.yml +98 -98
- disdrodb/l0/configs/{RD_80 → RD80}/bins_diameter.yml +40 -40
- disdrodb/l0/configs/RD80/l0a_encodings.yml +16 -0
- disdrodb/l0/configs/{RD_80 → RD80}/l0b_cf_attrs.yml +3 -3
- disdrodb/l0/configs/RD80/l0b_encodings.yml +135 -0
- disdrodb/l0/configs/{RD_80 → RD80}/raw_data_format.yml +48 -48
- disdrodb/l0/l0_reader.py +216 -340
- disdrodb/l0/l0a_processing.py +237 -208
- disdrodb/l0/l0b_nc_processing.py +227 -80
- disdrodb/l0/l0b_processing.py +93 -173
- disdrodb/l0/l0c_processing.py +627 -0
- disdrodb/l0/readers/{ARM → LPM/ARM}/ARM_LPM.py +36 -58
- disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +226 -0
- disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +185 -0
- disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +183 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +179 -0
- disdrodb/l0/readers/{UK → LPM/UK}/DIVEN.py +14 -35
- disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +157 -0
- disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +113 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/ARCTIC_2021.py +40 -57
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/COMMON_2011.py +37 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/DAVOS_2009_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_2009.py +34 -51
- disdrodb/l0/readers/{EPFL/PARADISO_2014.py → PARSIVEL/EPFL/EPFL_ROOF_2008.py} +38 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2012.py +33 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GENEPI_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007_2.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HPICONET_2010.py +34 -51
- disdrodb/l0/readers/{EPFL/EPFL_ROOF_2010.py → PARSIVEL/EPFL/HYMEX_LTE_SOP2.py} +37 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +111 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HYMEX_LTE_SOP4.py +36 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2018.py +34 -52
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2019.py +38 -56
- disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PARSIVEL_2007.py +27 -45
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PLATO_2019.py +24 -44
- disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +140 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RACLETS_2019_WJF.py +41 -59
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RIETHOLZBACH_2011.py +34 -51
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +117 -0
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +137 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/UNIL_2022.py +42 -55
- disdrodb/l0/readers/PARSIVEL/GPM/IFLOODS.py +104 -0
- disdrodb/l0/readers/{GPM → PARSIVEL/GPM}/LPVEX.py +29 -48
- disdrodb/l0/readers/PARSIVEL/GPM/MC3E.py +184 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +113 -0
- disdrodb/l0/readers/{NCAR/VORTEX_SE_2016_P1.py → PARSIVEL/NCAR/OWLES_MIPS.py} +46 -72
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +125 -0
- disdrodb/l0/readers/{NCAR/OWLES_MIPS.py → PARSIVEL/NCAR/PLOWS_MIPS.py} +45 -64
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +114 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +176 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +183 -0
- disdrodb/l0/readers/{ARM/ARM_LD.py → PARSIVEL2/ARM/ARM_PARSIVEL2.py} +27 -50
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/{DENMARK → PARSIVEL2/DENMARK}/EROSION_nc.py +14 -35
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +119 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +104 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/NSSTC.py +176 -0
- disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +32 -0
- disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +56 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +120 -0
- disdrodb/l0/readers/{NCAR → PARSIVEL2/NCAR}/PECAN_MIPS.py +45 -64
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +181 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +160 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +160 -0
- disdrodb/l0/readers/{NCAR/PLOWS_MIPS.py → PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py} +49 -66
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +118 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +152 -0
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT.py +166 -0
- disdrodb/l0/readers/{NCAR/RELAMPAGO_RD80.py → RD80/BRAZIL/CHUVA_RD80.py} +36 -60
- disdrodb/l0/readers/{BRAZIL → RD80/BRAZIL}/GOAMAZON_RD80.py +36 -55
- disdrodb/l0/readers/{NCAR → RD80/NCAR}/CINDY_2011_RD80.py +35 -54
- disdrodb/l0/readers/{BRAZIL/CHUVA_RD80.py → RD80/NCAR/RELAMPAGO_RD80.py} +40 -54
- disdrodb/l0/readers/template_reader_raw_netcdf_data.py +62 -0
- disdrodb/l0/readers/{reader_template.py → template_reader_raw_text_data.py} +20 -44
- disdrodb/l0/routines.py +885 -581
- disdrodb/l0/standards.py +72 -236
- disdrodb/l0/template_tools.py +104 -109
- disdrodb/l1/__init__.py +17 -0
- disdrodb/l1/beard_model.py +716 -0
- disdrodb/l1/encoding_attrs.py +620 -0
- disdrodb/l1/fall_velocity.py +260 -0
- disdrodb/l1/filters.py +192 -0
- disdrodb/l1/processing.py +200 -0
- disdrodb/l1/resampling.py +236 -0
- disdrodb/l1/routines.py +357 -0
- disdrodb/l1_env/__init__.py +17 -0
- disdrodb/l1_env/routines.py +38 -0
- disdrodb/l2/__init__.py +17 -0
- disdrodb/l2/empirical_dsd.py +1735 -0
- disdrodb/l2/event.py +388 -0
- disdrodb/l2/processing.py +519 -0
- disdrodb/l2/processing_options.py +213 -0
- disdrodb/l2/routines.py +868 -0
- disdrodb/metadata/__init__.py +9 -2
- disdrodb/metadata/checks.py +165 -118
- disdrodb/metadata/download.py +81 -0
- disdrodb/metadata/geolocation.py +146 -0
- disdrodb/metadata/info.py +20 -13
- disdrodb/metadata/manipulation.py +1 -1
- disdrodb/metadata/reader.py +59 -8
- disdrodb/metadata/search.py +77 -144
- disdrodb/metadata/standards.py +7 -8
- disdrodb/metadata/writer.py +8 -14
- disdrodb/psd/__init__.py +38 -0
- disdrodb/psd/fitting.py +2146 -0
- disdrodb/psd/models.py +774 -0
- disdrodb/routines.py +1176 -0
- disdrodb/scattering/__init__.py +28 -0
- disdrodb/scattering/axis_ratio.py +344 -0
- disdrodb/scattering/routines.py +456 -0
- disdrodb/utils/__init__.py +17 -0
- disdrodb/utils/attrs.py +208 -0
- disdrodb/utils/cli.py +269 -0
- disdrodb/utils/compression.py +60 -42
- disdrodb/utils/dask.py +62 -0
- disdrodb/utils/decorators.py +110 -0
- disdrodb/utils/directories.py +107 -46
- disdrodb/utils/encoding.py +127 -0
- disdrodb/utils/list.py +29 -0
- disdrodb/utils/logger.py +168 -46
- disdrodb/utils/time.py +657 -0
- disdrodb/utils/warnings.py +30 -0
- disdrodb/utils/writer.py +57 -0
- disdrodb/utils/xarray.py +138 -47
- disdrodb/utils/yaml.py +0 -1
- disdrodb/viz/__init__.py +17 -0
- disdrodb/viz/plots.py +17 -0
- disdrodb-0.1.0.dist-info/METADATA +321 -0
- disdrodb-0.1.0.dist-info/RECORD +216 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.0.dist-info}/WHEEL +1 -1
- disdrodb-0.1.0.dist-info/entry_points.txt +30 -0
- disdrodb/data_transfer/scripts/disdrodb_download_archive.py +0 -53
- disdrodb/data_transfer/scripts/disdrodb_upload_archive.py +0 -57
- disdrodb/l0/configs/OTT_Parsivel/l0a_encodings.yml +0 -32
- disdrodb/l0/configs/OTT_Parsivel2/l0a_encodings.yml +0 -39
- disdrodb/l0/configs/RD_80/l0a_encodings.yml +0 -16
- disdrodb/l0/configs/RD_80/l0b_encodings.yml +0 -135
- disdrodb/l0/configs/Thies_LPM/l0a_encodings.yml +0 -80
- disdrodb/l0/io.py +0 -257
- disdrodb/l0/l0_processing.py +0 -1091
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_OTT.py +0 -178
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_THIES.py +0 -247
- disdrodb/l0/readers/BRAZIL/CHUVA_LPM.py +0 -204
- disdrodb/l0/readers/BRAZIL/CHUVA_OTT.py +0 -183
- disdrodb/l0/readers/BRAZIL/GOAMAZON_LPM.py +0 -204
- disdrodb/l0/readers/BRAZIL/GOAMAZON_OTT.py +0 -183
- disdrodb/l0/readers/CHINA/CHONGQING.py +0 -131
- disdrodb/l0/readers/EPFL/EPFL_ROOF_2008.py +0 -128
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP2.py +0 -127
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP3.py +0 -129
- disdrodb/l0/readers/EPFL/RACLETS_2019.py +0 -158
- disdrodb/l0/readers/EPFL/SAMOYLOV_2017.py +0 -136
- disdrodb/l0/readers/EPFL/SAMOYLOV_2019.py +0 -158
- disdrodb/l0/readers/FRANCE/SIRTA_OTT2.py +0 -138
- disdrodb/l0/readers/GPM/GCPEX.py +0 -123
- disdrodb/l0/readers/GPM/IFLOODS.py +0 -123
- disdrodb/l0/readers/GPM/MC3E.py +0 -123
- disdrodb/l0/readers/GPM/NSSTC.py +0 -164
- disdrodb/l0/readers/ITALY/GID.py +0 -199
- disdrodb/l0/readers/MEXICO/OH_IIUNAM_nc.py +0 -92
- disdrodb/l0/readers/NCAR/CCOPE_2015.py +0 -133
- disdrodb/l0/readers/NCAR/PECAN_FP3.py +0 -137
- disdrodb/l0/readers/NCAR/PECAN_MOBILE.py +0 -144
- disdrodb/l0/readers/NCAR/RELAMPAGO_OTT.py +0 -195
- disdrodb/l0/readers/NCAR/SNOWIE_PJ.py +0 -172
- disdrodb/l0/readers/NCAR/SNOWIE_SB.py +0 -179
- disdrodb/l0/readers/NCAR/VORTEX2_2009.py +0 -133
- disdrodb/l0/readers/NCAR/VORTEX2_2010.py +0 -188
- disdrodb/l0/readers/NCAR/VORTEX2_2010_UF.py +0 -191
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_P2.py +0 -135
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_PIPS.py +0 -170
- disdrodb/l0/readers/NETHERLANDS/DELFT.py +0 -187
- disdrodb/l0/readers/SPAIN/SBEGUERIA.py +0 -179
- disdrodb/l0/scripts/disdrodb_run_l0b_concat.py +0 -93
- disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py +0 -85
- disdrodb/utils/netcdf.py +0 -452
- disdrodb/utils/scripts.py +0 -102
- disdrodb-0.0.21.dist-info/AUTHORS.md +0 -18
- disdrodb-0.0.21.dist-info/METADATA +0 -186
- disdrodb-0.0.21.dist-info/RECORD +0 -168
- disdrodb-0.0.21.dist-info/entry_points.txt +0 -15
- /disdrodb/l0/configs/{RD_80 → RD80}/bins_velocity.yml +0 -0
- /disdrodb/l0/manuals/{Thies_LPM.pdf → LPM.pdf} +0 -0
- /disdrodb/l0/manuals/{ODM_470.pdf → ODM470.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel.pdf → PARSIVEL.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel2.pdf → PARSIVEL2.pdf} +0 -0
- /disdrodb/l0/manuals/{PWS_100.pdf → PWS100.pdf} +0 -0
- /disdrodb/l0/manuals/{RD_80.pdf → RD80.pdf} +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.0.dist-info/licenses}/LICENSE +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.0.dist-info}/top_level.txt +0 -0
disdrodb/metadata/__init__.py
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
|
+
from disdrodb.metadata.download import download_metadata_archive
|
|
1
2
|
from disdrodb.metadata.info import get_archive_metadata_key_value
|
|
2
|
-
from disdrodb.metadata.reader import read_station_metadata
|
|
3
|
+
from disdrodb.metadata.reader import read_metadata_archive, read_station_metadata
|
|
3
4
|
from disdrodb.metadata.search import get_list_metadata
|
|
4
5
|
|
|
5
|
-
__all__ = [
|
|
6
|
+
__all__ = [
|
|
7
|
+
"download_metadata_archive",
|
|
8
|
+
"get_archive_metadata_key_value",
|
|
9
|
+
"get_list_metadata",
|
|
10
|
+
"read_metadata_archive",
|
|
11
|
+
"read_station_metadata",
|
|
12
|
+
]
|
disdrodb/metadata/checks.py
CHANGED
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
"""Check metadata."""
|
|
20
20
|
|
|
21
21
|
import os
|
|
22
|
-
from typing import Union
|
|
22
|
+
from typing import Optional, Union
|
|
23
23
|
|
|
24
24
|
import numpy as np
|
|
25
25
|
|
|
@@ -27,7 +27,7 @@ from disdrodb.api.info import (
|
|
|
27
27
|
infer_campaign_name_from_path,
|
|
28
28
|
infer_data_source_from_path,
|
|
29
29
|
)
|
|
30
|
-
from disdrodb.configs import
|
|
30
|
+
from disdrodb.configs import get_metadata_archive_dir
|
|
31
31
|
from disdrodb.metadata.reader import read_station_metadata
|
|
32
32
|
from disdrodb.metadata.search import get_list_metadata
|
|
33
33
|
from disdrodb.metadata.standards import get_valid_metadata_keys
|
|
@@ -67,22 +67,20 @@ def _check_metadata_keys(metadata):
|
|
|
67
67
|
missing_keys = get_metadata_missing_keys(metadata)
|
|
68
68
|
if len(missing_keys) > 0:
|
|
69
69
|
raise ValueError(f"Missing metadata keys: {missing_keys}")
|
|
70
|
-
return None
|
|
71
70
|
|
|
72
71
|
|
|
73
72
|
def _check_metadata_values(metadata):
|
|
74
|
-
"""Check validity of metadata values
|
|
73
|
+
"""Check validity of metadata values.
|
|
75
74
|
|
|
76
75
|
If null is specified in the YAML files (or None in the dict) raise error.
|
|
77
76
|
"""
|
|
78
77
|
for key, value in metadata.items():
|
|
79
78
|
if isinstance(value, type(None)):
|
|
80
79
|
raise ValueError(f"The metadata key {key} has None or null value. Use '' instead.")
|
|
81
|
-
return None
|
|
82
80
|
|
|
83
81
|
|
|
84
82
|
def _check_metadata_campaign_name(metadata, expected_name):
|
|
85
|
-
"""Check metadata campaign_name
|
|
83
|
+
"""Check metadata ``campaign_name``."""
|
|
86
84
|
if "campaign_name" not in metadata:
|
|
87
85
|
raise ValueError("The metadata file does not contain the 'campaign_name' key.")
|
|
88
86
|
campaign_name = metadata["campaign_name"]
|
|
@@ -90,13 +88,12 @@ def _check_metadata_campaign_name(metadata, expected_name):
|
|
|
90
88
|
raise ValueError("The 'campaign_name' key in the metadata is empty.")
|
|
91
89
|
if campaign_name != expected_name:
|
|
92
90
|
raise ValueError(
|
|
93
|
-
f"The campaign_name in the metadata is '{campaign_name}' but the campaign directory is '{expected_name}'"
|
|
91
|
+
f"The campaign_name in the metadata is '{campaign_name}' but the campaign directory is '{expected_name}'",
|
|
94
92
|
)
|
|
95
|
-
return None
|
|
96
93
|
|
|
97
94
|
|
|
98
95
|
def _check_metadata_data_source(metadata, expected_name):
|
|
99
|
-
"""Check metadata data_source
|
|
96
|
+
"""Check metadata ``data_source``."""
|
|
100
97
|
if "data_source" not in metadata:
|
|
101
98
|
raise ValueError("The metadata file does not contain the 'data_source' key.")
|
|
102
99
|
data_source = metadata["data_source"]
|
|
@@ -104,15 +101,15 @@ def _check_metadata_data_source(metadata, expected_name):
|
|
|
104
101
|
raise ValueError("The 'data_source' key in the metadata is empty.")
|
|
105
102
|
if data_source != expected_name:
|
|
106
103
|
raise ValueError(
|
|
107
|
-
f"The data_source in the metadata is '{data_source}' but the data_source directory is '{expected_name}'"
|
|
104
|
+
f"The data_source in the metadata is '{data_source}' but the data_source directory is '{expected_name}'",
|
|
108
105
|
)
|
|
109
|
-
return None
|
|
110
106
|
|
|
111
107
|
|
|
112
108
|
def _check_metadata_station_name(metadata, expected_name):
|
|
113
|
-
"""Check metadata
|
|
109
|
+
"""Check metadata ``station_name``.
|
|
114
110
|
|
|
115
|
-
This function does not check that data are available for the station!
|
|
111
|
+
This function does not check that data are available for the station!
|
|
112
|
+
"""
|
|
116
113
|
if "station_name" not in metadata:
|
|
117
114
|
raise ValueError("The metadata file does not contain the 'station_name' key.")
|
|
118
115
|
station_name = metadata["station_name"]
|
|
@@ -122,29 +119,37 @@ def _check_metadata_station_name(metadata, expected_name):
|
|
|
122
119
|
raise ValueError("The 'station_name' key in the metadata is empty.")
|
|
123
120
|
if station_name != expected_name:
|
|
124
121
|
raise ValueError(
|
|
125
|
-
f"The station_name in the metadata is '{station_name}' but the metadata file is named
|
|
122
|
+
f"The station_name in the metadata is '{station_name}' but the metadata file is named"
|
|
123
|
+
f" '{expected_name}.yml'",
|
|
126
124
|
)
|
|
127
|
-
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _check_metadata_measurement_interval(metadata):
|
|
128
|
+
"""Check metadata ``measurement_interval``."""
|
|
129
|
+
from disdrodb.api.checks import check_measurement_intervals
|
|
130
|
+
|
|
131
|
+
if "measurement_interval" not in metadata:
|
|
132
|
+
raise ValueError("The metadata file does not contain the 'measurement_interval' key.")
|
|
133
|
+
measurement_intervals = metadata["measurement_interval"]
|
|
134
|
+
_ = check_measurement_intervals(measurement_intervals)
|
|
128
135
|
|
|
129
136
|
|
|
130
137
|
def _check_metadata_sensor_name(metadata):
|
|
131
138
|
from disdrodb.api.checks import check_sensor_name
|
|
132
139
|
|
|
133
140
|
sensor_name = metadata["sensor_name"]
|
|
134
|
-
check_sensor_name(sensor_name
|
|
135
|
-
return None
|
|
141
|
+
check_sensor_name(sensor_name)
|
|
136
142
|
|
|
137
143
|
|
|
138
|
-
def
|
|
144
|
+
def check_station_metadata(data_source, campaign_name, station_name, metadata_archive_dir=None):
|
|
139
145
|
"""Check DISDRODB metadata compliance."""
|
|
140
|
-
from disdrodb.l0.l0_reader import
|
|
146
|
+
from disdrodb.l0.l0_reader import check_metadata_reader
|
|
141
147
|
|
|
142
148
|
metadata = read_station_metadata(
|
|
143
|
-
base_dir=base_dir,
|
|
144
|
-
product=product,
|
|
145
149
|
data_source=data_source,
|
|
146
150
|
campaign_name=campaign_name,
|
|
147
151
|
station_name=station_name,
|
|
152
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
148
153
|
)
|
|
149
154
|
_check_metadata_keys(metadata)
|
|
150
155
|
_check_metadata_values(metadata)
|
|
@@ -152,8 +157,8 @@ def check_metadata_compliance(data_source, campaign_name, station_name, base_dir
|
|
|
152
157
|
_check_metadata_data_source(metadata, expected_name=data_source)
|
|
153
158
|
_check_metadata_station_name(metadata, expected_name=station_name)
|
|
154
159
|
_check_metadata_sensor_name(metadata)
|
|
155
|
-
|
|
156
|
-
|
|
160
|
+
_check_metadata_measurement_interval(metadata)
|
|
161
|
+
check_metadata_reader(metadata)
|
|
157
162
|
|
|
158
163
|
|
|
159
164
|
#### --------------------------------------------------------------------------.
|
|
@@ -174,15 +179,13 @@ def _check_lonlat_type(longitude, latitude):
|
|
|
174
179
|
def _check_lonlat_validity(longitude, latitude):
|
|
175
180
|
if longitude == -9999 or latitude == -9999:
|
|
176
181
|
raise ValueError("Missing lat lon coordinates (-9999).")
|
|
177
|
-
|
|
182
|
+
if longitude > 180 or longitude < -180:
|
|
178
183
|
raise ValueError("Invalid longitude (outside [-180, 180])")
|
|
179
|
-
|
|
184
|
+
if latitude > 90 or latitude < -90:
|
|
180
185
|
raise ValueError("Invalid latitude (outside [-90, 90])")
|
|
181
|
-
else:
|
|
182
|
-
pass
|
|
183
186
|
|
|
184
187
|
|
|
185
|
-
def
|
|
188
|
+
def check_station_metadata_geolocation(metadata) -> None:
|
|
186
189
|
"""Identify metadata with missing or wrong geolocation."""
|
|
187
190
|
# Get longitude, latitude and platform type
|
|
188
191
|
longitude = metadata.get("longitude")
|
|
@@ -198,7 +201,6 @@ def check_metadata_geolocation(metadata) -> None:
|
|
|
198
201
|
# - If fixed platform
|
|
199
202
|
else:
|
|
200
203
|
_check_lonlat_validity(longitude=longitude, latitude=latitude)
|
|
201
|
-
return None
|
|
202
204
|
|
|
203
205
|
|
|
204
206
|
def identify_missing_metadata_coords(metadata_filepaths: str) -> None:
|
|
@@ -212,13 +214,12 @@ def identify_missing_metadata_coords(metadata_filepaths: str) -> None:
|
|
|
212
214
|
Raises
|
|
213
215
|
------
|
|
214
216
|
TypeError
|
|
215
|
-
Error if latitude or longitude coordinates are not present or are wrongly formatted.
|
|
217
|
+
Error if ``latitude`` or ``longitude`` coordinates are not present or are wrongly formatted.
|
|
216
218
|
|
|
217
219
|
"""
|
|
218
220
|
for filepath in metadata_filepaths:
|
|
219
221
|
metadata = read_yaml(filepath)
|
|
220
|
-
|
|
221
|
-
return None
|
|
222
|
+
check_station_metadata_geolocation(metadata)
|
|
222
223
|
|
|
223
224
|
|
|
224
225
|
def identify_empty_metadata_keys(metadata_filepaths: list, keys: Union[str, list]) -> None:
|
|
@@ -231,7 +232,6 @@ def identify_empty_metadata_keys(metadata_filepaths: list, keys: Union[str, list
|
|
|
231
232
|
keys : Union[str,list]
|
|
232
233
|
Attributes to verify the presence.
|
|
233
234
|
"""
|
|
234
|
-
|
|
235
235
|
if isinstance(keys, str):
|
|
236
236
|
keys = [keys]
|
|
237
237
|
|
|
@@ -240,31 +240,37 @@ def identify_empty_metadata_keys(metadata_filepaths: list, keys: Union[str, list
|
|
|
240
240
|
metadata = read_yaml(filepath)
|
|
241
241
|
if len(str(metadata.get(key, ""))) == 0: # ensure is string to avoid error
|
|
242
242
|
print(f"Empty {key} at: ", filepath)
|
|
243
|
-
return None
|
|
244
243
|
|
|
245
244
|
|
|
246
245
|
#### --------------------------------------------------------------------------.
|
|
247
246
|
#### Check Metadata Archive
|
|
248
247
|
|
|
249
248
|
|
|
250
|
-
def
|
|
251
|
-
"""Check that all metadata files have valid keys
|
|
249
|
+
def check_metadata_archive_keys(metadata_archive_dir: Optional[str] = None) -> bool:
|
|
250
|
+
"""Check that all metadata files have valid keys.
|
|
252
251
|
|
|
253
252
|
Parameters
|
|
254
253
|
----------
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
254
|
+
metadata_archive_dir : str (optional)
|
|
255
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
256
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
257
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
258
|
+
in the DISDRODB active configuration.
|
|
258
259
|
|
|
259
260
|
Returns
|
|
260
261
|
-------
|
|
261
262
|
bool
|
|
262
|
-
If the check succeeds, the result is True
|
|
263
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
263
264
|
"""
|
|
264
265
|
is_valid = True
|
|
265
|
-
|
|
266
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
266
267
|
list_metadata_paths = get_list_metadata(
|
|
267
|
-
|
|
268
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
269
|
+
data_sources=None,
|
|
270
|
+
campaign_names=None,
|
|
271
|
+
station_names=None,
|
|
272
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
273
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
268
274
|
)
|
|
269
275
|
for filepath in list_metadata_paths:
|
|
270
276
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -272,8 +278,7 @@ def check_archive_metadata_keys(base_dir: str = None) -> bool:
|
|
|
272
278
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
273
279
|
|
|
274
280
|
metadata = read_station_metadata(
|
|
275
|
-
|
|
276
|
-
product="RAW",
|
|
281
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
277
282
|
data_source=data_source,
|
|
278
283
|
campaign_name=campaign_name,
|
|
279
284
|
station_name=station_name,
|
|
@@ -288,24 +293,31 @@ def check_archive_metadata_keys(base_dir: str = None) -> bool:
|
|
|
288
293
|
return is_valid
|
|
289
294
|
|
|
290
295
|
|
|
291
|
-
def
|
|
292
|
-
"""Check metadata campaign_name
|
|
296
|
+
def check_metadata_archive_campaign_name(metadata_archive_dir: Optional[str] = None) -> bool:
|
|
297
|
+
"""Check metadata ``campaign_name``.
|
|
293
298
|
|
|
294
299
|
Parameters
|
|
295
300
|
----------
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
301
|
+
metadata_archive_dir : str (optional)
|
|
302
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
303
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
304
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
305
|
+
in the DISDRODB active configuration.
|
|
299
306
|
|
|
300
307
|
Returns
|
|
301
308
|
-------
|
|
302
309
|
bool
|
|
303
|
-
If the check succeeds, the result is True
|
|
310
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
304
311
|
"""
|
|
305
312
|
is_valid = True
|
|
306
|
-
|
|
313
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
307
314
|
list_metadata_paths = get_list_metadata(
|
|
308
|
-
|
|
315
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
316
|
+
data_sources=None,
|
|
317
|
+
campaign_names=None,
|
|
318
|
+
station_names=None,
|
|
319
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
320
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
309
321
|
)
|
|
310
322
|
for filepath in list_metadata_paths:
|
|
311
323
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -313,8 +325,7 @@ def check_archive_metadata_campaign_name(base_dir: str = None) -> bool:
|
|
|
313
325
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
314
326
|
|
|
315
327
|
metadata = read_station_metadata(
|
|
316
|
-
|
|
317
|
-
product="RAW",
|
|
328
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
318
329
|
data_source=data_source,
|
|
319
330
|
campaign_name=campaign_name,
|
|
320
331
|
station_name=station_name,
|
|
@@ -328,24 +339,31 @@ def check_archive_metadata_campaign_name(base_dir: str = None) -> bool:
|
|
|
328
339
|
return is_valid
|
|
329
340
|
|
|
330
341
|
|
|
331
|
-
def
|
|
332
|
-
"""Check metadata data_source
|
|
342
|
+
def check_metadata_archive_data_source(metadata_archive_dir: Optional[str] = None) -> bool:
|
|
343
|
+
"""Check metadata ``data_source``.
|
|
333
344
|
|
|
334
345
|
Parameters
|
|
335
346
|
----------
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
347
|
+
metadata_archive_dir : str (optional)
|
|
348
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
349
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
350
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
351
|
+
in the DISDRODB active configuration.
|
|
339
352
|
|
|
340
353
|
Returns
|
|
341
354
|
-------
|
|
342
355
|
bool
|
|
343
|
-
If the check succeeds, the result is True
|
|
356
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
344
357
|
"""
|
|
345
358
|
is_valid = True
|
|
346
|
-
|
|
359
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
347
360
|
list_metadata_paths = get_list_metadata(
|
|
348
|
-
|
|
361
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
362
|
+
data_sources=None,
|
|
363
|
+
campaign_names=None,
|
|
364
|
+
station_names=None,
|
|
365
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
366
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
349
367
|
)
|
|
350
368
|
for filepath in list_metadata_paths:
|
|
351
369
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -353,8 +371,7 @@ def check_archive_metadata_data_source(base_dir: str = None) -> bool:
|
|
|
353
371
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
354
372
|
|
|
355
373
|
metadata = read_station_metadata(
|
|
356
|
-
|
|
357
|
-
product="RAW",
|
|
374
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
358
375
|
data_source=data_source,
|
|
359
376
|
campaign_name=campaign_name,
|
|
360
377
|
station_name=station_name,
|
|
@@ -368,24 +385,31 @@ def check_archive_metadata_data_source(base_dir: str = None) -> bool:
|
|
|
368
385
|
return is_valid
|
|
369
386
|
|
|
370
387
|
|
|
371
|
-
def
|
|
372
|
-
"""Check metadata
|
|
388
|
+
def check_metadata_archive_sensor_name(metadata_archive_dir: Optional[str] = None) -> bool:
|
|
389
|
+
"""Check metadata ``sensor_name``.
|
|
373
390
|
|
|
374
391
|
Parameters
|
|
375
392
|
----------
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
393
|
+
metadata_archive_dir : str (optional)
|
|
394
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
395
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
396
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
397
|
+
in the DISDRODB active configuration.
|
|
379
398
|
|
|
380
399
|
Returns
|
|
381
400
|
-------
|
|
382
401
|
bool
|
|
383
|
-
If the check succeeds, the result is True
|
|
402
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
384
403
|
"""
|
|
385
404
|
is_valid = True
|
|
386
|
-
|
|
405
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
387
406
|
list_metadata_paths = get_list_metadata(
|
|
388
|
-
|
|
407
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
408
|
+
data_sources=None,
|
|
409
|
+
campaign_names=None,
|
|
410
|
+
station_names=None,
|
|
411
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
412
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
389
413
|
)
|
|
390
414
|
for filepath in list_metadata_paths:
|
|
391
415
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -393,8 +417,7 @@ def check_archive_metadata_sensor_name(base_dir: str = None) -> bool:
|
|
|
393
417
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
394
418
|
|
|
395
419
|
metadata = read_station_metadata(
|
|
396
|
-
|
|
397
|
-
product="RAW",
|
|
420
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
398
421
|
data_source=data_source,
|
|
399
422
|
campaign_name=campaign_name,
|
|
400
423
|
station_name=station_name,
|
|
@@ -408,24 +431,31 @@ def check_archive_metadata_sensor_name(base_dir: str = None) -> bool:
|
|
|
408
431
|
return is_valid
|
|
409
432
|
|
|
410
433
|
|
|
411
|
-
def
|
|
412
|
-
"""Check metadata
|
|
434
|
+
def check_metadata_archive_station_name(metadata_archive_dir: Optional[str] = None) -> bool:
|
|
435
|
+
"""Check metadata ``station_name``.
|
|
413
436
|
|
|
414
437
|
Parameters
|
|
415
438
|
----------
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
439
|
+
metadata_archive_dir : str (optional)
|
|
440
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
441
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
442
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
443
|
+
in the DISDRODB active configuration.
|
|
419
444
|
|
|
420
445
|
Returns
|
|
421
446
|
-------
|
|
422
447
|
bool
|
|
423
|
-
If the check succeeds, the result is True
|
|
448
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
424
449
|
"""
|
|
425
450
|
is_valid = True
|
|
426
|
-
|
|
451
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
427
452
|
list_metadata_paths = get_list_metadata(
|
|
428
|
-
|
|
453
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
454
|
+
data_sources=None,
|
|
455
|
+
campaign_names=None,
|
|
456
|
+
station_names=None,
|
|
457
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
458
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
429
459
|
)
|
|
430
460
|
for filepath in list_metadata_paths:
|
|
431
461
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -433,8 +463,7 @@ def check_archive_metadata_station_name(base_dir: str = None) -> bool:
|
|
|
433
463
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
434
464
|
|
|
435
465
|
metadata = read_station_metadata(
|
|
436
|
-
|
|
437
|
-
product="RAW",
|
|
466
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
438
467
|
data_source=data_source,
|
|
439
468
|
campaign_name=campaign_name,
|
|
440
469
|
station_name=station_name,
|
|
@@ -448,26 +477,33 @@ def check_archive_metadata_station_name(base_dir: str = None) -> bool:
|
|
|
448
477
|
return is_valid
|
|
449
478
|
|
|
450
479
|
|
|
451
|
-
def
|
|
452
|
-
"""Check if the reader key is available and there is the associated reader.
|
|
480
|
+
def check_metadata_archive_reader(metadata_archive_dir: Optional[str] = None) -> bool:
|
|
481
|
+
"""Check if the ``reader`` key is available and there is the associated reader.
|
|
453
482
|
|
|
454
483
|
Parameters
|
|
455
484
|
----------
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
485
|
+
metadata_archive_dir : str (optional)
|
|
486
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
487
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
488
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
489
|
+
in the DISDRODB active configuration.
|
|
459
490
|
|
|
460
491
|
Returns
|
|
461
492
|
-------
|
|
462
493
|
bool
|
|
463
|
-
If the check succeeds, the result is True
|
|
494
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
464
495
|
"""
|
|
465
|
-
from disdrodb.l0.l0_reader import
|
|
496
|
+
from disdrodb.l0.l0_reader import check_metadata_reader
|
|
466
497
|
|
|
467
498
|
is_valid = True
|
|
468
|
-
|
|
499
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
469
500
|
list_metadata_paths = get_list_metadata(
|
|
470
|
-
|
|
501
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
502
|
+
data_sources=None,
|
|
503
|
+
campaign_names=None,
|
|
504
|
+
station_names=None,
|
|
505
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
506
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
471
507
|
)
|
|
472
508
|
for filepath in list_metadata_paths:
|
|
473
509
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -475,14 +511,13 @@ def check_archive_metadata_reader(base_dir: str = None) -> bool:
|
|
|
475
511
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
476
512
|
|
|
477
513
|
metadata = read_station_metadata(
|
|
478
|
-
|
|
479
|
-
product="RAW",
|
|
514
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
480
515
|
data_source=data_source,
|
|
481
516
|
campaign_name=campaign_name,
|
|
482
517
|
station_name=station_name,
|
|
483
518
|
)
|
|
484
519
|
try:
|
|
485
|
-
|
|
520
|
+
check_metadata_reader(metadata)
|
|
486
521
|
except Exception as e:
|
|
487
522
|
is_valid = False
|
|
488
523
|
print(f"Error for {data_source} {campaign_name} {station_name}.")
|
|
@@ -490,27 +525,34 @@ def check_archive_metadata_reader(base_dir: str = None) -> bool:
|
|
|
490
525
|
return is_valid
|
|
491
526
|
|
|
492
527
|
|
|
493
|
-
def
|
|
528
|
+
def check_metadata_archive(metadata_archive_dir: Optional[str] = None, raise_error=False):
|
|
494
529
|
"""Check the archive metadata compliance.
|
|
495
530
|
|
|
496
531
|
Parameters
|
|
497
532
|
----------
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
533
|
+
metadata_archive_dir : str (optional)
|
|
534
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
535
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
536
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
537
|
+
in the DISDRODB active configuration.
|
|
501
538
|
raise_error: bool (optional)
|
|
502
539
|
Whether to raise an error and interrupt the archive check if a
|
|
503
|
-
metadata is not compliant. The default is False
|
|
540
|
+
metadata is not compliant. The default value is ``False``.
|
|
504
541
|
|
|
505
542
|
Returns
|
|
506
543
|
-------
|
|
507
544
|
bool
|
|
508
|
-
If the check succeeds, the result is True
|
|
545
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
509
546
|
"""
|
|
510
547
|
is_valid = True
|
|
511
|
-
|
|
548
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
512
549
|
list_metadata_paths = get_list_metadata(
|
|
513
|
-
|
|
550
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
551
|
+
data_sources=None,
|
|
552
|
+
campaign_names=None,
|
|
553
|
+
station_names=None,
|
|
554
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
555
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
514
556
|
)
|
|
515
557
|
for filepath in list_metadata_paths:
|
|
516
558
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -518,8 +560,8 @@ def check_archive_metadata_compliance(base_dir: str = None, raise_error=False):
|
|
|
518
560
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
519
561
|
# Check compliance
|
|
520
562
|
try:
|
|
521
|
-
|
|
522
|
-
|
|
563
|
+
check_station_metadata(
|
|
564
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
523
565
|
data_source=data_source,
|
|
524
566
|
campaign_name=campaign_name,
|
|
525
567
|
station_name=station_name,
|
|
@@ -530,30 +572,36 @@ def check_archive_metadata_compliance(base_dir: str = None, raise_error=False):
|
|
|
530
572
|
msg = msg + f"The error is: {e}."
|
|
531
573
|
if raise_error:
|
|
532
574
|
raise ValueError(msg)
|
|
533
|
-
|
|
534
|
-
print(msg)
|
|
575
|
+
print(msg)
|
|
535
576
|
|
|
536
577
|
return is_valid
|
|
537
578
|
|
|
538
579
|
|
|
539
|
-
def
|
|
580
|
+
def check_metadata_archive_geolocation(metadata_archive_dir: Optional[str] = None):
|
|
540
581
|
"""Check the metadata files have missing or wrong geolocation..
|
|
541
582
|
|
|
542
583
|
Parameters
|
|
543
584
|
----------
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
585
|
+
metadata_archive_dir : str (optional)
|
|
586
|
+
The directory path where the DISDRODB Metadata Archive is located.
|
|
587
|
+
The directory path must end with ``<...>/DISDRODB``.
|
|
588
|
+
If ``None``, it uses the ``metadata_archive_dir`` path specified
|
|
589
|
+
in the DISDRODB active configuration.
|
|
547
590
|
|
|
548
591
|
Returns
|
|
549
592
|
-------
|
|
550
593
|
bool
|
|
551
|
-
If the check succeeds, the result is True
|
|
594
|
+
If the check succeeds, the result is ``True``, otherwise ``False``.
|
|
552
595
|
"""
|
|
553
596
|
is_valid = True
|
|
554
|
-
|
|
597
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
555
598
|
list_metadata_paths = get_list_metadata(
|
|
556
|
-
|
|
599
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
600
|
+
data_sources=None,
|
|
601
|
+
campaign_names=None,
|
|
602
|
+
station_names=None,
|
|
603
|
+
product=None, # --> Search in DISDRODB Metadata Archive
|
|
604
|
+
available_data=False, # --> Select all metadata matching the filtering criteria
|
|
557
605
|
)
|
|
558
606
|
for filepath in list_metadata_paths:
|
|
559
607
|
data_source = infer_data_source_from_path(filepath)
|
|
@@ -561,14 +609,13 @@ def check_archive_metadata_geolocation(base_dir: str = None):
|
|
|
561
609
|
station_name = os.path.basename(filepath).replace(".yml", "")
|
|
562
610
|
|
|
563
611
|
metadata = read_station_metadata(
|
|
564
|
-
|
|
565
|
-
product="RAW",
|
|
612
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
566
613
|
data_source=data_source,
|
|
567
614
|
campaign_name=campaign_name,
|
|
568
615
|
station_name=station_name,
|
|
569
616
|
)
|
|
570
617
|
try:
|
|
571
|
-
|
|
618
|
+
check_station_metadata_geolocation(metadata)
|
|
572
619
|
except Exception as e:
|
|
573
620
|
is_valid = False
|
|
574
621
|
print(f"Missing information for {data_source} {campaign_name} {station_name}.")
|