disdrodb 0.0.21__py3-none-any.whl → 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +132 -15
- disdrodb/_config.py +4 -2
- disdrodb/_version.py +9 -4
- disdrodb/api/checks.py +264 -237
- disdrodb/api/configs.py +4 -8
- disdrodb/api/create_directories.py +235 -290
- disdrodb/api/info.py +217 -26
- disdrodb/api/io.py +295 -269
- disdrodb/api/path.py +597 -173
- disdrodb/api/search.py +486 -0
- disdrodb/{metadata/scripts → cli}/disdrodb_check_metadata_archive.py +12 -7
- disdrodb/{utils/pandas.py → cli/disdrodb_data_archive_directory.py} +9 -18
- disdrodb/cli/disdrodb_download_archive.py +86 -0
- disdrodb/cli/disdrodb_download_metadata_archive.py +53 -0
- disdrodb/cli/disdrodb_download_station.py +84 -0
- disdrodb/{api/scripts → cli}/disdrodb_initialize_station.py +22 -10
- disdrodb/cli/disdrodb_metadata_archive_directory.py +32 -0
- disdrodb/{data_transfer/scripts/disdrodb_download_station.py → cli/disdrodb_open_data_archive.py} +22 -22
- disdrodb/cli/disdrodb_open_logs_directory.py +69 -0
- disdrodb/{data_transfer/scripts/disdrodb_upload_station.py → cli/disdrodb_open_metadata_archive.py} +22 -24
- disdrodb/cli/disdrodb_open_metadata_directory.py +71 -0
- disdrodb/cli/disdrodb_open_product_directory.py +74 -0
- disdrodb/cli/disdrodb_open_readers_directory.py +32 -0
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0.py +38 -31
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0_station.py +32 -30
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a_station.py +24 -33
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b_station.py +25 -34
- disdrodb/cli/disdrodb_run_l0c.py +130 -0
- disdrodb/cli/disdrodb_run_l0c_station.py +129 -0
- disdrodb/cli/disdrodb_run_l1.py +122 -0
- disdrodb/cli/disdrodb_run_l1_station.py +121 -0
- disdrodb/cli/disdrodb_run_l2e.py +122 -0
- disdrodb/cli/disdrodb_run_l2e_station.py +122 -0
- disdrodb/cli/disdrodb_run_l2m.py +122 -0
- disdrodb/cli/disdrodb_run_l2m_station.py +122 -0
- disdrodb/cli/disdrodb_upload_archive.py +105 -0
- disdrodb/cli/disdrodb_upload_station.py +98 -0
- disdrodb/configs.py +90 -25
- disdrodb/data_transfer/__init__.py +22 -0
- disdrodb/data_transfer/download_data.py +87 -90
- disdrodb/data_transfer/upload_data.py +64 -37
- disdrodb/data_transfer/zenodo.py +15 -18
- disdrodb/docs.py +1 -1
- disdrodb/issue/__init__.py +17 -4
- disdrodb/issue/checks.py +10 -23
- disdrodb/issue/reader.py +9 -12
- disdrodb/issue/writer.py +14 -17
- disdrodb/l0/__init__.py +17 -26
- disdrodb/l0/check_configs.py +35 -23
- disdrodb/l0/check_standards.py +32 -42
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_diameter.yml +44 -44
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_velocity.yml +40 -40
- disdrodb/l0/configs/LPM/l0a_encodings.yml +80 -0
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_cf_attrs.yml +62 -59
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_encodings.yml +9 -9
- disdrodb/l0/configs/{Thies_LPM → LPM}/raw_data_format.yml +245 -245
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_diameter.yml +66 -66
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL/l0a_encodings.yml +32 -0
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_cf_attrs.yml +22 -20
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_encodings.yml +17 -17
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/raw_data_format.yml +77 -77
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_diameter.yml +64 -64
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +39 -0
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_cf_attrs.yml +24 -22
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_encodings.yml +20 -20
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/raw_data_format.yml +98 -98
- disdrodb/l0/configs/{RD_80 → RD80}/bins_diameter.yml +40 -40
- disdrodb/l0/configs/RD80/l0a_encodings.yml +16 -0
- disdrodb/l0/configs/{RD_80 → RD80}/l0b_cf_attrs.yml +3 -3
- disdrodb/l0/configs/RD80/l0b_encodings.yml +135 -0
- disdrodb/l0/configs/{RD_80 → RD80}/raw_data_format.yml +48 -48
- disdrodb/l0/l0_reader.py +216 -340
- disdrodb/l0/l0a_processing.py +237 -208
- disdrodb/l0/l0b_nc_processing.py +227 -80
- disdrodb/l0/l0b_processing.py +93 -173
- disdrodb/l0/l0c_processing.py +627 -0
- disdrodb/l0/readers/{ARM → LPM/ARM}/ARM_LPM.py +36 -58
- disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +226 -0
- disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +185 -0
- disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +183 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +179 -0
- disdrodb/l0/readers/{UK → LPM/UK}/DIVEN.py +14 -35
- disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +157 -0
- disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +113 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/ARCTIC_2021.py +40 -57
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/COMMON_2011.py +37 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/DAVOS_2009_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_2009.py +34 -51
- disdrodb/l0/readers/{EPFL/PARADISO_2014.py → PARSIVEL/EPFL/EPFL_ROOF_2008.py} +38 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2012.py +33 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GENEPI_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007_2.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HPICONET_2010.py +34 -51
- disdrodb/l0/readers/{EPFL/EPFL_ROOF_2010.py → PARSIVEL/EPFL/HYMEX_LTE_SOP2.py} +37 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +111 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HYMEX_LTE_SOP4.py +36 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2018.py +34 -52
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2019.py +38 -56
- disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PARSIVEL_2007.py +27 -45
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PLATO_2019.py +24 -44
- disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +140 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RACLETS_2019_WJF.py +41 -59
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RIETHOLZBACH_2011.py +34 -51
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +117 -0
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +137 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/UNIL_2022.py +42 -55
- disdrodb/l0/readers/PARSIVEL/GPM/IFLOODS.py +104 -0
- disdrodb/l0/readers/{GPM → PARSIVEL/GPM}/LPVEX.py +29 -48
- disdrodb/l0/readers/PARSIVEL/GPM/MC3E.py +184 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +113 -0
- disdrodb/l0/readers/{NCAR/VORTEX_SE_2016_P1.py → PARSIVEL/NCAR/OWLES_MIPS.py} +46 -72
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +125 -0
- disdrodb/l0/readers/{NCAR/OWLES_MIPS.py → PARSIVEL/NCAR/PLOWS_MIPS.py} +45 -64
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +114 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +176 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +183 -0
- disdrodb/l0/readers/{ARM/ARM_LD.py → PARSIVEL2/ARM/ARM_PARSIVEL2.py} +27 -50
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/{DENMARK → PARSIVEL2/DENMARK}/EROSION_nc.py +14 -35
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +119 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +104 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/NSSTC.py +176 -0
- disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +32 -0
- disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +56 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +120 -0
- disdrodb/l0/readers/{NCAR → PARSIVEL2/NCAR}/PECAN_MIPS.py +45 -64
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +181 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +160 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +160 -0
- disdrodb/l0/readers/{NCAR/PLOWS_MIPS.py → PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py} +49 -66
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +118 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +152 -0
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT.py +166 -0
- disdrodb/l0/readers/{NCAR/RELAMPAGO_RD80.py → RD80/BRAZIL/CHUVA_RD80.py} +36 -60
- disdrodb/l0/readers/{BRAZIL → RD80/BRAZIL}/GOAMAZON_RD80.py +36 -55
- disdrodb/l0/readers/{NCAR → RD80/NCAR}/CINDY_2011_RD80.py +35 -54
- disdrodb/l0/readers/{BRAZIL/CHUVA_RD80.py → RD80/NCAR/RELAMPAGO_RD80.py} +40 -54
- disdrodb/l0/readers/template_reader_raw_netcdf_data.py +62 -0
- disdrodb/l0/readers/{reader_template.py → template_reader_raw_text_data.py} +20 -44
- disdrodb/l0/routines.py +885 -581
- disdrodb/l0/standards.py +72 -236
- disdrodb/l0/template_tools.py +104 -109
- disdrodb/l1/__init__.py +17 -0
- disdrodb/l1/beard_model.py +716 -0
- disdrodb/l1/encoding_attrs.py +620 -0
- disdrodb/l1/fall_velocity.py +260 -0
- disdrodb/l1/filters.py +192 -0
- disdrodb/l1/processing.py +200 -0
- disdrodb/l1/resampling.py +236 -0
- disdrodb/l1/routines.py +357 -0
- disdrodb/l1_env/__init__.py +17 -0
- disdrodb/l1_env/routines.py +38 -0
- disdrodb/l2/__init__.py +17 -0
- disdrodb/l2/empirical_dsd.py +1735 -0
- disdrodb/l2/event.py +388 -0
- disdrodb/l2/processing.py +519 -0
- disdrodb/l2/processing_options.py +213 -0
- disdrodb/l2/routines.py +868 -0
- disdrodb/metadata/__init__.py +9 -2
- disdrodb/metadata/checks.py +165 -118
- disdrodb/metadata/download.py +81 -0
- disdrodb/metadata/geolocation.py +146 -0
- disdrodb/metadata/info.py +20 -13
- disdrodb/metadata/manipulation.py +1 -1
- disdrodb/metadata/reader.py +59 -8
- disdrodb/metadata/search.py +77 -144
- disdrodb/metadata/standards.py +7 -8
- disdrodb/metadata/writer.py +8 -14
- disdrodb/psd/__init__.py +38 -0
- disdrodb/psd/fitting.py +2146 -0
- disdrodb/psd/models.py +774 -0
- disdrodb/routines.py +1176 -0
- disdrodb/scattering/__init__.py +28 -0
- disdrodb/scattering/axis_ratio.py +344 -0
- disdrodb/scattering/routines.py +456 -0
- disdrodb/utils/__init__.py +17 -0
- disdrodb/utils/attrs.py +208 -0
- disdrodb/utils/cli.py +269 -0
- disdrodb/utils/compression.py +60 -42
- disdrodb/utils/dask.py +62 -0
- disdrodb/utils/decorators.py +110 -0
- disdrodb/utils/directories.py +107 -46
- disdrodb/utils/encoding.py +127 -0
- disdrodb/utils/list.py +29 -0
- disdrodb/utils/logger.py +168 -46
- disdrodb/utils/time.py +657 -0
- disdrodb/utils/warnings.py +30 -0
- disdrodb/utils/writer.py +57 -0
- disdrodb/utils/xarray.py +138 -47
- disdrodb/utils/yaml.py +0 -1
- disdrodb/viz/__init__.py +17 -0
- disdrodb/viz/plots.py +17 -0
- disdrodb-0.1.0.dist-info/METADATA +321 -0
- disdrodb-0.1.0.dist-info/RECORD +216 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.0.dist-info}/WHEEL +1 -1
- disdrodb-0.1.0.dist-info/entry_points.txt +30 -0
- disdrodb/data_transfer/scripts/disdrodb_download_archive.py +0 -53
- disdrodb/data_transfer/scripts/disdrodb_upload_archive.py +0 -57
- disdrodb/l0/configs/OTT_Parsivel/l0a_encodings.yml +0 -32
- disdrodb/l0/configs/OTT_Parsivel2/l0a_encodings.yml +0 -39
- disdrodb/l0/configs/RD_80/l0a_encodings.yml +0 -16
- disdrodb/l0/configs/RD_80/l0b_encodings.yml +0 -135
- disdrodb/l0/configs/Thies_LPM/l0a_encodings.yml +0 -80
- disdrodb/l0/io.py +0 -257
- disdrodb/l0/l0_processing.py +0 -1091
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_OTT.py +0 -178
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_THIES.py +0 -247
- disdrodb/l0/readers/BRAZIL/CHUVA_LPM.py +0 -204
- disdrodb/l0/readers/BRAZIL/CHUVA_OTT.py +0 -183
- disdrodb/l0/readers/BRAZIL/GOAMAZON_LPM.py +0 -204
- disdrodb/l0/readers/BRAZIL/GOAMAZON_OTT.py +0 -183
- disdrodb/l0/readers/CHINA/CHONGQING.py +0 -131
- disdrodb/l0/readers/EPFL/EPFL_ROOF_2008.py +0 -128
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP2.py +0 -127
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP3.py +0 -129
- disdrodb/l0/readers/EPFL/RACLETS_2019.py +0 -158
- disdrodb/l0/readers/EPFL/SAMOYLOV_2017.py +0 -136
- disdrodb/l0/readers/EPFL/SAMOYLOV_2019.py +0 -158
- disdrodb/l0/readers/FRANCE/SIRTA_OTT2.py +0 -138
- disdrodb/l0/readers/GPM/GCPEX.py +0 -123
- disdrodb/l0/readers/GPM/IFLOODS.py +0 -123
- disdrodb/l0/readers/GPM/MC3E.py +0 -123
- disdrodb/l0/readers/GPM/NSSTC.py +0 -164
- disdrodb/l0/readers/ITALY/GID.py +0 -199
- disdrodb/l0/readers/MEXICO/OH_IIUNAM_nc.py +0 -92
- disdrodb/l0/readers/NCAR/CCOPE_2015.py +0 -133
- disdrodb/l0/readers/NCAR/PECAN_FP3.py +0 -137
- disdrodb/l0/readers/NCAR/PECAN_MOBILE.py +0 -144
- disdrodb/l0/readers/NCAR/RELAMPAGO_OTT.py +0 -195
- disdrodb/l0/readers/NCAR/SNOWIE_PJ.py +0 -172
- disdrodb/l0/readers/NCAR/SNOWIE_SB.py +0 -179
- disdrodb/l0/readers/NCAR/VORTEX2_2009.py +0 -133
- disdrodb/l0/readers/NCAR/VORTEX2_2010.py +0 -188
- disdrodb/l0/readers/NCAR/VORTEX2_2010_UF.py +0 -191
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_P2.py +0 -135
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_PIPS.py +0 -170
- disdrodb/l0/readers/NETHERLANDS/DELFT.py +0 -187
- disdrodb/l0/readers/SPAIN/SBEGUERIA.py +0 -179
- disdrodb/l0/scripts/disdrodb_run_l0b_concat.py +0 -93
- disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py +0 -85
- disdrodb/utils/netcdf.py +0 -452
- disdrodb/utils/scripts.py +0 -102
- disdrodb-0.0.21.dist-info/AUTHORS.md +0 -18
- disdrodb-0.0.21.dist-info/METADATA +0 -186
- disdrodb-0.0.21.dist-info/RECORD +0 -168
- disdrodb-0.0.21.dist-info/entry_points.txt +0 -15
- /disdrodb/l0/configs/{RD_80 → RD80}/bins_velocity.yml +0 -0
- /disdrodb/l0/manuals/{Thies_LPM.pdf → LPM.pdf} +0 -0
- /disdrodb/l0/manuals/{ODM_470.pdf → ODM470.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel.pdf → PARSIVEL.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel2.pdf → PARSIVEL2.pdf} +0 -0
- /disdrodb/l0/manuals/{PWS_100.pdf → PWS100.pdf} +0 -0
- /disdrodb/l0/manuals/{RD_80.pdf → RD80.pdf} +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.0.dist-info/licenses}/LICENSE +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.0.dist-info}/top_level.txt +0 -0
disdrodb/api/search.py
ADDED
|
@@ -0,0 +1,486 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
from disdrodb.api.checks import (
|
|
6
|
+
check_invalid_fields_policy,
|
|
7
|
+
check_product,
|
|
8
|
+
check_product_kwargs,
|
|
9
|
+
check_valid_fields,
|
|
10
|
+
)
|
|
11
|
+
from disdrodb.api.path import (
|
|
12
|
+
define_data_dir,
|
|
13
|
+
define_data_source_dir,
|
|
14
|
+
define_metadata_dir,
|
|
15
|
+
define_metadata_filepath,
|
|
16
|
+
define_station_dir,
|
|
17
|
+
)
|
|
18
|
+
from disdrodb.configs import get_data_archive_dir, get_metadata_archive_dir
|
|
19
|
+
from disdrodb.utils.directories import contains_files, contains_netcdf_or_parquet_files
|
|
20
|
+
from disdrodb.utils.yaml import read_yaml
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def get_required_product(product):
|
|
24
|
+
"""Determine the required product for input product processing."""
|
|
25
|
+
from disdrodb import PRODUCTS_REQUIREMENTS
|
|
26
|
+
|
|
27
|
+
# Check input
|
|
28
|
+
check_product(product)
|
|
29
|
+
# Determine required product
|
|
30
|
+
required_product = PRODUCTS_REQUIREMENTS[product]
|
|
31
|
+
return required_product
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
####-------------------------------------------------------------------------
|
|
35
|
+
#### List DISDRODB infrastructure directories
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def list_data_sources(metadata_archive_dir, data_sources=None, invalid_fields_policy="raise"):
|
|
39
|
+
"""List data sources names in the DISDRODB Metadata Archive."""
|
|
40
|
+
available_data_sources = os.listdir(os.path.join(metadata_archive_dir, "METADATA"))
|
|
41
|
+
# Filter by optionally specified data_sources
|
|
42
|
+
if data_sources is not None:
|
|
43
|
+
available_data_sources = check_valid_fields(
|
|
44
|
+
fields=data_sources,
|
|
45
|
+
available_fields=available_data_sources,
|
|
46
|
+
field_name="data_sources",
|
|
47
|
+
invalid_fields_policy=invalid_fields_policy,
|
|
48
|
+
)
|
|
49
|
+
# Return the unique data_sources
|
|
50
|
+
return np.unique(available_data_sources).tolist()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _list_campaign_names(metadata_archive_dir, data_source):
|
|
54
|
+
data_source_dir = define_data_source_dir(metadata_archive_dir, product="METADATA", data_source=data_source)
|
|
55
|
+
campaign_names = os.listdir(data_source_dir)
|
|
56
|
+
return campaign_names
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def list_campaign_names(
|
|
60
|
+
metadata_archive_dir,
|
|
61
|
+
data_sources=None,
|
|
62
|
+
campaign_names=None,
|
|
63
|
+
invalid_fields_policy="raise",
|
|
64
|
+
return_tuple=False,
|
|
65
|
+
):
|
|
66
|
+
"""List campaign names in the DISDRODB Metadata Archive."""
|
|
67
|
+
# Retrieve available data sources
|
|
68
|
+
data_sources = list_data_sources(
|
|
69
|
+
metadata_archive_dir,
|
|
70
|
+
data_sources=data_sources,
|
|
71
|
+
invalid_fields_policy=invalid_fields_policy,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
# Retrieve (data_source, campaign_name) tuples
|
|
75
|
+
list_tuples = [
|
|
76
|
+
(data_source, campaign_name)
|
|
77
|
+
for data_source in data_sources
|
|
78
|
+
for campaign_name in _list_campaign_names(metadata_archive_dir=metadata_archive_dir, data_source=data_source)
|
|
79
|
+
]
|
|
80
|
+
# Filter by optionally specified campaign_names
|
|
81
|
+
if campaign_names is not None:
|
|
82
|
+
available_campaign_names = [campaign_name for _, campaign_name in list_tuples]
|
|
83
|
+
campaign_names = check_valid_fields(
|
|
84
|
+
fields=campaign_names,
|
|
85
|
+
available_fields=available_campaign_names,
|
|
86
|
+
field_name="campaign_names",
|
|
87
|
+
invalid_fields_policy=invalid_fields_policy,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
list_tuples = [
|
|
91
|
+
(data_source, campaign_name)
|
|
92
|
+
for data_source, campaign_name in list_tuples
|
|
93
|
+
if campaign_name in campaign_names
|
|
94
|
+
]
|
|
95
|
+
|
|
96
|
+
# If specified, return just the list of (data_source, campaign_name) tuples
|
|
97
|
+
if return_tuple:
|
|
98
|
+
return list_tuples
|
|
99
|
+
|
|
100
|
+
# Otherwise just return the unique campaign names
|
|
101
|
+
campaign_names = [campaign_name for _, campaign_name in list_tuples]
|
|
102
|
+
campaign_names = np.unique(campaign_names).tolist()
|
|
103
|
+
return campaign_names
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _list_station_names(metadata_archive_dir, data_source, campaign_name):
|
|
107
|
+
metadata_dir = define_metadata_dir(
|
|
108
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
109
|
+
data_source=data_source,
|
|
110
|
+
campaign_name=campaign_name,
|
|
111
|
+
)
|
|
112
|
+
metadata_filenames = os.listdir(metadata_dir)
|
|
113
|
+
station_names = [fname.replace(".yml", "").replace(".yaml", "") for fname in metadata_filenames]
|
|
114
|
+
return station_names
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def list_station_names(
|
|
118
|
+
metadata_archive_dir,
|
|
119
|
+
data_sources=None,
|
|
120
|
+
campaign_names=None,
|
|
121
|
+
station_names=None,
|
|
122
|
+
invalid_fields_policy="raise",
|
|
123
|
+
return_tuple=False,
|
|
124
|
+
):
|
|
125
|
+
"""List station names in the DISDRODB Metadata Archive."""
|
|
126
|
+
# Retrieve (data sources - campaign_names) tuples
|
|
127
|
+
list_tuples = list_campaign_names(
|
|
128
|
+
metadata_archive_dir,
|
|
129
|
+
data_sources=data_sources,
|
|
130
|
+
campaign_names=campaign_names,
|
|
131
|
+
invalid_fields_policy=invalid_fields_policy,
|
|
132
|
+
return_tuple=True,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
# Retrieve (data_source, campaign_name, station_name) tuples
|
|
136
|
+
list_info = [
|
|
137
|
+
(data_source, campaign_name, station_name)
|
|
138
|
+
for data_source, campaign_name in list_tuples
|
|
139
|
+
for station_name in _list_station_names(
|
|
140
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
141
|
+
data_source=data_source,
|
|
142
|
+
campaign_name=campaign_name,
|
|
143
|
+
)
|
|
144
|
+
]
|
|
145
|
+
# Filter by optionally specified station_names
|
|
146
|
+
if station_names is not None:
|
|
147
|
+
available_station_names = [station_name for data_source, campaign_name, station_name in list_info]
|
|
148
|
+
station_names = check_valid_fields(
|
|
149
|
+
fields=station_names,
|
|
150
|
+
available_fields=available_station_names,
|
|
151
|
+
field_name="station_names",
|
|
152
|
+
invalid_fields_policy=invalid_fields_policy,
|
|
153
|
+
)
|
|
154
|
+
list_info = [
|
|
155
|
+
(data_source, campaign_name, station_name)
|
|
156
|
+
for data_source, campaign_name, station_name in list_info
|
|
157
|
+
if station_name in station_names
|
|
158
|
+
]
|
|
159
|
+
|
|
160
|
+
# If specified, return just the list of (data_source, campaign_name, station_name) tuples
|
|
161
|
+
if return_tuple:
|
|
162
|
+
return list_info
|
|
163
|
+
|
|
164
|
+
# Otherwise just return the unique station_names
|
|
165
|
+
station_names = [station_name for _, _, station_name in list_info]
|
|
166
|
+
station_names = np.unique(station_names).tolist()
|
|
167
|
+
return station_names
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _finalize_output(list_info, return_tuple):
|
|
171
|
+
# - Return the (data_source, campaign_name, station_name) tuple
|
|
172
|
+
if return_tuple:
|
|
173
|
+
return list_info
|
|
174
|
+
# - Return list with the name of the available stations
|
|
175
|
+
return [info[2] for info in list_info]
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _raise_an_error_if_no_stations(list_info, raise_error_if_empty, msg):
|
|
179
|
+
if len(list_info) == 0 and raise_error_if_empty:
|
|
180
|
+
raise ValueError(msg)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def is_disdrodb_data_url_specified(metadata_filepath):
|
|
184
|
+
"""Check if the disdrodb_data_url is specified in the metadata file."""
|
|
185
|
+
disdrodb_data_url = read_yaml(metadata_filepath).get("disdrodb_data_url", "")
|
|
186
|
+
return isinstance(disdrodb_data_url, str) and len(disdrodb_data_url) > 1
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def keep_list_info_with_disdrodb_data_url(metadata_archive_dir, list_info):
|
|
190
|
+
"""Keep only the stations with disdrodb_data_url specified in the metadata file."""
|
|
191
|
+
list_info_with_data = []
|
|
192
|
+
for data_source, campaign_name, station_name in list_info:
|
|
193
|
+
# Define metadata filepath
|
|
194
|
+
metadata_filepath = define_metadata_filepath(
|
|
195
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
196
|
+
data_source=data_source,
|
|
197
|
+
campaign_name=campaign_name,
|
|
198
|
+
station_name=station_name,
|
|
199
|
+
)
|
|
200
|
+
# Add station if disdrodb_data_url is specified
|
|
201
|
+
if is_disdrodb_data_url_specified(metadata_filepath):
|
|
202
|
+
list_info_with_data.append((data_source, campaign_name, station_name))
|
|
203
|
+
return list_info_with_data
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def keep_list_info_elements_with_product_directory(data_archive_dir, product, list_info):
|
|
207
|
+
"""Keep only the stations with the product directory."""
|
|
208
|
+
list_info_with_product_directory = []
|
|
209
|
+
for data_source, campaign_name, station_name in list_info:
|
|
210
|
+
# Define station directory
|
|
211
|
+
station_dir = define_station_dir(
|
|
212
|
+
data_archive_dir=data_archive_dir,
|
|
213
|
+
product=product,
|
|
214
|
+
data_source=data_source,
|
|
215
|
+
campaign_name=campaign_name,
|
|
216
|
+
station_name=station_name,
|
|
217
|
+
check_exists=False,
|
|
218
|
+
)
|
|
219
|
+
# Add station if product station directory exists
|
|
220
|
+
if os.path.isdir(station_dir):
|
|
221
|
+
list_info_with_product_directory.append((data_source, campaign_name, station_name))
|
|
222
|
+
return list_info_with_product_directory
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def keep_list_info_elements_with_product_data(data_archive_dir, product, list_info, **product_kwargs):
|
|
226
|
+
"""Keep only the stations with product data."""
|
|
227
|
+
# Define file checking function
|
|
228
|
+
checking_function = contains_files if product == "RAW" else contains_netcdf_or_parquet_files
|
|
229
|
+
|
|
230
|
+
# Check presence of data for each station
|
|
231
|
+
# TODO: - In parallel over stations to speed up ?
|
|
232
|
+
list_info_with_product_data = []
|
|
233
|
+
for data_source, campaign_name, station_name in list_info:
|
|
234
|
+
data_dir = define_data_dir(
|
|
235
|
+
data_archive_dir=data_archive_dir,
|
|
236
|
+
product=product,
|
|
237
|
+
data_source=data_source,
|
|
238
|
+
campaign_name=campaign_name,
|
|
239
|
+
station_name=station_name,
|
|
240
|
+
check_exists=False,
|
|
241
|
+
**product_kwargs,
|
|
242
|
+
)
|
|
243
|
+
if checking_function(data_dir):
|
|
244
|
+
list_info_with_product_data.append((data_source, campaign_name, station_name))
|
|
245
|
+
return list_info_with_product_data
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
####-------------------------------------------------------------------------
|
|
249
|
+
#### DISDRODB Search Routines
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def available_stations(
|
|
253
|
+
product=None,
|
|
254
|
+
data_sources=None,
|
|
255
|
+
campaign_names=None,
|
|
256
|
+
station_names=None,
|
|
257
|
+
return_tuple=True,
|
|
258
|
+
available_data=False,
|
|
259
|
+
raise_error_if_empty=False,
|
|
260
|
+
invalid_fields_policy="raise",
|
|
261
|
+
data_archive_dir=None,
|
|
262
|
+
metadata_archive_dir=None,
|
|
263
|
+
**product_kwargs,
|
|
264
|
+
):
|
|
265
|
+
"""
|
|
266
|
+
Return stations information for which metadata or product data are available on disk.
|
|
267
|
+
|
|
268
|
+
This function queries the DISDRODB Metadata Archive and, optionally, the
|
|
269
|
+
local DISDRODB Data Archive to identify stations that satisfy the specified
|
|
270
|
+
filters.
|
|
271
|
+
|
|
272
|
+
If the DISDRODB product is not specified, it lists the stations present
|
|
273
|
+
in the DISDRODB Metadata Archive given the specified filtering criteria.
|
|
274
|
+
If the DISDRODB product is specified, it lists the stations present
|
|
275
|
+
in the local DISDRODB Data Archive given the specified filtering criteria.
|
|
276
|
+
|
|
277
|
+
Parameters
|
|
278
|
+
----------
|
|
279
|
+
product : str or None, optional
|
|
280
|
+
Name of the product to filter on (e.g., "RAW", "L0A", "L1").
|
|
281
|
+
|
|
282
|
+
If the DISDRODB product is not specified (default),
|
|
283
|
+
it lists the stations present in the DISDRODB Metadata Archive given the specified filtering criteria.
|
|
284
|
+
|
|
285
|
+
If the DISDRODB product is specified,
|
|
286
|
+
it lists the stations present in the local DISDRODB Data Archive given the specified filtering criteria.
|
|
287
|
+
The default is is None.
|
|
288
|
+
|
|
289
|
+
data_sources : str or sequence of str, optional
|
|
290
|
+
One or more data source identifiers to filter stations by.
|
|
291
|
+
The name(s) must be UPPER CASE.
|
|
292
|
+
If None, no filtering on data source is applied. The default is is ``None``.
|
|
293
|
+
campaign_names : str or sequence of str, optional
|
|
294
|
+
One or more campaign names to filter stations by.
|
|
295
|
+
The name(s) must be UPPER CASE.
|
|
296
|
+
If None, no filtering on campaign is applied. The default is is ``None``.
|
|
297
|
+
station_names : str or sequence of str, optional
|
|
298
|
+
One or more station names to include.
|
|
299
|
+
If None, all stations matching other filters are considered. The default is is ``None``.
|
|
300
|
+
available_data : bool, optional
|
|
301
|
+
|
|
302
|
+
If ``product`` is not specified:
|
|
303
|
+
|
|
304
|
+
- if available_data is False, return stations present in the DISDRODB Metadata Archive
|
|
305
|
+
- if available_data is True, return stations with data available on the
|
|
306
|
+
online DISDRODB Decentralized Data Archive (i.e., stations with the disdrodb_data_url in the metadata).
|
|
307
|
+
|
|
308
|
+
If ``product`` is specified:
|
|
309
|
+
|
|
310
|
+
- if available_data is False, return stations where the product directory exists in the
|
|
311
|
+
in the local DISDRODB Data Archive
|
|
312
|
+
- if available_data is True, return stations where product data exists in the
|
|
313
|
+
in the local DISDRODB Data Archive.
|
|
314
|
+
The default is is False.
|
|
315
|
+
|
|
316
|
+
return_tuple : bool, optional
|
|
317
|
+
If True, return a list of tuples ``(data_source, campaign_name, station_name)``.
|
|
318
|
+
If False, return only a list of station names
|
|
319
|
+
The default is True.
|
|
320
|
+
raise_error_if_empty : bool, optional
|
|
321
|
+
If True and no stations satisfy the criteria, raise a ``ValueError``.
|
|
322
|
+
If False, return an empty list/tuple. The default is False.
|
|
323
|
+
invalid_fields_policy : {'raise', 'warn', 'ignore'}, optional
|
|
324
|
+
How to handle invalid filter values for ``data_sources``, ``campaign_names``,
|
|
325
|
+
or ``station_names`` that are not present in the metadata archive:
|
|
326
|
+
|
|
327
|
+
- 'raise' : raise a ``ValueError`` (default)
|
|
328
|
+
- 'warn' : emit a warning, then ignore invalid entries
|
|
329
|
+
- 'ignore': silently drop invalid entries
|
|
330
|
+
|
|
331
|
+
data_archive_dir : str or Path-like, optional
|
|
332
|
+
Path to the root of the local DISDRODB Data Archive.
|
|
333
|
+
Required only if ``product``is specified.
|
|
334
|
+
If None, the default data archive base directory is used. Default is None.
|
|
335
|
+
metadata_archive_dir : str or Path-like, optional
|
|
336
|
+
Path to the root of the DISDRODB Metadata Archive.
|
|
337
|
+
If None, the default metadata base directory is used. Default is None.
|
|
338
|
+
**product_kwargs : dict, optional
|
|
339
|
+
Additional arguments required for some products.
|
|
340
|
+
For example, for the "L2E" product, you need to specify ``rolling`` and
|
|
341
|
+
``sample_interval``. For the "L2M" product, you need to specify also
|
|
342
|
+
the ``model_name``.
|
|
343
|
+
|
|
344
|
+
Returns
|
|
345
|
+
-------
|
|
346
|
+
list
|
|
347
|
+
If ``return_tuple=True``, return a list of tuples ``(data_source, campaign_name, station_name)``.
|
|
348
|
+
If ``return_tuple=True``,, return a list of station names.
|
|
349
|
+
|
|
350
|
+
Examples
|
|
351
|
+
--------
|
|
352
|
+
>>> # List all stations present in the DISDRODB Metadata Archive
|
|
353
|
+
>>> stations = available_stations()
|
|
354
|
+
>>> # List all stations present in the online DISDRODB Data Archive
|
|
355
|
+
>>> stations = available_stations(available_data=True)
|
|
356
|
+
>>> # List stations with raw data available in the local DISDRODB Data Archive
|
|
357
|
+
>>> raw_stations = available_stations(product="RAW", available_data=True)
|
|
358
|
+
>>> # List stations of specific data sources
|
|
359
|
+
>>> stations = available_stations(data_sources=["GPM", "EPFL"])
|
|
360
|
+
"""
|
|
361
|
+
# Retrieve DISDRODB Data and Metadata Archive directories
|
|
362
|
+
metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
|
|
363
|
+
product = check_product(product) if product is not None else None
|
|
364
|
+
invalid_fields_policy = check_invalid_fields_policy(invalid_fields_policy)
|
|
365
|
+
# Retrieve available stations from the Metadata Archive
|
|
366
|
+
# - Raise error if no stations availables !
|
|
367
|
+
list_info = list_station_names(
|
|
368
|
+
metadata_archive_dir,
|
|
369
|
+
data_sources=data_sources,
|
|
370
|
+
campaign_names=campaign_names,
|
|
371
|
+
station_names=station_names,
|
|
372
|
+
invalid_fields_policy=invalid_fields_policy,
|
|
373
|
+
return_tuple=True,
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
# Return stations in the Metadata Archive
|
|
377
|
+
if product is None and not available_data:
|
|
378
|
+
_raise_an_error_if_no_stations(
|
|
379
|
+
list_info,
|
|
380
|
+
raise_error_if_empty=raise_error_if_empty,
|
|
381
|
+
msg="No station available in the DISDRODB Metadata Archive.",
|
|
382
|
+
)
|
|
383
|
+
return _finalize_output(list_info, return_tuple=return_tuple)
|
|
384
|
+
|
|
385
|
+
# Return stations in the Metadata Archive with specified disdrodb_data_url
|
|
386
|
+
if product is None and available_data:
|
|
387
|
+
list_info = keep_list_info_with_disdrodb_data_url(metadata_archive_dir, list_info)
|
|
388
|
+
_raise_an_error_if_no_stations(
|
|
389
|
+
list_info,
|
|
390
|
+
raise_error_if_empty=raise_error_if_empty,
|
|
391
|
+
msg="No station has the disdrodb_data_url specified in the metadata.",
|
|
392
|
+
)
|
|
393
|
+
return _finalize_output(list_info, return_tuple=return_tuple)
|
|
394
|
+
|
|
395
|
+
# If product is specified, select stations available in the local DISDRODB Data Archive
|
|
396
|
+
# - If available_data=False, search for station with the existing product directory (do not check for data)
|
|
397
|
+
data_archive_dir = get_data_archive_dir(data_archive_dir)
|
|
398
|
+
product = check_product(product)
|
|
399
|
+
if not available_data:
|
|
400
|
+
list_info = keep_list_info_elements_with_product_directory(
|
|
401
|
+
data_archive_dir=data_archive_dir,
|
|
402
|
+
product=product,
|
|
403
|
+
list_info=list_info,
|
|
404
|
+
)
|
|
405
|
+
_raise_an_error_if_no_stations(
|
|
406
|
+
list_info,
|
|
407
|
+
raise_error_if_empty=raise_error_if_empty,
|
|
408
|
+
msg=f"No station product {product} directory available in the local DISDRODB Data Archive.",
|
|
409
|
+
)
|
|
410
|
+
return _finalize_output(list_info, return_tuple=return_tuple)
|
|
411
|
+
|
|
412
|
+
# - If available_data=True, search for station with product data
|
|
413
|
+
product_kwargs = check_product_kwargs(product, product_kwargs)
|
|
414
|
+
list_info = keep_list_info_elements_with_product_data(
|
|
415
|
+
data_archive_dir=data_archive_dir,
|
|
416
|
+
product=product,
|
|
417
|
+
list_info=list_info,
|
|
418
|
+
**product_kwargs,
|
|
419
|
+
)
|
|
420
|
+
product_kwargs = product_kwargs if product_kwargs else "" # if empty, set as ""
|
|
421
|
+
_raise_an_error_if_no_stations(
|
|
422
|
+
list_info,
|
|
423
|
+
raise_error_if_empty=raise_error_if_empty,
|
|
424
|
+
msg=f"No station has {product} {product_kwargs} data available in the local DISDRODB Data Archive.",
|
|
425
|
+
)
|
|
426
|
+
return _finalize_output(list_info, return_tuple=return_tuple)
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def available_data_sources(
|
|
430
|
+
product=None,
|
|
431
|
+
campaign_names=None,
|
|
432
|
+
station_names=None,
|
|
433
|
+
available_data=False,
|
|
434
|
+
raise_error_if_empty=False,
|
|
435
|
+
invalid_fields_policy="raise",
|
|
436
|
+
data_archive_dir=None,
|
|
437
|
+
metadata_archive_dir=None,
|
|
438
|
+
**product_kwargs,
|
|
439
|
+
):
|
|
440
|
+
"""Return data sources for which stations are available."""
|
|
441
|
+
list_info = available_stations(
|
|
442
|
+
product=product,
|
|
443
|
+
data_sources=None,
|
|
444
|
+
campaign_names=campaign_names,
|
|
445
|
+
station_names=station_names,
|
|
446
|
+
return_tuple=True,
|
|
447
|
+
available_data=available_data,
|
|
448
|
+
raise_error_if_empty=raise_error_if_empty,
|
|
449
|
+
invalid_fields_policy=invalid_fields_policy,
|
|
450
|
+
data_archive_dir=data_archive_dir,
|
|
451
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
452
|
+
**product_kwargs,
|
|
453
|
+
)
|
|
454
|
+
data_sources = [info[0] for info in list_info]
|
|
455
|
+
data_sources = np.unique(data_sources).tolist()
|
|
456
|
+
return data_sources
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def available_campaigns(
|
|
460
|
+
product=None,
|
|
461
|
+
data_sources=None,
|
|
462
|
+
station_names=None,
|
|
463
|
+
available_data=False,
|
|
464
|
+
raise_error_if_empty=False,
|
|
465
|
+
invalid_fields_policy="raise",
|
|
466
|
+
data_archive_dir=None,
|
|
467
|
+
metadata_archive_dir=None,
|
|
468
|
+
**product_kwargs,
|
|
469
|
+
):
|
|
470
|
+
"""Return campaigns names for which stations are available."""
|
|
471
|
+
list_info = available_stations(
|
|
472
|
+
product=product,
|
|
473
|
+
data_sources=data_sources,
|
|
474
|
+
campaign_names=None,
|
|
475
|
+
station_names=station_names,
|
|
476
|
+
return_tuple=True,
|
|
477
|
+
available_data=available_data,
|
|
478
|
+
raise_error_if_empty=raise_error_if_empty,
|
|
479
|
+
invalid_fields_policy=invalid_fields_policy,
|
|
480
|
+
data_archive_dir=data_archive_dir,
|
|
481
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
482
|
+
**product_kwargs,
|
|
483
|
+
)
|
|
484
|
+
campaign_names = [info[1] for info in list_info]
|
|
485
|
+
campaign_names = np.unique(campaign_names).tolist()
|
|
486
|
+
return campaign_names
|
|
@@ -19,18 +19,23 @@ import sys
|
|
|
19
19
|
|
|
20
20
|
import click
|
|
21
21
|
|
|
22
|
-
from disdrodb.utils.
|
|
22
|
+
from disdrodb.utils.cli import click_metadata_archive_dir_option, parse_archive_dir
|
|
23
23
|
|
|
24
24
|
sys.tracebacklimit = 0 # avoid full traceback error if occur
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
@click.command()
|
|
28
|
-
@
|
|
28
|
+
@click_metadata_archive_dir_option
|
|
29
29
|
@click.option(
|
|
30
|
-
"--raise_error",
|
|
30
|
+
"--raise_error",
|
|
31
|
+
type=bool,
|
|
32
|
+
show_default=True,
|
|
33
|
+
default=True,
|
|
34
|
+
help="Whether to raise error of finish the check",
|
|
31
35
|
)
|
|
32
|
-
def disdrodb_check_metadata_archive(
|
|
33
|
-
|
|
36
|
+
def disdrodb_check_metadata_archive(metadata_archive_dir=None, raise_error=True):
|
|
37
|
+
"""Run the DISDRODB Metadata Archive Checks."""
|
|
38
|
+
from disdrodb.metadata.checks import check_metadata_archive
|
|
34
39
|
|
|
35
|
-
|
|
36
|
-
|
|
40
|
+
metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
|
|
41
|
+
check_metadata_archive(metadata_archive_dir=metadata_archive_dir, raise_error=raise_error)
|
|
@@ -16,26 +16,17 @@
|
|
|
16
16
|
# You should have received a copy of the GNU General Public License
|
|
17
17
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
18
18
|
# -----------------------------------------------------------------------------.
|
|
19
|
-
"""
|
|
19
|
+
"""Routine to print the DISDRODB Data Archive directory."""
|
|
20
|
+
import sys
|
|
20
21
|
|
|
21
|
-
import
|
|
22
|
+
import click
|
|
22
23
|
|
|
24
|
+
sys.tracebacklimit = 0 # avoid full traceback error if occur
|
|
23
25
|
|
|
24
|
-
def get_dataframe_start_end_time(df: pd.DataFrame):
|
|
25
|
-
"""Retrieves dataframe starting and ending time.
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
27
|
+
@click.command()
|
|
28
|
+
def disdrodb_data_archive_directory():
|
|
29
|
+
"""Print the DISDRODB Data Archive directory."""
|
|
30
|
+
import disdrodb
|
|
31
31
|
|
|
32
|
-
|
|
33
|
-
-------
|
|
34
|
-
tuple
|
|
35
|
-
(starting_time, ending_time)
|
|
36
|
-
|
|
37
|
-
"""
|
|
38
|
-
|
|
39
|
-
starting_time = df["time"].iloc[0]
|
|
40
|
-
ending_time = df["time"].iloc[-1]
|
|
41
|
-
return (starting_time, ending_time)
|
|
32
|
+
print("The DISDRODB Data Archive Directory is: ", disdrodb.get_data_archive_dir(None))
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# -----------------------------------------------------------------------------.
|
|
2
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
3
|
+
#
|
|
4
|
+
# This program is free software: you can redistribute it and/or modify
|
|
5
|
+
# it under the terms of the GNU General Public License as published by
|
|
6
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
7
|
+
# (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This program is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
12
|
+
# GNU General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU General Public License
|
|
15
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
16
|
+
# -----------------------------------------------------------------------------.
|
|
17
|
+
"""Wrapper to download stations from the DISDRODB Decentralized Data Archive."""
|
|
18
|
+
|
|
19
|
+
import sys
|
|
20
|
+
from typing import Optional
|
|
21
|
+
|
|
22
|
+
import click
|
|
23
|
+
|
|
24
|
+
from disdrodb.data_transfer.download_data import click_download_archive_options, click_download_options
|
|
25
|
+
from disdrodb.utils.cli import (
|
|
26
|
+
click_data_archive_dir_option,
|
|
27
|
+
click_metadata_archive_dir_option,
|
|
28
|
+
parse_archive_dir,
|
|
29
|
+
parse_arg_to_list,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
sys.tracebacklimit = 0 # avoid full traceback error if occur
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@click.command()
|
|
36
|
+
@click_download_archive_options
|
|
37
|
+
@click_data_archive_dir_option
|
|
38
|
+
@click_metadata_archive_dir_option
|
|
39
|
+
@click_download_options
|
|
40
|
+
def disdrodb_download_archive(
|
|
41
|
+
data_sources: Optional[str] = None,
|
|
42
|
+
campaign_names: Optional[str] = None,
|
|
43
|
+
station_names: Optional[str] = None,
|
|
44
|
+
data_archive_dir: Optional[str] = None,
|
|
45
|
+
metadata_archive_dir: Optional[str] = None,
|
|
46
|
+
force: bool = False,
|
|
47
|
+
):
|
|
48
|
+
"""Download DISDRODB stations with the ``disdrodb_data_url`` in the metadata.
|
|
49
|
+
|
|
50
|
+
Parameters
|
|
51
|
+
----------
|
|
52
|
+
data_sources : str or list of str, optional
|
|
53
|
+
Data source name (eg : EPFL).
|
|
54
|
+
If not provided (``None``), all data sources will be downloaded.
|
|
55
|
+
The default value is ``data_source=None``.
|
|
56
|
+
campaign_names : str or list of str, optional
|
|
57
|
+
Campaign name (eg : EPFL_ROOF_2012).
|
|
58
|
+
If not provided (``None``), all campaigns will be downloaded.
|
|
59
|
+
The default value is ``campaign_name=None``.
|
|
60
|
+
station_names : str or list of str, optional
|
|
61
|
+
Station name.
|
|
62
|
+
If not provided (``None``), all stations will be downloaded.
|
|
63
|
+
The default value is ``station_name=None``.
|
|
64
|
+
force : bool, optional
|
|
65
|
+
If ``True``, overwrite the already existing raw data file.
|
|
66
|
+
The default value is ``False``.
|
|
67
|
+
data_archive_dir : str (optional)
|
|
68
|
+
DISDRODB Data Archive directory. Format: ``<...>/DISDRODB``.
|
|
69
|
+
If ``None`` (the default), the disdrodb config variable ``data_archive_dir`` is used.
|
|
70
|
+
"""
|
|
71
|
+
from disdrodb.data_transfer.download_data import download_archive
|
|
72
|
+
|
|
73
|
+
data_archive_dir = parse_archive_dir(data_archive_dir)
|
|
74
|
+
metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
|
|
75
|
+
data_sources = parse_arg_to_list(data_sources)
|
|
76
|
+
campaign_names = parse_arg_to_list(campaign_names)
|
|
77
|
+
station_names = parse_arg_to_list(station_names)
|
|
78
|
+
|
|
79
|
+
download_archive(
|
|
80
|
+
data_archive_dir=data_archive_dir,
|
|
81
|
+
metadata_archive_dir=metadata_archive_dir,
|
|
82
|
+
data_sources=data_sources,
|
|
83
|
+
campaign_names=campaign_names,
|
|
84
|
+
station_names=station_names,
|
|
85
|
+
force=force,
|
|
86
|
+
)
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# -----------------------------------------------------------------------------.
|
|
2
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
3
|
+
#
|
|
4
|
+
# This program is free software: you can redistribute it and/or modify
|
|
5
|
+
# it under the terms of the GNU General Public License as published by
|
|
6
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
7
|
+
# (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This program is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
12
|
+
# GNU General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU General Public License
|
|
15
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
16
|
+
# -----------------------------------------------------------------------------.
|
|
17
|
+
"""Routine to download the DISDRODB Metadata Data Archive."""
|
|
18
|
+
import sys
|
|
19
|
+
|
|
20
|
+
import click
|
|
21
|
+
|
|
22
|
+
from disdrodb.utils.cli import parse_archive_dir
|
|
23
|
+
|
|
24
|
+
sys.tracebacklimit = 0 # avoid full traceback error if occur
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@click.command()
|
|
28
|
+
@click.argument("directory_path", metavar="<station>")
|
|
29
|
+
@click.option("-f", "--force", type=bool, show_default=True, default=False, help="Force overwriting")
|
|
30
|
+
def disdrodb_download_metadata_archive(
|
|
31
|
+
directory_path,
|
|
32
|
+
force: bool = False,
|
|
33
|
+
):
|
|
34
|
+
"""Download the DISDRODB Metadata Archive to the specified directory.
|
|
35
|
+
|
|
36
|
+
Parameters
|
|
37
|
+
----------
|
|
38
|
+
directory_path : str
|
|
39
|
+
The directory path where the DISDRODB-METADATA directory will be downloaded.
|
|
40
|
+
force : bool, optional
|
|
41
|
+
If ``True``, the existing DISDRODB-METADATA directory will be removed
|
|
42
|
+
and a new one will be downloaded. The default value is ``False``.
|
|
43
|
+
|
|
44
|
+
Returns
|
|
45
|
+
-------
|
|
46
|
+
metadata_archive_dir
|
|
47
|
+
The DISDRODB Metadata Archive directory path.
|
|
48
|
+
"""
|
|
49
|
+
from disdrodb import download_metadata_archive
|
|
50
|
+
|
|
51
|
+
directory_path = parse_archive_dir(directory_path)
|
|
52
|
+
|
|
53
|
+
download_metadata_archive(directory_path, force=force)
|