disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +132 -15
- disdrodb/_config.py +4 -2
- disdrodb/_version.py +9 -4
- disdrodb/api/checks.py +264 -237
- disdrodb/api/configs.py +4 -8
- disdrodb/api/create_directories.py +235 -290
- disdrodb/api/info.py +217 -26
- disdrodb/api/io.py +306 -270
- disdrodb/api/path.py +597 -173
- disdrodb/api/search.py +486 -0
- disdrodb/{metadata/scripts → cli}/disdrodb_check_metadata_archive.py +12 -7
- disdrodb/{utils/pandas.py → cli/disdrodb_data_archive_directory.py} +9 -18
- disdrodb/cli/disdrodb_download_archive.py +86 -0
- disdrodb/cli/disdrodb_download_metadata_archive.py +53 -0
- disdrodb/cli/disdrodb_download_station.py +84 -0
- disdrodb/{api/scripts → cli}/disdrodb_initialize_station.py +22 -10
- disdrodb/cli/disdrodb_metadata_archive_directory.py +32 -0
- disdrodb/{data_transfer/scripts/disdrodb_download_station.py → cli/disdrodb_open_data_archive.py} +22 -22
- disdrodb/cli/disdrodb_open_logs_directory.py +69 -0
- disdrodb/{data_transfer/scripts/disdrodb_upload_station.py → cli/disdrodb_open_metadata_archive.py} +22 -24
- disdrodb/cli/disdrodb_open_metadata_directory.py +71 -0
- disdrodb/cli/disdrodb_open_product_directory.py +74 -0
- disdrodb/cli/disdrodb_open_readers_directory.py +32 -0
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0.py +38 -31
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0_station.py +32 -30
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a_station.py +24 -33
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b_station.py +25 -34
- disdrodb/cli/disdrodb_run_l0c.py +130 -0
- disdrodb/cli/disdrodb_run_l0c_station.py +129 -0
- disdrodb/cli/disdrodb_run_l1.py +122 -0
- disdrodb/cli/disdrodb_run_l1_station.py +121 -0
- disdrodb/cli/disdrodb_run_l2e.py +122 -0
- disdrodb/cli/disdrodb_run_l2e_station.py +122 -0
- disdrodb/cli/disdrodb_run_l2m.py +122 -0
- disdrodb/cli/disdrodb_run_l2m_station.py +122 -0
- disdrodb/cli/disdrodb_upload_archive.py +105 -0
- disdrodb/cli/disdrodb_upload_station.py +98 -0
- disdrodb/configs.py +90 -25
- disdrodb/data_transfer/__init__.py +22 -0
- disdrodb/data_transfer/download_data.py +87 -90
- disdrodb/data_transfer/upload_data.py +64 -37
- disdrodb/data_transfer/zenodo.py +15 -18
- disdrodb/docs.py +1 -1
- disdrodb/issue/__init__.py +17 -4
- disdrodb/issue/checks.py +10 -23
- disdrodb/issue/reader.py +9 -12
- disdrodb/issue/writer.py +14 -17
- disdrodb/l0/__init__.py +17 -26
- disdrodb/l0/check_configs.py +35 -23
- disdrodb/l0/check_standards.py +46 -51
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_diameter.yml +44 -44
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_velocity.yml +40 -40
- disdrodb/l0/configs/LPM/l0a_encodings.yml +80 -0
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_cf_attrs.yml +84 -65
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_encodings.yml +50 -9
- disdrodb/l0/configs/{Thies_LPM → LPM}/raw_data_format.yml +285 -245
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_diameter.yml +66 -66
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL/l0a_encodings.yml +32 -0
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_cf_attrs.yml +23 -21
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_encodings.yml +17 -17
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/raw_data_format.yml +77 -77
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_diameter.yml +64 -64
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +39 -0
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_cf_attrs.yml +28 -26
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_encodings.yml +20 -20
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/raw_data_format.yml +107 -107
- disdrodb/l0/configs/PWS100/bins_diameter.yml +173 -0
- disdrodb/l0/configs/PWS100/bins_velocity.yml +173 -0
- disdrodb/l0/configs/PWS100/l0a_encodings.yml +19 -0
- disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +76 -0
- disdrodb/l0/configs/PWS100/l0b_encodings.yml +176 -0
- disdrodb/l0/configs/PWS100/raw_data_format.yml +182 -0
- disdrodb/l0/configs/{RD_80 → RD80}/bins_diameter.yml +40 -40
- disdrodb/l0/configs/RD80/l0a_encodings.yml +16 -0
- disdrodb/l0/configs/{RD_80 → RD80}/l0b_cf_attrs.yml +3 -3
- disdrodb/l0/configs/RD80/l0b_encodings.yml +135 -0
- disdrodb/l0/configs/{RD_80 → RD80}/raw_data_format.yml +46 -50
- disdrodb/l0/l0_reader.py +216 -340
- disdrodb/l0/l0a_processing.py +237 -208
- disdrodb/l0/l0b_nc_processing.py +227 -80
- disdrodb/l0/l0b_processing.py +96 -174
- disdrodb/l0/l0c_processing.py +627 -0
- disdrodb/l0/readers/{ARM → LPM/ARM}/ARM_LPM.py +36 -58
- disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +236 -0
- disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +185 -0
- disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +185 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +195 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +210 -0
- disdrodb/l0/readers/{BRAZIL/GOAMAZON_LPM.py → LPM/KIT/CHWALA.py} +97 -76
- disdrodb/l0/readers/LPM/SLOVENIA/ARSO.py +197 -0
- disdrodb/l0/readers/LPM/SLOVENIA/CRNI_VRH.py +197 -0
- disdrodb/l0/readers/{UK → LPM/UK}/DIVEN.py +14 -35
- disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +157 -0
- disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +113 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/ARCTIC_2021.py +40 -57
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/COMMON_2011.py +37 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/DAVOS_2009_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_2009.py +34 -51
- disdrodb/l0/readers/{EPFL/PARADISO_2014.py → PARSIVEL/EPFL/EPFL_ROOF_2008.py} +38 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2012.py +33 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GENEPI_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007_2.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HPICONET_2010.py +34 -51
- disdrodb/l0/readers/{EPFL/EPFL_ROOF_2010.py → PARSIVEL/EPFL/HYMEX_LTE_SOP2.py} +37 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +111 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HYMEX_LTE_SOP4.py +36 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2018.py +34 -52
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2019.py +38 -56
- disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PARSIVEL_2007.py +27 -45
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PLATO_2019.py +24 -44
- disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +140 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RACLETS_2019_WJF.py +41 -59
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RIETHOLZBACH_2011.py +34 -51
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +117 -0
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +137 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/UNIL_2022.py +42 -55
- disdrodb/l0/readers/PARSIVEL/GPM/IFLOODS.py +104 -0
- disdrodb/l0/readers/{GPM → PARSIVEL/GPM}/LPVEX.py +29 -48
- disdrodb/l0/readers/PARSIVEL/GPM/MC3E.py +184 -0
- disdrodb/l0/readers/PARSIVEL/KIT/BURKINA_FASO.py +133 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +113 -0
- disdrodb/l0/readers/{NCAR/VORTEX_SE_2016_P1.py → PARSIVEL/NCAR/OWLES_MIPS.py} +46 -72
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +125 -0
- disdrodb/l0/readers/{NCAR/OWLES_MIPS.py → PARSIVEL/NCAR/PLOWS_MIPS.py} +45 -64
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +114 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +176 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +183 -0
- disdrodb/l0/readers/PARSIVEL/SLOVENIA/UL_FGG.py +121 -0
- disdrodb/l0/readers/{ARM/ARM_LD.py → PARSIVEL2/ARM/ARM_PARSIVEL2.py} +27 -50
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/{DENMARK → PARSIVEL2/DENMARK}/EROSION_nc.py +14 -35
- disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +189 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +119 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +104 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/NSSTC.py +176 -0
- disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +32 -0
- disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +56 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +120 -0
- disdrodb/l0/readers/{NCAR → PARSIVEL2/NCAR}/PECAN_MIPS.py +45 -64
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +181 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +160 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +160 -0
- disdrodb/l0/readers/{NCAR/PLOWS_MIPS.py → PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py} +49 -66
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +118 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +152 -0
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT.py +166 -0
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +150 -0
- disdrodb/l0/readers/{NCAR/RELAMPAGO_RD80.py → RD80/BRAZIL/CHUVA_RD80.py} +36 -60
- disdrodb/l0/readers/{BRAZIL → RD80/BRAZIL}/GOAMAZON_RD80.py +36 -55
- disdrodb/l0/readers/{NCAR → RD80/NCAR}/CINDY_2011_RD80.py +35 -54
- disdrodb/l0/readers/{BRAZIL/CHUVA_RD80.py → RD80/NCAR/RELAMPAGO_RD80.py} +40 -54
- disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py +274 -0
- disdrodb/l0/readers/template_reader_raw_netcdf_data.py +62 -0
- disdrodb/l0/readers/{reader_template.py → template_reader_raw_text_data.py} +20 -44
- disdrodb/l0/routines.py +885 -581
- disdrodb/l0/standards.py +77 -238
- disdrodb/l0/template_tools.py +105 -110
- disdrodb/l1/__init__.py +17 -0
- disdrodb/l1/beard_model.py +716 -0
- disdrodb/l1/encoding_attrs.py +635 -0
- disdrodb/l1/fall_velocity.py +260 -0
- disdrodb/l1/filters.py +192 -0
- disdrodb/l1/processing.py +202 -0
- disdrodb/l1/resampling.py +236 -0
- disdrodb/l1/routines.py +358 -0
- disdrodb/l1_env/__init__.py +17 -0
- disdrodb/l1_env/routines.py +38 -0
- disdrodb/l2/__init__.py +17 -0
- disdrodb/l2/empirical_dsd.py +1833 -0
- disdrodb/l2/event.py +388 -0
- disdrodb/l2/processing.py +528 -0
- disdrodb/l2/processing_options.py +213 -0
- disdrodb/l2/routines.py +868 -0
- disdrodb/metadata/__init__.py +9 -2
- disdrodb/metadata/checks.py +180 -124
- disdrodb/metadata/download.py +81 -0
- disdrodb/metadata/geolocation.py +146 -0
- disdrodb/metadata/info.py +20 -13
- disdrodb/metadata/manipulation.py +3 -3
- disdrodb/metadata/reader.py +59 -8
- disdrodb/metadata/search.py +77 -144
- disdrodb/metadata/standards.py +83 -80
- disdrodb/metadata/writer.py +10 -16
- disdrodb/psd/__init__.py +38 -0
- disdrodb/psd/fitting.py +2146 -0
- disdrodb/psd/models.py +774 -0
- disdrodb/routines.py +1412 -0
- disdrodb/scattering/__init__.py +28 -0
- disdrodb/scattering/axis_ratio.py +344 -0
- disdrodb/scattering/routines.py +456 -0
- disdrodb/utils/__init__.py +17 -0
- disdrodb/utils/attrs.py +208 -0
- disdrodb/utils/cli.py +269 -0
- disdrodb/utils/compression.py +60 -42
- disdrodb/utils/dask.py +62 -0
- disdrodb/utils/dataframe.py +342 -0
- disdrodb/utils/decorators.py +110 -0
- disdrodb/utils/directories.py +107 -46
- disdrodb/utils/encoding.py +127 -0
- disdrodb/utils/list.py +29 -0
- disdrodb/utils/logger.py +168 -46
- disdrodb/utils/time.py +657 -0
- disdrodb/utils/warnings.py +30 -0
- disdrodb/utils/writer.py +57 -0
- disdrodb/utils/xarray.py +138 -47
- disdrodb/utils/yaml.py +0 -1
- disdrodb/viz/__init__.py +17 -0
- disdrodb/viz/plots.py +17 -0
- disdrodb-0.1.1.dist-info/METADATA +294 -0
- disdrodb-0.1.1.dist-info/RECORD +232 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/WHEEL +1 -1
- disdrodb-0.1.1.dist-info/entry_points.txt +30 -0
- disdrodb/data_transfer/scripts/disdrodb_download_archive.py +0 -53
- disdrodb/data_transfer/scripts/disdrodb_upload_archive.py +0 -57
- disdrodb/l0/configs/OTT_Parsivel/l0a_encodings.yml +0 -32
- disdrodb/l0/configs/OTT_Parsivel2/l0a_encodings.yml +0 -39
- disdrodb/l0/configs/RD_80/l0a_encodings.yml +0 -16
- disdrodb/l0/configs/RD_80/l0b_encodings.yml +0 -135
- disdrodb/l0/configs/Thies_LPM/l0a_encodings.yml +0 -80
- disdrodb/l0/io.py +0 -257
- disdrodb/l0/l0_processing.py +0 -1091
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_OTT.py +0 -178
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_THIES.py +0 -247
- disdrodb/l0/readers/BRAZIL/CHUVA_LPM.py +0 -204
- disdrodb/l0/readers/BRAZIL/CHUVA_OTT.py +0 -183
- disdrodb/l0/readers/BRAZIL/GOAMAZON_OTT.py +0 -183
- disdrodb/l0/readers/CHINA/CHONGQING.py +0 -131
- disdrodb/l0/readers/EPFL/EPFL_ROOF_2008.py +0 -128
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP2.py +0 -127
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP3.py +0 -129
- disdrodb/l0/readers/EPFL/RACLETS_2019.py +0 -158
- disdrodb/l0/readers/EPFL/SAMOYLOV_2017.py +0 -136
- disdrodb/l0/readers/EPFL/SAMOYLOV_2019.py +0 -158
- disdrodb/l0/readers/FRANCE/SIRTA_OTT2.py +0 -138
- disdrodb/l0/readers/GPM/GCPEX.py +0 -123
- disdrodb/l0/readers/GPM/IFLOODS.py +0 -123
- disdrodb/l0/readers/GPM/MC3E.py +0 -123
- disdrodb/l0/readers/GPM/NSSTC.py +0 -164
- disdrodb/l0/readers/ITALY/GID.py +0 -199
- disdrodb/l0/readers/MEXICO/OH_IIUNAM_nc.py +0 -92
- disdrodb/l0/readers/NCAR/CCOPE_2015.py +0 -133
- disdrodb/l0/readers/NCAR/PECAN_FP3.py +0 -137
- disdrodb/l0/readers/NCAR/PECAN_MOBILE.py +0 -144
- disdrodb/l0/readers/NCAR/RELAMPAGO_OTT.py +0 -195
- disdrodb/l0/readers/NCAR/SNOWIE_PJ.py +0 -172
- disdrodb/l0/readers/NCAR/SNOWIE_SB.py +0 -179
- disdrodb/l0/readers/NCAR/VORTEX2_2009.py +0 -133
- disdrodb/l0/readers/NCAR/VORTEX2_2010.py +0 -188
- disdrodb/l0/readers/NCAR/VORTEX2_2010_UF.py +0 -191
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_P2.py +0 -135
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_PIPS.py +0 -170
- disdrodb/l0/readers/NETHERLANDS/DELFT.py +0 -187
- disdrodb/l0/readers/SPAIN/SBEGUERIA.py +0 -179
- disdrodb/l0/scripts/disdrodb_run_l0b_concat.py +0 -93
- disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py +0 -85
- disdrodb/utils/netcdf.py +0 -452
- disdrodb/utils/scripts.py +0 -102
- disdrodb-0.0.21.dist-info/AUTHORS.md +0 -18
- disdrodb-0.0.21.dist-info/METADATA +0 -186
- disdrodb-0.0.21.dist-info/RECORD +0 -168
- disdrodb-0.0.21.dist-info/entry_points.txt +0 -15
- /disdrodb/l0/configs/{RD_80 → RD80}/bins_velocity.yml +0 -0
- /disdrodb/l0/manuals/{Thies_LPM.pdf → LPM.pdf} +0 -0
- /disdrodb/l0/manuals/{ODM_470.pdf → ODM470.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel.pdf → PARSIVEL.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel2.pdf → PARSIVEL2.pdf} +0 -0
- /disdrodb/l0/manuals/{PWS_100.pdf → PWS100.pdf} +0 -0
- /disdrodb/l0/manuals/{RD_80.pdf → RD80.pdf} +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info/licenses}/LICENSE +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/top_level.txt +0 -0
|
@@ -1,135 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
# -----------------------------------------------------------------------------.
|
|
3
|
-
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
-
#
|
|
5
|
-
# This program is free software: you can redistribute it and/or modify
|
|
6
|
-
# it under the terms of the GNU General Public License as published by
|
|
7
|
-
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
-
# (at your option) any later version.
|
|
9
|
-
#
|
|
10
|
-
# This program is distributed in the hope that it will be useful,
|
|
11
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
-
# GNU General Public License for more details.
|
|
14
|
-
#
|
|
15
|
-
# You should have received a copy of the GNU General Public License
|
|
16
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
-
# -----------------------------------------------------------------------------.
|
|
18
|
-
from disdrodb.l0 import run_l0a
|
|
19
|
-
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
@is_documented_by(reader_generic_docstring)
|
|
23
|
-
def reader(
|
|
24
|
-
raw_dir,
|
|
25
|
-
processed_dir,
|
|
26
|
-
station_name,
|
|
27
|
-
# Processing options
|
|
28
|
-
force=False,
|
|
29
|
-
verbose=False,
|
|
30
|
-
parallel=False,
|
|
31
|
-
debugging_mode=False,
|
|
32
|
-
):
|
|
33
|
-
##------------------------------------------------------------------------.
|
|
34
|
-
#### - Define column names
|
|
35
|
-
column_names = [
|
|
36
|
-
"date",
|
|
37
|
-
"time",
|
|
38
|
-
"sensor_battery_voltage",
|
|
39
|
-
"laser_amplitude",
|
|
40
|
-
"sensor_heating_current",
|
|
41
|
-
"sensor_temperature",
|
|
42
|
-
"number_particles",
|
|
43
|
-
"mor_visibility",
|
|
44
|
-
"rainfall_rate_32bit",
|
|
45
|
-
"rainfall_accumulated_32bit",
|
|
46
|
-
"weather_code_synop_4680",
|
|
47
|
-
"reflectivity_32bit",
|
|
48
|
-
"weather_code_nws",
|
|
49
|
-
"weather_code_metar_4678",
|
|
50
|
-
"raw_drop_number",
|
|
51
|
-
]
|
|
52
|
-
|
|
53
|
-
##------------------------------------------------------------------------.
|
|
54
|
-
#### - Define reader options
|
|
55
|
-
reader_kwargs = {}
|
|
56
|
-
# - Define delimiter
|
|
57
|
-
reader_kwargs["delimiter"] = ","
|
|
58
|
-
|
|
59
|
-
# - Avoid first column to become df index !!!
|
|
60
|
-
reader_kwargs["index_col"] = False
|
|
61
|
-
|
|
62
|
-
# - Define behaviour when encountering bad lines
|
|
63
|
-
reader_kwargs["on_bad_lines"] = "skip"
|
|
64
|
-
|
|
65
|
-
# - Define encoding
|
|
66
|
-
reader_kwargs["encoding"] = "ISO-8859-1"
|
|
67
|
-
|
|
68
|
-
# - Define reader engine
|
|
69
|
-
# - C engine is faster
|
|
70
|
-
# - Python engine is more feature-complete
|
|
71
|
-
reader_kwargs["engine"] = "python"
|
|
72
|
-
|
|
73
|
-
# - Define on-the-fly decompression of on-disk data
|
|
74
|
-
# - Available: gzip, bz2, zip
|
|
75
|
-
reader_kwargs["compression"] = "infer"
|
|
76
|
-
|
|
77
|
-
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
78
|
-
# - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
|
|
79
|
-
# ‘-NaN’, ‘-nan’, ‘1.#IND’, ‘1.#QNAN’, ‘<NA>’, ‘N/A’,
|
|
80
|
-
# ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
|
|
81
|
-
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
82
|
-
|
|
83
|
-
# Skip first row as columns names
|
|
84
|
-
reader_kwargs["header"] = None
|
|
85
|
-
|
|
86
|
-
##------------------------------------------------------------------------.
|
|
87
|
-
#### - Define dataframe sanitizer function for L0 processing
|
|
88
|
-
def df_sanitizer_fun(df):
|
|
89
|
-
# - Import pandas
|
|
90
|
-
import pandas as pd
|
|
91
|
-
|
|
92
|
-
# - Define datetime 'time' column
|
|
93
|
-
df["time"] = df["date"] + "-" + df["time"]
|
|
94
|
-
df["time"] = pd.to_datetime(df["time"], format="%Y%m%d-%H:%M:%S", errors="coerce")
|
|
95
|
-
|
|
96
|
-
# Preprocess the raw spectrum
|
|
97
|
-
# - The '<SPECTRUM>ZERO</SPECTRUM>' indicates no drops detected
|
|
98
|
-
# --> "" generates an array of zeros in L0B processing
|
|
99
|
-
df["raw_drop_number"] = df["raw_drop_number"].replace("<SPECTRUM>ZERO</SPECTRUM>", "")
|
|
100
|
-
|
|
101
|
-
# Remove <SPECTRUM> and </SPECTRUM>" acronyms from the raw_drop_number field
|
|
102
|
-
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
|
|
103
|
-
df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
|
|
104
|
-
|
|
105
|
-
# Preprocess the raw spectrum and raw_drop_average_velocity
|
|
106
|
-
# - Add 0 before every ; if ; not preceded by a digit
|
|
107
|
-
# - Example: ';;1;;' --> '0;0;1;0;'
|
|
108
|
-
df["raw_drop_number"] = df["raw_drop_number"].str.replace(r"(?<!\d);", "0;", regex=True)
|
|
109
|
-
|
|
110
|
-
# - Drop columns not agreeing with DISDRODB L0 standards
|
|
111
|
-
df = df.drop(columns=["date"])
|
|
112
|
-
|
|
113
|
-
return df
|
|
114
|
-
|
|
115
|
-
##------------------------------------------------------------------------.
|
|
116
|
-
#### - Define glob pattern to search data files in <raw_dir>/data/<station_name>
|
|
117
|
-
glob_patterns = "*.csv"
|
|
118
|
-
|
|
119
|
-
####----------------------------------------------------------------------.
|
|
120
|
-
#### - Create L0A products
|
|
121
|
-
run_l0a(
|
|
122
|
-
raw_dir=raw_dir,
|
|
123
|
-
processed_dir=processed_dir,
|
|
124
|
-
station_name=station_name,
|
|
125
|
-
# Custom arguments of the reader for L0A processing
|
|
126
|
-
glob_patterns=glob_patterns,
|
|
127
|
-
column_names=column_names,
|
|
128
|
-
reader_kwargs=reader_kwargs,
|
|
129
|
-
df_sanitizer_fun=df_sanitizer_fun,
|
|
130
|
-
# Processing options
|
|
131
|
-
force=force,
|
|
132
|
-
verbose=verbose,
|
|
133
|
-
parallel=parallel,
|
|
134
|
-
debugging_mode=debugging_mode,
|
|
135
|
-
)
|
|
@@ -1,170 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
# -----------------------------------------------------------------------------.
|
|
3
|
-
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
-
#
|
|
5
|
-
# This program is free software: you can redistribute it and/or modify
|
|
6
|
-
# it under the terms of the GNU General Public License as published by
|
|
7
|
-
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
-
# (at your option) any later version.
|
|
9
|
-
#
|
|
10
|
-
# This program is distributed in the hope that it will be useful,
|
|
11
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
-
# GNU General Public License for more details.
|
|
14
|
-
#
|
|
15
|
-
# You should have received a copy of the GNU General Public License
|
|
16
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
-
# -----------------------------------------------------------------------------.
|
|
18
|
-
from disdrodb.l0 import run_l0a
|
|
19
|
-
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
@is_documented_by(reader_generic_docstring)
|
|
23
|
-
def reader(
|
|
24
|
-
raw_dir,
|
|
25
|
-
processed_dir,
|
|
26
|
-
station_name,
|
|
27
|
-
# Processing options
|
|
28
|
-
force=False,
|
|
29
|
-
verbose=False,
|
|
30
|
-
parallel=False,
|
|
31
|
-
debugging_mode=False,
|
|
32
|
-
):
|
|
33
|
-
##------------------------------------------------------------------------.
|
|
34
|
-
#### - Define column names
|
|
35
|
-
column_names = [
|
|
36
|
-
"logger_time",
|
|
37
|
-
"record_number",
|
|
38
|
-
"battery_voltage",
|
|
39
|
-
"logger_temperature",
|
|
40
|
-
"wind_direction",
|
|
41
|
-
"wind_speed",
|
|
42
|
-
"wind_flag",
|
|
43
|
-
"fast_temperature",
|
|
44
|
-
"slow_temperature",
|
|
45
|
-
"relative_humidity",
|
|
46
|
-
"pressure",
|
|
47
|
-
"compass_direction",
|
|
48
|
-
"gps_time",
|
|
49
|
-
"gps_status",
|
|
50
|
-
"gps_latitude", # DD.DM, DM = decimal minutes/100, DD = degrees
|
|
51
|
-
"gps_latitude_hemisphere", # N/S
|
|
52
|
-
"gps_longitude",
|
|
53
|
-
"gps_longitude_hemisphere", # W/E
|
|
54
|
-
"gps_speed",
|
|
55
|
-
"gps_direction",
|
|
56
|
-
"gps_date",
|
|
57
|
-
"gps_magnetic_version",
|
|
58
|
-
"gps_altitude",
|
|
59
|
-
"relative_wind_direction",
|
|
60
|
-
"dew_point_temperature",
|
|
61
|
-
"rh",
|
|
62
|
-
"disdrometer_data",
|
|
63
|
-
]
|
|
64
|
-
|
|
65
|
-
##------------------------------------------------------------------------.
|
|
66
|
-
#### - Define reader options
|
|
67
|
-
reader_kwargs = {}
|
|
68
|
-
# - Define delimiter
|
|
69
|
-
reader_kwargs["delimiter"] = ","
|
|
70
|
-
|
|
71
|
-
# - Avoid first column to become df index !!!
|
|
72
|
-
reader_kwargs["index_col"] = False
|
|
73
|
-
|
|
74
|
-
# - Define behaviour when encountering bad lines
|
|
75
|
-
reader_kwargs["on_bad_lines"] = "skip"
|
|
76
|
-
|
|
77
|
-
# - Define encoding
|
|
78
|
-
reader_kwargs["encoding"] = "ISO-8859-1"
|
|
79
|
-
|
|
80
|
-
# - Define reader engine
|
|
81
|
-
# - C engine is faster
|
|
82
|
-
# - Python engine is more feature-complete
|
|
83
|
-
reader_kwargs["engine"] = "python"
|
|
84
|
-
|
|
85
|
-
# - Define on-the-fly decompression of on-disk data
|
|
86
|
-
# - Available: gzip, bz2, zip
|
|
87
|
-
reader_kwargs["compression"] = "infer"
|
|
88
|
-
|
|
89
|
-
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
90
|
-
# - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
|
|
91
|
-
# ‘-NaN’, ‘-nan’, ‘1.#IND’, ‘1.#QNAN’, ‘<NA>’, ‘N/A’,
|
|
92
|
-
# ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
|
|
93
|
-
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
94
|
-
|
|
95
|
-
# Skip first row as columns names
|
|
96
|
-
reader_kwargs["header"] = None
|
|
97
|
-
|
|
98
|
-
##------------------------------------------------------------------------.
|
|
99
|
-
#### - Define dataframe sanitizer function for L0 processing
|
|
100
|
-
def df_sanitizer_fun(df):
|
|
101
|
-
# - Import pandas
|
|
102
|
-
import pandas as pd
|
|
103
|
-
|
|
104
|
-
# - Drop timesteps without disdrometer data
|
|
105
|
-
df = df[~df["disdrometer_data"].isna()]
|
|
106
|
-
|
|
107
|
-
# - Retrieve disdrometer data
|
|
108
|
-
df_data = df["disdrometer_data"].str.split(";", expand=True, n=11)
|
|
109
|
-
# - Assign column names
|
|
110
|
-
column_names = [
|
|
111
|
-
"serial_number",
|
|
112
|
-
"rainfall_rate_32bit",
|
|
113
|
-
"rainfall_accumulated_32bit",
|
|
114
|
-
"reflectivity_32bit",
|
|
115
|
-
"sample_interval",
|
|
116
|
-
"laser_amplitude",
|
|
117
|
-
"number_particles",
|
|
118
|
-
"sensor_temperature",
|
|
119
|
-
"sensor_battery_voltage",
|
|
120
|
-
"sensor_time", # Note: logger_time is currently used !
|
|
121
|
-
"sensor_date",
|
|
122
|
-
"raw_drop_number",
|
|
123
|
-
]
|
|
124
|
-
df_data.columns = column_names
|
|
125
|
-
|
|
126
|
-
# - Retrieve time and coordinates information
|
|
127
|
-
# --> Latitude in degrees_north
|
|
128
|
-
# --> Longitude in degrees_east
|
|
129
|
-
df_time = pd.to_datetime(df["logger_time"], errors="coerce")
|
|
130
|
-
df_lat_sign = df["gps_latitude_hemisphere"].str.replace("N", "1").str.replace("S", "-1")
|
|
131
|
-
df_lon_sign = df["gps_longitude_hemisphere"].str.replace("E", "1").str.replace("W", "-1")
|
|
132
|
-
df_lat_sign = df_lat_sign.astype(float)
|
|
133
|
-
df_lon_sign = df_lon_sign.astype(float)
|
|
134
|
-
df_lon = df["gps_longitude"].astype(float)
|
|
135
|
-
df_lat = df["gps_latitude"].astype(float)
|
|
136
|
-
df_lon = df_lon * df_lon_sign
|
|
137
|
-
df_lat = df_lat * df_lat_sign
|
|
138
|
-
|
|
139
|
-
# Create dataframe
|
|
140
|
-
df = df_data
|
|
141
|
-
df["time"] = df_time
|
|
142
|
-
df["latitude"] = df_lat
|
|
143
|
-
df["longitude"] = df_lon
|
|
144
|
-
|
|
145
|
-
# - Drop columns not agreeing with DISDRODB L0 standards
|
|
146
|
-
df = df.drop(columns=["serial_number", "sensor_time", "serial_number"])
|
|
147
|
-
|
|
148
|
-
return df
|
|
149
|
-
|
|
150
|
-
##------------------------------------------------------------------------.
|
|
151
|
-
#### - Define glob pattern to search data files in <raw_dir>/data/<station_name>
|
|
152
|
-
glob_patterns = "*.txt"
|
|
153
|
-
|
|
154
|
-
####----------------------------------------------------------------------.
|
|
155
|
-
#### - Create L0A products
|
|
156
|
-
run_l0a(
|
|
157
|
-
raw_dir=raw_dir,
|
|
158
|
-
processed_dir=processed_dir,
|
|
159
|
-
station_name=station_name,
|
|
160
|
-
# Custom arguments of the reader for L0A processing
|
|
161
|
-
glob_patterns=glob_patterns,
|
|
162
|
-
column_names=column_names,
|
|
163
|
-
reader_kwargs=reader_kwargs,
|
|
164
|
-
df_sanitizer_fun=df_sanitizer_fun,
|
|
165
|
-
# Processing options
|
|
166
|
-
force=force,
|
|
167
|
-
verbose=verbose,
|
|
168
|
-
parallel=parallel,
|
|
169
|
-
debugging_mode=debugging_mode,
|
|
170
|
-
)
|
|
@@ -1,187 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
# -----------------------------------------------------------------------------.
|
|
3
|
-
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
-
#
|
|
5
|
-
# This program is free software: you can redistribute it and/or modify
|
|
6
|
-
# it under the terms of the GNU General Public License as published by
|
|
7
|
-
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
-
# (at your option) any later version.
|
|
9
|
-
#
|
|
10
|
-
# This program is distributed in the hope that it will be useful,
|
|
11
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
-
# GNU General Public License for more details.
|
|
14
|
-
#
|
|
15
|
-
# You should have received a copy of the GNU General Public License
|
|
16
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
-
# -----------------------------------------------------------------------------.
|
|
18
|
-
from disdrodb.l0 import run_l0a
|
|
19
|
-
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
@is_documented_by(reader_generic_docstring)
|
|
23
|
-
def reader(
|
|
24
|
-
raw_dir,
|
|
25
|
-
processed_dir,
|
|
26
|
-
station_name,
|
|
27
|
-
# Processing options
|
|
28
|
-
force=False,
|
|
29
|
-
verbose=False,
|
|
30
|
-
parallel=False,
|
|
31
|
-
debugging_mode=False,
|
|
32
|
-
):
|
|
33
|
-
##------------------------------------------------------------------------.
|
|
34
|
-
#### - Define column names
|
|
35
|
-
column_names = ["time", "epoch_time", "TO_BE_PARSED"]
|
|
36
|
-
|
|
37
|
-
##------------------------------------------------------------------------.
|
|
38
|
-
#### - Define reader options
|
|
39
|
-
reader_kwargs = {}
|
|
40
|
-
# - Define delimiter
|
|
41
|
-
reader_kwargs["delimiter"] = ";"
|
|
42
|
-
# - Skip first row as columns names
|
|
43
|
-
reader_kwargs["header"] = None
|
|
44
|
-
# - Avoid first column to become df index
|
|
45
|
-
reader_kwargs["index_col"] = False
|
|
46
|
-
# - Define behaviour when encountering bad lines
|
|
47
|
-
reader_kwargs["on_bad_lines"] = "skip"
|
|
48
|
-
# - Define parser engine
|
|
49
|
-
# - C engine is faster
|
|
50
|
-
# - Python engine is more feature-complete
|
|
51
|
-
reader_kwargs["engine"] = "python"
|
|
52
|
-
# - Define on-the-fly decompression of on-disk data
|
|
53
|
-
# - Available: gzip, bz2, zip
|
|
54
|
-
reader_kwargs["compression"] = "infer"
|
|
55
|
-
# reader_kwargs['zipped'] = False
|
|
56
|
-
# reader_kwargs['zipped'] = True
|
|
57
|
-
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
58
|
-
# - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
|
|
59
|
-
# ‘-NaN’, ‘-nan’, ‘1.#IND’, ‘1.#QNAN’, ‘<NA>’, ‘N/A’,
|
|
60
|
-
# ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
|
|
61
|
-
reader_kwargs["na_values"] = ["na", "", "error", "-.-", " NA"]
|
|
62
|
-
##------------------------------------------------------------------------.
|
|
63
|
-
#### - Define dataframe sanitizer function for L0 processing
|
|
64
|
-
# Station 8 has all raw_drop_number corrupted, so it can't be used
|
|
65
|
-
|
|
66
|
-
def df_sanitizer_fun(df):
|
|
67
|
-
# Import pandas
|
|
68
|
-
import pandas as pd
|
|
69
|
-
|
|
70
|
-
# - Remove rows that have a corrupted "TO_BE_PARSED" column
|
|
71
|
-
# - PAR001, PAR002 have length 3726 (no station_name)
|
|
72
|
-
# - PAR007 have length 3736 ()
|
|
73
|
-
df = df.loc[df["TO_BE_PARSED"].astype(str).str.len() >= 3726]
|
|
74
|
-
df = df.loc[df["TO_BE_PARSED"].astype(str).str.len() <= 3736]
|
|
75
|
-
|
|
76
|
-
# - Convert 'time' column to datetime
|
|
77
|
-
df_time = pd.to_datetime(df["time"], format="%Y%m%d-%H%M%S", errors="coerce")
|
|
78
|
-
|
|
79
|
-
# - Strip values from start and end of the string
|
|
80
|
-
df["TO_BE_PARSED"] = df["TO_BE_PARSED"].str.lstrip("b'").str.rstrip("'").str.rstrip("\\r\\n'")
|
|
81
|
-
|
|
82
|
-
# - Split the column 'TO_BE_PARSED'
|
|
83
|
-
df_to_parse = df["TO_BE_PARSED"].str.split(";", expand=True, n=99)
|
|
84
|
-
|
|
85
|
-
# - Retrieve DISDRODB compliant columns
|
|
86
|
-
df = df_to_parse.iloc[:, 0:35]
|
|
87
|
-
|
|
88
|
-
# - Assign column names
|
|
89
|
-
column_names = [
|
|
90
|
-
"rainfall_rate_32bit",
|
|
91
|
-
"rainfall_accumulated_32bit",
|
|
92
|
-
"weather_code_synop_4680",
|
|
93
|
-
"weather_code_synop_4677",
|
|
94
|
-
"weather_code_metar_4678",
|
|
95
|
-
"weather_code_nws",
|
|
96
|
-
"reflectivity_32bit",
|
|
97
|
-
"mor_visibility",
|
|
98
|
-
"sample_interval",
|
|
99
|
-
"laser_amplitude",
|
|
100
|
-
"number_particles",
|
|
101
|
-
"sensor_temperature",
|
|
102
|
-
"sensor_serial_number",
|
|
103
|
-
"firmware_iop",
|
|
104
|
-
"firmware_dsp",
|
|
105
|
-
"sensor_heating_current",
|
|
106
|
-
"sensor_battery_voltage",
|
|
107
|
-
"sensor_status",
|
|
108
|
-
"sensor_time_measurement_start",
|
|
109
|
-
"sensor_time",
|
|
110
|
-
"sensor_date",
|
|
111
|
-
"station_name",
|
|
112
|
-
"station_number",
|
|
113
|
-
"rainfall_amount_absolute_32bit",
|
|
114
|
-
"error_code",
|
|
115
|
-
"sensor_temperature_pcb",
|
|
116
|
-
"sensor_temperature_receiver",
|
|
117
|
-
"sensor_temperature_trasmitter",
|
|
118
|
-
"rainfall_rate_16_bit_30",
|
|
119
|
-
"rainfall_rate_16_bit_1200",
|
|
120
|
-
"rainfall_accumulated_16bit",
|
|
121
|
-
"reflectivity_16bit",
|
|
122
|
-
"rain_kinetic_energy",
|
|
123
|
-
"snowfall_rate",
|
|
124
|
-
"number_particles_all",
|
|
125
|
-
# "number_particles_all_detected",
|
|
126
|
-
]
|
|
127
|
-
df.columns = column_names
|
|
128
|
-
|
|
129
|
-
# - Add time column
|
|
130
|
-
df["time"] = df_time
|
|
131
|
-
|
|
132
|
-
# - Retrieve raw_drop_concentration
|
|
133
|
-
df["raw_drop_concentration"] = df_to_parse.iloc[:, 35:67].apply(
|
|
134
|
-
lambda x: ",".join(x.dropna().astype(str)),
|
|
135
|
-
axis=1,
|
|
136
|
-
)
|
|
137
|
-
# - Retrieve raw_drop_average_velocity
|
|
138
|
-
df["raw_drop_average_velocity"] = df_to_parse.iloc[:, 67:99].apply(
|
|
139
|
-
lambda x: ",".join(x.dropna().astype(str)),
|
|
140
|
-
axis=1,
|
|
141
|
-
)
|
|
142
|
-
|
|
143
|
-
# - Retrieve raw_drop_number
|
|
144
|
-
df_raw_drop_number = df_to_parse.iloc[:, 99].squeeze()
|
|
145
|
-
df_raw_drop_number = df_raw_drop_number.str.replace(r"(\w{3})", r"\1,", regex=True)
|
|
146
|
-
df["raw_drop_number"] = df_raw_drop_number
|
|
147
|
-
|
|
148
|
-
# - Drop columns not agreeing with DISDRODB L0 standards
|
|
149
|
-
columns_to_drop = [
|
|
150
|
-
"firmware_iop",
|
|
151
|
-
"firmware_dsp",
|
|
152
|
-
"sensor_time_measurement_start",
|
|
153
|
-
"sensor_time",
|
|
154
|
-
"sensor_date",
|
|
155
|
-
"station_name",
|
|
156
|
-
"station_number",
|
|
157
|
-
"sensor_serial_number",
|
|
158
|
-
"sample_interval",
|
|
159
|
-
"sensor_serial_number",
|
|
160
|
-
# "epoch_time",
|
|
161
|
-
# "number_particles_all_detected",
|
|
162
|
-
]
|
|
163
|
-
df = df.drop(columns=columns_to_drop)
|
|
164
|
-
|
|
165
|
-
return df
|
|
166
|
-
|
|
167
|
-
##------------------------------------------------------------------------.
|
|
168
|
-
#### - Define glob pattern to search data files in <raw_dir>/data/<station_name>
|
|
169
|
-
glob_patterns = "*/*/*.csv" # <year>/<yearmonth>/*csv
|
|
170
|
-
|
|
171
|
-
####----------------------------------------------------------------------.
|
|
172
|
-
#### - Create L0A products
|
|
173
|
-
run_l0a(
|
|
174
|
-
raw_dir=raw_dir,
|
|
175
|
-
processed_dir=processed_dir,
|
|
176
|
-
station_name=station_name,
|
|
177
|
-
# Custom arguments of the reader for L0A processing
|
|
178
|
-
glob_patterns=glob_patterns,
|
|
179
|
-
column_names=column_names,
|
|
180
|
-
reader_kwargs=reader_kwargs,
|
|
181
|
-
df_sanitizer_fun=df_sanitizer_fun,
|
|
182
|
-
# Processing options
|
|
183
|
-
force=force,
|
|
184
|
-
verbose=verbose,
|
|
185
|
-
parallel=parallel,
|
|
186
|
-
debugging_mode=debugging_mode,
|
|
187
|
-
)
|
|
@@ -1,179 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
# -----------------------------------------------------------------------------.
|
|
3
|
-
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
-
#
|
|
5
|
-
# This program is free software: you can redistribute it and/or modify
|
|
6
|
-
# it under the terms of the GNU General Public License as published by
|
|
7
|
-
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
-
# (at your option) any later version.
|
|
9
|
-
#
|
|
10
|
-
# This program is distributed in the hope that it will be useful,
|
|
11
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
-
# GNU General Public License for more details.
|
|
14
|
-
#
|
|
15
|
-
# You should have received a copy of the GNU General Public License
|
|
16
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
-
# -----------------------------------------------------------------------------.
|
|
18
|
-
from disdrodb.l0 import run_l0a
|
|
19
|
-
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
@is_documented_by(reader_generic_docstring)
|
|
23
|
-
def reader(
|
|
24
|
-
raw_dir,
|
|
25
|
-
processed_dir,
|
|
26
|
-
station_name,
|
|
27
|
-
# Processing options
|
|
28
|
-
force=False,
|
|
29
|
-
verbose=False,
|
|
30
|
-
parallel=False,
|
|
31
|
-
debugging_mode=False,
|
|
32
|
-
):
|
|
33
|
-
##------------------------------------------------------------------------.
|
|
34
|
-
#### - Define column names
|
|
35
|
-
# These are the variables included in the datasets:
|
|
36
|
-
|
|
37
|
-
# var full name units
|
|
38
|
-
|
|
39
|
-
# Time time of the record Y-m-d hh:mm:ss
|
|
40
|
-
# Event event ID (factor)
|
|
41
|
-
# ID disdromter ID (factor: T1, T2, P1, P2)
|
|
42
|
-
# Serial disdrometer serial number (factor)
|
|
43
|
-
# Type disdrometer type (factor: Thi, Par)
|
|
44
|
-
# Mast mast ID (factor: 1, 2)
|
|
45
|
-
# NP_meas number of particles detected (-)
|
|
46
|
-
# R_meas rainfall intensity, as outputted by the device mm h−1
|
|
47
|
-
# Z_meas radar reflectivity, as outputted by the device dB mm6 m−3
|
|
48
|
-
# E_meas erosivity, as outputted by the device J m−2 mm−1
|
|
49
|
-
# Pcum_meas cumulative rainfall amount mm
|
|
50
|
-
# Ecum_meas cumulative kinetic energy J m−2 mm−1
|
|
51
|
-
# NP number of particles detected (-)
|
|
52
|
-
# ND particle density m−3 mm−1
|
|
53
|
-
# R rainfall intensity mm h−1
|
|
54
|
-
# P rainfall amount mm
|
|
55
|
-
# Z radar reflectivity dB mm6 m−3
|
|
56
|
-
# M water content gm−3
|
|
57
|
-
# E kinetic energy J m−2 mm−1
|
|
58
|
-
# Pcum cumulative rainfall amount mm
|
|
59
|
-
# Ecum cumulative kinetic energy J m−2 mm−1
|
|
60
|
-
# D10 drop diameter’s 10th percentile mm
|
|
61
|
-
# D25 drop diameter’s 25th percentile mm
|
|
62
|
-
# D50 drop diameter’s 50th percentile mm
|
|
63
|
-
# D75 drop diameter’s 75th percentile mm
|
|
64
|
-
# D90 drop diameter’s 90th percentile mm
|
|
65
|
-
# Dm mean drop diameter mm
|
|
66
|
-
# V10 drop velocity’s 10th percentile m s−1
|
|
67
|
-
# V25 drop velocity’s 25th percentile m s−1
|
|
68
|
-
# V50 drop velocity’s 50th percentile m s−1
|
|
69
|
-
# V75 drop velocity’s 75th percentile m s−1
|
|
70
|
-
# V90 drop velocity’s 90th percentile m s−1
|
|
71
|
-
# Vm mean drop velocity m s−1
|
|
72
|
-
|
|
73
|
-
column_names = [
|
|
74
|
-
"time",
|
|
75
|
-
"event_id",
|
|
76
|
-
"disdrometer_ID",
|
|
77
|
-
"disdrometer_serial",
|
|
78
|
-
"disdrometer_type",
|
|
79
|
-
"mast_ID",
|
|
80
|
-
"number_particles",
|
|
81
|
-
"rainfall_rate_32bit",
|
|
82
|
-
"reflectivity_32bit",
|
|
83
|
-
"unknown", # rain_kinetic_energy ?
|
|
84
|
-
"rainfall_accumulated_32bit",
|
|
85
|
-
"rain_kinetic_energy", # unknown ?
|
|
86
|
-
"NP",
|
|
87
|
-
"NDRPZMEPcumEcumD10",
|
|
88
|
-
"D25",
|
|
89
|
-
"D50",
|
|
90
|
-
"D75",
|
|
91
|
-
"D90",
|
|
92
|
-
"Dm",
|
|
93
|
-
"V10",
|
|
94
|
-
"V25",
|
|
95
|
-
"V50",
|
|
96
|
-
"V75",
|
|
97
|
-
"V90",
|
|
98
|
-
"Vm",
|
|
99
|
-
]
|
|
100
|
-
|
|
101
|
-
##------------------------------------------------------------------------.
|
|
102
|
-
#### - Define reader options
|
|
103
|
-
reader_kwargs = {}
|
|
104
|
-
# - Define delimiter
|
|
105
|
-
reader_kwargs["delimiter"] = ","
|
|
106
|
-
# Skip first row as columns names
|
|
107
|
-
reader_kwargs["header"] = None
|
|
108
|
-
# - Avoid first column to become df index !!!
|
|
109
|
-
reader_kwargs["index_col"] = False
|
|
110
|
-
# - Define behaviour when encountering bad lines
|
|
111
|
-
reader_kwargs["on_bad_lines"] = "skip"
|
|
112
|
-
# - Define reader engine
|
|
113
|
-
# - C engine is faster
|
|
114
|
-
# - Python engine is more feature-complete
|
|
115
|
-
reader_kwargs["engine"] = "python"
|
|
116
|
-
# - Define on-the-fly decompression of on-disk data
|
|
117
|
-
# - Available: gzip, bz2, zip
|
|
118
|
-
reader_kwargs["compression"] = "infer"
|
|
119
|
-
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
120
|
-
# - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
|
|
121
|
-
# ‘-NaN’, ‘-nan’, ‘1.#IND’, ‘1.#QNAN’, ‘<NA>’, ‘N/A’,
|
|
122
|
-
# ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
|
|
123
|
-
reader_kwargs["na_values"] = ["na", "", "error", "NA", "-.-"] # Cast all to string
|
|
124
|
-
|
|
125
|
-
##------------------------------------------------------------------------.
|
|
126
|
-
#### - Define dataframe sanitizer function for L0 processing
|
|
127
|
-
def df_sanitizer_fun(df):
|
|
128
|
-
# - Import pandas
|
|
129
|
-
import pandas as pd
|
|
130
|
-
|
|
131
|
-
# - Convert time column to datetime
|
|
132
|
-
df["time"] = pd.to_datetime(df["time"], format="%Y-%m-%d %H:%M:%S", errors="coerce")
|
|
133
|
-
|
|
134
|
-
# - Drop columns not agreeing with DISDRODB L0 standards
|
|
135
|
-
columns_to_drop = [
|
|
136
|
-
"event_id",
|
|
137
|
-
"disdrometer_ID",
|
|
138
|
-
"disdrometer_serial",
|
|
139
|
-
"disdrometer_type",
|
|
140
|
-
"mast_ID",
|
|
141
|
-
"NP",
|
|
142
|
-
"NDRPZMEPcumEcumD10",
|
|
143
|
-
"D25",
|
|
144
|
-
"D50",
|
|
145
|
-
"D75",
|
|
146
|
-
"D90",
|
|
147
|
-
"Dm",
|
|
148
|
-
"V10",
|
|
149
|
-
"V25",
|
|
150
|
-
"V50",
|
|
151
|
-
"V75",
|
|
152
|
-
"V90",
|
|
153
|
-
"Vm",
|
|
154
|
-
]
|
|
155
|
-
df = df.drop(columns=columns_to_drop)
|
|
156
|
-
|
|
157
|
-
return df
|
|
158
|
-
|
|
159
|
-
##------------------------------------------------------------------------.
|
|
160
|
-
#### - Define glob pattern to search data files in <raw_dir>/data/<station_name>
|
|
161
|
-
glob_patterns = "*.csv*"
|
|
162
|
-
|
|
163
|
-
####----------------------------------------------------------------------.
|
|
164
|
-
#### - Create L0A products
|
|
165
|
-
run_l0a(
|
|
166
|
-
raw_dir=raw_dir,
|
|
167
|
-
processed_dir=processed_dir,
|
|
168
|
-
station_name=station_name,
|
|
169
|
-
# Custom arguments of the reader for L0A processing
|
|
170
|
-
glob_patterns=glob_patterns,
|
|
171
|
-
column_names=column_names,
|
|
172
|
-
reader_kwargs=reader_kwargs,
|
|
173
|
-
df_sanitizer_fun=df_sanitizer_fun,
|
|
174
|
-
# Processing options
|
|
175
|
-
force=force,
|
|
176
|
-
verbose=verbose,
|
|
177
|
-
parallel=parallel,
|
|
178
|
-
debugging_mode=debugging_mode,
|
|
179
|
-
)
|