disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +132 -15
- disdrodb/_config.py +4 -2
- disdrodb/_version.py +9 -4
- disdrodb/api/checks.py +264 -237
- disdrodb/api/configs.py +4 -8
- disdrodb/api/create_directories.py +235 -290
- disdrodb/api/info.py +217 -26
- disdrodb/api/io.py +306 -270
- disdrodb/api/path.py +597 -173
- disdrodb/api/search.py +486 -0
- disdrodb/{metadata/scripts → cli}/disdrodb_check_metadata_archive.py +12 -7
- disdrodb/{utils/pandas.py → cli/disdrodb_data_archive_directory.py} +9 -18
- disdrodb/cli/disdrodb_download_archive.py +86 -0
- disdrodb/cli/disdrodb_download_metadata_archive.py +53 -0
- disdrodb/cli/disdrodb_download_station.py +84 -0
- disdrodb/{api/scripts → cli}/disdrodb_initialize_station.py +22 -10
- disdrodb/cli/disdrodb_metadata_archive_directory.py +32 -0
- disdrodb/{data_transfer/scripts/disdrodb_download_station.py → cli/disdrodb_open_data_archive.py} +22 -22
- disdrodb/cli/disdrodb_open_logs_directory.py +69 -0
- disdrodb/{data_transfer/scripts/disdrodb_upload_station.py → cli/disdrodb_open_metadata_archive.py} +22 -24
- disdrodb/cli/disdrodb_open_metadata_directory.py +71 -0
- disdrodb/cli/disdrodb_open_product_directory.py +74 -0
- disdrodb/cli/disdrodb_open_readers_directory.py +32 -0
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0.py +38 -31
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0_station.py +32 -30
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a_station.py +24 -33
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b_station.py +25 -34
- disdrodb/cli/disdrodb_run_l0c.py +130 -0
- disdrodb/cli/disdrodb_run_l0c_station.py +129 -0
- disdrodb/cli/disdrodb_run_l1.py +122 -0
- disdrodb/cli/disdrodb_run_l1_station.py +121 -0
- disdrodb/cli/disdrodb_run_l2e.py +122 -0
- disdrodb/cli/disdrodb_run_l2e_station.py +122 -0
- disdrodb/cli/disdrodb_run_l2m.py +122 -0
- disdrodb/cli/disdrodb_run_l2m_station.py +122 -0
- disdrodb/cli/disdrodb_upload_archive.py +105 -0
- disdrodb/cli/disdrodb_upload_station.py +98 -0
- disdrodb/configs.py +90 -25
- disdrodb/data_transfer/__init__.py +22 -0
- disdrodb/data_transfer/download_data.py +87 -90
- disdrodb/data_transfer/upload_data.py +64 -37
- disdrodb/data_transfer/zenodo.py +15 -18
- disdrodb/docs.py +1 -1
- disdrodb/issue/__init__.py +17 -4
- disdrodb/issue/checks.py +10 -23
- disdrodb/issue/reader.py +9 -12
- disdrodb/issue/writer.py +14 -17
- disdrodb/l0/__init__.py +17 -26
- disdrodb/l0/check_configs.py +35 -23
- disdrodb/l0/check_standards.py +46 -51
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_diameter.yml +44 -44
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_velocity.yml +40 -40
- disdrodb/l0/configs/LPM/l0a_encodings.yml +80 -0
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_cf_attrs.yml +84 -65
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_encodings.yml +50 -9
- disdrodb/l0/configs/{Thies_LPM → LPM}/raw_data_format.yml +285 -245
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_diameter.yml +66 -66
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL/l0a_encodings.yml +32 -0
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_cf_attrs.yml +23 -21
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_encodings.yml +17 -17
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/raw_data_format.yml +77 -77
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_diameter.yml +64 -64
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +39 -0
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_cf_attrs.yml +28 -26
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_encodings.yml +20 -20
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/raw_data_format.yml +107 -107
- disdrodb/l0/configs/PWS100/bins_diameter.yml +173 -0
- disdrodb/l0/configs/PWS100/bins_velocity.yml +173 -0
- disdrodb/l0/configs/PWS100/l0a_encodings.yml +19 -0
- disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +76 -0
- disdrodb/l0/configs/PWS100/l0b_encodings.yml +176 -0
- disdrodb/l0/configs/PWS100/raw_data_format.yml +182 -0
- disdrodb/l0/configs/{RD_80 → RD80}/bins_diameter.yml +40 -40
- disdrodb/l0/configs/RD80/l0a_encodings.yml +16 -0
- disdrodb/l0/configs/{RD_80 → RD80}/l0b_cf_attrs.yml +3 -3
- disdrodb/l0/configs/RD80/l0b_encodings.yml +135 -0
- disdrodb/l0/configs/{RD_80 → RD80}/raw_data_format.yml +46 -50
- disdrodb/l0/l0_reader.py +216 -340
- disdrodb/l0/l0a_processing.py +237 -208
- disdrodb/l0/l0b_nc_processing.py +227 -80
- disdrodb/l0/l0b_processing.py +96 -174
- disdrodb/l0/l0c_processing.py +627 -0
- disdrodb/l0/readers/{ARM → LPM/ARM}/ARM_LPM.py +36 -58
- disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +236 -0
- disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +185 -0
- disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +185 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +195 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +210 -0
- disdrodb/l0/readers/{BRAZIL/GOAMAZON_LPM.py → LPM/KIT/CHWALA.py} +97 -76
- disdrodb/l0/readers/LPM/SLOVENIA/ARSO.py +197 -0
- disdrodb/l0/readers/LPM/SLOVENIA/CRNI_VRH.py +197 -0
- disdrodb/l0/readers/{UK → LPM/UK}/DIVEN.py +14 -35
- disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +157 -0
- disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +113 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/ARCTIC_2021.py +40 -57
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/COMMON_2011.py +37 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/DAVOS_2009_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_2009.py +34 -51
- disdrodb/l0/readers/{EPFL/PARADISO_2014.py → PARSIVEL/EPFL/EPFL_ROOF_2008.py} +38 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2012.py +33 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GENEPI_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007_2.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HPICONET_2010.py +34 -51
- disdrodb/l0/readers/{EPFL/EPFL_ROOF_2010.py → PARSIVEL/EPFL/HYMEX_LTE_SOP2.py} +37 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +111 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HYMEX_LTE_SOP4.py +36 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2018.py +34 -52
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2019.py +38 -56
- disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PARSIVEL_2007.py +27 -45
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PLATO_2019.py +24 -44
- disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +140 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RACLETS_2019_WJF.py +41 -59
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RIETHOLZBACH_2011.py +34 -51
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +117 -0
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +137 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/UNIL_2022.py +42 -55
- disdrodb/l0/readers/PARSIVEL/GPM/IFLOODS.py +104 -0
- disdrodb/l0/readers/{GPM → PARSIVEL/GPM}/LPVEX.py +29 -48
- disdrodb/l0/readers/PARSIVEL/GPM/MC3E.py +184 -0
- disdrodb/l0/readers/PARSIVEL/KIT/BURKINA_FASO.py +133 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +113 -0
- disdrodb/l0/readers/{NCAR/VORTEX_SE_2016_P1.py → PARSIVEL/NCAR/OWLES_MIPS.py} +46 -72
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +125 -0
- disdrodb/l0/readers/{NCAR/OWLES_MIPS.py → PARSIVEL/NCAR/PLOWS_MIPS.py} +45 -64
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +114 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +176 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +183 -0
- disdrodb/l0/readers/PARSIVEL/SLOVENIA/UL_FGG.py +121 -0
- disdrodb/l0/readers/{ARM/ARM_LD.py → PARSIVEL2/ARM/ARM_PARSIVEL2.py} +27 -50
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/{DENMARK → PARSIVEL2/DENMARK}/EROSION_nc.py +14 -35
- disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +189 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +119 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +104 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/NSSTC.py +176 -0
- disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +32 -0
- disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +56 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +120 -0
- disdrodb/l0/readers/{NCAR → PARSIVEL2/NCAR}/PECAN_MIPS.py +45 -64
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +181 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +160 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +160 -0
- disdrodb/l0/readers/{NCAR/PLOWS_MIPS.py → PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py} +49 -66
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +118 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +152 -0
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT.py +166 -0
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +150 -0
- disdrodb/l0/readers/{NCAR/RELAMPAGO_RD80.py → RD80/BRAZIL/CHUVA_RD80.py} +36 -60
- disdrodb/l0/readers/{BRAZIL → RD80/BRAZIL}/GOAMAZON_RD80.py +36 -55
- disdrodb/l0/readers/{NCAR → RD80/NCAR}/CINDY_2011_RD80.py +35 -54
- disdrodb/l0/readers/{BRAZIL/CHUVA_RD80.py → RD80/NCAR/RELAMPAGO_RD80.py} +40 -54
- disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py +274 -0
- disdrodb/l0/readers/template_reader_raw_netcdf_data.py +62 -0
- disdrodb/l0/readers/{reader_template.py → template_reader_raw_text_data.py} +20 -44
- disdrodb/l0/routines.py +885 -581
- disdrodb/l0/standards.py +77 -238
- disdrodb/l0/template_tools.py +105 -110
- disdrodb/l1/__init__.py +17 -0
- disdrodb/l1/beard_model.py +716 -0
- disdrodb/l1/encoding_attrs.py +635 -0
- disdrodb/l1/fall_velocity.py +260 -0
- disdrodb/l1/filters.py +192 -0
- disdrodb/l1/processing.py +202 -0
- disdrodb/l1/resampling.py +236 -0
- disdrodb/l1/routines.py +358 -0
- disdrodb/l1_env/__init__.py +17 -0
- disdrodb/l1_env/routines.py +38 -0
- disdrodb/l2/__init__.py +17 -0
- disdrodb/l2/empirical_dsd.py +1833 -0
- disdrodb/l2/event.py +388 -0
- disdrodb/l2/processing.py +528 -0
- disdrodb/l2/processing_options.py +213 -0
- disdrodb/l2/routines.py +868 -0
- disdrodb/metadata/__init__.py +9 -2
- disdrodb/metadata/checks.py +180 -124
- disdrodb/metadata/download.py +81 -0
- disdrodb/metadata/geolocation.py +146 -0
- disdrodb/metadata/info.py +20 -13
- disdrodb/metadata/manipulation.py +3 -3
- disdrodb/metadata/reader.py +59 -8
- disdrodb/metadata/search.py +77 -144
- disdrodb/metadata/standards.py +83 -80
- disdrodb/metadata/writer.py +10 -16
- disdrodb/psd/__init__.py +38 -0
- disdrodb/psd/fitting.py +2146 -0
- disdrodb/psd/models.py +774 -0
- disdrodb/routines.py +1412 -0
- disdrodb/scattering/__init__.py +28 -0
- disdrodb/scattering/axis_ratio.py +344 -0
- disdrodb/scattering/routines.py +456 -0
- disdrodb/utils/__init__.py +17 -0
- disdrodb/utils/attrs.py +208 -0
- disdrodb/utils/cli.py +269 -0
- disdrodb/utils/compression.py +60 -42
- disdrodb/utils/dask.py +62 -0
- disdrodb/utils/dataframe.py +342 -0
- disdrodb/utils/decorators.py +110 -0
- disdrodb/utils/directories.py +107 -46
- disdrodb/utils/encoding.py +127 -0
- disdrodb/utils/list.py +29 -0
- disdrodb/utils/logger.py +168 -46
- disdrodb/utils/time.py +657 -0
- disdrodb/utils/warnings.py +30 -0
- disdrodb/utils/writer.py +57 -0
- disdrodb/utils/xarray.py +138 -47
- disdrodb/utils/yaml.py +0 -1
- disdrodb/viz/__init__.py +17 -0
- disdrodb/viz/plots.py +17 -0
- disdrodb-0.1.1.dist-info/METADATA +294 -0
- disdrodb-0.1.1.dist-info/RECORD +232 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/WHEEL +1 -1
- disdrodb-0.1.1.dist-info/entry_points.txt +30 -0
- disdrodb/data_transfer/scripts/disdrodb_download_archive.py +0 -53
- disdrodb/data_transfer/scripts/disdrodb_upload_archive.py +0 -57
- disdrodb/l0/configs/OTT_Parsivel/l0a_encodings.yml +0 -32
- disdrodb/l0/configs/OTT_Parsivel2/l0a_encodings.yml +0 -39
- disdrodb/l0/configs/RD_80/l0a_encodings.yml +0 -16
- disdrodb/l0/configs/RD_80/l0b_encodings.yml +0 -135
- disdrodb/l0/configs/Thies_LPM/l0a_encodings.yml +0 -80
- disdrodb/l0/io.py +0 -257
- disdrodb/l0/l0_processing.py +0 -1091
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_OTT.py +0 -178
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_THIES.py +0 -247
- disdrodb/l0/readers/BRAZIL/CHUVA_LPM.py +0 -204
- disdrodb/l0/readers/BRAZIL/CHUVA_OTT.py +0 -183
- disdrodb/l0/readers/BRAZIL/GOAMAZON_OTT.py +0 -183
- disdrodb/l0/readers/CHINA/CHONGQING.py +0 -131
- disdrodb/l0/readers/EPFL/EPFL_ROOF_2008.py +0 -128
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP2.py +0 -127
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP3.py +0 -129
- disdrodb/l0/readers/EPFL/RACLETS_2019.py +0 -158
- disdrodb/l0/readers/EPFL/SAMOYLOV_2017.py +0 -136
- disdrodb/l0/readers/EPFL/SAMOYLOV_2019.py +0 -158
- disdrodb/l0/readers/FRANCE/SIRTA_OTT2.py +0 -138
- disdrodb/l0/readers/GPM/GCPEX.py +0 -123
- disdrodb/l0/readers/GPM/IFLOODS.py +0 -123
- disdrodb/l0/readers/GPM/MC3E.py +0 -123
- disdrodb/l0/readers/GPM/NSSTC.py +0 -164
- disdrodb/l0/readers/ITALY/GID.py +0 -199
- disdrodb/l0/readers/MEXICO/OH_IIUNAM_nc.py +0 -92
- disdrodb/l0/readers/NCAR/CCOPE_2015.py +0 -133
- disdrodb/l0/readers/NCAR/PECAN_FP3.py +0 -137
- disdrodb/l0/readers/NCAR/PECAN_MOBILE.py +0 -144
- disdrodb/l0/readers/NCAR/RELAMPAGO_OTT.py +0 -195
- disdrodb/l0/readers/NCAR/SNOWIE_PJ.py +0 -172
- disdrodb/l0/readers/NCAR/SNOWIE_SB.py +0 -179
- disdrodb/l0/readers/NCAR/VORTEX2_2009.py +0 -133
- disdrodb/l0/readers/NCAR/VORTEX2_2010.py +0 -188
- disdrodb/l0/readers/NCAR/VORTEX2_2010_UF.py +0 -191
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_P2.py +0 -135
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_PIPS.py +0 -170
- disdrodb/l0/readers/NETHERLANDS/DELFT.py +0 -187
- disdrodb/l0/readers/SPAIN/SBEGUERIA.py +0 -179
- disdrodb/l0/scripts/disdrodb_run_l0b_concat.py +0 -93
- disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py +0 -85
- disdrodb/utils/netcdf.py +0 -452
- disdrodb/utils/scripts.py +0 -102
- disdrodb-0.0.21.dist-info/AUTHORS.md +0 -18
- disdrodb-0.0.21.dist-info/METADATA +0 -186
- disdrodb-0.0.21.dist-info/RECORD +0 -168
- disdrodb-0.0.21.dist-info/entry_points.txt +0 -15
- /disdrodb/l0/configs/{RD_80 → RD80}/bins_velocity.yml +0 -0
- /disdrodb/l0/manuals/{Thies_LPM.pdf → LPM.pdf} +0 -0
- /disdrodb/l0/manuals/{ODM_470.pdf → ODM470.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel.pdf → PARSIVEL.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel2.pdf → PARSIVEL2.pdf} +0 -0
- /disdrodb/l0/manuals/{PWS_100.pdf → PWS100.pdf} +0 -0
- /disdrodb/l0/manuals/{RD_80.pdf → RD80.pdf} +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info/licenses}/LICENSE +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/top_level.txt +0 -0
|
@@ -1,158 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
# -----------------------------------------------------------------------------.
|
|
3
|
-
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
-
#
|
|
5
|
-
# This program is free software: you can redistribute it and/or modify
|
|
6
|
-
# it under the terms of the GNU General Public License as published by
|
|
7
|
-
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
-
# (at your option) any later version.
|
|
9
|
-
#
|
|
10
|
-
# This program is distributed in the hope that it will be useful,
|
|
11
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
-
# GNU General Public License for more details.
|
|
14
|
-
#
|
|
15
|
-
# You should have received a copy of the GNU General Public License
|
|
16
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
-
# -----------------------------------------------------------------------------.
|
|
18
|
-
from disdrodb.l0 import run_l0a
|
|
19
|
-
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
@is_documented_by(reader_generic_docstring)
|
|
23
|
-
def reader(
|
|
24
|
-
raw_dir,
|
|
25
|
-
processed_dir,
|
|
26
|
-
station_name,
|
|
27
|
-
# Processing options
|
|
28
|
-
force=False,
|
|
29
|
-
verbose=False,
|
|
30
|
-
parallel=False,
|
|
31
|
-
debugging_mode=False,
|
|
32
|
-
):
|
|
33
|
-
##------------------------------------------------------------------------.
|
|
34
|
-
#### - Define column names
|
|
35
|
-
column_names = [
|
|
36
|
-
"id",
|
|
37
|
-
"latitude",
|
|
38
|
-
"longitude",
|
|
39
|
-
"time",
|
|
40
|
-
"datalogger_temperature",
|
|
41
|
-
"TO_BE_SPLITTED", # datalogger_voltage and rainfall_rate_32bit
|
|
42
|
-
"rainfall_accumulated_32bit",
|
|
43
|
-
"weather_code_synop_4680",
|
|
44
|
-
"weather_code_synop_4677",
|
|
45
|
-
"reflectivity_32bit",
|
|
46
|
-
"mor_visibility",
|
|
47
|
-
"laser_amplitude",
|
|
48
|
-
"number_particles",
|
|
49
|
-
"sensor_temperature",
|
|
50
|
-
"sensor_heating_current",
|
|
51
|
-
"sensor_battery_voltage",
|
|
52
|
-
"sensor_status",
|
|
53
|
-
"rainfall_amount_absolute_32bit",
|
|
54
|
-
"error_code",
|
|
55
|
-
"raw_drop_concentration",
|
|
56
|
-
"raw_drop_average_velocity",
|
|
57
|
-
"raw_drop_number",
|
|
58
|
-
"datalogger_error",
|
|
59
|
-
]
|
|
60
|
-
|
|
61
|
-
##------------------------------------------------------------------------.
|
|
62
|
-
#### - Define reader options
|
|
63
|
-
reader_kwargs = {}
|
|
64
|
-
|
|
65
|
-
# - Define delimiter
|
|
66
|
-
reader_kwargs["delimiter"] = ";"
|
|
67
|
-
|
|
68
|
-
# Skip first row as columns names
|
|
69
|
-
reader_kwargs["header"] = None
|
|
70
|
-
|
|
71
|
-
# - Avoid first column to become df index !!!
|
|
72
|
-
reader_kwargs["index_col"] = False
|
|
73
|
-
|
|
74
|
-
# - Define behaviour when encountering bad lines
|
|
75
|
-
reader_kwargs["on_bad_lines"] = "skip"
|
|
76
|
-
|
|
77
|
-
# - Define encoding
|
|
78
|
-
# --> Avoid UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf0
|
|
79
|
-
reader_kwargs["encoding"] = "latin-1"
|
|
80
|
-
|
|
81
|
-
# - Define reader engine
|
|
82
|
-
# - C engine is faster
|
|
83
|
-
# - Python engine is more feature-complete
|
|
84
|
-
reader_kwargs["engine"] = "python"
|
|
85
|
-
|
|
86
|
-
# - Define on-the-fly decompression of on-disk data
|
|
87
|
-
# - Available: gzip, bz2, zip
|
|
88
|
-
reader_kwargs["compression"] = "infer"
|
|
89
|
-
|
|
90
|
-
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
91
|
-
# - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
|
|
92
|
-
# ‘-NaN’, ‘-nan’, ‘1.#IND’, ‘1.#QNAN’, ‘<NA>’, ‘N/A’,
|
|
93
|
-
# ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
|
|
94
|
-
reader_kwargs["na_values"] = [
|
|
95
|
-
"na",
|
|
96
|
-
"",
|
|
97
|
-
"error",
|
|
98
|
-
"NA",
|
|
99
|
-
]
|
|
100
|
-
|
|
101
|
-
##------------------------------------------------------------------------.
|
|
102
|
-
#### - Define facultative dataframe sanitizer function for L0 processing
|
|
103
|
-
# - Enable to deal with bad raw data files
|
|
104
|
-
# - Enable to standardize raw data files to L0 standards (i.e. time to datetime)
|
|
105
|
-
|
|
106
|
-
def df_sanitizer_fun(df):
|
|
107
|
-
# Import dask or pandas
|
|
108
|
-
import pandas as pd
|
|
109
|
-
|
|
110
|
-
# - Convert time column to datetime
|
|
111
|
-
df["time"] = pd.to_datetime(df["time"], format="%d-%m-%Y %H:%M:%S", errors="coerce")
|
|
112
|
-
|
|
113
|
-
# Drop rows with None value
|
|
114
|
-
df = df[~df["TO_BE_SPLITTED"].isnull()]
|
|
115
|
-
|
|
116
|
-
# - Drop rows when 'Error in data reading' in TO_BE_SPLITTED column
|
|
117
|
-
bad_indices = df[df["TO_BE_SPLITTED"].str.contains("Error in data reading!")].index
|
|
118
|
-
df = df.drop(bad_indices)
|
|
119
|
-
|
|
120
|
-
# - Split TO_BE_SPLITTED columns
|
|
121
|
-
df_splitted = df["TO_BE_SPLITTED"].str.split(",", expand=True, n=1)
|
|
122
|
-
df_splitted.columns = ["datalogger_voltage", "rainfall_rate_32bit"]
|
|
123
|
-
df["rainfall_rate_32bit"] = df_splitted["rainfall_rate_32bit"]
|
|
124
|
-
|
|
125
|
-
# - Drop columns not agreeing with DISDRODB L0 standards
|
|
126
|
-
columns_to_drop = [
|
|
127
|
-
"id",
|
|
128
|
-
"TO_BE_SPLITTED",
|
|
129
|
-
"datalogger_temperature",
|
|
130
|
-
"datalogger_error",
|
|
131
|
-
"latitude",
|
|
132
|
-
"longitude",
|
|
133
|
-
]
|
|
134
|
-
df = df.drop(columns=columns_to_drop)
|
|
135
|
-
|
|
136
|
-
return df
|
|
137
|
-
|
|
138
|
-
##------------------------------------------------------------------------.
|
|
139
|
-
#### - Define glob pattern to search data files in raw_dir/data/<station_name>
|
|
140
|
-
glob_patterns = "*.log*"
|
|
141
|
-
|
|
142
|
-
####----------------------------------------------------------------------.
|
|
143
|
-
#### - Create L0A products
|
|
144
|
-
run_l0a(
|
|
145
|
-
raw_dir=raw_dir,
|
|
146
|
-
processed_dir=processed_dir,
|
|
147
|
-
station_name=station_name,
|
|
148
|
-
# Custom arguments of the reader for L0A processing
|
|
149
|
-
glob_patterns=glob_patterns,
|
|
150
|
-
column_names=column_names,
|
|
151
|
-
reader_kwargs=reader_kwargs,
|
|
152
|
-
df_sanitizer_fun=df_sanitizer_fun,
|
|
153
|
-
# Processing options
|
|
154
|
-
force=force,
|
|
155
|
-
verbose=verbose,
|
|
156
|
-
parallel=parallel,
|
|
157
|
-
debugging_mode=debugging_mode,
|
|
158
|
-
)
|
|
@@ -1,138 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
|
|
3
|
-
# -----------------------------------------------------------------------------.
|
|
4
|
-
# Copyright (c) 2021-2023 DISDRODB developers
|
|
5
|
-
#
|
|
6
|
-
# This program is free software: you can redistribute it and/or modify
|
|
7
|
-
# it under the terms of the GNU General Public License as published by
|
|
8
|
-
# the Free Software Foundation, either version 3 of the License, or
|
|
9
|
-
# (at your option) any later version.
|
|
10
|
-
#
|
|
11
|
-
# This program is distributed in the hope that it will be useful,
|
|
12
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
13
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
|
-
# GNU General Public License for more details.
|
|
15
|
-
#
|
|
16
|
-
# You should have received a copy of the GNU General Public License
|
|
17
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
18
|
-
# -----------------------------------------------------------------------------.
|
|
19
|
-
from disdrodb.l0 import run_l0a
|
|
20
|
-
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
@is_documented_by(reader_generic_docstring)
|
|
24
|
-
def reader(
|
|
25
|
-
raw_dir,
|
|
26
|
-
processed_dir,
|
|
27
|
-
station_name,
|
|
28
|
-
# Processing options
|
|
29
|
-
force=False,
|
|
30
|
-
verbose=False,
|
|
31
|
-
parallel=False,
|
|
32
|
-
debugging_mode=False,
|
|
33
|
-
):
|
|
34
|
-
##------------------------------------------------------------------------.
|
|
35
|
-
#### - Define column names
|
|
36
|
-
column_names = ["TO_SPLIT"]
|
|
37
|
-
|
|
38
|
-
##------------------------------------------------------------------------.
|
|
39
|
-
#### - Define reader options
|
|
40
|
-
reader_kwargs = {}
|
|
41
|
-
# - Define delimiter
|
|
42
|
-
reader_kwargs["delimiter"] = "\\n"
|
|
43
|
-
# - Avoid first column to become df index !!!
|
|
44
|
-
reader_kwargs["index_col"] = False
|
|
45
|
-
# Skip first row as columns names
|
|
46
|
-
reader_kwargs["header"] = None
|
|
47
|
-
# Skip the first row (header)
|
|
48
|
-
reader_kwargs["skiprows"] = 1
|
|
49
|
-
# - Define behaviour when encountering bad lines
|
|
50
|
-
reader_kwargs["on_bad_lines"] = "skip"
|
|
51
|
-
# Define encoding
|
|
52
|
-
reader_kwargs["encoding"] = "latin1"
|
|
53
|
-
# - Define reader engine
|
|
54
|
-
# - C engine is faster
|
|
55
|
-
# - Python engine is more feature-complete
|
|
56
|
-
reader_kwargs["engine"] = "python"
|
|
57
|
-
# - Define on-the-fly decompression of on-disk data
|
|
58
|
-
# - Available: gzip, bz2, zip
|
|
59
|
-
reader_kwargs["compression"] = "infer"
|
|
60
|
-
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
61
|
-
# - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
|
|
62
|
-
# ‘-NaN’, ‘-nan’, ‘1.#IND’, ‘1.#QNAN’, ‘<NA>’, ‘N/A’,
|
|
63
|
-
# ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
|
|
64
|
-
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
65
|
-
|
|
66
|
-
##------------------------------------------------------------------------.
|
|
67
|
-
#### - Define dataframe sanitizer function for L0 processing
|
|
68
|
-
def df_sanitizer_fun(df):
|
|
69
|
-
# Import pandas
|
|
70
|
-
import pandas as pd
|
|
71
|
-
|
|
72
|
-
# The delimiter ; is used for separating both the variables and the
|
|
73
|
-
# values of the raw spectrum. So we need to retrieve the columns
|
|
74
|
-
# inside the sanitizer assuming a fixed number of columns.
|
|
75
|
-
df = df["TO_SPLIT"].str.split(";", expand=True, n=16)
|
|
76
|
-
|
|
77
|
-
# Define the column names
|
|
78
|
-
column_names = [
|
|
79
|
-
"date",
|
|
80
|
-
"time",
|
|
81
|
-
"rainfall_rate_32bit",
|
|
82
|
-
"rainfall_accumulated_32bit",
|
|
83
|
-
"reflectivity_32bit",
|
|
84
|
-
"mor_visibility",
|
|
85
|
-
"laser_amplitude",
|
|
86
|
-
"number_particles",
|
|
87
|
-
"sensor_temperature",
|
|
88
|
-
"sensor_heating_current",
|
|
89
|
-
"sensor_battery_voltage",
|
|
90
|
-
"rain_kinetic_energy",
|
|
91
|
-
"snowfall_rate",
|
|
92
|
-
"weather_code_synop_4680",
|
|
93
|
-
"weather_code_metar_4678",
|
|
94
|
-
"weather_code_nws",
|
|
95
|
-
"raw_drop_number",
|
|
96
|
-
]
|
|
97
|
-
df.columns = column_names
|
|
98
|
-
|
|
99
|
-
# Define the time column
|
|
100
|
-
df["time"] = df["date"] + "-" + df["time"]
|
|
101
|
-
df["time"] = pd.to_datetime(df["time"], format="%Y/%m/%d-%H:%M:%S", errors="coerce")
|
|
102
|
-
df = df.drop(columns=["date"])
|
|
103
|
-
|
|
104
|
-
# Preprocess the raw spectrum
|
|
105
|
-
# - The '<SPECTRUM>ZERO</SPECTRUM>' indicates no drops detected
|
|
106
|
-
# --> "" generates an array of zeros in L0B processing
|
|
107
|
-
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
|
|
108
|
-
# Remove <SPECTRUM> and </SPECTRUM>" acronyms from the raw_drop_number field
|
|
109
|
-
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
|
|
110
|
-
df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
|
|
111
|
-
|
|
112
|
-
# Add 0 before every ; if ; not preceded by a digit
|
|
113
|
-
# Example: ';;1;;' --> '0;0;1;0;'
|
|
114
|
-
df["raw_drop_number"] = df["raw_drop_number"].str.replace(r"(?<!\d);", "0;", regex=True)
|
|
115
|
-
|
|
116
|
-
return df
|
|
117
|
-
|
|
118
|
-
##------------------------------------------------------------------------.
|
|
119
|
-
#### - Define glob pattern to search data files in <raw_dir>/data/<station_name>
|
|
120
|
-
glob_patterns = "*.txt" # There is only one file without extension
|
|
121
|
-
|
|
122
|
-
####----------------------------------------------------------------------.
|
|
123
|
-
#### - Create L0A products
|
|
124
|
-
run_l0a(
|
|
125
|
-
raw_dir=raw_dir,
|
|
126
|
-
processed_dir=processed_dir,
|
|
127
|
-
station_name=station_name,
|
|
128
|
-
# Custom arguments of the reader for L0A processing
|
|
129
|
-
glob_patterns=glob_patterns,
|
|
130
|
-
column_names=column_names,
|
|
131
|
-
reader_kwargs=reader_kwargs,
|
|
132
|
-
df_sanitizer_fun=df_sanitizer_fun,
|
|
133
|
-
# Processing options
|
|
134
|
-
force=force,
|
|
135
|
-
verbose=verbose,
|
|
136
|
-
parallel=parallel,
|
|
137
|
-
debugging_mode=debugging_mode,
|
|
138
|
-
)
|
disdrodb/l0/readers/GPM/GCPEX.py
DELETED
|
@@ -1,123 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
# -----------------------------------------------------------------------------.
|
|
3
|
-
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
-
#
|
|
5
|
-
# This program is free software: you can redistribute it and/or modify
|
|
6
|
-
# it under the terms of the GNU General Public License as published by
|
|
7
|
-
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
-
# (at your option) any later version.
|
|
9
|
-
#
|
|
10
|
-
# This program is distributed in the hope that it will be useful,
|
|
11
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
-
# GNU General Public License for more details.
|
|
14
|
-
#
|
|
15
|
-
# You should have received a copy of the GNU General Public License
|
|
16
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
-
# -----------------------------------------------------------------------------.
|
|
18
|
-
from disdrodb.l0 import run_l0a
|
|
19
|
-
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
@is_documented_by(reader_generic_docstring)
|
|
23
|
-
def reader(
|
|
24
|
-
raw_dir,
|
|
25
|
-
processed_dir,
|
|
26
|
-
station_name,
|
|
27
|
-
# Processing options
|
|
28
|
-
force=False,
|
|
29
|
-
verbose=False,
|
|
30
|
-
parallel=False,
|
|
31
|
-
debugging_mode=False,
|
|
32
|
-
):
|
|
33
|
-
##------------------------------------------------------------------------.
|
|
34
|
-
#### - Define column names
|
|
35
|
-
column_names = ["time", "TO_BE_SPLITTED"]
|
|
36
|
-
|
|
37
|
-
##------------------------------------------------------------------------.
|
|
38
|
-
#### - Define reader options
|
|
39
|
-
reader_kwargs = {}
|
|
40
|
-
# Skip first row as columns names
|
|
41
|
-
reader_kwargs["header"] = None
|
|
42
|
-
# Skip file with encoding errors
|
|
43
|
-
reader_kwargs["encoding_errors"] = "ignore"
|
|
44
|
-
# - Define delimiter
|
|
45
|
-
reader_kwargs["delimiter"] = ";"
|
|
46
|
-
# - Avoid first column to become df index !!!
|
|
47
|
-
reader_kwargs["index_col"] = False
|
|
48
|
-
# - Define behaviour when encountering bad lines
|
|
49
|
-
reader_kwargs["on_bad_lines"] = "skip"
|
|
50
|
-
# - Define reader engine
|
|
51
|
-
# - C engine is faster
|
|
52
|
-
# - Python engine is more feature-complete
|
|
53
|
-
reader_kwargs["engine"] = "python"
|
|
54
|
-
# - Define on-the-fly decompression of on-disk data
|
|
55
|
-
# - Available: gzip, bz2, zip
|
|
56
|
-
reader_kwargs["compression"] = "infer"
|
|
57
|
-
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
58
|
-
# - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
|
|
59
|
-
# ‘-NaN’, ‘-nan’, ‘1.#IND’, ‘1.#QNAN’, ‘<NA>’, ‘N/A’,
|
|
60
|
-
# ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
|
|
61
|
-
reader_kwargs["na_values"] = ["na", "", "error", "NA", "-.-"]
|
|
62
|
-
|
|
63
|
-
##------------------------------------------------------------------------.
|
|
64
|
-
#### - Define dataframe sanitizer function for L0 processing
|
|
65
|
-
def df_sanitizer_fun(df):
|
|
66
|
-
# - Import pandas
|
|
67
|
-
import pandas as pd
|
|
68
|
-
|
|
69
|
-
# - Define 'time' datetime
|
|
70
|
-
df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
|
|
71
|
-
|
|
72
|
-
# - Split the 'TO_BE_SPLITTED' column
|
|
73
|
-
df = df["TO_BE_SPLITTED"].str.split(",", n=9, expand=True)
|
|
74
|
-
|
|
75
|
-
# - Assign column names
|
|
76
|
-
column_names = [
|
|
77
|
-
"station_name",
|
|
78
|
-
"sensor_status",
|
|
79
|
-
"sensor_temperature",
|
|
80
|
-
"number_particles",
|
|
81
|
-
"rainfall_rate_32bit",
|
|
82
|
-
"reflectivity_16bit",
|
|
83
|
-
"mor_visibility",
|
|
84
|
-
"weather_code_synop_4680",
|
|
85
|
-
"weather_code_synop_4677",
|
|
86
|
-
"raw_drop_number",
|
|
87
|
-
]
|
|
88
|
-
df.columns = column_names
|
|
89
|
-
|
|
90
|
-
# - Add the time column
|
|
91
|
-
df["time"] = df_time
|
|
92
|
-
|
|
93
|
-
# - Drop columns not agreeing with DISDRODB L0 standards
|
|
94
|
-
df = df.drop(columns=["station_name"])
|
|
95
|
-
|
|
96
|
-
# - Drop rows with invalid values
|
|
97
|
-
# --> Ensure that weather_code_synop_4677 has length 2
|
|
98
|
-
# --> If a previous column is missing it will have 000
|
|
99
|
-
df = df[df["weather_code_synop_4677"].str.len() == 2]
|
|
100
|
-
|
|
101
|
-
return df
|
|
102
|
-
|
|
103
|
-
##------------------------------------------------------------------------.
|
|
104
|
-
#### - Define glob pattern to search data files in <raw_dir>/data/<station_name>
|
|
105
|
-
glob_patterns = "*_raw.txt"
|
|
106
|
-
|
|
107
|
-
####----------------------------------------------------------------------.
|
|
108
|
-
#### - Create L0A products
|
|
109
|
-
run_l0a(
|
|
110
|
-
raw_dir=raw_dir,
|
|
111
|
-
processed_dir=processed_dir,
|
|
112
|
-
station_name=station_name,
|
|
113
|
-
# Custom arguments of the reader for L0A processing
|
|
114
|
-
glob_patterns=glob_patterns,
|
|
115
|
-
column_names=column_names,
|
|
116
|
-
reader_kwargs=reader_kwargs,
|
|
117
|
-
df_sanitizer_fun=df_sanitizer_fun,
|
|
118
|
-
# Processing options
|
|
119
|
-
force=force,
|
|
120
|
-
verbose=verbose,
|
|
121
|
-
parallel=parallel,
|
|
122
|
-
debugging_mode=debugging_mode,
|
|
123
|
-
)
|
|
@@ -1,123 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
# -----------------------------------------------------------------------------.
|
|
3
|
-
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
-
#
|
|
5
|
-
# This program is free software: you can redistribute it and/or modify
|
|
6
|
-
# it under the terms of the GNU General Public License as published by
|
|
7
|
-
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
-
# (at your option) any later version.
|
|
9
|
-
#
|
|
10
|
-
# This program is distributed in the hope that it will be useful,
|
|
11
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
-
# GNU General Public License for more details.
|
|
14
|
-
#
|
|
15
|
-
# You should have received a copy of the GNU General Public License
|
|
16
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
-
# -----------------------------------------------------------------------------.
|
|
18
|
-
from disdrodb.l0 import run_l0a
|
|
19
|
-
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
@is_documented_by(reader_generic_docstring)
|
|
23
|
-
def reader(
|
|
24
|
-
raw_dir,
|
|
25
|
-
processed_dir,
|
|
26
|
-
station_name,
|
|
27
|
-
# Processing options
|
|
28
|
-
force=False,
|
|
29
|
-
verbose=False,
|
|
30
|
-
parallel=False,
|
|
31
|
-
debugging_mode=False,
|
|
32
|
-
):
|
|
33
|
-
##------------------------------------------------------------------------.
|
|
34
|
-
#### - Define column names
|
|
35
|
-
column_names = ["time", "TO_BE_SPLITTED"]
|
|
36
|
-
|
|
37
|
-
##------------------------------------------------------------------------.
|
|
38
|
-
#### - Define reader options
|
|
39
|
-
reader_kwargs = {}
|
|
40
|
-
# - Define delimiter
|
|
41
|
-
reader_kwargs["delimiter"] = ";"
|
|
42
|
-
# - Skip first row as columns names
|
|
43
|
-
reader_kwargs["header"] = None
|
|
44
|
-
# - Skip file with encoding errors
|
|
45
|
-
reader_kwargs["encoding_errors"] = "ignore"
|
|
46
|
-
# - Avoid first column to become df index !!!
|
|
47
|
-
reader_kwargs["index_col"] = False
|
|
48
|
-
# - Define behaviour when encountering bad lines
|
|
49
|
-
reader_kwargs["on_bad_lines"] = "skip"
|
|
50
|
-
# - Define reader engine
|
|
51
|
-
# - C engine is faster
|
|
52
|
-
# - Python engine is more feature-complete
|
|
53
|
-
reader_kwargs["engine"] = "python"
|
|
54
|
-
# - Define on-the-fly decompression of on-disk data
|
|
55
|
-
# - Available: gzip, bz2, zip
|
|
56
|
-
reader_kwargs["compression"] = "infer"
|
|
57
|
-
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
58
|
-
# - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
|
|
59
|
-
# ‘-NaN’, ‘-nan’, ‘1.#IND’, ‘1.#QNAN’, ‘<NA>’, ‘N/A’,
|
|
60
|
-
# ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
|
|
61
|
-
reader_kwargs["na_values"] = ["na", "", "error", "NA", "-.-"]
|
|
62
|
-
|
|
63
|
-
##------------------------------------------------------------------------.
|
|
64
|
-
#### - Define dataframe sanitizer function for L0 processing
|
|
65
|
-
def df_sanitizer_fun(df):
|
|
66
|
-
# - Import pandas
|
|
67
|
-
import pandas as pd
|
|
68
|
-
|
|
69
|
-
# - Convert time column to datetime
|
|
70
|
-
df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
|
|
71
|
-
|
|
72
|
-
# - Split the 'TO_BE_SPLITTED' column
|
|
73
|
-
df = df["TO_BE_SPLITTED"].str.split(",", expand=True, n=9)
|
|
74
|
-
|
|
75
|
-
# - Assign column names
|
|
76
|
-
columns_names = [
|
|
77
|
-
"station_name",
|
|
78
|
-
"sensor_status",
|
|
79
|
-
"sensor_temperature",
|
|
80
|
-
"number_particles",
|
|
81
|
-
"rainfall_rate_32bit",
|
|
82
|
-
"reflectivity_16bit",
|
|
83
|
-
"mor_visibility",
|
|
84
|
-
"weather_code_synop_4680",
|
|
85
|
-
"weather_code_synop_4677",
|
|
86
|
-
"raw_drop_number",
|
|
87
|
-
]
|
|
88
|
-
df.columns = columns_names
|
|
89
|
-
|
|
90
|
-
# - Add the time column
|
|
91
|
-
df["time"] = df_time
|
|
92
|
-
|
|
93
|
-
# - Drop columns not agreeing with DISDRODB L0 standards
|
|
94
|
-
df = df.drop(columns=["station_name"])
|
|
95
|
-
|
|
96
|
-
# - Drop rows with invalid values
|
|
97
|
-
# --> Ensure that weather_code_synop_4677 has length 2
|
|
98
|
-
# --> If a previous column is missing it will have 000
|
|
99
|
-
df = df[df["weather_code_synop_4677"].str.len() == 2]
|
|
100
|
-
|
|
101
|
-
return df
|
|
102
|
-
|
|
103
|
-
##------------------------------------------------------------------------.
|
|
104
|
-
#### - Define glob pattern to search data files in <raw_dir>/data/<station_name>
|
|
105
|
-
glob_patterns = "*_raw.txt"
|
|
106
|
-
|
|
107
|
-
####----------------------------------------------------------------------.
|
|
108
|
-
#### - Create L0A products
|
|
109
|
-
run_l0a(
|
|
110
|
-
raw_dir=raw_dir,
|
|
111
|
-
processed_dir=processed_dir,
|
|
112
|
-
station_name=station_name,
|
|
113
|
-
# Custom arguments of the reader for L0A processing
|
|
114
|
-
glob_patterns=glob_patterns,
|
|
115
|
-
column_names=column_names,
|
|
116
|
-
reader_kwargs=reader_kwargs,
|
|
117
|
-
df_sanitizer_fun=df_sanitizer_fun,
|
|
118
|
-
# Processing options
|
|
119
|
-
force=force,
|
|
120
|
-
verbose=verbose,
|
|
121
|
-
parallel=parallel,
|
|
122
|
-
debugging_mode=debugging_mode,
|
|
123
|
-
)
|
disdrodb/l0/readers/GPM/MC3E.py
DELETED
|
@@ -1,123 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
# -----------------------------------------------------------------------------.
|
|
3
|
-
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
-
#
|
|
5
|
-
# This program is free software: you can redistribute it and/or modify
|
|
6
|
-
# it under the terms of the GNU General Public License as published by
|
|
7
|
-
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
-
# (at your option) any later version.
|
|
9
|
-
#
|
|
10
|
-
# This program is distributed in the hope that it will be useful,
|
|
11
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
-
# GNU General Public License for more details.
|
|
14
|
-
#
|
|
15
|
-
# You should have received a copy of the GNU General Public License
|
|
16
|
-
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
-
# -----------------------------------------------------------------------------.
|
|
18
|
-
from disdrodb.l0 import run_l0a
|
|
19
|
-
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
@is_documented_by(reader_generic_docstring)
|
|
23
|
-
def reader(
|
|
24
|
-
raw_dir,
|
|
25
|
-
processed_dir,
|
|
26
|
-
station_name,
|
|
27
|
-
# Processing options
|
|
28
|
-
force=False,
|
|
29
|
-
verbose=False,
|
|
30
|
-
parallel=False,
|
|
31
|
-
debugging_mode=False,
|
|
32
|
-
):
|
|
33
|
-
##------------------------------------------------------------------------.
|
|
34
|
-
#### - Define column name
|
|
35
|
-
column_names = ["time", "TO_BE_SPLITTED"]
|
|
36
|
-
|
|
37
|
-
##------------------------------------------------------------------------.
|
|
38
|
-
#### - Define reader options
|
|
39
|
-
reader_kwargs = {}
|
|
40
|
-
# - Define delimiter
|
|
41
|
-
reader_kwargs["delimiter"] = ";"
|
|
42
|
-
# - Skip first row as columns names
|
|
43
|
-
reader_kwargs["header"] = None
|
|
44
|
-
# - Skip file with encoding errors
|
|
45
|
-
reader_kwargs["encoding_errors"] = "ignore"
|
|
46
|
-
# - Avoid first column to become df index
|
|
47
|
-
reader_kwargs["index_col"] = False
|
|
48
|
-
# - Define behaviour when encountering bad lines
|
|
49
|
-
reader_kwargs["on_bad_lines"] = "skip"
|
|
50
|
-
# - Define reader engine
|
|
51
|
-
# - C engine is faster
|
|
52
|
-
# - Python engine is more feature-complete
|
|
53
|
-
reader_kwargs["engine"] = "python"
|
|
54
|
-
# - Define on-the-fly decompression of on-disk data
|
|
55
|
-
# - Available: gzip, bz2, zip
|
|
56
|
-
reader_kwargs["compression"] = "infer"
|
|
57
|
-
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
58
|
-
# - Already included: ‘#N/A’, ‘#N/A N/A’, ‘#NA’, ‘-1.#IND’, ‘-1.#QNAN’,
|
|
59
|
-
# ‘-NaN’, ‘-nan’, ‘1.#IND’, ‘1.#QNAN’, ‘<NA>’, ‘N/A’,
|
|
60
|
-
# ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
|
|
61
|
-
reader_kwargs["na_values"] = ["na", "", "error", "-.-"]
|
|
62
|
-
|
|
63
|
-
##------------------------------------------------------------------------.
|
|
64
|
-
#### - Define dataframe sanitizer function for L0 processing
|
|
65
|
-
def df_sanitizer_fun(df):
|
|
66
|
-
# - Import pandas
|
|
67
|
-
import pandas as pd
|
|
68
|
-
|
|
69
|
-
# - Define 'time' datetime
|
|
70
|
-
df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
|
|
71
|
-
|
|
72
|
-
# - Split the 'TO_BE_SPLITTED' column
|
|
73
|
-
df = df["TO_BE_SPLITTED"].str.split(",", n=9, expand=True)
|
|
74
|
-
|
|
75
|
-
# - Assign column names
|
|
76
|
-
column_names = [
|
|
77
|
-
"station_name",
|
|
78
|
-
"sensor_status",
|
|
79
|
-
"sensor_temperature",
|
|
80
|
-
"number_particles",
|
|
81
|
-
"rainfall_rate_32bit",
|
|
82
|
-
"reflectivity_16bit",
|
|
83
|
-
"mor_visibility",
|
|
84
|
-
"weather_code_synop_4680",
|
|
85
|
-
"weather_code_synop_4677",
|
|
86
|
-
"raw_drop_number",
|
|
87
|
-
]
|
|
88
|
-
df.columns = column_names
|
|
89
|
-
|
|
90
|
-
# - Add the time column
|
|
91
|
-
df["time"] = df_time
|
|
92
|
-
|
|
93
|
-
# - Drop columns not agreeing with DISDRODB L0 standards
|
|
94
|
-
df = df.drop(columns=["station_name"])
|
|
95
|
-
|
|
96
|
-
# - Drop rows with invalid values
|
|
97
|
-
# --> Ensure that weather_code_synop_4677 has length 2
|
|
98
|
-
# --> If a previous column is missing it will have 000
|
|
99
|
-
df = df[df["weather_code_synop_4677"].str.len() == 2]
|
|
100
|
-
|
|
101
|
-
return df
|
|
102
|
-
|
|
103
|
-
##------------------------------------------------------------------------.
|
|
104
|
-
#### - Define glob pattern to search data files in <raw_dir>/data/<station_name>
|
|
105
|
-
glob_patterns = "*_raw.txt"
|
|
106
|
-
|
|
107
|
-
####----------------------------------------------------------------------.
|
|
108
|
-
#### - Create L0A products
|
|
109
|
-
run_l0a(
|
|
110
|
-
raw_dir=raw_dir,
|
|
111
|
-
processed_dir=processed_dir,
|
|
112
|
-
station_name=station_name,
|
|
113
|
-
# Custom arguments of the reader for L0A processing
|
|
114
|
-
glob_patterns=glob_patterns,
|
|
115
|
-
column_names=column_names,
|
|
116
|
-
reader_kwargs=reader_kwargs,
|
|
117
|
-
df_sanitizer_fun=df_sanitizer_fun,
|
|
118
|
-
# Processing options
|
|
119
|
-
force=force,
|
|
120
|
-
verbose=verbose,
|
|
121
|
-
parallel=parallel,
|
|
122
|
-
debugging_mode=debugging_mode,
|
|
123
|
-
)
|