disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +132 -15
- disdrodb/_config.py +4 -2
- disdrodb/_version.py +9 -4
- disdrodb/api/checks.py +264 -237
- disdrodb/api/configs.py +4 -8
- disdrodb/api/create_directories.py +235 -290
- disdrodb/api/info.py +217 -26
- disdrodb/api/io.py +306 -270
- disdrodb/api/path.py +597 -173
- disdrodb/api/search.py +486 -0
- disdrodb/{metadata/scripts → cli}/disdrodb_check_metadata_archive.py +12 -7
- disdrodb/{utils/pandas.py → cli/disdrodb_data_archive_directory.py} +9 -18
- disdrodb/cli/disdrodb_download_archive.py +86 -0
- disdrodb/cli/disdrodb_download_metadata_archive.py +53 -0
- disdrodb/cli/disdrodb_download_station.py +84 -0
- disdrodb/{api/scripts → cli}/disdrodb_initialize_station.py +22 -10
- disdrodb/cli/disdrodb_metadata_archive_directory.py +32 -0
- disdrodb/{data_transfer/scripts/disdrodb_download_station.py → cli/disdrodb_open_data_archive.py} +22 -22
- disdrodb/cli/disdrodb_open_logs_directory.py +69 -0
- disdrodb/{data_transfer/scripts/disdrodb_upload_station.py → cli/disdrodb_open_metadata_archive.py} +22 -24
- disdrodb/cli/disdrodb_open_metadata_directory.py +71 -0
- disdrodb/cli/disdrodb_open_product_directory.py +74 -0
- disdrodb/cli/disdrodb_open_readers_directory.py +32 -0
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0.py +38 -31
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0_station.py +32 -30
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a_station.py +24 -33
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b_station.py +25 -34
- disdrodb/cli/disdrodb_run_l0c.py +130 -0
- disdrodb/cli/disdrodb_run_l0c_station.py +129 -0
- disdrodb/cli/disdrodb_run_l1.py +122 -0
- disdrodb/cli/disdrodb_run_l1_station.py +121 -0
- disdrodb/cli/disdrodb_run_l2e.py +122 -0
- disdrodb/cli/disdrodb_run_l2e_station.py +122 -0
- disdrodb/cli/disdrodb_run_l2m.py +122 -0
- disdrodb/cli/disdrodb_run_l2m_station.py +122 -0
- disdrodb/cli/disdrodb_upload_archive.py +105 -0
- disdrodb/cli/disdrodb_upload_station.py +98 -0
- disdrodb/configs.py +90 -25
- disdrodb/data_transfer/__init__.py +22 -0
- disdrodb/data_transfer/download_data.py +87 -90
- disdrodb/data_transfer/upload_data.py +64 -37
- disdrodb/data_transfer/zenodo.py +15 -18
- disdrodb/docs.py +1 -1
- disdrodb/issue/__init__.py +17 -4
- disdrodb/issue/checks.py +10 -23
- disdrodb/issue/reader.py +9 -12
- disdrodb/issue/writer.py +14 -17
- disdrodb/l0/__init__.py +17 -26
- disdrodb/l0/check_configs.py +35 -23
- disdrodb/l0/check_standards.py +46 -51
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_diameter.yml +44 -44
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_velocity.yml +40 -40
- disdrodb/l0/configs/LPM/l0a_encodings.yml +80 -0
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_cf_attrs.yml +84 -65
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_encodings.yml +50 -9
- disdrodb/l0/configs/{Thies_LPM → LPM}/raw_data_format.yml +285 -245
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_diameter.yml +66 -66
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL/l0a_encodings.yml +32 -0
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_cf_attrs.yml +23 -21
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_encodings.yml +17 -17
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/raw_data_format.yml +77 -77
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_diameter.yml +64 -64
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +39 -0
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_cf_attrs.yml +28 -26
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_encodings.yml +20 -20
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/raw_data_format.yml +107 -107
- disdrodb/l0/configs/PWS100/bins_diameter.yml +173 -0
- disdrodb/l0/configs/PWS100/bins_velocity.yml +173 -0
- disdrodb/l0/configs/PWS100/l0a_encodings.yml +19 -0
- disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +76 -0
- disdrodb/l0/configs/PWS100/l0b_encodings.yml +176 -0
- disdrodb/l0/configs/PWS100/raw_data_format.yml +182 -0
- disdrodb/l0/configs/{RD_80 → RD80}/bins_diameter.yml +40 -40
- disdrodb/l0/configs/RD80/l0a_encodings.yml +16 -0
- disdrodb/l0/configs/{RD_80 → RD80}/l0b_cf_attrs.yml +3 -3
- disdrodb/l0/configs/RD80/l0b_encodings.yml +135 -0
- disdrodb/l0/configs/{RD_80 → RD80}/raw_data_format.yml +46 -50
- disdrodb/l0/l0_reader.py +216 -340
- disdrodb/l0/l0a_processing.py +237 -208
- disdrodb/l0/l0b_nc_processing.py +227 -80
- disdrodb/l0/l0b_processing.py +96 -174
- disdrodb/l0/l0c_processing.py +627 -0
- disdrodb/l0/readers/{ARM → LPM/ARM}/ARM_LPM.py +36 -58
- disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +236 -0
- disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +185 -0
- disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +185 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +195 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +210 -0
- disdrodb/l0/readers/{BRAZIL/GOAMAZON_LPM.py → LPM/KIT/CHWALA.py} +97 -76
- disdrodb/l0/readers/LPM/SLOVENIA/ARSO.py +197 -0
- disdrodb/l0/readers/LPM/SLOVENIA/CRNI_VRH.py +197 -0
- disdrodb/l0/readers/{UK → LPM/UK}/DIVEN.py +14 -35
- disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +157 -0
- disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +113 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/ARCTIC_2021.py +40 -57
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/COMMON_2011.py +37 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/DAVOS_2009_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_2009.py +34 -51
- disdrodb/l0/readers/{EPFL/PARADISO_2014.py → PARSIVEL/EPFL/EPFL_ROOF_2008.py} +38 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2012.py +33 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GENEPI_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007_2.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HPICONET_2010.py +34 -51
- disdrodb/l0/readers/{EPFL/EPFL_ROOF_2010.py → PARSIVEL/EPFL/HYMEX_LTE_SOP2.py} +37 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +111 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HYMEX_LTE_SOP4.py +36 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2018.py +34 -52
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2019.py +38 -56
- disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PARSIVEL_2007.py +27 -45
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PLATO_2019.py +24 -44
- disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +140 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RACLETS_2019_WJF.py +41 -59
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RIETHOLZBACH_2011.py +34 -51
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +117 -0
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +137 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/UNIL_2022.py +42 -55
- disdrodb/l0/readers/PARSIVEL/GPM/IFLOODS.py +104 -0
- disdrodb/l0/readers/{GPM → PARSIVEL/GPM}/LPVEX.py +29 -48
- disdrodb/l0/readers/PARSIVEL/GPM/MC3E.py +184 -0
- disdrodb/l0/readers/PARSIVEL/KIT/BURKINA_FASO.py +133 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +113 -0
- disdrodb/l0/readers/{NCAR/VORTEX_SE_2016_P1.py → PARSIVEL/NCAR/OWLES_MIPS.py} +46 -72
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +125 -0
- disdrodb/l0/readers/{NCAR/OWLES_MIPS.py → PARSIVEL/NCAR/PLOWS_MIPS.py} +45 -64
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +114 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +176 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +183 -0
- disdrodb/l0/readers/PARSIVEL/SLOVENIA/UL_FGG.py +121 -0
- disdrodb/l0/readers/{ARM/ARM_LD.py → PARSIVEL2/ARM/ARM_PARSIVEL2.py} +27 -50
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/{DENMARK → PARSIVEL2/DENMARK}/EROSION_nc.py +14 -35
- disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +189 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +119 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +104 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/NSSTC.py +176 -0
- disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +32 -0
- disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +56 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +120 -0
- disdrodb/l0/readers/{NCAR → PARSIVEL2/NCAR}/PECAN_MIPS.py +45 -64
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +181 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +160 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +160 -0
- disdrodb/l0/readers/{NCAR/PLOWS_MIPS.py → PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py} +49 -66
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +118 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +152 -0
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT.py +166 -0
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +150 -0
- disdrodb/l0/readers/{NCAR/RELAMPAGO_RD80.py → RD80/BRAZIL/CHUVA_RD80.py} +36 -60
- disdrodb/l0/readers/{BRAZIL → RD80/BRAZIL}/GOAMAZON_RD80.py +36 -55
- disdrodb/l0/readers/{NCAR → RD80/NCAR}/CINDY_2011_RD80.py +35 -54
- disdrodb/l0/readers/{BRAZIL/CHUVA_RD80.py → RD80/NCAR/RELAMPAGO_RD80.py} +40 -54
- disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py +274 -0
- disdrodb/l0/readers/template_reader_raw_netcdf_data.py +62 -0
- disdrodb/l0/readers/{reader_template.py → template_reader_raw_text_data.py} +20 -44
- disdrodb/l0/routines.py +885 -581
- disdrodb/l0/standards.py +77 -238
- disdrodb/l0/template_tools.py +105 -110
- disdrodb/l1/__init__.py +17 -0
- disdrodb/l1/beard_model.py +716 -0
- disdrodb/l1/encoding_attrs.py +635 -0
- disdrodb/l1/fall_velocity.py +260 -0
- disdrodb/l1/filters.py +192 -0
- disdrodb/l1/processing.py +202 -0
- disdrodb/l1/resampling.py +236 -0
- disdrodb/l1/routines.py +358 -0
- disdrodb/l1_env/__init__.py +17 -0
- disdrodb/l1_env/routines.py +38 -0
- disdrodb/l2/__init__.py +17 -0
- disdrodb/l2/empirical_dsd.py +1833 -0
- disdrodb/l2/event.py +388 -0
- disdrodb/l2/processing.py +528 -0
- disdrodb/l2/processing_options.py +213 -0
- disdrodb/l2/routines.py +868 -0
- disdrodb/metadata/__init__.py +9 -2
- disdrodb/metadata/checks.py +180 -124
- disdrodb/metadata/download.py +81 -0
- disdrodb/metadata/geolocation.py +146 -0
- disdrodb/metadata/info.py +20 -13
- disdrodb/metadata/manipulation.py +3 -3
- disdrodb/metadata/reader.py +59 -8
- disdrodb/metadata/search.py +77 -144
- disdrodb/metadata/standards.py +83 -80
- disdrodb/metadata/writer.py +10 -16
- disdrodb/psd/__init__.py +38 -0
- disdrodb/psd/fitting.py +2146 -0
- disdrodb/psd/models.py +774 -0
- disdrodb/routines.py +1412 -0
- disdrodb/scattering/__init__.py +28 -0
- disdrodb/scattering/axis_ratio.py +344 -0
- disdrodb/scattering/routines.py +456 -0
- disdrodb/utils/__init__.py +17 -0
- disdrodb/utils/attrs.py +208 -0
- disdrodb/utils/cli.py +269 -0
- disdrodb/utils/compression.py +60 -42
- disdrodb/utils/dask.py +62 -0
- disdrodb/utils/dataframe.py +342 -0
- disdrodb/utils/decorators.py +110 -0
- disdrodb/utils/directories.py +107 -46
- disdrodb/utils/encoding.py +127 -0
- disdrodb/utils/list.py +29 -0
- disdrodb/utils/logger.py +168 -46
- disdrodb/utils/time.py +657 -0
- disdrodb/utils/warnings.py +30 -0
- disdrodb/utils/writer.py +57 -0
- disdrodb/utils/xarray.py +138 -47
- disdrodb/utils/yaml.py +0 -1
- disdrodb/viz/__init__.py +17 -0
- disdrodb/viz/plots.py +17 -0
- disdrodb-0.1.1.dist-info/METADATA +294 -0
- disdrodb-0.1.1.dist-info/RECORD +232 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/WHEEL +1 -1
- disdrodb-0.1.1.dist-info/entry_points.txt +30 -0
- disdrodb/data_transfer/scripts/disdrodb_download_archive.py +0 -53
- disdrodb/data_transfer/scripts/disdrodb_upload_archive.py +0 -57
- disdrodb/l0/configs/OTT_Parsivel/l0a_encodings.yml +0 -32
- disdrodb/l0/configs/OTT_Parsivel2/l0a_encodings.yml +0 -39
- disdrodb/l0/configs/RD_80/l0a_encodings.yml +0 -16
- disdrodb/l0/configs/RD_80/l0b_encodings.yml +0 -135
- disdrodb/l0/configs/Thies_LPM/l0a_encodings.yml +0 -80
- disdrodb/l0/io.py +0 -257
- disdrodb/l0/l0_processing.py +0 -1091
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_OTT.py +0 -178
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_THIES.py +0 -247
- disdrodb/l0/readers/BRAZIL/CHUVA_LPM.py +0 -204
- disdrodb/l0/readers/BRAZIL/CHUVA_OTT.py +0 -183
- disdrodb/l0/readers/BRAZIL/GOAMAZON_OTT.py +0 -183
- disdrodb/l0/readers/CHINA/CHONGQING.py +0 -131
- disdrodb/l0/readers/EPFL/EPFL_ROOF_2008.py +0 -128
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP2.py +0 -127
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP3.py +0 -129
- disdrodb/l0/readers/EPFL/RACLETS_2019.py +0 -158
- disdrodb/l0/readers/EPFL/SAMOYLOV_2017.py +0 -136
- disdrodb/l0/readers/EPFL/SAMOYLOV_2019.py +0 -158
- disdrodb/l0/readers/FRANCE/SIRTA_OTT2.py +0 -138
- disdrodb/l0/readers/GPM/GCPEX.py +0 -123
- disdrodb/l0/readers/GPM/IFLOODS.py +0 -123
- disdrodb/l0/readers/GPM/MC3E.py +0 -123
- disdrodb/l0/readers/GPM/NSSTC.py +0 -164
- disdrodb/l0/readers/ITALY/GID.py +0 -199
- disdrodb/l0/readers/MEXICO/OH_IIUNAM_nc.py +0 -92
- disdrodb/l0/readers/NCAR/CCOPE_2015.py +0 -133
- disdrodb/l0/readers/NCAR/PECAN_FP3.py +0 -137
- disdrodb/l0/readers/NCAR/PECAN_MOBILE.py +0 -144
- disdrodb/l0/readers/NCAR/RELAMPAGO_OTT.py +0 -195
- disdrodb/l0/readers/NCAR/SNOWIE_PJ.py +0 -172
- disdrodb/l0/readers/NCAR/SNOWIE_SB.py +0 -179
- disdrodb/l0/readers/NCAR/VORTEX2_2009.py +0 -133
- disdrodb/l0/readers/NCAR/VORTEX2_2010.py +0 -188
- disdrodb/l0/readers/NCAR/VORTEX2_2010_UF.py +0 -191
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_P2.py +0 -135
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_PIPS.py +0 -170
- disdrodb/l0/readers/NETHERLANDS/DELFT.py +0 -187
- disdrodb/l0/readers/SPAIN/SBEGUERIA.py +0 -179
- disdrodb/l0/scripts/disdrodb_run_l0b_concat.py +0 -93
- disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py +0 -85
- disdrodb/utils/netcdf.py +0 -452
- disdrodb/utils/scripts.py +0 -102
- disdrodb-0.0.21.dist-info/AUTHORS.md +0 -18
- disdrodb-0.0.21.dist-info/METADATA +0 -186
- disdrodb-0.0.21.dist-info/RECORD +0 -168
- disdrodb-0.0.21.dist-info/entry_points.txt +0 -15
- /disdrodb/l0/configs/{RD_80 → RD80}/bins_velocity.yml +0 -0
- /disdrodb/l0/manuals/{Thies_LPM.pdf → LPM.pdf} +0 -0
- /disdrodb/l0/manuals/{ODM_470.pdf → ODM470.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel.pdf → PARSIVEL.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel2.pdf → PARSIVEL2.pdf} +0 -0
- /disdrodb/l0/manuals/{PWS_100.pdf → PWS100.pdf} +0 -0
- /disdrodb/l0/manuals/{RD_80.pdf → RD80.pdf} +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info/licenses}/LICENSE +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
20
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
21
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@is_documented_by(reader_generic_docstring)
|
|
25
|
+
def reader(
|
|
26
|
+
filepath,
|
|
27
|
+
logger=None,
|
|
28
|
+
):
|
|
29
|
+
"""Reader."""
|
|
30
|
+
##------------------------------------------------------------------------.
|
|
31
|
+
#### Define column names
|
|
32
|
+
column_names = ["TO_SPLIT"]
|
|
33
|
+
|
|
34
|
+
##------------------------------------------------------------------------.
|
|
35
|
+
#### Define reader options
|
|
36
|
+
reader_kwargs = {}
|
|
37
|
+
|
|
38
|
+
# - Define delimiter
|
|
39
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
40
|
+
|
|
41
|
+
# - Skip first row as columns names
|
|
42
|
+
reader_kwargs["header"] = None
|
|
43
|
+
|
|
44
|
+
# - Skip header
|
|
45
|
+
reader_kwargs["skiprows"] = 7
|
|
46
|
+
|
|
47
|
+
# - Define encoding
|
|
48
|
+
reader_kwargs["encoding"] = "ISO-8859-1"
|
|
49
|
+
|
|
50
|
+
# - Avoid first column to become df index !!!
|
|
51
|
+
reader_kwargs["index_col"] = False
|
|
52
|
+
|
|
53
|
+
# - Define behaviour when encountering bad lines
|
|
54
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
55
|
+
|
|
56
|
+
# - Define reader engine
|
|
57
|
+
# - C engine is faster
|
|
58
|
+
# - Python engine is more feature-complete
|
|
59
|
+
reader_kwargs["engine"] = "python"
|
|
60
|
+
|
|
61
|
+
# - Define on-the-fly decompression of on-disk data
|
|
62
|
+
# - Available: gzip, bz2, zip
|
|
63
|
+
# reader_kwargs['compression'] = 'xz'
|
|
64
|
+
|
|
65
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
66
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
67
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
68
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
69
|
+
reader_kwargs["na_values"] = ["na", "error", "-.-", " NA"]
|
|
70
|
+
|
|
71
|
+
##------------------------------------------------------------------------.
|
|
72
|
+
#### Read the data
|
|
73
|
+
df = read_raw_text_file(
|
|
74
|
+
filepath=filepath,
|
|
75
|
+
column_names=column_names,
|
|
76
|
+
reader_kwargs=reader_kwargs,
|
|
77
|
+
logger=logger,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
##------------------------------------------------------------------------.
|
|
81
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
82
|
+
# Remove repeated headers (with length 301)
|
|
83
|
+
# df = df[df["TO_SPLIT"].str.len() != 301]
|
|
84
|
+
|
|
85
|
+
# Split into columns and assign name
|
|
86
|
+
df = df["TO_SPLIT"].str.split(";", expand=True, n=16)
|
|
87
|
+
columns = [
|
|
88
|
+
"date",
|
|
89
|
+
"time",
|
|
90
|
+
"rainfall_rate_32bit",
|
|
91
|
+
"rainfall_accumulated_32bit",
|
|
92
|
+
"weather_code_synop_4680",
|
|
93
|
+
"weather_code_metar_4678",
|
|
94
|
+
"weather_code_nws",
|
|
95
|
+
"reflectivity_32bit",
|
|
96
|
+
"mor_visibility",
|
|
97
|
+
"laser_amplitude",
|
|
98
|
+
"number_particles",
|
|
99
|
+
"sensor_temperature",
|
|
100
|
+
"sensor_status",
|
|
101
|
+
"sensor_battery_voltage",
|
|
102
|
+
"unknown", # "rainfall_amount_absolute_32bit", ?
|
|
103
|
+
"error_code",
|
|
104
|
+
"raw_drop_number",
|
|
105
|
+
]
|
|
106
|
+
df.columns = columns
|
|
107
|
+
|
|
108
|
+
# Add datetime time column
|
|
109
|
+
df["time"] = df["date"] + "-" + df["time"]
|
|
110
|
+
df["time"] = pd.to_datetime(df["time"], format="%d.%m.%Y-%H:%M:%S", errors="coerce")
|
|
111
|
+
df = df.drop(columns=["date"])
|
|
112
|
+
|
|
113
|
+
# Preprocess the raw spectrum
|
|
114
|
+
# - The '<SPECTRUM>ZERO</SPECTRUM>' indicates no drops detected
|
|
115
|
+
# --> "" generates an array of zeros in L0B processing
|
|
116
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
|
|
117
|
+
|
|
118
|
+
# Remove <SPECTRUM> and </SPECTRUM>" acronyms from the raw_drop_number field
|
|
119
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
|
|
120
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
|
|
121
|
+
|
|
122
|
+
# Add 0 before every , if , not preceded by a digit
|
|
123
|
+
# Example: ',,1,,' --> '0,0,1,0,'
|
|
124
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace(r"(?<!\d);", "0;", regex=True)
|
|
125
|
+
|
|
126
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
127
|
+
columns_to_drop = [
|
|
128
|
+
"unknown",
|
|
129
|
+
]
|
|
130
|
+
df = df.drop(columns=columns_to_drop)
|
|
131
|
+
|
|
132
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
133
|
+
return df
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
20
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
21
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@is_documented_by(reader_generic_docstring)
|
|
25
|
+
def reader(
|
|
26
|
+
filepath,
|
|
27
|
+
logger=None,
|
|
28
|
+
):
|
|
29
|
+
"""Reader."""
|
|
30
|
+
##------------------------------------------------------------------------.
|
|
31
|
+
#### Define column names
|
|
32
|
+
column_names = ["TO_PARSE"]
|
|
33
|
+
|
|
34
|
+
##------------------------------------------------------------------------.
|
|
35
|
+
#### Define reader options
|
|
36
|
+
reader_kwargs = {}
|
|
37
|
+
# - Define delimiter
|
|
38
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
39
|
+
|
|
40
|
+
# Skip first row as columns names
|
|
41
|
+
reader_kwargs["header"] = None
|
|
42
|
+
|
|
43
|
+
# - Avoid first column to become df index !!!
|
|
44
|
+
reader_kwargs["index_col"] = False
|
|
45
|
+
|
|
46
|
+
# - Define behaviour when encountering bad lines
|
|
47
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
48
|
+
|
|
49
|
+
# - Define encoding
|
|
50
|
+
reader_kwargs["encoding"] = "ISO-8859-1"
|
|
51
|
+
|
|
52
|
+
# - Define reader engine
|
|
53
|
+
# - C engine is faster
|
|
54
|
+
# - Python engine is more feature-complete
|
|
55
|
+
reader_kwargs["engine"] = "python"
|
|
56
|
+
|
|
57
|
+
# - Define on-the-fly decompression of on-disk data
|
|
58
|
+
# - Available: gzip, bz2, zip
|
|
59
|
+
reader_kwargs["compression"] = "infer"
|
|
60
|
+
|
|
61
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
62
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
63
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
64
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
65
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
66
|
+
|
|
67
|
+
##------------------------------------------------------------------------.
|
|
68
|
+
#### Read the data
|
|
69
|
+
df = read_raw_text_file(
|
|
70
|
+
filepath=filepath,
|
|
71
|
+
column_names=column_names,
|
|
72
|
+
reader_kwargs=reader_kwargs,
|
|
73
|
+
logger=logger,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
##------------------------------------------------------------------------.
|
|
77
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
78
|
+
# Retrieve file start time (date hour:minute)
|
|
79
|
+
start_time = df["TO_PARSE"].iloc[0]
|
|
80
|
+
start_time = start_time[0:16]
|
|
81
|
+
start_time = pd.to_datetime(start_time, format="%m/%d/%Y %H:%M", errors="coerce")
|
|
82
|
+
df = df.iloc[1:] # remove start_time row
|
|
83
|
+
|
|
84
|
+
# Replace heterogeneous number of spaces with ;
|
|
85
|
+
df["TO_PARSE"] = df["TO_PARSE"].str.replace(r" +", ";", regex=True)
|
|
86
|
+
|
|
87
|
+
# Split into columns and assign name
|
|
88
|
+
df = df["TO_PARSE"].str.split(";", expand=True)
|
|
89
|
+
columns = [
|
|
90
|
+
"MMSSmmm",
|
|
91
|
+
"rainfall_rate_32bit",
|
|
92
|
+
"rainfall_accumulated_32bit",
|
|
93
|
+
"reflectivity_32bit",
|
|
94
|
+
"number_particles",
|
|
95
|
+
"sensor_status",
|
|
96
|
+
"error_code",
|
|
97
|
+
"raw_drop_concentration",
|
|
98
|
+
"raw_drop_average_velocity",
|
|
99
|
+
"raw_drop_number",
|
|
100
|
+
]
|
|
101
|
+
df.columns = columns
|
|
102
|
+
|
|
103
|
+
# Define datetime 'time' column
|
|
104
|
+
dt_minute = df["MMSSmmm"].str[0:2].astype(int).astype("<m8[m]")
|
|
105
|
+
dt_second = df["MMSSmmm"].str[2:4].astype(int).astype("<m8[s]")
|
|
106
|
+
df_time = start_time + dt_minute + dt_second
|
|
107
|
+
df["time"] = df_time
|
|
108
|
+
|
|
109
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
110
|
+
df = df.drop(columns=["MMSSmmm"])
|
|
111
|
+
|
|
112
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
113
|
+
return df
|
|
@@ -1,11 +1,4 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
-
"""
|
|
3
|
-
Created on Fri Feb 17 20:47:10 2023
|
|
4
|
-
|
|
5
|
-
@author: ghiggi
|
|
6
|
-
"""
|
|
7
|
-
#!/usr/bin/env python3
|
|
8
|
-
# -*- coding: utf-8 -*-
|
|
9
2
|
# -----------------------------------------------------------------------------.
|
|
10
3
|
# Copyright (c) 2021-2023 DISDRODB developers
|
|
11
4
|
#
|
|
@@ -22,23 +15,20 @@ Created on Fri Feb 17 20:47:10 2023
|
|
|
22
15
|
# You should have received a copy of the GNU General Public License
|
|
23
16
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
24
17
|
# -----------------------------------------------------------------------------.
|
|
25
|
-
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
26
20
|
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
21
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
27
22
|
|
|
28
23
|
|
|
29
24
|
@is_documented_by(reader_generic_docstring)
|
|
30
25
|
def reader(
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
station_name,
|
|
34
|
-
# Processing options
|
|
35
|
-
force=False,
|
|
36
|
-
verbose=False,
|
|
37
|
-
parallel=False,
|
|
38
|
-
debugging_mode=False,
|
|
26
|
+
filepath,
|
|
27
|
+
logger=None,
|
|
39
28
|
):
|
|
29
|
+
"""Reader."""
|
|
40
30
|
##------------------------------------------------------------------------.
|
|
41
|
-
####
|
|
31
|
+
#### Define column names
|
|
42
32
|
column_names = [
|
|
43
33
|
"day",
|
|
44
34
|
"month",
|
|
@@ -57,7 +47,7 @@ def reader(
|
|
|
57
47
|
]
|
|
58
48
|
|
|
59
49
|
##------------------------------------------------------------------------.
|
|
60
|
-
####
|
|
50
|
+
#### Define reader options
|
|
61
51
|
reader_kwargs = {}
|
|
62
52
|
# - Define delimiter
|
|
63
53
|
reader_kwargs["delimiter"] = ","
|
|
@@ -68,9 +58,6 @@ def reader(
|
|
|
68
58
|
# - Define behaviour when encountering bad lines
|
|
69
59
|
reader_kwargs["on_bad_lines"] = "skip"
|
|
70
60
|
|
|
71
|
-
# - Define encoding
|
|
72
|
-
reader_kwargs["encoding"] = "ISO-8859-1"
|
|
73
|
-
|
|
74
61
|
# - Define reader engine
|
|
75
62
|
# - C engine is faster
|
|
76
63
|
# - Python engine is more feature-complete
|
|
@@ -81,64 +68,51 @@ def reader(
|
|
|
81
68
|
reader_kwargs["compression"] = "infer"
|
|
82
69
|
|
|
83
70
|
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
84
|
-
# - Already included:
|
|
85
|
-
#
|
|
86
|
-
#
|
|
71
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
72
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
73
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
87
74
|
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
88
75
|
|
|
89
76
|
# Skip first row as columns names
|
|
90
77
|
reader_kwargs["header"] = None
|
|
91
78
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
def df_sanitizer_fun(df):
|
|
95
|
-
# - Import pandas
|
|
96
|
-
import pandas as pd
|
|
97
|
-
|
|
98
|
-
# - Define datetime 'time' column
|
|
99
|
-
df["time"] = (
|
|
100
|
-
df["day"].astype(str)
|
|
101
|
-
+ "-"
|
|
102
|
-
+ df["month"].astype(str)
|
|
103
|
-
+ "-"
|
|
104
|
-
+ df["year"].astype(str)
|
|
105
|
-
+ " "
|
|
106
|
-
+ df["hour_minute"].astype(str)
|
|
107
|
-
+ ":"
|
|
108
|
-
+ df["second"].astype(str)
|
|
109
|
-
)
|
|
110
|
-
df["time"] = pd.to_datetime(df["time"], format="%d-%m-%Y %H%M:%S")
|
|
111
|
-
|
|
112
|
-
# - Drop columns not agreeing with DISDRODB L0 standards
|
|
113
|
-
columns_to_drop = [
|
|
114
|
-
"day",
|
|
115
|
-
"month",
|
|
116
|
-
"year",
|
|
117
|
-
"hour_minute",
|
|
118
|
-
"second",
|
|
119
|
-
]
|
|
120
|
-
df = df.drop(columns=columns_to_drop)
|
|
121
|
-
|
|
122
|
-
return df
|
|
79
|
+
# - Define encoding
|
|
80
|
+
reader_kwargs["encoding"] = "ISO-8859-1"
|
|
123
81
|
|
|
124
82
|
##------------------------------------------------------------------------.
|
|
125
|
-
####
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
####----------------------------------------------------------------------.
|
|
129
|
-
#### - Create L0A products
|
|
130
|
-
run_l0a(
|
|
131
|
-
raw_dir=raw_dir,
|
|
132
|
-
processed_dir=processed_dir,
|
|
133
|
-
station_name=station_name,
|
|
134
|
-
# Custom arguments of the reader for L0A processing
|
|
135
|
-
glob_patterns=glob_patterns,
|
|
83
|
+
#### Read the data
|
|
84
|
+
df = read_raw_text_file(
|
|
85
|
+
filepath=filepath,
|
|
136
86
|
column_names=column_names,
|
|
137
87
|
reader_kwargs=reader_kwargs,
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
88
|
+
logger=logger,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
##------------------------------------------------------------------------.
|
|
92
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
93
|
+
# Define datetime time column
|
|
94
|
+
df["time"] = (
|
|
95
|
+
df["day"].astype(str)
|
|
96
|
+
+ "-"
|
|
97
|
+
+ df["month"].astype(str)
|
|
98
|
+
+ "-"
|
|
99
|
+
+ df["year"].astype(str)
|
|
100
|
+
+ " "
|
|
101
|
+
+ df["hour_minute"].astype(str)
|
|
102
|
+
+ ":"
|
|
103
|
+
+ df["second"].astype(str)
|
|
144
104
|
)
|
|
105
|
+
df["time"] = pd.to_datetime(df["time"], format="%d-%m-%Y %H%M:%S", errors="coerce")
|
|
106
|
+
|
|
107
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
108
|
+
columns_to_drop = [
|
|
109
|
+
"day",
|
|
110
|
+
"month",
|
|
111
|
+
"year",
|
|
112
|
+
"hour_minute",
|
|
113
|
+
"second",
|
|
114
|
+
]
|
|
115
|
+
df = df.drop(columns=columns_to_drop)
|
|
116
|
+
|
|
117
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
118
|
+
return df
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
20
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
21
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@is_documented_by(reader_generic_docstring)
|
|
25
|
+
def reader(
|
|
26
|
+
filepath,
|
|
27
|
+
logger=None,
|
|
28
|
+
):
|
|
29
|
+
"""Reader."""
|
|
30
|
+
##------------------------------------------------------------------------.
|
|
31
|
+
#### Define column names
|
|
32
|
+
column_names = ["TO_SPLIT"]
|
|
33
|
+
|
|
34
|
+
##------------------------------------------------------------------------.
|
|
35
|
+
#### Define reader options
|
|
36
|
+
reader_kwargs = {}
|
|
37
|
+
|
|
38
|
+
# - Define delimiter
|
|
39
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
40
|
+
|
|
41
|
+
# - Skip first row as columns names
|
|
42
|
+
reader_kwargs["header"] = None
|
|
43
|
+
|
|
44
|
+
# - Skip header
|
|
45
|
+
reader_kwargs["skiprows"] = 7
|
|
46
|
+
|
|
47
|
+
# - Define encoding
|
|
48
|
+
reader_kwargs["encoding"] = "ISO-8859-1"
|
|
49
|
+
|
|
50
|
+
# - Avoid first column to become df index !!!
|
|
51
|
+
reader_kwargs["index_col"] = False
|
|
52
|
+
|
|
53
|
+
# - Define behaviour when encountering bad lines
|
|
54
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
55
|
+
|
|
56
|
+
# - Define reader engine
|
|
57
|
+
# - C engine is faster
|
|
58
|
+
# - Python engine is more feature-complete
|
|
59
|
+
reader_kwargs["engine"] = "python"
|
|
60
|
+
|
|
61
|
+
# - Define on-the-fly decompression of on-disk data
|
|
62
|
+
# - Available: gzip, bz2, zip
|
|
63
|
+
# reader_kwargs['compression'] = 'xz'
|
|
64
|
+
|
|
65
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
66
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
67
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
68
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
69
|
+
reader_kwargs["na_values"] = ["na", "error", "-.-", " NA"]
|
|
70
|
+
|
|
71
|
+
##------------------------------------------------------------------------.
|
|
72
|
+
#### Read the data
|
|
73
|
+
df = read_raw_text_file(
|
|
74
|
+
filepath=filepath,
|
|
75
|
+
column_names=column_names,
|
|
76
|
+
reader_kwargs=reader_kwargs,
|
|
77
|
+
logger=logger,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
##------------------------------------------------------------------------.
|
|
81
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
82
|
+
# Remove repeated headers (with length 301)
|
|
83
|
+
df = df[df["TO_SPLIT"].str.len() != 301]
|
|
84
|
+
|
|
85
|
+
# Split into columns and assign name
|
|
86
|
+
df = df["TO_SPLIT"].str.split(",", expand=True, n=14)
|
|
87
|
+
columns = [
|
|
88
|
+
"date",
|
|
89
|
+
"time",
|
|
90
|
+
"rainfall_rate_32bit",
|
|
91
|
+
"rainfall_accumulated_32bit",
|
|
92
|
+
"weather_code_synop_4680",
|
|
93
|
+
"reflectivity_32bit",
|
|
94
|
+
"mor_visibility",
|
|
95
|
+
"laser_amplitude",
|
|
96
|
+
"sensor_temperature",
|
|
97
|
+
"weather_code_metar_4678",
|
|
98
|
+
"weather_code_nws",
|
|
99
|
+
"number_particles",
|
|
100
|
+
"sensor_heating_current",
|
|
101
|
+
"sensor_battery_voltage",
|
|
102
|
+
"raw_drop_number",
|
|
103
|
+
]
|
|
104
|
+
df.columns = columns
|
|
105
|
+
|
|
106
|
+
# Add datetime time column
|
|
107
|
+
df["time"] = df["date"] + "-" + df["time"]
|
|
108
|
+
df["time"] = pd.to_datetime(df["time"], format="%d.%m.%Y-%H:%M:%S", errors="coerce")
|
|
109
|
+
df = df.drop(columns=["date"])
|
|
110
|
+
|
|
111
|
+
# Preprocess the raw spectrum
|
|
112
|
+
# - The '<SPECTRUM>ZERO</SPECTRUM>' indicates no drops detected
|
|
113
|
+
# --> "" generates an array of zeros in L0B processing
|
|
114
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
|
|
115
|
+
|
|
116
|
+
# Remove <SPECTRUM> and </SPECTRUM>" acronyms from the raw_drop_number field
|
|
117
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
|
|
118
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
|
|
119
|
+
|
|
120
|
+
# Add 0 before every , if , not preceded by a digit
|
|
121
|
+
# Example: ',,1,,' --> '0,0,1,0,'
|
|
122
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace(r"(?<!\d),", "0,", regex=True)
|
|
123
|
+
|
|
124
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
125
|
+
return df
|
|
@@ -15,23 +15,20 @@
|
|
|
15
15
|
# You should have received a copy of the GNU General Public License
|
|
16
16
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
17
|
# -----------------------------------------------------------------------------.
|
|
18
|
-
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
19
20
|
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
21
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
20
22
|
|
|
21
23
|
|
|
22
24
|
@is_documented_by(reader_generic_docstring)
|
|
23
25
|
def reader(
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
station_name,
|
|
27
|
-
# Processing options
|
|
28
|
-
force=False,
|
|
29
|
-
verbose=False,
|
|
30
|
-
parallel=False,
|
|
31
|
-
debugging_mode=False,
|
|
26
|
+
filepath,
|
|
27
|
+
logger=None,
|
|
32
28
|
):
|
|
33
|
-
|
|
34
|
-
|
|
29
|
+
"""Reader."""
|
|
30
|
+
##------------------------------------------------------------------------.
|
|
31
|
+
#### Define column names
|
|
35
32
|
column_names = [
|
|
36
33
|
"day",
|
|
37
34
|
"month",
|
|
@@ -50,7 +47,7 @@ def reader(
|
|
|
50
47
|
]
|
|
51
48
|
|
|
52
49
|
##------------------------------------------------------------------------.
|
|
53
|
-
####
|
|
50
|
+
#### Define reader options
|
|
54
51
|
reader_kwargs = {}
|
|
55
52
|
# - Define delimiter
|
|
56
53
|
reader_kwargs["delimiter"] = ","
|
|
@@ -71,9 +68,9 @@ def reader(
|
|
|
71
68
|
reader_kwargs["compression"] = "infer"
|
|
72
69
|
|
|
73
70
|
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
74
|
-
# - Already included:
|
|
75
|
-
#
|
|
76
|
-
#
|
|
71
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
72
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
73
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
77
74
|
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
78
75
|
|
|
79
76
|
# Skip first row as columns names
|
|
@@ -83,55 +80,39 @@ def reader(
|
|
|
83
80
|
reader_kwargs["encoding"] = "ISO-8859-1"
|
|
84
81
|
|
|
85
82
|
##------------------------------------------------------------------------.
|
|
86
|
-
####
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
import pandas as pd
|
|
90
|
-
|
|
91
|
-
# Define datetime time column
|
|
92
|
-
df["time"] = (
|
|
93
|
-
df["day"].astype(str)
|
|
94
|
-
+ "-"
|
|
95
|
-
+ df["month"].astype(str)
|
|
96
|
-
+ "-"
|
|
97
|
-
+ df["year"].astype(str)
|
|
98
|
-
+ " "
|
|
99
|
-
+ df["hour_minute"].astype(str)
|
|
100
|
-
+ ":"
|
|
101
|
-
+ df["second"].astype(str)
|
|
102
|
-
)
|
|
103
|
-
df["time"] = pd.to_datetime(df["time"], format="%d-%m-%Y %H%M:%S", errors="coerce")
|
|
104
|
-
|
|
105
|
-
# - Drop columns not agreeing with DISDRODB L0 standards
|
|
106
|
-
columns_to_drop = [
|
|
107
|
-
"day",
|
|
108
|
-
"month",
|
|
109
|
-
"year",
|
|
110
|
-
"hour_minute",
|
|
111
|
-
"second",
|
|
112
|
-
]
|
|
113
|
-
df = df.drop(columns=columns_to_drop)
|
|
114
|
-
|
|
115
|
-
return df
|
|
116
|
-
|
|
117
|
-
##------------------------------------------------------------------------.
|
|
118
|
-
#### - Define glob pattern to search data files in raw_dir/data/<station_name>
|
|
119
|
-
glob_patterns = "*.dat"
|
|
120
|
-
|
|
121
|
-
####----------------------------------------------------------------------.
|
|
122
|
-
#### - Create L0A products
|
|
123
|
-
run_l0a(
|
|
124
|
-
raw_dir=raw_dir,
|
|
125
|
-
processed_dir=processed_dir,
|
|
126
|
-
station_name=station_name,
|
|
127
|
-
# Custom arguments of the reader for L0A processing
|
|
128
|
-
glob_patterns=glob_patterns,
|
|
83
|
+
#### Read the data
|
|
84
|
+
df = read_raw_text_file(
|
|
85
|
+
filepath=filepath,
|
|
129
86
|
column_names=column_names,
|
|
130
87
|
reader_kwargs=reader_kwargs,
|
|
131
|
-
|
|
132
|
-
# Processing options
|
|
133
|
-
force=force,
|
|
134
|
-
verbose=verbose,
|
|
135
|
-
parallel=parallel,
|
|
136
|
-
debugging_mode=debugging_mode,
|
|
88
|
+
logger=logger,
|
|
137
89
|
)
|
|
90
|
+
|
|
91
|
+
##------------------------------------------------------------------------.
|
|
92
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
93
|
+
# Define datetime 'time' column
|
|
94
|
+
df["time"] = (
|
|
95
|
+
df["day"].astype(str)
|
|
96
|
+
+ "-"
|
|
97
|
+
+ df["month"].astype(str)
|
|
98
|
+
+ "-"
|
|
99
|
+
+ df["year"].astype(str)
|
|
100
|
+
+ " "
|
|
101
|
+
+ df["hour_minute"].astype(str)
|
|
102
|
+
+ ":"
|
|
103
|
+
+ df["second"].astype(str)
|
|
104
|
+
)
|
|
105
|
+
df["time"] = pd.to_datetime(df["time"], format="%d-%m-%Y %H%M:%S", errors="coerce")
|
|
106
|
+
|
|
107
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
108
|
+
columns_to_drop = [
|
|
109
|
+
"day",
|
|
110
|
+
"month",
|
|
111
|
+
"year",
|
|
112
|
+
"hour_minute",
|
|
113
|
+
"second",
|
|
114
|
+
]
|
|
115
|
+
df = df.drop(columns=columns_to_drop)
|
|
116
|
+
|
|
117
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
118
|
+
return df
|