disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +132 -15
- disdrodb/_config.py +4 -2
- disdrodb/_version.py +9 -4
- disdrodb/api/checks.py +264 -237
- disdrodb/api/configs.py +4 -8
- disdrodb/api/create_directories.py +235 -290
- disdrodb/api/info.py +217 -26
- disdrodb/api/io.py +306 -270
- disdrodb/api/path.py +597 -173
- disdrodb/api/search.py +486 -0
- disdrodb/{metadata/scripts → cli}/disdrodb_check_metadata_archive.py +12 -7
- disdrodb/{utils/pandas.py → cli/disdrodb_data_archive_directory.py} +9 -18
- disdrodb/cli/disdrodb_download_archive.py +86 -0
- disdrodb/cli/disdrodb_download_metadata_archive.py +53 -0
- disdrodb/cli/disdrodb_download_station.py +84 -0
- disdrodb/{api/scripts → cli}/disdrodb_initialize_station.py +22 -10
- disdrodb/cli/disdrodb_metadata_archive_directory.py +32 -0
- disdrodb/{data_transfer/scripts/disdrodb_download_station.py → cli/disdrodb_open_data_archive.py} +22 -22
- disdrodb/cli/disdrodb_open_logs_directory.py +69 -0
- disdrodb/{data_transfer/scripts/disdrodb_upload_station.py → cli/disdrodb_open_metadata_archive.py} +22 -24
- disdrodb/cli/disdrodb_open_metadata_directory.py +71 -0
- disdrodb/cli/disdrodb_open_product_directory.py +74 -0
- disdrodb/cli/disdrodb_open_readers_directory.py +32 -0
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0.py +38 -31
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0_station.py +32 -30
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a_station.py +24 -33
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b_station.py +25 -34
- disdrodb/cli/disdrodb_run_l0c.py +130 -0
- disdrodb/cli/disdrodb_run_l0c_station.py +129 -0
- disdrodb/cli/disdrodb_run_l1.py +122 -0
- disdrodb/cli/disdrodb_run_l1_station.py +121 -0
- disdrodb/cli/disdrodb_run_l2e.py +122 -0
- disdrodb/cli/disdrodb_run_l2e_station.py +122 -0
- disdrodb/cli/disdrodb_run_l2m.py +122 -0
- disdrodb/cli/disdrodb_run_l2m_station.py +122 -0
- disdrodb/cli/disdrodb_upload_archive.py +105 -0
- disdrodb/cli/disdrodb_upload_station.py +98 -0
- disdrodb/configs.py +90 -25
- disdrodb/data_transfer/__init__.py +22 -0
- disdrodb/data_transfer/download_data.py +87 -90
- disdrodb/data_transfer/upload_data.py +64 -37
- disdrodb/data_transfer/zenodo.py +15 -18
- disdrodb/docs.py +1 -1
- disdrodb/issue/__init__.py +17 -4
- disdrodb/issue/checks.py +10 -23
- disdrodb/issue/reader.py +9 -12
- disdrodb/issue/writer.py +14 -17
- disdrodb/l0/__init__.py +17 -26
- disdrodb/l0/check_configs.py +35 -23
- disdrodb/l0/check_standards.py +46 -51
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_diameter.yml +44 -44
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_velocity.yml +40 -40
- disdrodb/l0/configs/LPM/l0a_encodings.yml +80 -0
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_cf_attrs.yml +84 -65
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_encodings.yml +50 -9
- disdrodb/l0/configs/{Thies_LPM → LPM}/raw_data_format.yml +285 -245
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_diameter.yml +66 -66
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL/l0a_encodings.yml +32 -0
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_cf_attrs.yml +23 -21
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_encodings.yml +17 -17
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/raw_data_format.yml +77 -77
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_diameter.yml +64 -64
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +39 -0
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_cf_attrs.yml +28 -26
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_encodings.yml +20 -20
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/raw_data_format.yml +107 -107
- disdrodb/l0/configs/PWS100/bins_diameter.yml +173 -0
- disdrodb/l0/configs/PWS100/bins_velocity.yml +173 -0
- disdrodb/l0/configs/PWS100/l0a_encodings.yml +19 -0
- disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +76 -0
- disdrodb/l0/configs/PWS100/l0b_encodings.yml +176 -0
- disdrodb/l0/configs/PWS100/raw_data_format.yml +182 -0
- disdrodb/l0/configs/{RD_80 → RD80}/bins_diameter.yml +40 -40
- disdrodb/l0/configs/RD80/l0a_encodings.yml +16 -0
- disdrodb/l0/configs/{RD_80 → RD80}/l0b_cf_attrs.yml +3 -3
- disdrodb/l0/configs/RD80/l0b_encodings.yml +135 -0
- disdrodb/l0/configs/{RD_80 → RD80}/raw_data_format.yml +46 -50
- disdrodb/l0/l0_reader.py +216 -340
- disdrodb/l0/l0a_processing.py +237 -208
- disdrodb/l0/l0b_nc_processing.py +227 -80
- disdrodb/l0/l0b_processing.py +96 -174
- disdrodb/l0/l0c_processing.py +627 -0
- disdrodb/l0/readers/{ARM → LPM/ARM}/ARM_LPM.py +36 -58
- disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +236 -0
- disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +185 -0
- disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +185 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +195 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +210 -0
- disdrodb/l0/readers/{BRAZIL/GOAMAZON_LPM.py → LPM/KIT/CHWALA.py} +97 -76
- disdrodb/l0/readers/LPM/SLOVENIA/ARSO.py +197 -0
- disdrodb/l0/readers/LPM/SLOVENIA/CRNI_VRH.py +197 -0
- disdrodb/l0/readers/{UK → LPM/UK}/DIVEN.py +14 -35
- disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +157 -0
- disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +113 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/ARCTIC_2021.py +40 -57
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/COMMON_2011.py +37 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/DAVOS_2009_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_2009.py +34 -51
- disdrodb/l0/readers/{EPFL/PARADISO_2014.py → PARSIVEL/EPFL/EPFL_ROOF_2008.py} +38 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2012.py +33 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GENEPI_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007_2.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HPICONET_2010.py +34 -51
- disdrodb/l0/readers/{EPFL/EPFL_ROOF_2010.py → PARSIVEL/EPFL/HYMEX_LTE_SOP2.py} +37 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +111 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HYMEX_LTE_SOP4.py +36 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2018.py +34 -52
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2019.py +38 -56
- disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PARSIVEL_2007.py +27 -45
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PLATO_2019.py +24 -44
- disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +140 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RACLETS_2019_WJF.py +41 -59
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RIETHOLZBACH_2011.py +34 -51
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +117 -0
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +137 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/UNIL_2022.py +42 -55
- disdrodb/l0/readers/PARSIVEL/GPM/IFLOODS.py +104 -0
- disdrodb/l0/readers/{GPM → PARSIVEL/GPM}/LPVEX.py +29 -48
- disdrodb/l0/readers/PARSIVEL/GPM/MC3E.py +184 -0
- disdrodb/l0/readers/PARSIVEL/KIT/BURKINA_FASO.py +133 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +113 -0
- disdrodb/l0/readers/{NCAR/VORTEX_SE_2016_P1.py → PARSIVEL/NCAR/OWLES_MIPS.py} +46 -72
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +125 -0
- disdrodb/l0/readers/{NCAR/OWLES_MIPS.py → PARSIVEL/NCAR/PLOWS_MIPS.py} +45 -64
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +114 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +176 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +183 -0
- disdrodb/l0/readers/PARSIVEL/SLOVENIA/UL_FGG.py +121 -0
- disdrodb/l0/readers/{ARM/ARM_LD.py → PARSIVEL2/ARM/ARM_PARSIVEL2.py} +27 -50
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/{DENMARK → PARSIVEL2/DENMARK}/EROSION_nc.py +14 -35
- disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +189 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +119 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +104 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/NSSTC.py +176 -0
- disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +32 -0
- disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +56 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +120 -0
- disdrodb/l0/readers/{NCAR → PARSIVEL2/NCAR}/PECAN_MIPS.py +45 -64
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +181 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +160 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +160 -0
- disdrodb/l0/readers/{NCAR/PLOWS_MIPS.py → PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py} +49 -66
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +118 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +152 -0
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT.py +166 -0
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +150 -0
- disdrodb/l0/readers/{NCAR/RELAMPAGO_RD80.py → RD80/BRAZIL/CHUVA_RD80.py} +36 -60
- disdrodb/l0/readers/{BRAZIL → RD80/BRAZIL}/GOAMAZON_RD80.py +36 -55
- disdrodb/l0/readers/{NCAR → RD80/NCAR}/CINDY_2011_RD80.py +35 -54
- disdrodb/l0/readers/{BRAZIL/CHUVA_RD80.py → RD80/NCAR/RELAMPAGO_RD80.py} +40 -54
- disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py +274 -0
- disdrodb/l0/readers/template_reader_raw_netcdf_data.py +62 -0
- disdrodb/l0/readers/{reader_template.py → template_reader_raw_text_data.py} +20 -44
- disdrodb/l0/routines.py +885 -581
- disdrodb/l0/standards.py +77 -238
- disdrodb/l0/template_tools.py +105 -110
- disdrodb/l1/__init__.py +17 -0
- disdrodb/l1/beard_model.py +716 -0
- disdrodb/l1/encoding_attrs.py +635 -0
- disdrodb/l1/fall_velocity.py +260 -0
- disdrodb/l1/filters.py +192 -0
- disdrodb/l1/processing.py +202 -0
- disdrodb/l1/resampling.py +236 -0
- disdrodb/l1/routines.py +358 -0
- disdrodb/l1_env/__init__.py +17 -0
- disdrodb/l1_env/routines.py +38 -0
- disdrodb/l2/__init__.py +17 -0
- disdrodb/l2/empirical_dsd.py +1833 -0
- disdrodb/l2/event.py +388 -0
- disdrodb/l2/processing.py +528 -0
- disdrodb/l2/processing_options.py +213 -0
- disdrodb/l2/routines.py +868 -0
- disdrodb/metadata/__init__.py +9 -2
- disdrodb/metadata/checks.py +180 -124
- disdrodb/metadata/download.py +81 -0
- disdrodb/metadata/geolocation.py +146 -0
- disdrodb/metadata/info.py +20 -13
- disdrodb/metadata/manipulation.py +3 -3
- disdrodb/metadata/reader.py +59 -8
- disdrodb/metadata/search.py +77 -144
- disdrodb/metadata/standards.py +83 -80
- disdrodb/metadata/writer.py +10 -16
- disdrodb/psd/__init__.py +38 -0
- disdrodb/psd/fitting.py +2146 -0
- disdrodb/psd/models.py +774 -0
- disdrodb/routines.py +1412 -0
- disdrodb/scattering/__init__.py +28 -0
- disdrodb/scattering/axis_ratio.py +344 -0
- disdrodb/scattering/routines.py +456 -0
- disdrodb/utils/__init__.py +17 -0
- disdrodb/utils/attrs.py +208 -0
- disdrodb/utils/cli.py +269 -0
- disdrodb/utils/compression.py +60 -42
- disdrodb/utils/dask.py +62 -0
- disdrodb/utils/dataframe.py +342 -0
- disdrodb/utils/decorators.py +110 -0
- disdrodb/utils/directories.py +107 -46
- disdrodb/utils/encoding.py +127 -0
- disdrodb/utils/list.py +29 -0
- disdrodb/utils/logger.py +168 -46
- disdrodb/utils/time.py +657 -0
- disdrodb/utils/warnings.py +30 -0
- disdrodb/utils/writer.py +57 -0
- disdrodb/utils/xarray.py +138 -47
- disdrodb/utils/yaml.py +0 -1
- disdrodb/viz/__init__.py +17 -0
- disdrodb/viz/plots.py +17 -0
- disdrodb-0.1.1.dist-info/METADATA +294 -0
- disdrodb-0.1.1.dist-info/RECORD +232 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/WHEEL +1 -1
- disdrodb-0.1.1.dist-info/entry_points.txt +30 -0
- disdrodb/data_transfer/scripts/disdrodb_download_archive.py +0 -53
- disdrodb/data_transfer/scripts/disdrodb_upload_archive.py +0 -57
- disdrodb/l0/configs/OTT_Parsivel/l0a_encodings.yml +0 -32
- disdrodb/l0/configs/OTT_Parsivel2/l0a_encodings.yml +0 -39
- disdrodb/l0/configs/RD_80/l0a_encodings.yml +0 -16
- disdrodb/l0/configs/RD_80/l0b_encodings.yml +0 -135
- disdrodb/l0/configs/Thies_LPM/l0a_encodings.yml +0 -80
- disdrodb/l0/io.py +0 -257
- disdrodb/l0/l0_processing.py +0 -1091
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_OTT.py +0 -178
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_THIES.py +0 -247
- disdrodb/l0/readers/BRAZIL/CHUVA_LPM.py +0 -204
- disdrodb/l0/readers/BRAZIL/CHUVA_OTT.py +0 -183
- disdrodb/l0/readers/BRAZIL/GOAMAZON_OTT.py +0 -183
- disdrodb/l0/readers/CHINA/CHONGQING.py +0 -131
- disdrodb/l0/readers/EPFL/EPFL_ROOF_2008.py +0 -128
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP2.py +0 -127
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP3.py +0 -129
- disdrodb/l0/readers/EPFL/RACLETS_2019.py +0 -158
- disdrodb/l0/readers/EPFL/SAMOYLOV_2017.py +0 -136
- disdrodb/l0/readers/EPFL/SAMOYLOV_2019.py +0 -158
- disdrodb/l0/readers/FRANCE/SIRTA_OTT2.py +0 -138
- disdrodb/l0/readers/GPM/GCPEX.py +0 -123
- disdrodb/l0/readers/GPM/IFLOODS.py +0 -123
- disdrodb/l0/readers/GPM/MC3E.py +0 -123
- disdrodb/l0/readers/GPM/NSSTC.py +0 -164
- disdrodb/l0/readers/ITALY/GID.py +0 -199
- disdrodb/l0/readers/MEXICO/OH_IIUNAM_nc.py +0 -92
- disdrodb/l0/readers/NCAR/CCOPE_2015.py +0 -133
- disdrodb/l0/readers/NCAR/PECAN_FP3.py +0 -137
- disdrodb/l0/readers/NCAR/PECAN_MOBILE.py +0 -144
- disdrodb/l0/readers/NCAR/RELAMPAGO_OTT.py +0 -195
- disdrodb/l0/readers/NCAR/SNOWIE_PJ.py +0 -172
- disdrodb/l0/readers/NCAR/SNOWIE_SB.py +0 -179
- disdrodb/l0/readers/NCAR/VORTEX2_2009.py +0 -133
- disdrodb/l0/readers/NCAR/VORTEX2_2010.py +0 -188
- disdrodb/l0/readers/NCAR/VORTEX2_2010_UF.py +0 -191
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_P2.py +0 -135
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_PIPS.py +0 -170
- disdrodb/l0/readers/NETHERLANDS/DELFT.py +0 -187
- disdrodb/l0/readers/SPAIN/SBEGUERIA.py +0 -179
- disdrodb/l0/scripts/disdrodb_run_l0b_concat.py +0 -93
- disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py +0 -85
- disdrodb/utils/netcdf.py +0 -452
- disdrodb/utils/scripts.py +0 -102
- disdrodb-0.0.21.dist-info/AUTHORS.md +0 -18
- disdrodb-0.0.21.dist-info/METADATA +0 -186
- disdrodb-0.0.21.dist-info/RECORD +0 -168
- disdrodb-0.0.21.dist-info/entry_points.txt +0 -15
- /disdrodb/l0/configs/{RD_80 → RD80}/bins_velocity.yml +0 -0
- /disdrodb/l0/manuals/{Thies_LPM.pdf → LPM.pdf} +0 -0
- /disdrodb/l0/manuals/{ODM_470.pdf → ODM470.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel.pdf → PARSIVEL.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel2.pdf → PARSIVEL2.pdf} +0 -0
- /disdrodb/l0/manuals/{PWS_100.pdf → PWS100.pdf} +0 -0
- /disdrodb/l0/manuals/{RD_80.pdf → RD80.pdf} +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info/licenses}/LICENSE +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
20
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
21
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@is_documented_by(reader_generic_docstring)
|
|
25
|
+
def reader(
|
|
26
|
+
filepath,
|
|
27
|
+
logger=None,
|
|
28
|
+
):
|
|
29
|
+
"""Reader."""
|
|
30
|
+
##------------------------------------------------------------------------.
|
|
31
|
+
#### Define column names
|
|
32
|
+
column_names = ["TO_PARSE"]
|
|
33
|
+
|
|
34
|
+
##------------------------------------------------------------------------.
|
|
35
|
+
#### Define reader options
|
|
36
|
+
reader_kwargs = {}
|
|
37
|
+
# - Define delimiter
|
|
38
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
39
|
+
|
|
40
|
+
# - Avoid first column to become df index !!!
|
|
41
|
+
reader_kwargs["index_col"] = False
|
|
42
|
+
|
|
43
|
+
# - Define behaviour when encountering bad lines
|
|
44
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
45
|
+
|
|
46
|
+
# - Define encoding
|
|
47
|
+
reader_kwargs["encoding"] = "ISO-8859-1"
|
|
48
|
+
|
|
49
|
+
# - Define reader engine
|
|
50
|
+
# - C engine is faster
|
|
51
|
+
# - Python engine is more feature-complete
|
|
52
|
+
reader_kwargs["engine"] = "python"
|
|
53
|
+
|
|
54
|
+
# - Define on-the-fly decompression of on-disk data
|
|
55
|
+
# - Available: gzip, bz2, zip
|
|
56
|
+
reader_kwargs["compression"] = "infer"
|
|
57
|
+
|
|
58
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
59
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
60
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
61
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
62
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
63
|
+
|
|
64
|
+
# Skip first row as columns names
|
|
65
|
+
reader_kwargs["header"] = None
|
|
66
|
+
|
|
67
|
+
##------------------------------------------------------------------------.
|
|
68
|
+
#### Read the data
|
|
69
|
+
df = read_raw_text_file(
|
|
70
|
+
filepath=filepath,
|
|
71
|
+
column_names=column_names,
|
|
72
|
+
reader_kwargs=reader_kwargs,
|
|
73
|
+
logger=logger,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
##------------------------------------------------------------------------.
|
|
77
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
78
|
+
# Split into columns and assign name
|
|
79
|
+
df = df["TO_PARSE"].str.split(";", expand=True, n=9)
|
|
80
|
+
|
|
81
|
+
columns = [
|
|
82
|
+
"date",
|
|
83
|
+
"time",
|
|
84
|
+
"rainfall_rate_32bit",
|
|
85
|
+
"rainfall_accumulated_32bit",
|
|
86
|
+
"weather_code_synop_4680",
|
|
87
|
+
"reflectivity_32bit",
|
|
88
|
+
"mor_visibility",
|
|
89
|
+
"number_particles",
|
|
90
|
+
"sensor_temperature",
|
|
91
|
+
"raw_drop_number",
|
|
92
|
+
]
|
|
93
|
+
df.columns = columns
|
|
94
|
+
|
|
95
|
+
# Add datetime time column
|
|
96
|
+
df["time"] = df["date"] + "-" + df["time"]
|
|
97
|
+
df["time"] = pd.to_datetime(df["time"], format="%d.%m.%Y-%H:%M:%S", errors="coerce")
|
|
98
|
+
df = df.drop(columns=["date"])
|
|
99
|
+
|
|
100
|
+
# Preprocess the raw spectrum
|
|
101
|
+
# - The '<SPECTRUM>ZERO</SPECTRUM>' indicates no drops detected
|
|
102
|
+
# --> "" generates an array of zeros in L0B processing
|
|
103
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
|
|
104
|
+
|
|
105
|
+
# Remove <SPECTRUM> and </SPECTRUM>" acronyms from the raw_drop_number field
|
|
106
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
|
|
107
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
|
|
108
|
+
|
|
109
|
+
# Add 0 before every ; if ; not preceded by a digit
|
|
110
|
+
# Example: ';;1;;' --> '0;0;1;0;'
|
|
111
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace(r"(?<!\d);", "0;", regex=True)
|
|
112
|
+
|
|
113
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
114
|
+
return df
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
# -----------------------------------------------------------------------------.
|
|
4
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
5
|
+
#
|
|
6
|
+
# This program is free software: you can redistribute it and/or modify
|
|
7
|
+
# it under the terms of the GNU General Public License as published by
|
|
8
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
9
|
+
# (at your option) any later version.
|
|
10
|
+
#
|
|
11
|
+
# This program is distributed in the hope that it will be useful,
|
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
|
+
# GNU General Public License for more details.
|
|
15
|
+
#
|
|
16
|
+
# You should have received a copy of the GNU General Public License
|
|
17
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
18
|
+
# -----------------------------------------------------------------------------.
|
|
19
|
+
import numpy as np
|
|
20
|
+
import pandas as pd
|
|
21
|
+
|
|
22
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
23
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@is_documented_by(reader_generic_docstring)
|
|
27
|
+
def reader(
|
|
28
|
+
filepath,
|
|
29
|
+
logger=None,
|
|
30
|
+
):
|
|
31
|
+
"""Reader."""
|
|
32
|
+
##------------------------------------------------------------------------.
|
|
33
|
+
#### Define column names
|
|
34
|
+
column_names = ["TO_PARSE"]
|
|
35
|
+
|
|
36
|
+
##------------------------------------------------------------------------.
|
|
37
|
+
#### Define reader options
|
|
38
|
+
reader_kwargs = {}
|
|
39
|
+
# - Define delimiter
|
|
40
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
41
|
+
|
|
42
|
+
# - Avoid first column to become df index !!!
|
|
43
|
+
reader_kwargs["index_col"] = False
|
|
44
|
+
|
|
45
|
+
# - Define behaviour when encountering bad lines
|
|
46
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
47
|
+
|
|
48
|
+
# Skip the first row (header)
|
|
49
|
+
reader_kwargs["skiprows"] = 0
|
|
50
|
+
|
|
51
|
+
# - Define encoding
|
|
52
|
+
reader_kwargs["encoding"] = "latin"
|
|
53
|
+
|
|
54
|
+
# - Define reader engine
|
|
55
|
+
# - C engine is faster
|
|
56
|
+
# - Python engine is more feature-complete
|
|
57
|
+
reader_kwargs["engine"] = "python"
|
|
58
|
+
|
|
59
|
+
# - Define on-the-fly decompression of on-disk data
|
|
60
|
+
# - Available: gzip, bz2, zip
|
|
61
|
+
reader_kwargs["compression"] = "infer"
|
|
62
|
+
|
|
63
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
64
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
65
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
66
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
67
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
68
|
+
|
|
69
|
+
# Skip first row as columns names
|
|
70
|
+
reader_kwargs["header"] = None
|
|
71
|
+
|
|
72
|
+
##------------------------------------------------------------------------.
|
|
73
|
+
#### Read the data
|
|
74
|
+
df = read_raw_text_file(
|
|
75
|
+
filepath=filepath,
|
|
76
|
+
column_names=column_names,
|
|
77
|
+
reader_kwargs=reader_kwargs,
|
|
78
|
+
logger=logger,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
##------------------------------------------------------------------------.
|
|
82
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
83
|
+
# Create ID and Value columns
|
|
84
|
+
df = df["TO_PARSE"].str.split(":", expand=True, n=1)
|
|
85
|
+
df.columns = ["ID", "Value"]
|
|
86
|
+
|
|
87
|
+
# Select only rows with values
|
|
88
|
+
df = df[df["Value"].astype(bool)]
|
|
89
|
+
df = df[df["Value"].apply(lambda x: x is not None)]
|
|
90
|
+
|
|
91
|
+
# Drop rows with invalid IDs
|
|
92
|
+
# - Corrupted rows
|
|
93
|
+
valid_id_str = np.char.rjust(np.arange(0, 94).astype(str), width=2, fillchar="0")
|
|
94
|
+
df = df[df["ID"].astype(str).isin(valid_id_str)]
|
|
95
|
+
|
|
96
|
+
# Create the dataframe with each row corresponding to a timestep
|
|
97
|
+
# - Group rows based on when ID values restart
|
|
98
|
+
groups = df.groupby((df["ID"].astype(int).diff() <= 0).cumsum())
|
|
99
|
+
|
|
100
|
+
# Reshape the dataframe
|
|
101
|
+
group_dfs = []
|
|
102
|
+
for _, group in groups:
|
|
103
|
+
group_df = group.set_index("ID").T
|
|
104
|
+
group_dfs.append(group_df)
|
|
105
|
+
|
|
106
|
+
# Merge each timestep dataframe
|
|
107
|
+
# --> Missing columns are infilled by NaN
|
|
108
|
+
df = pd.concat(group_dfs, axis=0)
|
|
109
|
+
df.columns = df.columns.astype(str).str.pad(width=2, side="left", fillchar="0")
|
|
110
|
+
|
|
111
|
+
# Define column names
|
|
112
|
+
column_dict = {
|
|
113
|
+
"01": "rainfall_rate_32bit",
|
|
114
|
+
"02": "rainfall_accumulated_32bit",
|
|
115
|
+
"03": "weather_code_synop_4680",
|
|
116
|
+
"04": "weather_code_synop_4677",
|
|
117
|
+
"05": "weather_code_metar_4678",
|
|
118
|
+
"06": "weather_code_nws",
|
|
119
|
+
"07": "reflectivity_32bit",
|
|
120
|
+
"08": "mor_visibility",
|
|
121
|
+
"09": "sample_interval",
|
|
122
|
+
"10": "laser_amplitude",
|
|
123
|
+
"11": "number_particles",
|
|
124
|
+
"12": "sensor_temperature",
|
|
125
|
+
# "13": "sensor_serial_number",
|
|
126
|
+
# "14": "firmware_iop",
|
|
127
|
+
# "15": "firmware_dsp",
|
|
128
|
+
"16": "sensor_heating_current",
|
|
129
|
+
"17": "sensor_battery_voltage",
|
|
130
|
+
"18": "sensor_status",
|
|
131
|
+
# "19": "start_time",
|
|
132
|
+
"20": "sensor_time",
|
|
133
|
+
"21": "sensor_date",
|
|
134
|
+
# "22": "station_name",
|
|
135
|
+
# "23": "station_number",
|
|
136
|
+
"24": "rainfall_amount_absolute_32bit",
|
|
137
|
+
"25": "error_code",
|
|
138
|
+
"30": "rainfall_rate_16bit",
|
|
139
|
+
"31": "rainfall_rate_12bit",
|
|
140
|
+
"32": "rainfall_accumulated_16bit",
|
|
141
|
+
"90": "raw_drop_concentration",
|
|
142
|
+
"91": "raw_drop_average_velocity",
|
|
143
|
+
"93": "raw_drop_number",
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
# Identify missing columns and add NaN
|
|
147
|
+
expected_columns = np.array(list(column_dict.keys()))
|
|
148
|
+
missing_columns = expected_columns[np.isin(expected_columns, df.columns, invert=True)].tolist()
|
|
149
|
+
if len(missing_columns) > 0:
|
|
150
|
+
for column in missing_columns:
|
|
151
|
+
df[column] = "NaN"
|
|
152
|
+
|
|
153
|
+
# Rename columns
|
|
154
|
+
df = df.rename(column_dict, axis=1)
|
|
155
|
+
|
|
156
|
+
# Keep only columns defined in the dictionary
|
|
157
|
+
df = df[list(column_dict.values())]
|
|
158
|
+
|
|
159
|
+
# Define datetime "time" column
|
|
160
|
+
df["time"] = df["sensor_date"] + "-" + df["sensor_time"]
|
|
161
|
+
df["time"] = pd.to_datetime(df["time"], format="%d.%m.%Y-%H:%M:%S", errors="coerce")
|
|
162
|
+
|
|
163
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
164
|
+
columns_to_drop = [
|
|
165
|
+
"sensor_date",
|
|
166
|
+
"sensor_time",
|
|
167
|
+
# "firmware_iop",
|
|
168
|
+
# "firmware_dsp",
|
|
169
|
+
# "sensor_serial_number",
|
|
170
|
+
# "station_name",
|
|
171
|
+
# "station_number",
|
|
172
|
+
]
|
|
173
|
+
df = df.drop(columns=columns_to_drop)
|
|
174
|
+
|
|
175
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
176
|
+
return df
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
# -----------------------------------------------------------------------------.
|
|
4
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
5
|
+
#
|
|
6
|
+
# This program is free software: you can redistribute it and/or modify
|
|
7
|
+
# it under the terms of the GNU General Public License as published by
|
|
8
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
9
|
+
# (at your option) any later version.
|
|
10
|
+
#
|
|
11
|
+
# This program is distributed in the hope that it will be useful,
|
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
|
+
# GNU General Public License for more details.
|
|
15
|
+
#
|
|
16
|
+
# You should have received a copy of the GNU General Public License
|
|
17
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
18
|
+
# -----------------------------------------------------------------------------.
|
|
19
|
+
import numpy as np
|
|
20
|
+
import pandas as pd
|
|
21
|
+
|
|
22
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
23
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@is_documented_by(reader_generic_docstring)
|
|
27
|
+
def reader(
|
|
28
|
+
filepath,
|
|
29
|
+
logger=None,
|
|
30
|
+
):
|
|
31
|
+
"""Reader."""
|
|
32
|
+
##------------------------------------------------------------------------.
|
|
33
|
+
#### Define column names
|
|
34
|
+
column_names = ["TO_PARSE"]
|
|
35
|
+
|
|
36
|
+
##------------------------------------------------------------------------.
|
|
37
|
+
#### Define reader options
|
|
38
|
+
reader_kwargs = {}
|
|
39
|
+
# - Define delimiter
|
|
40
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
41
|
+
|
|
42
|
+
# - Avoid first column to become df index !!!
|
|
43
|
+
reader_kwargs["index_col"] = False
|
|
44
|
+
|
|
45
|
+
# - Define behaviour when encountering bad lines
|
|
46
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
47
|
+
|
|
48
|
+
# Skip the first row (header)
|
|
49
|
+
reader_kwargs["skiprows"] = 0
|
|
50
|
+
|
|
51
|
+
# - Define encoding
|
|
52
|
+
reader_kwargs["encoding"] = "latin"
|
|
53
|
+
|
|
54
|
+
# - Define reader engine
|
|
55
|
+
# - C engine is faster
|
|
56
|
+
# - Python engine is more feature-complete
|
|
57
|
+
reader_kwargs["engine"] = "python"
|
|
58
|
+
|
|
59
|
+
# - Define on-the-fly decompression of on-disk data
|
|
60
|
+
# - Available: gzip, bz2, zip
|
|
61
|
+
reader_kwargs["compression"] = "infer"
|
|
62
|
+
|
|
63
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
64
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
65
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
66
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
67
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
68
|
+
|
|
69
|
+
# Skip first row as columns names
|
|
70
|
+
reader_kwargs["header"] = None
|
|
71
|
+
|
|
72
|
+
##------------------------------------------------------------------------.
|
|
73
|
+
#### Read the data
|
|
74
|
+
df = read_raw_text_file(
|
|
75
|
+
filepath=filepath,
|
|
76
|
+
column_names=column_names,
|
|
77
|
+
reader_kwargs=reader_kwargs,
|
|
78
|
+
logger=logger,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
##------------------------------------------------------------------------.
|
|
82
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
83
|
+
# Create ID and Value columns
|
|
84
|
+
df = df["TO_PARSE"].str.split(":", expand=True, n=1)
|
|
85
|
+
df.columns = ["ID", "Value"]
|
|
86
|
+
|
|
87
|
+
# Select only rows with values
|
|
88
|
+
df = df[df["Value"].astype(bool)]
|
|
89
|
+
df = df[df["Value"].apply(lambda x: x is not None)]
|
|
90
|
+
|
|
91
|
+
# Drop rows with invalid IDs
|
|
92
|
+
# - Corrupted rows
|
|
93
|
+
valid_id_str = np.char.rjust(np.arange(0, 94).astype(str), width=2, fillchar="0")
|
|
94
|
+
df = df[df["ID"].astype(str).isin(valid_id_str)]
|
|
95
|
+
|
|
96
|
+
# Raise error if no more rows after removed corrupted ones
|
|
97
|
+
if len(df) == 0:
|
|
98
|
+
raise ValueError("No rows left after removing corrupted ones.")
|
|
99
|
+
|
|
100
|
+
# Create the dataframe with each row corresponding to a timestep
|
|
101
|
+
# - Group rows based on when ID values restart
|
|
102
|
+
groups = df.groupby((df["ID"].astype(int).diff() <= 0).cumsum())
|
|
103
|
+
|
|
104
|
+
# Reshape the dataframe
|
|
105
|
+
group_dfs = []
|
|
106
|
+
for _, group in groups:
|
|
107
|
+
group_df = group.set_index("ID").T
|
|
108
|
+
group_dfs.append(group_df)
|
|
109
|
+
|
|
110
|
+
# Merge each timestep dataframe
|
|
111
|
+
# --> Missing columns are infilled by NaN
|
|
112
|
+
df = pd.concat(group_dfs, axis=0)
|
|
113
|
+
df.columns = df.columns.astype(str).str.pad(width=2, side="left", fillchar="0")
|
|
114
|
+
|
|
115
|
+
# Define column names
|
|
116
|
+
column_dict = {
|
|
117
|
+
"01": "rainfall_rate_32bit",
|
|
118
|
+
# "02": "rainfall_accumulated_32bit",
|
|
119
|
+
# "03": "weather_code_synop_4680",
|
|
120
|
+
# "04": "weather_code_synop_4677",
|
|
121
|
+
# "05": "weather_code_metar_4678",
|
|
122
|
+
"06": "weather_code_nws",
|
|
123
|
+
"07": "reflectivity_32bit",
|
|
124
|
+
"08": "mor_visibility",
|
|
125
|
+
"09": "sample_interval",
|
|
126
|
+
# "10": "laser_amplitude",
|
|
127
|
+
"11": "number_particles",
|
|
128
|
+
"12": "sensor_temperature",
|
|
129
|
+
# "13": "sensor_serial_number",
|
|
130
|
+
# "14": "firmware_iop",
|
|
131
|
+
# "15": "firmware_dsp",
|
|
132
|
+
# "16": "sensor_heating_current",
|
|
133
|
+
# "17": "sensor_battery_voltage",
|
|
134
|
+
# "18": "sensor_status",
|
|
135
|
+
# "19": "start_time",
|
|
136
|
+
"20": "sensor_time",
|
|
137
|
+
"21": "sensor_date",
|
|
138
|
+
# "22": "station_name",
|
|
139
|
+
# "23": "station_number",
|
|
140
|
+
"24": "rainfall_amount_absolute_32bit",
|
|
141
|
+
# "25": "error_code",
|
|
142
|
+
# "30": "rainfall_rate_16_bit_30",
|
|
143
|
+
# "31": "rainfall_rate_16_bit_1200",
|
|
144
|
+
"32": "rainfall_accumulated_16bit",
|
|
145
|
+
# "90": "raw_drop_concentration",
|
|
146
|
+
# "91": "raw_drop_average_velocity",
|
|
147
|
+
"93": "raw_drop_number",
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
# Identify missing columns and add NaN
|
|
151
|
+
expected_columns = np.array(list(column_dict.keys()))
|
|
152
|
+
missing_columns = expected_columns[np.isin(expected_columns, df.columns, invert=True)].tolist()
|
|
153
|
+
if len(missing_columns) > 0:
|
|
154
|
+
for column in missing_columns:
|
|
155
|
+
df[column] = "NaN"
|
|
156
|
+
|
|
157
|
+
# Rename columns
|
|
158
|
+
df = df.rename(column_dict, axis=1)
|
|
159
|
+
|
|
160
|
+
# Keep only columns defined in the dictionary
|
|
161
|
+
df = df[list(column_dict.values())]
|
|
162
|
+
|
|
163
|
+
# Define datetime "time" column
|
|
164
|
+
df["time"] = df["sensor_date"] + "-" + df["sensor_time"]
|
|
165
|
+
df["time"] = pd.to_datetime(df["time"], format="%d.%m.%Y-%H:%M:%S", errors="coerce")
|
|
166
|
+
|
|
167
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
168
|
+
columns_to_drop = [
|
|
169
|
+
"sensor_date",
|
|
170
|
+
"sensor_time",
|
|
171
|
+
# "firmware_iop",
|
|
172
|
+
# "firmware_dsp",
|
|
173
|
+
# "sensor_serial_number",
|
|
174
|
+
# "station_name",
|
|
175
|
+
# "station_number",
|
|
176
|
+
]
|
|
177
|
+
df = df.drop(columns=columns_to_drop)
|
|
178
|
+
|
|
179
|
+
# Stations UF4-7 have NAN at the end of the raw drop number
|
|
180
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("NaN;", "").iloc[0]
|
|
181
|
+
|
|
182
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
183
|
+
return df
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
"""Reader for EPFL 2009 campaign."""
|
|
19
|
+
import pandas as pd
|
|
20
|
+
|
|
21
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
22
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@is_documented_by(reader_generic_docstring)
|
|
26
|
+
def reader(
|
|
27
|
+
filepath,
|
|
28
|
+
logger=None,
|
|
29
|
+
):
|
|
30
|
+
"""Reader."""
|
|
31
|
+
##------------------------------------------------------------------------.
|
|
32
|
+
#### Define column names
|
|
33
|
+
column_names = ["TO_PARSE"]
|
|
34
|
+
|
|
35
|
+
##------------------------------------------------------------------------.
|
|
36
|
+
#### Define reader options
|
|
37
|
+
reader_kwargs = {}
|
|
38
|
+
# - Define delimiter
|
|
39
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
40
|
+
|
|
41
|
+
# - Avoid first column to become df index !!!
|
|
42
|
+
reader_kwargs["index_col"] = False
|
|
43
|
+
|
|
44
|
+
# - Define behaviour when encountering bad lines
|
|
45
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
46
|
+
|
|
47
|
+
# Skip the first row (header)
|
|
48
|
+
reader_kwargs["skiprows"] = 0
|
|
49
|
+
|
|
50
|
+
# - Define encoding
|
|
51
|
+
reader_kwargs["encoding"] = "latin"
|
|
52
|
+
|
|
53
|
+
# - Define reader engine
|
|
54
|
+
# - C engine is faster
|
|
55
|
+
# - Python engine is more feature-complete
|
|
56
|
+
reader_kwargs["engine"] = "python"
|
|
57
|
+
|
|
58
|
+
# - Define on-the-fly decompression of on-disk data
|
|
59
|
+
# - Available: gzip, bz2, zip
|
|
60
|
+
reader_kwargs["compression"] = "infer"
|
|
61
|
+
|
|
62
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
63
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
64
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
65
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
66
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
67
|
+
|
|
68
|
+
# Skip first row as columns names
|
|
69
|
+
reader_kwargs["header"] = None
|
|
70
|
+
|
|
71
|
+
##------------------------------------------------------------------------.
|
|
72
|
+
#### Read the data
|
|
73
|
+
df = read_raw_text_file(
|
|
74
|
+
filepath=filepath,
|
|
75
|
+
column_names=column_names,
|
|
76
|
+
reader_kwargs=reader_kwargs,
|
|
77
|
+
logger=logger,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
##------------------------------------------------------------------------.
|
|
81
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
82
|
+
# Create ID and Value columns
|
|
83
|
+
df = df["TO_PARSE"].str.split(";", expand=True, n=14)
|
|
84
|
+
|
|
85
|
+
# Assign column names
|
|
86
|
+
column_names = [
|
|
87
|
+
"id",
|
|
88
|
+
"time",
|
|
89
|
+
"rainfall_rate_32bit",
|
|
90
|
+
"rainfall_accumulated_32bit",
|
|
91
|
+
"weather_code_synop_4680",
|
|
92
|
+
"reflectivity_32bit",
|
|
93
|
+
"mor_visibility",
|
|
94
|
+
"sensor_temperature", # maybe
|
|
95
|
+
"laser_amplitude", # probably
|
|
96
|
+
"number_particles",
|
|
97
|
+
"sensor_status",
|
|
98
|
+
"sensor_heating_current",
|
|
99
|
+
"sensor_battery_voltage",
|
|
100
|
+
"error_code",
|
|
101
|
+
"raw_drop_number",
|
|
102
|
+
]
|
|
103
|
+
df.columns = column_names
|
|
104
|
+
|
|
105
|
+
# Convert time column to datetime
|
|
106
|
+
df["time"] = pd.to_datetime(df["time"], format="%d/%m/%Y %H.%M.%S", errors="coerce")
|
|
107
|
+
|
|
108
|
+
# Preprocess the raw spectrum
|
|
109
|
+
# - Add 0 before every ; if ; not preceded by a digit
|
|
110
|
+
# Example: ';;1;;' --> '0;0;1;0;'
|
|
111
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("R;", "")
|
|
112
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace(r"(?<!\d);", "0;", regex=True)
|
|
113
|
+
|
|
114
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
115
|
+
columns_to_drop = [
|
|
116
|
+
"id",
|
|
117
|
+
]
|
|
118
|
+
df = df.drop(columns=columns_to_drop)
|
|
119
|
+
|
|
120
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
121
|
+
return df
|
|
@@ -15,22 +15,23 @@
|
|
|
15
15
|
# You should have received a copy of the GNU General Public License
|
|
16
16
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
17
|
# -----------------------------------------------------------------------------.
|
|
18
|
-
"""Reader for ARM OTT
|
|
19
|
-
from disdrodb.l0 import run_l0b_from_nc
|
|
18
|
+
"""Reader for ARM OTT PARSIVEL2 sensor."""
|
|
20
19
|
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
20
|
+
from disdrodb.l0.l0b_nc_processing import open_raw_netcdf_file, replace_custom_nan_flags, standardize_raw_dataset
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
@is_documented_by(reader_generic_docstring)
|
|
24
24
|
def reader(
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
station_name,
|
|
28
|
-
# Processing options
|
|
29
|
-
force=False,
|
|
30
|
-
verbose=False,
|
|
31
|
-
parallel=False,
|
|
32
|
-
debugging_mode=False,
|
|
25
|
+
filepath,
|
|
26
|
+
logger=None,
|
|
33
27
|
):
|
|
28
|
+
"""Reader."""
|
|
29
|
+
##------------------------------------------------------------------------.
|
|
30
|
+
#### Open the netCDF
|
|
31
|
+
ds = open_raw_netcdf_file(filepath=filepath, logger=logger)
|
|
32
|
+
|
|
33
|
+
##------------------------------------------------------------------------.
|
|
34
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
34
35
|
# Define dictionary mapping dataset variables to select and rename
|
|
35
36
|
dict_names = {
|
|
36
37
|
### Dimensions
|
|
@@ -84,45 +85,21 @@ def reader(
|
|
|
84
85
|
# 'qc_sensor_voltage':
|
|
85
86
|
}
|
|
86
87
|
|
|
87
|
-
#
|
|
88
|
-
|
|
89
|
-
from disdrodb.l0.l0b_nc_processing import replace_custom_nan_flags
|
|
90
|
-
|
|
91
|
-
# Replace nan flags with np.nan
|
|
92
|
-
# - ARM use the -9999 flags
|
|
93
|
-
nan_flags_variables = [
|
|
94
|
-
"sensor_temperature",
|
|
95
|
-
"laser_amplitude",
|
|
96
|
-
"mor_visibility",
|
|
97
|
-
"number_particles",
|
|
98
|
-
"weather_code_synop_4680",
|
|
99
|
-
"raw_drop_number",
|
|
100
|
-
]
|
|
101
|
-
dict_nan_flags = {var: [-9999] for var in nan_flags_variables}
|
|
102
|
-
ds = replace_custom_nan_flags(ds, dict_nan_flags=dict_nan_flags)
|
|
103
|
-
|
|
104
|
-
# Return dataset
|
|
105
|
-
return ds
|
|
106
|
-
|
|
107
|
-
# Define glob pattern to search data files in <raw_dir>/data/<station_name>
|
|
108
|
-
glob_patterns = "*.cdf"
|
|
109
|
-
|
|
110
|
-
####----------------------------------------------------------------------.
|
|
111
|
-
#### - Create L0A products
|
|
112
|
-
run_l0b_from_nc(
|
|
113
|
-
raw_dir=raw_dir,
|
|
114
|
-
processed_dir=processed_dir,
|
|
115
|
-
station_name=station_name,
|
|
116
|
-
# Custom arguments of the reader
|
|
117
|
-
glob_patterns=glob_patterns,
|
|
118
|
-
dict_names=dict_names,
|
|
119
|
-
ds_sanitizer_fun=ds_sanitizer_fun,
|
|
120
|
-
# Processing options
|
|
121
|
-
force=force,
|
|
122
|
-
verbose=verbose,
|
|
123
|
-
parallel=parallel,
|
|
124
|
-
debugging_mode=debugging_mode,
|
|
125
|
-
)
|
|
88
|
+
# Rename dataset variables and columns and infill missing variables
|
|
89
|
+
ds = standardize_raw_dataset(ds=ds, dict_names=dict_names, sensor_name="PARSIVEL2")
|
|
126
90
|
|
|
91
|
+
# Replace nan flags with np.nan
|
|
92
|
+
# - ARM use the -9999 flags
|
|
93
|
+
nan_flags_variables = [
|
|
94
|
+
"sensor_temperature",
|
|
95
|
+
"laser_amplitude",
|
|
96
|
+
"mor_visibility",
|
|
97
|
+
"number_particles",
|
|
98
|
+
"weather_code_synop_4680",
|
|
99
|
+
"raw_drop_number",
|
|
100
|
+
]
|
|
101
|
+
dict_nan_flags = {var: [-9999] for var in nan_flags_variables}
|
|
102
|
+
ds = replace_custom_nan_flags(ds, dict_nan_flags=dict_nan_flags, logger=logger)
|
|
127
103
|
|
|
128
|
-
#
|
|
104
|
+
# Return the dataset adhering to DISDRODB L0B standards
|
|
105
|
+
return ds
|