disdrodb 0.0.20__py3-none-any.whl → 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +132 -15
- disdrodb/_config.py +4 -2
- disdrodb/_version.py +9 -4
- disdrodb/api/checks.py +264 -237
- disdrodb/api/configs.py +4 -8
- disdrodb/api/create_directories.py +235 -290
- disdrodb/api/info.py +217 -26
- disdrodb/api/io.py +295 -269
- disdrodb/api/path.py +597 -173
- disdrodb/api/search.py +486 -0
- disdrodb/{metadata/scripts → cli}/disdrodb_check_metadata_archive.py +12 -7
- disdrodb/{utils/pandas.py → cli/disdrodb_data_archive_directory.py} +9 -18
- disdrodb/cli/disdrodb_download_archive.py +86 -0
- disdrodb/cli/disdrodb_download_metadata_archive.py +53 -0
- disdrodb/cli/disdrodb_download_station.py +84 -0
- disdrodb/{api/scripts → cli}/disdrodb_initialize_station.py +22 -10
- disdrodb/cli/disdrodb_metadata_archive_directory.py +32 -0
- disdrodb/{data_transfer/scripts/disdrodb_download_station.py → cli/disdrodb_open_data_archive.py} +22 -22
- disdrodb/cli/disdrodb_open_logs_directory.py +69 -0
- disdrodb/{data_transfer/scripts/disdrodb_upload_station.py → cli/disdrodb_open_metadata_archive.py} +22 -24
- disdrodb/cli/disdrodb_open_metadata_directory.py +71 -0
- disdrodb/cli/disdrodb_open_product_directory.py +74 -0
- disdrodb/cli/disdrodb_open_readers_directory.py +32 -0
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0.py +38 -31
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0_station.py +32 -30
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a_station.py +24 -33
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b_station.py +25 -34
- disdrodb/cli/disdrodb_run_l0c.py +130 -0
- disdrodb/cli/disdrodb_run_l0c_station.py +129 -0
- disdrodb/cli/disdrodb_run_l1.py +122 -0
- disdrodb/cli/disdrodb_run_l1_station.py +121 -0
- disdrodb/cli/disdrodb_run_l2e.py +122 -0
- disdrodb/cli/disdrodb_run_l2e_station.py +122 -0
- disdrodb/cli/disdrodb_run_l2m.py +122 -0
- disdrodb/cli/disdrodb_run_l2m_station.py +122 -0
- disdrodb/cli/disdrodb_upload_archive.py +105 -0
- disdrodb/cli/disdrodb_upload_station.py +98 -0
- disdrodb/configs.py +90 -25
- disdrodb/data_transfer/__init__.py +22 -0
- disdrodb/data_transfer/download_data.py +87 -90
- disdrodb/data_transfer/upload_data.py +64 -37
- disdrodb/data_transfer/zenodo.py +15 -18
- disdrodb/docs.py +1 -1
- disdrodb/issue/__init__.py +17 -4
- disdrodb/issue/checks.py +10 -23
- disdrodb/issue/reader.py +9 -12
- disdrodb/issue/writer.py +14 -17
- disdrodb/l0/__init__.py +17 -26
- disdrodb/l0/check_configs.py +35 -23
- disdrodb/l0/check_standards.py +32 -42
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_diameter.yml +44 -44
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_velocity.yml +40 -40
- disdrodb/l0/configs/LPM/l0a_encodings.yml +80 -0
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_cf_attrs.yml +62 -59
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_encodings.yml +9 -9
- disdrodb/l0/configs/{Thies_LPM → LPM}/raw_data_format.yml +245 -245
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_diameter.yml +66 -66
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL/l0a_encodings.yml +32 -0
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_cf_attrs.yml +22 -20
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_encodings.yml +17 -17
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/raw_data_format.yml +77 -77
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_diameter.yml +64 -64
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +39 -0
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_cf_attrs.yml +24 -22
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_encodings.yml +20 -20
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/raw_data_format.yml +98 -98
- disdrodb/l0/configs/{RD_80 → RD80}/bins_diameter.yml +40 -40
- disdrodb/l0/configs/RD80/l0a_encodings.yml +16 -0
- disdrodb/l0/configs/{RD_80 → RD80}/l0b_cf_attrs.yml +3 -3
- disdrodb/l0/configs/RD80/l0b_encodings.yml +135 -0
- disdrodb/l0/configs/{RD_80 → RD80}/raw_data_format.yml +48 -48
- disdrodb/l0/l0_reader.py +216 -340
- disdrodb/l0/l0a_processing.py +237 -208
- disdrodb/l0/l0b_nc_processing.py +227 -80
- disdrodb/l0/l0b_processing.py +93 -173
- disdrodb/l0/l0c_processing.py +627 -0
- disdrodb/l0/readers/{ARM → LPM/ARM}/ARM_LPM.py +36 -58
- disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +226 -0
- disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +185 -0
- disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +183 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +179 -0
- disdrodb/l0/readers/{UK → LPM/UK}/DIVEN.py +14 -35
- disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +157 -0
- disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +113 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/ARCTIC_2021.py +40 -57
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/COMMON_2011.py +37 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/DAVOS_2009_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_2009.py +34 -51
- disdrodb/l0/readers/{EPFL/PARADISO_2014.py → PARSIVEL/EPFL/EPFL_ROOF_2008.py} +38 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2012.py +33 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GENEPI_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007_2.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HPICONET_2010.py +34 -51
- disdrodb/l0/readers/{EPFL/EPFL_ROOF_2010.py → PARSIVEL/EPFL/HYMEX_LTE_SOP2.py} +37 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +111 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HYMEX_LTE_SOP4.py +36 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2018.py +34 -52
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2019.py +38 -56
- disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PARSIVEL_2007.py +27 -45
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PLATO_2019.py +24 -44
- disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +140 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RACLETS_2019_WJF.py +41 -59
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RIETHOLZBACH_2011.py +34 -51
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +117 -0
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +137 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/UNIL_2022.py +42 -55
- disdrodb/l0/readers/PARSIVEL/GPM/IFLOODS.py +104 -0
- disdrodb/l0/readers/{GPM → PARSIVEL/GPM}/LPVEX.py +29 -48
- disdrodb/l0/readers/PARSIVEL/GPM/MC3E.py +184 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +113 -0
- disdrodb/l0/readers/{NCAR/VORTEX_SE_2016_P1.py → PARSIVEL/NCAR/OWLES_MIPS.py} +46 -72
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +125 -0
- disdrodb/l0/readers/{NCAR/OWLES_MIPS.py → PARSIVEL/NCAR/PLOWS_MIPS.py} +45 -64
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +114 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +176 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +183 -0
- disdrodb/l0/readers/{ARM/ARM_LD.py → PARSIVEL2/ARM/ARM_PARSIVEL2.py} +27 -50
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/{DENMARK → PARSIVEL2/DENMARK}/EROSION_nc.py +14 -35
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +119 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +104 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/NSSTC.py +176 -0
- disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +32 -0
- disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +56 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +120 -0
- disdrodb/l0/readers/{NCAR → PARSIVEL2/NCAR}/PECAN_MIPS.py +45 -64
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +181 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +160 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +160 -0
- disdrodb/l0/readers/{NCAR/PLOWS_MIPS.py → PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py} +49 -66
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +118 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +152 -0
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT.py +166 -0
- disdrodb/l0/readers/{NCAR/RELAMPAGO_RD80.py → RD80/BRAZIL/CHUVA_RD80.py} +36 -60
- disdrodb/l0/readers/{BRAZIL → RD80/BRAZIL}/GOAMAZON_RD80.py +36 -55
- disdrodb/l0/readers/{NCAR → RD80/NCAR}/CINDY_2011_RD80.py +35 -54
- disdrodb/l0/readers/{BRAZIL/CHUVA_RD80.py → RD80/NCAR/RELAMPAGO_RD80.py} +40 -54
- disdrodb/l0/readers/template_reader_raw_netcdf_data.py +62 -0
- disdrodb/l0/readers/{reader_template.py → template_reader_raw_text_data.py} +20 -44
- disdrodb/l0/routines.py +885 -581
- disdrodb/l0/standards.py +72 -236
- disdrodb/l0/template_tools.py +104 -109
- disdrodb/l1/__init__.py +17 -0
- disdrodb/l1/beard_model.py +716 -0
- disdrodb/l1/encoding_attrs.py +620 -0
- disdrodb/l1/fall_velocity.py +260 -0
- disdrodb/l1/filters.py +192 -0
- disdrodb/l1/processing.py +200 -0
- disdrodb/l1/resampling.py +236 -0
- disdrodb/l1/routines.py +357 -0
- disdrodb/l1_env/__init__.py +17 -0
- disdrodb/l1_env/routines.py +38 -0
- disdrodb/l2/__init__.py +17 -0
- disdrodb/l2/empirical_dsd.py +1735 -0
- disdrodb/l2/event.py +388 -0
- disdrodb/l2/processing.py +519 -0
- disdrodb/l2/processing_options.py +213 -0
- disdrodb/l2/routines.py +868 -0
- disdrodb/metadata/__init__.py +9 -2
- disdrodb/metadata/checks.py +165 -118
- disdrodb/metadata/download.py +81 -0
- disdrodb/metadata/geolocation.py +146 -0
- disdrodb/metadata/info.py +20 -13
- disdrodb/metadata/manipulation.py +1 -1
- disdrodb/metadata/reader.py +59 -8
- disdrodb/metadata/search.py +77 -144
- disdrodb/metadata/standards.py +7 -8
- disdrodb/metadata/writer.py +8 -14
- disdrodb/psd/__init__.py +38 -0
- disdrodb/psd/fitting.py +2146 -0
- disdrodb/psd/models.py +774 -0
- disdrodb/routines.py +1176 -0
- disdrodb/scattering/__init__.py +28 -0
- disdrodb/scattering/axis_ratio.py +344 -0
- disdrodb/scattering/routines.py +456 -0
- disdrodb/utils/__init__.py +17 -0
- disdrodb/utils/attrs.py +208 -0
- disdrodb/utils/cli.py +269 -0
- disdrodb/utils/compression.py +60 -42
- disdrodb/utils/dask.py +62 -0
- disdrodb/utils/decorators.py +110 -0
- disdrodb/utils/directories.py +107 -46
- disdrodb/utils/encoding.py +127 -0
- disdrodb/utils/list.py +29 -0
- disdrodb/utils/logger.py +168 -46
- disdrodb/utils/time.py +657 -0
- disdrodb/utils/warnings.py +30 -0
- disdrodb/utils/writer.py +57 -0
- disdrodb/utils/xarray.py +138 -47
- disdrodb/utils/yaml.py +0 -1
- disdrodb/viz/__init__.py +17 -0
- disdrodb/viz/plots.py +17 -0
- disdrodb-0.1.0.dist-info/METADATA +321 -0
- disdrodb-0.1.0.dist-info/RECORD +216 -0
- {disdrodb-0.0.20.dist-info → disdrodb-0.1.0.dist-info}/WHEEL +1 -1
- disdrodb-0.1.0.dist-info/entry_points.txt +30 -0
- disdrodb/data_transfer/scripts/disdrodb_download_archive.py +0 -53
- disdrodb/data_transfer/scripts/disdrodb_upload_archive.py +0 -57
- disdrodb/l0/configs/OTT_Parsivel/l0a_encodings.yml +0 -32
- disdrodb/l0/configs/OTT_Parsivel2/l0a_encodings.yml +0 -39
- disdrodb/l0/configs/RD_80/l0a_encodings.yml +0 -16
- disdrodb/l0/configs/RD_80/l0b_encodings.yml +0 -135
- disdrodb/l0/configs/Thies_LPM/l0a_encodings.yml +0 -80
- disdrodb/l0/io.py +0 -257
- disdrodb/l0/l0_processing.py +0 -1091
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_OTT.py +0 -178
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_THIES.py +0 -247
- disdrodb/l0/readers/BRAZIL/CHUVA_LPM.py +0 -204
- disdrodb/l0/readers/BRAZIL/CHUVA_OTT.py +0 -183
- disdrodb/l0/readers/BRAZIL/GOAMAZON_LPM.py +0 -204
- disdrodb/l0/readers/BRAZIL/GOAMAZON_OTT.py +0 -183
- disdrodb/l0/readers/CHINA/CHONGQING.py +0 -131
- disdrodb/l0/readers/EPFL/EPFL_ROOF_2008.py +0 -128
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP2.py +0 -127
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP3.py +0 -129
- disdrodb/l0/readers/EPFL/RACLETS_2019.py +0 -158
- disdrodb/l0/readers/EPFL/SAMOYLOV_2017.py +0 -136
- disdrodb/l0/readers/EPFL/SAMOYLOV_2019.py +0 -158
- disdrodb/l0/readers/FRANCE/SIRTA_OTT2.py +0 -138
- disdrodb/l0/readers/GPM/GCPEX.py +0 -123
- disdrodb/l0/readers/GPM/IFLOODS.py +0 -123
- disdrodb/l0/readers/GPM/MC3E.py +0 -123
- disdrodb/l0/readers/GPM/NSSTC.py +0 -164
- disdrodb/l0/readers/ITALY/GID.py +0 -199
- disdrodb/l0/readers/MEXICO/OH_IIUNAM_nc.py +0 -92
- disdrodb/l0/readers/NCAR/CCOPE_2015.py +0 -133
- disdrodb/l0/readers/NCAR/PECAN_FP3.py +0 -137
- disdrodb/l0/readers/NCAR/PECAN_MOBILE.py +0 -144
- disdrodb/l0/readers/NCAR/RELAMPAGO_OTT.py +0 -195
- disdrodb/l0/readers/NCAR/SNOWIE_PJ.py +0 -172
- disdrodb/l0/readers/NCAR/SNOWIE_SB.py +0 -179
- disdrodb/l0/readers/NCAR/VORTEX2_2009.py +0 -133
- disdrodb/l0/readers/NCAR/VORTEX2_2010.py +0 -188
- disdrodb/l0/readers/NCAR/VORTEX2_2010_UF.py +0 -191
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_P2.py +0 -135
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_PIPS.py +0 -170
- disdrodb/l0/readers/NETHERLANDS/DELFT.py +0 -187
- disdrodb/l0/readers/SPAIN/SBEGUERIA.py +0 -179
- disdrodb/l0/scripts/disdrodb_run_l0b_concat.py +0 -93
- disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py +0 -85
- disdrodb/utils/netcdf.py +0 -452
- disdrodb/utils/scripts.py +0 -102
- disdrodb-0.0.20.dist-info/AUTHORS.md +0 -18
- disdrodb-0.0.20.dist-info/METADATA +0 -186
- disdrodb-0.0.20.dist-info/RECORD +0 -168
- disdrodb-0.0.20.dist-info/entry_points.txt +0 -15
- /disdrodb/l0/configs/{RD_80 → RD80}/bins_velocity.yml +0 -0
- /disdrodb/l0/manuals/{Thies_LPM.pdf → LPM.pdf} +0 -0
- /disdrodb/l0/manuals/{ODM_470.pdf → ODM470.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel.pdf → PARSIVEL.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel2.pdf → PARSIVEL2.pdf} +0 -0
- /disdrodb/l0/manuals/{PWS_100.pdf → PWS100.pdf} +0 -0
- /disdrodb/l0/manuals/{RD_80.pdf → RD80.pdf} +0 -0
- {disdrodb-0.0.20.dist-info → disdrodb-0.1.0.dist-info/licenses}/LICENSE +0 -0
- {disdrodb-0.0.20.dist-info → disdrodb-0.1.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
import numpy as np
|
|
19
|
+
import pandas as pd
|
|
20
|
+
|
|
21
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
22
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@is_documented_by(reader_generic_docstring)
|
|
26
|
+
def reader(
|
|
27
|
+
filepath,
|
|
28
|
+
logger=None,
|
|
29
|
+
):
|
|
30
|
+
"""Reader."""
|
|
31
|
+
##------------------------------------------------------------------------.
|
|
32
|
+
#### Define column names
|
|
33
|
+
column_names = ["TO_PARSE"]
|
|
34
|
+
|
|
35
|
+
##------------------------------------------------------------------------.
|
|
36
|
+
#### Define reader options
|
|
37
|
+
reader_kwargs = {}
|
|
38
|
+
# - Define delimiter
|
|
39
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
40
|
+
# - Skip first row as columns names
|
|
41
|
+
# - Define encoding
|
|
42
|
+
reader_kwargs["encoding"] = "latin" # "ISO-8859-1"
|
|
43
|
+
# - Avoid first column to become df index !!!
|
|
44
|
+
reader_kwargs["index_col"] = False
|
|
45
|
+
# - Define behaviour when encountering bad lines
|
|
46
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
47
|
+
# - Define reader engine
|
|
48
|
+
# - C engine is faster
|
|
49
|
+
# - Python engine is more feature-complete
|
|
50
|
+
reader_kwargs["engine"] = "python"
|
|
51
|
+
# - Define on-the-fly decompression of on-disk data
|
|
52
|
+
# - Available: gzip, bz2, zip
|
|
53
|
+
reader_kwargs["compression"] = "infer"
|
|
54
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
55
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
56
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
57
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
58
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
59
|
+
|
|
60
|
+
##------------------------------------------------------------------------.
|
|
61
|
+
#### Read the data
|
|
62
|
+
df = read_raw_text_file(
|
|
63
|
+
filepath=filepath,
|
|
64
|
+
column_names=column_names,
|
|
65
|
+
reader_kwargs=reader_kwargs,
|
|
66
|
+
logger=logger,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
##------------------------------------------------------------------------.
|
|
70
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
71
|
+
# Create ID and Value columns
|
|
72
|
+
df = df["TO_PARSE"].str.split(":", expand=True, n=1)
|
|
73
|
+
df.columns = ["ID", "Value"]
|
|
74
|
+
|
|
75
|
+
# Select only rows with values
|
|
76
|
+
df = df[df["Value"].astype(bool)]
|
|
77
|
+
df = df[df["Value"].apply(lambda x: x is not None)]
|
|
78
|
+
|
|
79
|
+
# Drop rows with invalid IDs
|
|
80
|
+
# - Corrupted rows
|
|
81
|
+
valid_id_str = np.char.rjust(np.arange(0, 94).astype(str), width=2, fillchar="0")
|
|
82
|
+
df = df[df["ID"].astype(str).isin(valid_id_str)]
|
|
83
|
+
|
|
84
|
+
# Create the dataframe with each row corresponding to a timestep
|
|
85
|
+
# - Group rows based on when ID values restart
|
|
86
|
+
groups = df.groupby((df["ID"].astype(int).diff() <= 0).cumsum())
|
|
87
|
+
|
|
88
|
+
# Reshape the dataframe
|
|
89
|
+
group_dfs = []
|
|
90
|
+
for _, group in groups:
|
|
91
|
+
group_df = group.set_index("ID").T
|
|
92
|
+
group_dfs.append(group_df)
|
|
93
|
+
|
|
94
|
+
# Merge each timestep dataframe
|
|
95
|
+
# --> Missing columns are infilled by NaN
|
|
96
|
+
df = pd.concat(group_dfs, axis=0)
|
|
97
|
+
df.columns = df.columns.astype(str).str.pad(width=2, side="left", fillchar="0")
|
|
98
|
+
|
|
99
|
+
# Assign column names
|
|
100
|
+
column_dict = {
|
|
101
|
+
"01": "rainfall_rate_32bit",
|
|
102
|
+
"02": "rainfall_accumulated_32bit",
|
|
103
|
+
"03": "weather_code_synop_4680",
|
|
104
|
+
"04": "weather_code_synop_4677",
|
|
105
|
+
"05": "weather_code_metar_4678",
|
|
106
|
+
"06": "weather_code_nws",
|
|
107
|
+
"07": "reflectivity_32bit",
|
|
108
|
+
"08": "mor_visibility",
|
|
109
|
+
"09": "sample_interval",
|
|
110
|
+
"10": "laser_amplitude",
|
|
111
|
+
"11": "number_particles",
|
|
112
|
+
"12": "sensor_temperature",
|
|
113
|
+
# "13": "sensor_serial_number",
|
|
114
|
+
# "14": "firmware_iop",
|
|
115
|
+
# "15": "firmware_dsp",
|
|
116
|
+
"16": "sensor_heating_current",
|
|
117
|
+
"17": "sensor_battery_voltage",
|
|
118
|
+
"18": "sensor_status",
|
|
119
|
+
# "19": "start_time",
|
|
120
|
+
"20": "sensor_time",
|
|
121
|
+
"21": "sensor_date",
|
|
122
|
+
# "22": "station_name",
|
|
123
|
+
# "23": "station_number",
|
|
124
|
+
"24": "rainfall_amount_absolute_32bit",
|
|
125
|
+
"25": "error_code",
|
|
126
|
+
"30": "rainfall_rate_16_bit_30",
|
|
127
|
+
"31": "rainfall_rate_16_bit_1200",
|
|
128
|
+
"32": "rainfall_accumulated_16bit",
|
|
129
|
+
"90": "raw_drop_concentration",
|
|
130
|
+
"91": "raw_drop_average_velocity",
|
|
131
|
+
"93": "raw_drop_number",
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
# Identify missing columns and add NaN
|
|
135
|
+
expected_columns = np.array(list(column_dict.keys()))
|
|
136
|
+
missing_columns = expected_columns[np.isin(expected_columns, df.columns, invert=True)].tolist()
|
|
137
|
+
if len(missing_columns) > 0:
|
|
138
|
+
for column in missing_columns:
|
|
139
|
+
df[column] = "NaN"
|
|
140
|
+
|
|
141
|
+
# Rename columns
|
|
142
|
+
df = df.rename(column_dict, axis=1)
|
|
143
|
+
|
|
144
|
+
# Keep only columns defined in the dictionary
|
|
145
|
+
df = df[list(column_dict.values())]
|
|
146
|
+
|
|
147
|
+
# Define datetime "time" column
|
|
148
|
+
df["time"] = df["sensor_date"] + "-" + df["sensor_time"]
|
|
149
|
+
df["time"] = pd.to_datetime(df["time"], format="%d.%m.%Y-%H:%M:%S", errors="coerce")
|
|
150
|
+
|
|
151
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
152
|
+
columns_to_drop = [
|
|
153
|
+
"sensor_date",
|
|
154
|
+
"sensor_time",
|
|
155
|
+
# "firmware_iop",
|
|
156
|
+
# "firmware_dsp",
|
|
157
|
+
# "sensor_serial_number",
|
|
158
|
+
# "station_name",
|
|
159
|
+
# "station_number",
|
|
160
|
+
]
|
|
161
|
+
df = df.drop(columns=columns_to_drop)
|
|
162
|
+
|
|
163
|
+
return df
|
|
@@ -16,21 +16,22 @@
|
|
|
16
16
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
17
|
# -----------------------------------------------------------------------------.
|
|
18
18
|
"""Reader for the EROSION campaign in Denmark."""
|
|
19
|
-
from disdrodb.l0 import run_l0b_from_nc
|
|
20
19
|
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
20
|
+
from disdrodb.l0.l0b_nc_processing import open_raw_netcdf_file, standardize_raw_dataset
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
@is_documented_by(reader_generic_docstring)
|
|
24
24
|
def reader(
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
station_name,
|
|
28
|
-
# Processing options
|
|
29
|
-
force=False,
|
|
30
|
-
verbose=False,
|
|
31
|
-
parallel=False,
|
|
32
|
-
debugging_mode=False,
|
|
25
|
+
filepath,
|
|
26
|
+
logger=None,
|
|
33
27
|
):
|
|
28
|
+
"""Reader."""
|
|
29
|
+
##------------------------------------------------------------------------.
|
|
30
|
+
#### Open the netCDF
|
|
31
|
+
ds = open_raw_netcdf_file(filepath=filepath, logger=logger)
|
|
32
|
+
|
|
33
|
+
##------------------------------------------------------------------------.
|
|
34
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
34
35
|
# Define dictionary mapping dataset variables to select and rename
|
|
35
36
|
dict_names = {
|
|
36
37
|
### Dimensions
|
|
@@ -54,30 +55,8 @@ def reader(
|
|
|
54
55
|
"number_of_particles_classified": "raw_drop_number",
|
|
55
56
|
}
|
|
56
57
|
|
|
57
|
-
#
|
|
58
|
-
|
|
59
|
-
# Return dataset
|
|
60
|
-
return ds
|
|
61
|
-
|
|
62
|
-
# Define glob pattern to search data files in <raw_dir>/data/<station_name>
|
|
63
|
-
glob_patterns = "*.nc"
|
|
64
|
-
|
|
65
|
-
####----------------------------------------------------------------------.
|
|
66
|
-
#### - Create L0A products
|
|
67
|
-
run_l0b_from_nc(
|
|
68
|
-
raw_dir=raw_dir,
|
|
69
|
-
processed_dir=processed_dir,
|
|
70
|
-
station_name=station_name,
|
|
71
|
-
# Custom arguments of the reader
|
|
72
|
-
glob_patterns=glob_patterns,
|
|
73
|
-
dict_names=dict_names,
|
|
74
|
-
ds_sanitizer_fun=ds_sanitizer_fun,
|
|
75
|
-
# Processing options
|
|
76
|
-
force=force,
|
|
77
|
-
verbose=verbose,
|
|
78
|
-
parallel=parallel,
|
|
79
|
-
debugging_mode=debugging_mode,
|
|
80
|
-
)
|
|
81
|
-
|
|
58
|
+
# Rename dataset variables and columns and infill missing variables
|
|
59
|
+
ds = standardize_raw_dataset(ds=ds, dict_names=dict_names, sensor_name="PARSIVEL2")
|
|
82
60
|
|
|
83
|
-
#
|
|
61
|
+
# Return the dataset adhering to DISDRODB L0B standards
|
|
62
|
+
return ds
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
# -----------------------------------------------------------------------------.
|
|
4
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
5
|
+
#
|
|
6
|
+
# This program is free software: you can redistribute it and/or modify
|
|
7
|
+
# it under the terms of the GNU General Public License as published by
|
|
8
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
9
|
+
# (at your option) any later version.
|
|
10
|
+
#
|
|
11
|
+
# This program is distributed in the hope that it will be useful,
|
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
|
+
# GNU General Public License for more details.
|
|
15
|
+
#
|
|
16
|
+
# You should have received a copy of the GNU General Public License
|
|
17
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
18
|
+
# -----------------------------------------------------------------------------.
|
|
19
|
+
import pandas as pd
|
|
20
|
+
|
|
21
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
22
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@is_documented_by(reader_generic_docstring)
|
|
26
|
+
def reader(
|
|
27
|
+
filepath,
|
|
28
|
+
logger=None,
|
|
29
|
+
):
|
|
30
|
+
"""Reader."""
|
|
31
|
+
##------------------------------------------------------------------------.
|
|
32
|
+
#### Define column names
|
|
33
|
+
column_names = ["TO_SPLIT"]
|
|
34
|
+
|
|
35
|
+
##------------------------------------------------------------------------.
|
|
36
|
+
#### Define reader options
|
|
37
|
+
reader_kwargs = {}
|
|
38
|
+
# - Define delimiter
|
|
39
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
40
|
+
# - Avoid first column to become df index !!!
|
|
41
|
+
reader_kwargs["index_col"] = False
|
|
42
|
+
# Skip first row as columns names
|
|
43
|
+
reader_kwargs["header"] = None
|
|
44
|
+
# Skip the first row (header)
|
|
45
|
+
reader_kwargs["skiprows"] = 1
|
|
46
|
+
# - Define behaviour when encountering bad lines
|
|
47
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
48
|
+
# Define encoding
|
|
49
|
+
reader_kwargs["encoding"] = "latin1"
|
|
50
|
+
# - Define reader engine
|
|
51
|
+
# - C engine is faster
|
|
52
|
+
# - Python engine is more feature-complete
|
|
53
|
+
reader_kwargs["engine"] = "python"
|
|
54
|
+
# - Define on-the-fly decompression of on-disk data
|
|
55
|
+
# - Available: gzip, bz2, zip
|
|
56
|
+
reader_kwargs["compression"] = "infer"
|
|
57
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
58
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
59
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
60
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
61
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
62
|
+
|
|
63
|
+
##------------------------------------------------------------------------.
|
|
64
|
+
#### Read the data
|
|
65
|
+
df = read_raw_text_file(
|
|
66
|
+
filepath=filepath,
|
|
67
|
+
column_names=column_names,
|
|
68
|
+
reader_kwargs=reader_kwargs,
|
|
69
|
+
logger=logger,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
##------------------------------------------------------------------------.
|
|
73
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
74
|
+
# The delimiter ; is used for separating both the variables and the
|
|
75
|
+
# values of the raw spectrum. So we need to retrieve the columns
|
|
76
|
+
# inside the sanitizer assuming a fixed number of columns.
|
|
77
|
+
df = df["TO_SPLIT"].str.split(";", expand=True, n=16)
|
|
78
|
+
|
|
79
|
+
# Define the column names
|
|
80
|
+
column_names = [
|
|
81
|
+
"date",
|
|
82
|
+
"time",
|
|
83
|
+
"rainfall_rate_32bit",
|
|
84
|
+
"rainfall_accumulated_32bit",
|
|
85
|
+
"reflectivity_32bit",
|
|
86
|
+
"mor_visibility",
|
|
87
|
+
"laser_amplitude",
|
|
88
|
+
"number_particles",
|
|
89
|
+
"sensor_temperature",
|
|
90
|
+
"sensor_heating_current",
|
|
91
|
+
"sensor_battery_voltage",
|
|
92
|
+
"rain_kinetic_energy",
|
|
93
|
+
"snowfall_rate",
|
|
94
|
+
"weather_code_synop_4680",
|
|
95
|
+
"weather_code_metar_4678",
|
|
96
|
+
"weather_code_nws",
|
|
97
|
+
"raw_drop_number",
|
|
98
|
+
]
|
|
99
|
+
df.columns = column_names
|
|
100
|
+
|
|
101
|
+
# Define the time column
|
|
102
|
+
df["time"] = df["date"] + "-" + df["time"]
|
|
103
|
+
df["time"] = pd.to_datetime(df["time"], format="%Y/%m/%d-%H:%M:%S", errors="coerce")
|
|
104
|
+
df = df.drop(columns=["date"])
|
|
105
|
+
|
|
106
|
+
# Preprocess the raw spectrum
|
|
107
|
+
# - The '<SPECTRUM>ZERO</SPECTRUM>' indicates no drops detected
|
|
108
|
+
# --> "" generates an array of zeros in L0B processing
|
|
109
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
|
|
110
|
+
# Remove <SPECTRUM> and </SPECTRUM>" acronyms from the raw_drop_number field
|
|
111
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
|
|
112
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
|
|
113
|
+
|
|
114
|
+
# Add 0 before every ; if ; not preceded by a digit
|
|
115
|
+
# Example: ';;1;;' --> '0;0;1;0;'
|
|
116
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace(r"(?<!\d);", "0;", regex=True)
|
|
117
|
+
|
|
118
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
119
|
+
return df
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
20
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
21
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@is_documented_by(reader_generic_docstring)
|
|
25
|
+
def reader(
|
|
26
|
+
filepath,
|
|
27
|
+
logger=None,
|
|
28
|
+
):
|
|
29
|
+
"""Reader."""
|
|
30
|
+
##------------------------------------------------------------------------.
|
|
31
|
+
#### Define column names
|
|
32
|
+
column_names = ["time", "TO_BE_SPLITTED"]
|
|
33
|
+
|
|
34
|
+
##------------------------------------------------------------------------.
|
|
35
|
+
#### Define reader options
|
|
36
|
+
reader_kwargs = {}
|
|
37
|
+
# Skip first row as columns names
|
|
38
|
+
reader_kwargs["header"] = None
|
|
39
|
+
# Skip file with encoding errors
|
|
40
|
+
reader_kwargs["encoding_errors"] = "ignore"
|
|
41
|
+
# - Define delimiter
|
|
42
|
+
reader_kwargs["delimiter"] = ";"
|
|
43
|
+
# - Avoid first column to become df index !!!
|
|
44
|
+
reader_kwargs["index_col"] = False
|
|
45
|
+
# - Define behaviour when encountering bad lines
|
|
46
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
47
|
+
# - Define reader engine
|
|
48
|
+
# - C engine is faster
|
|
49
|
+
# - Python engine is more feature-complete
|
|
50
|
+
reader_kwargs["engine"] = "python"
|
|
51
|
+
# - Define on-the-fly decompression of on-disk data
|
|
52
|
+
# - Available: gzip, bz2, zip
|
|
53
|
+
reader_kwargs["compression"] = "infer"
|
|
54
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
55
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
56
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
57
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
58
|
+
reader_kwargs["na_values"] = ["na", "", "error", "NA", "-.-"]
|
|
59
|
+
|
|
60
|
+
##------------------------------------------------------------------------.
|
|
61
|
+
#### Read the data
|
|
62
|
+
df = read_raw_text_file(
|
|
63
|
+
filepath=filepath,
|
|
64
|
+
column_names=column_names,
|
|
65
|
+
reader_kwargs=reader_kwargs,
|
|
66
|
+
logger=logger,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
##------------------------------------------------------------------------.
|
|
70
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
71
|
+
# Define 'time' datetime
|
|
72
|
+
df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
|
|
73
|
+
|
|
74
|
+
# Split the 'TO_BE_SPLITTED' column
|
|
75
|
+
df = df["TO_BE_SPLITTED"].str.split(",", n=9, expand=True)
|
|
76
|
+
|
|
77
|
+
# Assign column names
|
|
78
|
+
column_names = [
|
|
79
|
+
"station_name",
|
|
80
|
+
"sensor_status",
|
|
81
|
+
"sensor_temperature",
|
|
82
|
+
"number_particles",
|
|
83
|
+
"rainfall_rate_32bit",
|
|
84
|
+
"reflectivity_16bit",
|
|
85
|
+
"mor_visibility",
|
|
86
|
+
"weather_code_synop_4680",
|
|
87
|
+
"weather_code_synop_4677",
|
|
88
|
+
"raw_drop_number",
|
|
89
|
+
]
|
|
90
|
+
df.columns = column_names
|
|
91
|
+
|
|
92
|
+
# Add the time column
|
|
93
|
+
df["time"] = df_time
|
|
94
|
+
|
|
95
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
96
|
+
df = df.drop(columns=["station_name"])
|
|
97
|
+
|
|
98
|
+
# Drop rows with invalid values
|
|
99
|
+
# --> Ensure that weather_code_synop_4677 has length 2
|
|
100
|
+
# --> If a previous column is missing it will have 000
|
|
101
|
+
df = df[df["weather_code_synop_4677"].str.len() == 2]
|
|
102
|
+
|
|
103
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
104
|
+
return df
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
import numpy as np
|
|
19
|
+
import pandas as pd
|
|
20
|
+
|
|
21
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
22
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@is_documented_by(reader_generic_docstring)
|
|
26
|
+
def reader(
|
|
27
|
+
filepath,
|
|
28
|
+
logger=None,
|
|
29
|
+
):
|
|
30
|
+
"""Reader."""
|
|
31
|
+
##------------------------------------------------------------------------.
|
|
32
|
+
#### Define column names
|
|
33
|
+
column_names = ["time", "TO_BE_SPLITTED"]
|
|
34
|
+
|
|
35
|
+
##------------------------------------------------------------------------.
|
|
36
|
+
#### Define reader options
|
|
37
|
+
reader_kwargs = {}
|
|
38
|
+
# - Define delimiter
|
|
39
|
+
reader_kwargs["delimiter"] = ";"
|
|
40
|
+
# - Skip first row as columns names
|
|
41
|
+
reader_kwargs["header"] = None
|
|
42
|
+
# - Skip file with encoding errors
|
|
43
|
+
reader_kwargs["encoding_errors"] = "ignore"
|
|
44
|
+
# - Avoid first column to become df index !!!
|
|
45
|
+
reader_kwargs["index_col"] = False
|
|
46
|
+
# - Define behaviour when encountering bad lines
|
|
47
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
48
|
+
# - Define reader engine
|
|
49
|
+
# - C engine is faster
|
|
50
|
+
# - Python engine is more feature-complete
|
|
51
|
+
reader_kwargs["engine"] = "python"
|
|
52
|
+
# - Define on-the-fly decompression of on-disk data
|
|
53
|
+
# - Available: gzip, bz2, zip
|
|
54
|
+
reader_kwargs["compression"] = "infer"
|
|
55
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
56
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
57
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
58
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
59
|
+
reader_kwargs["na_values"] = ["na", "", "error", "-.-"]
|
|
60
|
+
|
|
61
|
+
##------------------------------------------------------------------------.
|
|
62
|
+
#### Read the data
|
|
63
|
+
df = read_raw_text_file(
|
|
64
|
+
filepath=filepath,
|
|
65
|
+
column_names=column_names,
|
|
66
|
+
reader_kwargs=reader_kwargs,
|
|
67
|
+
logger=logger,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
##------------------------------------------------------------------------.
|
|
71
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
72
|
+
# NOTE: Has to deal with changing file format after 25 feb 2011 by the documentation
|
|
73
|
+
# --> https://ghrc.nsstc.nasa.gov/pub/fieldCampaigns/gpmValidation/relatedProjects/nsstc/parsivel/doc/gpm_parsivel_nsstc_dataset.html).
|
|
74
|
+
|
|
75
|
+
# Check 'time' string length to detect corrupted rows
|
|
76
|
+
df = df[df["time"].str.len() == 14]
|
|
77
|
+
|
|
78
|
+
# Convert time column to datetime
|
|
79
|
+
df["time"] = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
|
|
80
|
+
|
|
81
|
+
# Count number of delimiters in the column to be parsed
|
|
82
|
+
# --> Some first rows are corrupted, so count the most frequent occurrence
|
|
83
|
+
possible_delimiters, counts = np.unique(df["TO_BE_SPLITTED"].str.count(","), return_counts=True)
|
|
84
|
+
n_delimiters = possible_delimiters[np.argmax(counts)]
|
|
85
|
+
|
|
86
|
+
if n_delimiters == 1027: # APU 2010
|
|
87
|
+
# Select valid rows
|
|
88
|
+
df = df.loc[df["TO_BE_SPLITTED"].str.count(",") == 1027]
|
|
89
|
+
# Get time column
|
|
90
|
+
df_time = df["time"]
|
|
91
|
+
# Split the 'TO_BE_SPLITTED' column
|
|
92
|
+
df = df["TO_BE_SPLITTED"].str.split(",", expand=True, n=3)
|
|
93
|
+
# Assign column names
|
|
94
|
+
column_names = [
|
|
95
|
+
"station_name",
|
|
96
|
+
"sensor_status",
|
|
97
|
+
"sensor_temperature",
|
|
98
|
+
"raw_drop_number",
|
|
99
|
+
]
|
|
100
|
+
df.columns = column_names
|
|
101
|
+
# Add time column
|
|
102
|
+
df["time"] = df_time
|
|
103
|
+
# Add missing columns and set NaN value
|
|
104
|
+
missing_columns = [
|
|
105
|
+
"number_particles",
|
|
106
|
+
"rainfall_rate_32bit",
|
|
107
|
+
"reflectivity_16bit",
|
|
108
|
+
"mor_visibility",
|
|
109
|
+
"weather_code_synop_4680",
|
|
110
|
+
"weather_code_synop_4677",
|
|
111
|
+
]
|
|
112
|
+
for column in missing_columns:
|
|
113
|
+
df[column] = "NaN"
|
|
114
|
+
elif n_delimiters == 1031: # APU08 (2011)
|
|
115
|
+
# Select valid rows
|
|
116
|
+
df = df.loc[df["TO_BE_SPLITTED"].str.count(",") == 1031]
|
|
117
|
+
# Get time column
|
|
118
|
+
df_time = df["time"]
|
|
119
|
+
# Split the 'TO_BE_SPLITTED' column
|
|
120
|
+
df = df["TO_BE_SPLITTED"].str.split(",", expand=True, n=7)
|
|
121
|
+
# Assign column names
|
|
122
|
+
column_names = [
|
|
123
|
+
"station_name",
|
|
124
|
+
"sensor_status",
|
|
125
|
+
"sensor_temperature",
|
|
126
|
+
"reflectivity_32bit",
|
|
127
|
+
"mor_visibility",
|
|
128
|
+
"weather_code_synop_4680",
|
|
129
|
+
"weather_code_synop_4677",
|
|
130
|
+
"raw_drop_number",
|
|
131
|
+
]
|
|
132
|
+
df.columns = column_names
|
|
133
|
+
# Add time column
|
|
134
|
+
df["time"] = df_time
|
|
135
|
+
# Remove columns not in other files
|
|
136
|
+
df = df.drop(columns="reflectivity_32bit")
|
|
137
|
+
# Add missing columns and set NaN value
|
|
138
|
+
missing_columns = [
|
|
139
|
+
"number_particles",
|
|
140
|
+
"rainfall_rate_32bit",
|
|
141
|
+
"reflectivity_16bit",
|
|
142
|
+
]
|
|
143
|
+
for column in missing_columns:
|
|
144
|
+
df[column] = "NaN"
|
|
145
|
+
elif n_delimiters == 1033:
|
|
146
|
+
# Select valid rows
|
|
147
|
+
df = df.loc[df["TO_BE_SPLITTED"].str.count(",") == 1033]
|
|
148
|
+
# Get time column
|
|
149
|
+
df_time = df["time"]
|
|
150
|
+
# Split the column be parsed
|
|
151
|
+
df = df["TO_BE_SPLITTED"].str.split(",", expand=True, n=9)
|
|
152
|
+
# Assign column names
|
|
153
|
+
column_names = [
|
|
154
|
+
"station_name",
|
|
155
|
+
"sensor_status",
|
|
156
|
+
"sensor_temperature",
|
|
157
|
+
"number_particles",
|
|
158
|
+
"rainfall_rate_32bit",
|
|
159
|
+
"reflectivity_16bit",
|
|
160
|
+
"mor_visibility",
|
|
161
|
+
"weather_code_synop_4680",
|
|
162
|
+
"weather_code_synop_4677",
|
|
163
|
+
"raw_drop_number",
|
|
164
|
+
]
|
|
165
|
+
df.columns = column_names
|
|
166
|
+
# Add time column
|
|
167
|
+
df["time"] = df_time
|
|
168
|
+
else:
|
|
169
|
+
# Wrong number of delimiters ... likely a corrupted file
|
|
170
|
+
raise ValueError("Unexpected number of comma delimiters !")
|
|
171
|
+
|
|
172
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
173
|
+
df = df.drop(columns=["station_name"])
|
|
174
|
+
|
|
175
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
176
|
+
return df
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
# -----------------------------------------------------------------------------.
|
|
4
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
5
|
+
#
|
|
6
|
+
# This program is free software: you can redistribute it and/or modify
|
|
7
|
+
# it under the terms of the GNU General Public License as published by
|
|
8
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
9
|
+
# (at your option) any later version.
|
|
10
|
+
#
|
|
11
|
+
# This program is distributed in the hope that it will be useful,
|
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
|
+
# GNU General Public License for more details.
|
|
15
|
+
#
|
|
16
|
+
# You should have received a copy of the GNU General Public License
|
|
17
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
18
|
+
# -----------------------------------------------------------------------------.
|
|
19
|
+
import pandas as pd # noqa
|
|
20
|
+
|
|
21
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
22
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file # noqa
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@is_documented_by(reader_generic_docstring)
|
|
26
|
+
def reader(
|
|
27
|
+
filepath,
|
|
28
|
+
logger=None,
|
|
29
|
+
):
|
|
30
|
+
"""Reader."""
|
|
31
|
+
##------------------------------------------------------------------------.
|
|
32
|
+
raise NotImplementedError
|