disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +132 -15
- disdrodb/_config.py +4 -2
- disdrodb/_version.py +9 -4
- disdrodb/api/checks.py +264 -237
- disdrodb/api/configs.py +4 -8
- disdrodb/api/create_directories.py +235 -290
- disdrodb/api/info.py +217 -26
- disdrodb/api/io.py +306 -270
- disdrodb/api/path.py +597 -173
- disdrodb/api/search.py +486 -0
- disdrodb/{metadata/scripts → cli}/disdrodb_check_metadata_archive.py +12 -7
- disdrodb/{utils/pandas.py → cli/disdrodb_data_archive_directory.py} +9 -18
- disdrodb/cli/disdrodb_download_archive.py +86 -0
- disdrodb/cli/disdrodb_download_metadata_archive.py +53 -0
- disdrodb/cli/disdrodb_download_station.py +84 -0
- disdrodb/{api/scripts → cli}/disdrodb_initialize_station.py +22 -10
- disdrodb/cli/disdrodb_metadata_archive_directory.py +32 -0
- disdrodb/{data_transfer/scripts/disdrodb_download_station.py → cli/disdrodb_open_data_archive.py} +22 -22
- disdrodb/cli/disdrodb_open_logs_directory.py +69 -0
- disdrodb/{data_transfer/scripts/disdrodb_upload_station.py → cli/disdrodb_open_metadata_archive.py} +22 -24
- disdrodb/cli/disdrodb_open_metadata_directory.py +71 -0
- disdrodb/cli/disdrodb_open_product_directory.py +74 -0
- disdrodb/cli/disdrodb_open_readers_directory.py +32 -0
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0.py +38 -31
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0_station.py +32 -30
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0a_station.py +24 -33
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b.py +30 -21
- disdrodb/{l0/scripts → cli}/disdrodb_run_l0b_station.py +25 -34
- disdrodb/cli/disdrodb_run_l0c.py +130 -0
- disdrodb/cli/disdrodb_run_l0c_station.py +129 -0
- disdrodb/cli/disdrodb_run_l1.py +122 -0
- disdrodb/cli/disdrodb_run_l1_station.py +121 -0
- disdrodb/cli/disdrodb_run_l2e.py +122 -0
- disdrodb/cli/disdrodb_run_l2e_station.py +122 -0
- disdrodb/cli/disdrodb_run_l2m.py +122 -0
- disdrodb/cli/disdrodb_run_l2m_station.py +122 -0
- disdrodb/cli/disdrodb_upload_archive.py +105 -0
- disdrodb/cli/disdrodb_upload_station.py +98 -0
- disdrodb/configs.py +90 -25
- disdrodb/data_transfer/__init__.py +22 -0
- disdrodb/data_transfer/download_data.py +87 -90
- disdrodb/data_transfer/upload_data.py +64 -37
- disdrodb/data_transfer/zenodo.py +15 -18
- disdrodb/docs.py +1 -1
- disdrodb/issue/__init__.py +17 -4
- disdrodb/issue/checks.py +10 -23
- disdrodb/issue/reader.py +9 -12
- disdrodb/issue/writer.py +14 -17
- disdrodb/l0/__init__.py +17 -26
- disdrodb/l0/check_configs.py +35 -23
- disdrodb/l0/check_standards.py +46 -51
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_diameter.yml +44 -44
- disdrodb/l0/configs/{Thies_LPM → LPM}/bins_velocity.yml +40 -40
- disdrodb/l0/configs/LPM/l0a_encodings.yml +80 -0
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_cf_attrs.yml +84 -65
- disdrodb/l0/configs/{Thies_LPM → LPM}/l0b_encodings.yml +50 -9
- disdrodb/l0/configs/{Thies_LPM → LPM}/raw_data_format.yml +285 -245
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_diameter.yml +66 -66
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL/l0a_encodings.yml +32 -0
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_cf_attrs.yml +23 -21
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/l0b_encodings.yml +17 -17
- disdrodb/l0/configs/{OTT_Parsivel → PARSIVEL}/raw_data_format.yml +77 -77
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_diameter.yml +64 -64
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/bins_velocity.yml +64 -64
- disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +39 -0
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_cf_attrs.yml +28 -26
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/l0b_encodings.yml +20 -20
- disdrodb/l0/configs/{OTT_Parsivel2 → PARSIVEL2}/raw_data_format.yml +107 -107
- disdrodb/l0/configs/PWS100/bins_diameter.yml +173 -0
- disdrodb/l0/configs/PWS100/bins_velocity.yml +173 -0
- disdrodb/l0/configs/PWS100/l0a_encodings.yml +19 -0
- disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +76 -0
- disdrodb/l0/configs/PWS100/l0b_encodings.yml +176 -0
- disdrodb/l0/configs/PWS100/raw_data_format.yml +182 -0
- disdrodb/l0/configs/{RD_80 → RD80}/bins_diameter.yml +40 -40
- disdrodb/l0/configs/RD80/l0a_encodings.yml +16 -0
- disdrodb/l0/configs/{RD_80 → RD80}/l0b_cf_attrs.yml +3 -3
- disdrodb/l0/configs/RD80/l0b_encodings.yml +135 -0
- disdrodb/l0/configs/{RD_80 → RD80}/raw_data_format.yml +46 -50
- disdrodb/l0/l0_reader.py +216 -340
- disdrodb/l0/l0a_processing.py +237 -208
- disdrodb/l0/l0b_nc_processing.py +227 -80
- disdrodb/l0/l0b_processing.py +96 -174
- disdrodb/l0/l0c_processing.py +627 -0
- disdrodb/l0/readers/{ARM → LPM/ARM}/ARM_LPM.py +36 -58
- disdrodb/l0/readers/LPM/AUSTRALIA/MELBOURNE_2007_LPM.py +236 -0
- disdrodb/l0/readers/LPM/BRAZIL/CHUVA_LPM.py +185 -0
- disdrodb/l0/readers/LPM/BRAZIL/GOAMAZON_LPM.py +185 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +195 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +210 -0
- disdrodb/l0/readers/{BRAZIL/GOAMAZON_LPM.py → LPM/KIT/CHWALA.py} +97 -76
- disdrodb/l0/readers/LPM/SLOVENIA/ARSO.py +197 -0
- disdrodb/l0/readers/LPM/SLOVENIA/CRNI_VRH.py +197 -0
- disdrodb/l0/readers/{UK → LPM/UK}/DIVEN.py +14 -35
- disdrodb/l0/readers/PARSIVEL/AUSTRALIA/MELBOURNE_2007_PARSIVEL.py +157 -0
- disdrodb/l0/readers/PARSIVEL/CHINA/CHONGQING.py +113 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/ARCTIC_2021.py +40 -57
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/COMMON_2011.py +37 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/DAVOS_2009_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_2009.py +34 -51
- disdrodb/l0/readers/{EPFL/PARADISO_2014.py → PARSIVEL/EPFL/EPFL_ROOF_2008.py} +38 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/EPFL_ROOF_2010.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2011.py +34 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/EPFL_ROOF_2012.py +33 -51
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GENEPI_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/GRAND_ST_BERNARD_2007_2.py +25 -44
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HPICONET_2010.py +34 -51
- disdrodb/l0/readers/{EPFL/EPFL_ROOF_2010.py → PARSIVEL/EPFL/HYMEX_LTE_SOP2.py} +37 -50
- disdrodb/l0/readers/PARSIVEL/EPFL/HYMEX_LTE_SOP3.py +111 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/HYMEX_LTE_SOP4.py +36 -54
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2018.py +34 -52
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/LOCARNO_2019.py +38 -56
- disdrodb/l0/readers/PARSIVEL/EPFL/PARADISO_2014.py +105 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PARSIVEL_2007.py +27 -45
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/PLATO_2019.py +24 -44
- disdrodb/l0/readers/PARSIVEL/EPFL/RACLETS_2019.py +140 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RACLETS_2019_WJF.py +41 -59
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/RIETHOLZBACH_2011.py +34 -51
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2017.py +117 -0
- disdrodb/l0/readers/PARSIVEL/EPFL/SAMOYLOV_2019.py +137 -0
- disdrodb/l0/readers/{EPFL → PARSIVEL/EPFL}/UNIL_2022.py +42 -55
- disdrodb/l0/readers/PARSIVEL/GPM/IFLOODS.py +104 -0
- disdrodb/l0/readers/{GPM → PARSIVEL/GPM}/LPVEX.py +29 -48
- disdrodb/l0/readers/PARSIVEL/GPM/MC3E.py +184 -0
- disdrodb/l0/readers/PARSIVEL/KIT/BURKINA_FASO.py +133 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/CCOPE_2015.py +113 -0
- disdrodb/l0/readers/{NCAR/VORTEX_SE_2016_P1.py → PARSIVEL/NCAR/OWLES_MIPS.py} +46 -72
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +125 -0
- disdrodb/l0/readers/{NCAR/OWLES_MIPS.py → PARSIVEL/NCAR/PLOWS_MIPS.py} +45 -64
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +114 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010.py +176 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2010_UF.py +183 -0
- disdrodb/l0/readers/PARSIVEL/SLOVENIA/UL_FGG.py +121 -0
- disdrodb/l0/readers/{ARM/ARM_LD.py → PARSIVEL2/ARM/ARM_PARSIVEL2.py} +27 -50
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/CHUVA_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/PARSIVEL2/BRAZIL/GOAMAZON_PARSIVEL2.py +163 -0
- disdrodb/l0/readers/{DENMARK → PARSIVEL2/DENMARK}/EROSION_nc.py +14 -35
- disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +189 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +119 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +104 -0
- disdrodb/l0/readers/PARSIVEL2/GPM/NSSTC.py +176 -0
- disdrodb/l0/readers/PARSIVEL2/ITALY/GID_PARSIVEL2.py +32 -0
- disdrodb/l0/readers/PARSIVEL2/MEXICO/OH_IIUNAM_nc.py +56 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +120 -0
- disdrodb/l0/readers/{NCAR → PARSIVEL2/NCAR}/PECAN_MIPS.py +45 -64
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +181 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_PJ.py +160 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/SNOWIE_SB.py +160 -0
- disdrodb/l0/readers/{NCAR/PLOWS_MIPS.py → PARSIVEL2/NCAR/VORTEX_SE_2016_P1.py} +49 -66
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +118 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +152 -0
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT.py +166 -0
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +150 -0
- disdrodb/l0/readers/{NCAR/RELAMPAGO_RD80.py → RD80/BRAZIL/CHUVA_RD80.py} +36 -60
- disdrodb/l0/readers/{BRAZIL → RD80/BRAZIL}/GOAMAZON_RD80.py +36 -55
- disdrodb/l0/readers/{NCAR → RD80/NCAR}/CINDY_2011_RD80.py +35 -54
- disdrodb/l0/readers/{BRAZIL/CHUVA_RD80.py → RD80/NCAR/RELAMPAGO_RD80.py} +40 -54
- disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py +274 -0
- disdrodb/l0/readers/template_reader_raw_netcdf_data.py +62 -0
- disdrodb/l0/readers/{reader_template.py → template_reader_raw_text_data.py} +20 -44
- disdrodb/l0/routines.py +885 -581
- disdrodb/l0/standards.py +77 -238
- disdrodb/l0/template_tools.py +105 -110
- disdrodb/l1/__init__.py +17 -0
- disdrodb/l1/beard_model.py +716 -0
- disdrodb/l1/encoding_attrs.py +635 -0
- disdrodb/l1/fall_velocity.py +260 -0
- disdrodb/l1/filters.py +192 -0
- disdrodb/l1/processing.py +202 -0
- disdrodb/l1/resampling.py +236 -0
- disdrodb/l1/routines.py +358 -0
- disdrodb/l1_env/__init__.py +17 -0
- disdrodb/l1_env/routines.py +38 -0
- disdrodb/l2/__init__.py +17 -0
- disdrodb/l2/empirical_dsd.py +1833 -0
- disdrodb/l2/event.py +388 -0
- disdrodb/l2/processing.py +528 -0
- disdrodb/l2/processing_options.py +213 -0
- disdrodb/l2/routines.py +868 -0
- disdrodb/metadata/__init__.py +9 -2
- disdrodb/metadata/checks.py +180 -124
- disdrodb/metadata/download.py +81 -0
- disdrodb/metadata/geolocation.py +146 -0
- disdrodb/metadata/info.py +20 -13
- disdrodb/metadata/manipulation.py +3 -3
- disdrodb/metadata/reader.py +59 -8
- disdrodb/metadata/search.py +77 -144
- disdrodb/metadata/standards.py +83 -80
- disdrodb/metadata/writer.py +10 -16
- disdrodb/psd/__init__.py +38 -0
- disdrodb/psd/fitting.py +2146 -0
- disdrodb/psd/models.py +774 -0
- disdrodb/routines.py +1412 -0
- disdrodb/scattering/__init__.py +28 -0
- disdrodb/scattering/axis_ratio.py +344 -0
- disdrodb/scattering/routines.py +456 -0
- disdrodb/utils/__init__.py +17 -0
- disdrodb/utils/attrs.py +208 -0
- disdrodb/utils/cli.py +269 -0
- disdrodb/utils/compression.py +60 -42
- disdrodb/utils/dask.py +62 -0
- disdrodb/utils/dataframe.py +342 -0
- disdrodb/utils/decorators.py +110 -0
- disdrodb/utils/directories.py +107 -46
- disdrodb/utils/encoding.py +127 -0
- disdrodb/utils/list.py +29 -0
- disdrodb/utils/logger.py +168 -46
- disdrodb/utils/time.py +657 -0
- disdrodb/utils/warnings.py +30 -0
- disdrodb/utils/writer.py +57 -0
- disdrodb/utils/xarray.py +138 -47
- disdrodb/utils/yaml.py +0 -1
- disdrodb/viz/__init__.py +17 -0
- disdrodb/viz/plots.py +17 -0
- disdrodb-0.1.1.dist-info/METADATA +294 -0
- disdrodb-0.1.1.dist-info/RECORD +232 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/WHEEL +1 -1
- disdrodb-0.1.1.dist-info/entry_points.txt +30 -0
- disdrodb/data_transfer/scripts/disdrodb_download_archive.py +0 -53
- disdrodb/data_transfer/scripts/disdrodb_upload_archive.py +0 -57
- disdrodb/l0/configs/OTT_Parsivel/l0a_encodings.yml +0 -32
- disdrodb/l0/configs/OTT_Parsivel2/l0a_encodings.yml +0 -39
- disdrodb/l0/configs/RD_80/l0a_encodings.yml +0 -16
- disdrodb/l0/configs/RD_80/l0b_encodings.yml +0 -135
- disdrodb/l0/configs/Thies_LPM/l0a_encodings.yml +0 -80
- disdrodb/l0/io.py +0 -257
- disdrodb/l0/l0_processing.py +0 -1091
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_OTT.py +0 -178
- disdrodb/l0/readers/AUSTRALIA/MELBOURNE_2007_THIES.py +0 -247
- disdrodb/l0/readers/BRAZIL/CHUVA_LPM.py +0 -204
- disdrodb/l0/readers/BRAZIL/CHUVA_OTT.py +0 -183
- disdrodb/l0/readers/BRAZIL/GOAMAZON_OTT.py +0 -183
- disdrodb/l0/readers/CHINA/CHONGQING.py +0 -131
- disdrodb/l0/readers/EPFL/EPFL_ROOF_2008.py +0 -128
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP2.py +0 -127
- disdrodb/l0/readers/EPFL/HYMEX_LTE_SOP3.py +0 -129
- disdrodb/l0/readers/EPFL/RACLETS_2019.py +0 -158
- disdrodb/l0/readers/EPFL/SAMOYLOV_2017.py +0 -136
- disdrodb/l0/readers/EPFL/SAMOYLOV_2019.py +0 -158
- disdrodb/l0/readers/FRANCE/SIRTA_OTT2.py +0 -138
- disdrodb/l0/readers/GPM/GCPEX.py +0 -123
- disdrodb/l0/readers/GPM/IFLOODS.py +0 -123
- disdrodb/l0/readers/GPM/MC3E.py +0 -123
- disdrodb/l0/readers/GPM/NSSTC.py +0 -164
- disdrodb/l0/readers/ITALY/GID.py +0 -199
- disdrodb/l0/readers/MEXICO/OH_IIUNAM_nc.py +0 -92
- disdrodb/l0/readers/NCAR/CCOPE_2015.py +0 -133
- disdrodb/l0/readers/NCAR/PECAN_FP3.py +0 -137
- disdrodb/l0/readers/NCAR/PECAN_MOBILE.py +0 -144
- disdrodb/l0/readers/NCAR/RELAMPAGO_OTT.py +0 -195
- disdrodb/l0/readers/NCAR/SNOWIE_PJ.py +0 -172
- disdrodb/l0/readers/NCAR/SNOWIE_SB.py +0 -179
- disdrodb/l0/readers/NCAR/VORTEX2_2009.py +0 -133
- disdrodb/l0/readers/NCAR/VORTEX2_2010.py +0 -188
- disdrodb/l0/readers/NCAR/VORTEX2_2010_UF.py +0 -191
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_P2.py +0 -135
- disdrodb/l0/readers/NCAR/VORTEX_SE_2016_PIPS.py +0 -170
- disdrodb/l0/readers/NETHERLANDS/DELFT.py +0 -187
- disdrodb/l0/readers/SPAIN/SBEGUERIA.py +0 -179
- disdrodb/l0/scripts/disdrodb_run_l0b_concat.py +0 -93
- disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py +0 -85
- disdrodb/utils/netcdf.py +0 -452
- disdrodb/utils/scripts.py +0 -102
- disdrodb-0.0.21.dist-info/AUTHORS.md +0 -18
- disdrodb-0.0.21.dist-info/METADATA +0 -186
- disdrodb-0.0.21.dist-info/RECORD +0 -168
- disdrodb-0.0.21.dist-info/entry_points.txt +0 -15
- /disdrodb/l0/configs/{RD_80 → RD80}/bins_velocity.yml +0 -0
- /disdrodb/l0/manuals/{Thies_LPM.pdf → LPM.pdf} +0 -0
- /disdrodb/l0/manuals/{ODM_470.pdf → ODM470.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel.pdf → PARSIVEL.pdf} +0 -0
- /disdrodb/l0/manuals/{OTT_Parsivel2.pdf → PARSIVEL2.pdf} +0 -0
- /disdrodb/l0/manuals/{PWS_100.pdf → PWS100.pdf} +0 -0
- /disdrodb/l0/manuals/{RD_80.pdf → RD80.pdf} +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info/licenses}/LICENSE +0 -0
- {disdrodb-0.0.21.dist-info → disdrodb-0.1.1.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
+
|
|
2
3
|
# -----------------------------------------------------------------------------.
|
|
3
4
|
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
5
|
#
|
|
@@ -15,61 +16,78 @@
|
|
|
15
16
|
# You should have received a copy of the GNU General Public License
|
|
16
17
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
18
|
# -----------------------------------------------------------------------------.
|
|
18
|
-
|
|
19
|
+
"""DISDRODB reader for GID LPM sensors not measuring wind."""
|
|
20
|
+
import os
|
|
21
|
+
|
|
22
|
+
import pandas as pd
|
|
23
|
+
|
|
19
24
|
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
25
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
20
26
|
|
|
21
27
|
|
|
22
28
|
@is_documented_by(reader_generic_docstring)
|
|
23
29
|
def reader(
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
station_name,
|
|
27
|
-
# Processing options
|
|
28
|
-
force=False,
|
|
29
|
-
verbose=False,
|
|
30
|
-
parallel=False,
|
|
31
|
-
debugging_mode=False,
|
|
30
|
+
filepath,
|
|
31
|
+
logger=None,
|
|
32
32
|
):
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
# ‘NA’, ‘NULL’, ‘NaN’, ‘n/a’, ‘nan’, ‘null’
|
|
60
|
-
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
61
|
-
|
|
62
|
-
##------------------------------------------------------------------------.
|
|
63
|
-
#### - Define dataframe sanitizer function for L0 processing
|
|
64
|
-
def df_sanitizer_fun(df):
|
|
65
|
-
# - Import pandas
|
|
66
|
-
import pandas as pd
|
|
33
|
+
"""Reader."""
|
|
34
|
+
|
|
35
|
+
def read_txt_file(file, filename):
|
|
36
|
+
##------------------------------------------------------------------------.
|
|
37
|
+
#### - Define raw data headers
|
|
38
|
+
column_names = ["TO_BE_SPLITTED"]
|
|
39
|
+
|
|
40
|
+
##------------------------------------------------------------------------.
|
|
41
|
+
#### Define reader options
|
|
42
|
+
# - For more info: https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
|
|
43
|
+
reader_kwargs = {}
|
|
44
|
+
|
|
45
|
+
# - Define delimiter
|
|
46
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
47
|
+
|
|
48
|
+
# - Avoid first column to become df index !!!
|
|
49
|
+
reader_kwargs["index_col"] = False
|
|
50
|
+
|
|
51
|
+
# Since column names are expected to be passed explicitly, header is set to None
|
|
52
|
+
reader_kwargs["header"] = None
|
|
53
|
+
|
|
54
|
+
# - Number of rows to be skipped at the beginning of the file
|
|
55
|
+
reader_kwargs["skiprows"] = None
|
|
56
|
+
|
|
57
|
+
# - Define behaviour when encountering bad lines
|
|
58
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
67
59
|
|
|
60
|
+
# - Define reader engine
|
|
61
|
+
# - C engine is faster
|
|
62
|
+
# - Python engine is more feature-complete
|
|
63
|
+
reader_kwargs["engine"] = "python"
|
|
64
|
+
|
|
65
|
+
# - Define on-the-fly decompression of on-disk data
|
|
66
|
+
# - Available: gzip, bz2, zip
|
|
67
|
+
reader_kwargs["compression"] = "infer"
|
|
68
|
+
|
|
69
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
70
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
71
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
72
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
73
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
74
|
+
|
|
75
|
+
##------------------------------------------------------------------------.
|
|
76
|
+
#### Read the data
|
|
77
|
+
df = read_raw_text_file(
|
|
78
|
+
filepath=file,
|
|
79
|
+
column_names=column_names,
|
|
80
|
+
reader_kwargs=reader_kwargs,
|
|
81
|
+
logger=logger,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
##------------------------------------------------------------------------.
|
|
85
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
68
86
|
# Count number of delimiters to identify valid rows
|
|
69
|
-
df = df[df["
|
|
87
|
+
df = df[df["TO_BE_SPLITTED"].str.count(";") == 520]
|
|
70
88
|
|
|
71
|
-
# Split by ; delimiter
|
|
72
|
-
df = df["
|
|
89
|
+
# Split by ; delimiter (before raw drop number)
|
|
90
|
+
df = df["TO_BE_SPLITTED"].str.split(";", expand=True, n=79)
|
|
73
91
|
|
|
74
92
|
# Assign column names
|
|
75
93
|
column_names = [
|
|
@@ -156,20 +174,20 @@ def reader(
|
|
|
156
174
|
]
|
|
157
175
|
df.columns = column_names
|
|
158
176
|
|
|
159
|
-
#
|
|
160
|
-
df["
|
|
161
|
-
|
|
177
|
+
# Remove checksum from raw_drop_number
|
|
178
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.rsplit(";", n=2, expand=True)[0]
|
|
179
|
+
|
|
180
|
+
# Define datetime "time" column from file name
|
|
181
|
+
datetime_str = "".join(filename.split("_")[1:3])
|
|
182
|
+
df["time"] = pd.to_datetime(datetime_str, format="%Y%m%d%H%M", errors="coerce")
|
|
162
183
|
|
|
163
184
|
# Drop row if start_identifier different than 00
|
|
164
185
|
df = df[df["start_identifier"].astype(str) == "00"]
|
|
165
186
|
|
|
166
|
-
# Clean raw_drop_number (ignore last "AC" character)
|
|
167
|
-
df["raw_drop_number"] = df["raw_drop_number"].str[:1760]
|
|
168
|
-
|
|
169
187
|
# Drop rows with invalid raw_drop_number
|
|
170
|
-
df = df[df["raw_drop_number"].astype(str).str.len() ==
|
|
188
|
+
df = df[df["raw_drop_number"].astype(str).str.len() == 1759]
|
|
171
189
|
|
|
172
|
-
#
|
|
190
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
173
191
|
columns_to_drop = [
|
|
174
192
|
"start_identifier",
|
|
175
193
|
"device_address",
|
|
@@ -178,27 +196,30 @@ def reader(
|
|
|
178
196
|
"sensor_time",
|
|
179
197
|
]
|
|
180
198
|
df = df.drop(columns=columns_to_drop)
|
|
181
|
-
|
|
199
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
182
200
|
return df
|
|
183
201
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
)
|
|
202
|
+
#### TEMPORARY: to read just a single 1-min timestep
|
|
203
|
+
df = read_txt_file(file=filepath, filename=os.path.basename(filepath))
|
|
204
|
+
|
|
205
|
+
#### FUTURE: Iterate over all files (aka 1-min timesteps) in the daily zip archive
|
|
206
|
+
# - Each file contain a single timestep !
|
|
207
|
+
# list_df = []
|
|
208
|
+
# with zipfile.ZipFile(filepath, "r") as zip_ref:
|
|
209
|
+
# filenames = sorted(zip_ref.namelist())
|
|
210
|
+
# for filename in filenames:
|
|
211
|
+
# if filename.endswith(".dat"):
|
|
212
|
+
# # Open file
|
|
213
|
+
# with zip_ref.open(filename) as file:
|
|
214
|
+
# try:
|
|
215
|
+
# df = read_txt_file(file=file, filename=filename)
|
|
216
|
+
# list_df.append(df)
|
|
217
|
+
# except Exception as e:
|
|
218
|
+
# msg = f"An error occurred while reading {filename}. The error is: {e}."
|
|
219
|
+
# log_error(logger=logger, msg=msg, verbose=True)
|
|
220
|
+
|
|
221
|
+
# Concatenate all dataframes into a single one
|
|
222
|
+
# df = pd.concat(list_df)
|
|
223
|
+
|
|
224
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
225
|
+
return df
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
# -----------------------------------------------------------------------------.
|
|
4
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
5
|
+
#
|
|
6
|
+
# This program is free software: you can redistribute it and/or modify
|
|
7
|
+
# it under the terms of the GNU General Public License as published by
|
|
8
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
9
|
+
# (at your option) any later version.
|
|
10
|
+
#
|
|
11
|
+
# This program is distributed in the hope that it will be useful,
|
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
|
+
# GNU General Public License for more details.
|
|
15
|
+
#
|
|
16
|
+
# You should have received a copy of the GNU General Public License
|
|
17
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
18
|
+
# -----------------------------------------------------------------------------.
|
|
19
|
+
"""DISDRODB reader for GID LPM sensors not measuring wind."""
|
|
20
|
+
import pandas as pd
|
|
21
|
+
|
|
22
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
23
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@is_documented_by(reader_generic_docstring)
|
|
27
|
+
def reader(
|
|
28
|
+
filepath,
|
|
29
|
+
logger=None,
|
|
30
|
+
):
|
|
31
|
+
"""Reader."""
|
|
32
|
+
##------------------------------------------------------------------------.
|
|
33
|
+
#### - Define raw data headers
|
|
34
|
+
column_names = ["TO_PARSE"]
|
|
35
|
+
|
|
36
|
+
##------------------------------------------------------------------------.
|
|
37
|
+
#### Define reader options
|
|
38
|
+
# - For more info: https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
|
|
39
|
+
reader_kwargs = {}
|
|
40
|
+
|
|
41
|
+
# - Define delimiter
|
|
42
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
43
|
+
|
|
44
|
+
# - Avoid first column to become df index !!!
|
|
45
|
+
reader_kwargs["index_col"] = False
|
|
46
|
+
|
|
47
|
+
# Since column names are expected to be passed explicitly, header is set to None
|
|
48
|
+
reader_kwargs["header"] = None
|
|
49
|
+
|
|
50
|
+
# - Number of rows to be skipped at the beginning of the file
|
|
51
|
+
reader_kwargs["skiprows"] = None
|
|
52
|
+
|
|
53
|
+
# - Define behaviour when encountering bad lines
|
|
54
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
55
|
+
|
|
56
|
+
# - Define reader engine
|
|
57
|
+
# - C engine is faster
|
|
58
|
+
# - Python engine is more feature-complete
|
|
59
|
+
reader_kwargs["engine"] = "python"
|
|
60
|
+
|
|
61
|
+
# - Define on-the-fly decompression of on-disk data
|
|
62
|
+
# - Available: gzip, bz2, zip
|
|
63
|
+
reader_kwargs["compression"] = "infer"
|
|
64
|
+
|
|
65
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
66
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
67
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
68
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
69
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
70
|
+
|
|
71
|
+
##------------------------------------------------------------------------.
|
|
72
|
+
#### Read the data
|
|
73
|
+
df = read_raw_text_file(
|
|
74
|
+
filepath=filepath,
|
|
75
|
+
column_names=column_names,
|
|
76
|
+
reader_kwargs=reader_kwargs,
|
|
77
|
+
logger=logger,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
##------------------------------------------------------------------------.
|
|
81
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
82
|
+
# Count number of delimiters to identify valid rows
|
|
83
|
+
df = df[df["TO_PARSE"].str.count(";") == 521]
|
|
84
|
+
|
|
85
|
+
# Split by ; delimiter (before raw drop number)
|
|
86
|
+
df = df["TO_PARSE"].str.split(";", expand=True, n=80)
|
|
87
|
+
|
|
88
|
+
# Assign column names
|
|
89
|
+
column_names = [
|
|
90
|
+
"time",
|
|
91
|
+
"start_identifier",
|
|
92
|
+
"device_address",
|
|
93
|
+
"sensor_serial_number",
|
|
94
|
+
"sensor_date",
|
|
95
|
+
"sensor_time",
|
|
96
|
+
"weather_code_synop_4677_5min",
|
|
97
|
+
"weather_code_synop_4680_5min",
|
|
98
|
+
"weather_code_metar_4678_5min",
|
|
99
|
+
"precipitation_rate_5min",
|
|
100
|
+
"weather_code_synop_4677",
|
|
101
|
+
"weather_code_synop_4680",
|
|
102
|
+
"weather_code_metar_4678",
|
|
103
|
+
"precipitation_rate",
|
|
104
|
+
"rainfall_rate",
|
|
105
|
+
"snowfall_rate",
|
|
106
|
+
"precipitation_accumulated",
|
|
107
|
+
"mor_visibility",
|
|
108
|
+
"reflectivity",
|
|
109
|
+
"quality_index",
|
|
110
|
+
"max_hail_diameter",
|
|
111
|
+
"laser_status",
|
|
112
|
+
"static_signal",
|
|
113
|
+
"laser_temperature_analog_status",
|
|
114
|
+
"laser_temperature_digital_status",
|
|
115
|
+
"laser_current_analog_status",
|
|
116
|
+
"laser_current_digital_status",
|
|
117
|
+
"sensor_voltage_supply_status",
|
|
118
|
+
"current_heating_pane_transmitter_head_status",
|
|
119
|
+
"current_heating_pane_receiver_head_status",
|
|
120
|
+
"temperature_sensor_status",
|
|
121
|
+
"current_heating_voltage_supply_status",
|
|
122
|
+
"current_heating_house_status",
|
|
123
|
+
"current_heating_heads_status",
|
|
124
|
+
"current_heating_carriers_status",
|
|
125
|
+
"control_output_laser_power_status",
|
|
126
|
+
"reserve_status",
|
|
127
|
+
"temperature_interior",
|
|
128
|
+
"laser_temperature",
|
|
129
|
+
"laser_current_average",
|
|
130
|
+
"control_voltage",
|
|
131
|
+
"optical_control_voltage_output",
|
|
132
|
+
"sensor_voltage_supply",
|
|
133
|
+
"current_heating_pane_transmitter_head",
|
|
134
|
+
"current_heating_pane_receiver_head",
|
|
135
|
+
"temperature_ambient",
|
|
136
|
+
"current_heating_voltage_supply",
|
|
137
|
+
"current_heating_house",
|
|
138
|
+
"current_heating_heads",
|
|
139
|
+
"current_heating_carriers",
|
|
140
|
+
"number_particles",
|
|
141
|
+
"number_particles_internal_data",
|
|
142
|
+
"number_particles_min_speed",
|
|
143
|
+
"number_particles_min_speed_internal_data",
|
|
144
|
+
"number_particles_max_speed",
|
|
145
|
+
"number_particles_max_speed_internal_data",
|
|
146
|
+
"number_particles_min_diameter",
|
|
147
|
+
"number_particles_min_diameter_internal_data",
|
|
148
|
+
"number_particles_no_hydrometeor",
|
|
149
|
+
"number_particles_no_hydrometeor_internal_data",
|
|
150
|
+
"number_particles_unknown_classification",
|
|
151
|
+
"number_particles_unknown_classification_internal_data",
|
|
152
|
+
"number_particles_class_1",
|
|
153
|
+
"number_particles_class_1_internal_data",
|
|
154
|
+
"number_particles_class_2",
|
|
155
|
+
"number_particles_class_2_internal_data",
|
|
156
|
+
"number_particles_class_3",
|
|
157
|
+
"number_particles_class_3_internal_data",
|
|
158
|
+
"number_particles_class_4",
|
|
159
|
+
"number_particles_class_4_internal_data",
|
|
160
|
+
"number_particles_class_5",
|
|
161
|
+
"number_particles_class_5_internal_data",
|
|
162
|
+
"number_particles_class_6",
|
|
163
|
+
"number_particles_class_6_internal_data",
|
|
164
|
+
"number_particles_class_7",
|
|
165
|
+
"number_particles_class_7_internal_data",
|
|
166
|
+
"number_particles_class_8",
|
|
167
|
+
"number_particles_class_8_internal_data",
|
|
168
|
+
"number_particles_class_9",
|
|
169
|
+
"number_particles_class_9_internal_data",
|
|
170
|
+
"raw_drop_number",
|
|
171
|
+
]
|
|
172
|
+
df.columns = column_names
|
|
173
|
+
|
|
174
|
+
# Remove checksum from raw_drop_number
|
|
175
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.rsplit(";", n=2, expand=True)[0]
|
|
176
|
+
|
|
177
|
+
# Define datetime "time" column
|
|
178
|
+
time = df["time"].str.extract(r"(\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2})")[0]
|
|
179
|
+
df["time"] = pd.to_datetime(time, format="%d/%m/%Y %H:%M:%S", errors="coerce")
|
|
180
|
+
|
|
181
|
+
# Drop row if start_identifier different than 00
|
|
182
|
+
df["start_identifier"] = df["start_identifier"].astype(str).str[-2:]
|
|
183
|
+
df = df[df["start_identifier"] == "00"]
|
|
184
|
+
|
|
185
|
+
# Drop rows with invalid raw_drop_number
|
|
186
|
+
df = df[df["raw_drop_number"].astype(str).str.len() == 1759]
|
|
187
|
+
|
|
188
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
189
|
+
columns_to_drop = [
|
|
190
|
+
"start_identifier",
|
|
191
|
+
"device_address",
|
|
192
|
+
"sensor_serial_number",
|
|
193
|
+
"sensor_date",
|
|
194
|
+
"sensor_time",
|
|
195
|
+
]
|
|
196
|
+
df = df.drop(columns=columns_to_drop)
|
|
197
|
+
return df
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
# -----------------------------------------------------------------------------.
|
|
4
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
5
|
+
#
|
|
6
|
+
# This program is free software: you can redistribute it and/or modify
|
|
7
|
+
# it under the terms of the GNU General Public License as published by
|
|
8
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
9
|
+
# (at your option) any later version.
|
|
10
|
+
#
|
|
11
|
+
# This program is distributed in the hope that it will be useful,
|
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
|
+
# GNU General Public License for more details.
|
|
15
|
+
#
|
|
16
|
+
# You should have received a copy of the GNU General Public License
|
|
17
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
18
|
+
# -----------------------------------------------------------------------------.
|
|
19
|
+
"""DISDRODB reader for GID LPM sensors not measuring wind."""
|
|
20
|
+
import pandas as pd
|
|
21
|
+
|
|
22
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
23
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@is_documented_by(reader_generic_docstring)
|
|
27
|
+
def reader(
|
|
28
|
+
filepath,
|
|
29
|
+
logger=None,
|
|
30
|
+
):
|
|
31
|
+
"""Reader."""
|
|
32
|
+
##------------------------------------------------------------------------.
|
|
33
|
+
#### - Define raw data headers
|
|
34
|
+
column_names = ["TO_BE_SPLITTED"]
|
|
35
|
+
|
|
36
|
+
##------------------------------------------------------------------------.
|
|
37
|
+
#### Define reader options
|
|
38
|
+
# - For more info: https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
|
|
39
|
+
reader_kwargs = {}
|
|
40
|
+
|
|
41
|
+
# - Define delimiter
|
|
42
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
43
|
+
|
|
44
|
+
# - Avoid first column to become df index !!!
|
|
45
|
+
reader_kwargs["index_col"] = False
|
|
46
|
+
|
|
47
|
+
# Since column names are expected to be passed explicitly, header is set to None
|
|
48
|
+
reader_kwargs["header"] = None
|
|
49
|
+
|
|
50
|
+
# - Number of rows to be skipped at the beginning of the file
|
|
51
|
+
reader_kwargs["skiprows"] = None
|
|
52
|
+
|
|
53
|
+
# - Define behaviour when encountering bad lines
|
|
54
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
55
|
+
|
|
56
|
+
# - Define reader engine
|
|
57
|
+
# - C engine is faster
|
|
58
|
+
# - Python engine is more feature-complete
|
|
59
|
+
reader_kwargs["engine"] = "python"
|
|
60
|
+
|
|
61
|
+
# - Define on-the-fly decompression of on-disk data
|
|
62
|
+
# - Available: gzip, bz2, zip
|
|
63
|
+
reader_kwargs["compression"] = "infer"
|
|
64
|
+
|
|
65
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
66
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
67
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
68
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
69
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
70
|
+
|
|
71
|
+
##------------------------------------------------------------------------.
|
|
72
|
+
#### Read the data
|
|
73
|
+
df = read_raw_text_file(
|
|
74
|
+
filepath=filepath,
|
|
75
|
+
column_names=column_names,
|
|
76
|
+
reader_kwargs=reader_kwargs,
|
|
77
|
+
logger=logger,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
##------------------------------------------------------------------------.
|
|
81
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
82
|
+
# Count number of delimiters to identify valid rows
|
|
83
|
+
df = df[df["TO_BE_SPLITTED"].str.count(";") == 521]
|
|
84
|
+
|
|
85
|
+
# Split by ; delimiter (before raw drop number)
|
|
86
|
+
df = df["TO_BE_SPLITTED"].str.split(";", expand=True, n=81)
|
|
87
|
+
|
|
88
|
+
# Assign column names
|
|
89
|
+
column_names = [
|
|
90
|
+
"id",
|
|
91
|
+
"time",
|
|
92
|
+
"start_identifier",
|
|
93
|
+
"device_address",
|
|
94
|
+
"sensor_serial_number",
|
|
95
|
+
"sensor_date",
|
|
96
|
+
"sensor_time",
|
|
97
|
+
"weather_code_synop_4677_5min",
|
|
98
|
+
"weather_code_synop_4680_5min",
|
|
99
|
+
"weather_code_metar_4678_5min",
|
|
100
|
+
"precipitation_rate_5min",
|
|
101
|
+
"weather_code_synop_4677",
|
|
102
|
+
"weather_code_synop_4680",
|
|
103
|
+
"weather_code_metar_4678",
|
|
104
|
+
"precipitation_rate",
|
|
105
|
+
"rainfall_rate",
|
|
106
|
+
"snowfall_rate",
|
|
107
|
+
"precipitation_accumulated",
|
|
108
|
+
"mor_visibility",
|
|
109
|
+
"reflectivity",
|
|
110
|
+
"quality_index",
|
|
111
|
+
"max_hail_diameter",
|
|
112
|
+
"laser_status",
|
|
113
|
+
"static_signal",
|
|
114
|
+
"laser_temperature_analog_status",
|
|
115
|
+
"laser_temperature_digital_status",
|
|
116
|
+
"laser_current_analog_status",
|
|
117
|
+
"laser_current_digital_status",
|
|
118
|
+
"sensor_voltage_supply_status",
|
|
119
|
+
"current_heating_pane_transmitter_head_status",
|
|
120
|
+
"current_heating_pane_receiver_head_status",
|
|
121
|
+
"temperature_sensor_status",
|
|
122
|
+
"current_heating_voltage_supply_status",
|
|
123
|
+
"current_heating_house_status",
|
|
124
|
+
"current_heating_heads_status",
|
|
125
|
+
"current_heating_carriers_status",
|
|
126
|
+
"control_output_laser_power_status",
|
|
127
|
+
"reserve_status",
|
|
128
|
+
"temperature_interior",
|
|
129
|
+
"laser_temperature",
|
|
130
|
+
"laser_current_average",
|
|
131
|
+
"control_voltage",
|
|
132
|
+
"optical_control_voltage_output",
|
|
133
|
+
"sensor_voltage_supply",
|
|
134
|
+
"current_heating_pane_transmitter_head",
|
|
135
|
+
"current_heating_pane_receiver_head",
|
|
136
|
+
"temperature_ambient",
|
|
137
|
+
"current_heating_voltage_supply",
|
|
138
|
+
"current_heating_house",
|
|
139
|
+
"current_heating_heads",
|
|
140
|
+
"current_heating_carriers",
|
|
141
|
+
"number_particles",
|
|
142
|
+
"number_particles_internal_data",
|
|
143
|
+
"number_particles_min_speed",
|
|
144
|
+
"number_particles_min_speed_internal_data",
|
|
145
|
+
"number_particles_max_speed",
|
|
146
|
+
"number_particles_max_speed_internal_data",
|
|
147
|
+
"number_particles_min_diameter",
|
|
148
|
+
"number_particles_min_diameter_internal_data",
|
|
149
|
+
"number_particles_no_hydrometeor",
|
|
150
|
+
"number_particles_no_hydrometeor_internal_data",
|
|
151
|
+
"number_particles_unknown_classification",
|
|
152
|
+
"number_particles_unknown_classification_internal_data",
|
|
153
|
+
"number_particles_class_1",
|
|
154
|
+
"number_particles_class_1_internal_data",
|
|
155
|
+
"number_particles_class_2",
|
|
156
|
+
"number_particles_class_2_internal_data",
|
|
157
|
+
"number_particles_class_3",
|
|
158
|
+
"number_particles_class_3_internal_data",
|
|
159
|
+
"number_particles_class_4",
|
|
160
|
+
"number_particles_class_4_internal_data",
|
|
161
|
+
"number_particles_class_5",
|
|
162
|
+
"number_particles_class_5_internal_data",
|
|
163
|
+
"number_particles_class_6",
|
|
164
|
+
"number_particles_class_6_internal_data",
|
|
165
|
+
"number_particles_class_7",
|
|
166
|
+
"number_particles_class_7_internal_data",
|
|
167
|
+
"number_particles_class_8",
|
|
168
|
+
"number_particles_class_8_internal_data",
|
|
169
|
+
"number_particles_class_9",
|
|
170
|
+
"number_particles_class_9_internal_data",
|
|
171
|
+
"raw_drop_number",
|
|
172
|
+
]
|
|
173
|
+
df.columns = column_names
|
|
174
|
+
|
|
175
|
+
# Remove checksum from raw_drop_number
|
|
176
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.rsplit(";", n=1, expand=True)[0]
|
|
177
|
+
|
|
178
|
+
# Define datetime "time" column
|
|
179
|
+
df["time"] = pd.to_datetime(df["time"], format="%d/%m/%Y %H.%M.%S", errors="coerce")
|
|
180
|
+
|
|
181
|
+
# Drop row if start_identifier different than 00
|
|
182
|
+
df = df[df["start_identifier"].astype(str) == "00"]
|
|
183
|
+
|
|
184
|
+
# Drop rows with invalid raw_drop_number
|
|
185
|
+
df = df[df["raw_drop_number"].astype(str).str.len() == 1759]
|
|
186
|
+
|
|
187
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
188
|
+
columns_to_drop = [
|
|
189
|
+
"id",
|
|
190
|
+
"start_identifier",
|
|
191
|
+
"device_address",
|
|
192
|
+
"sensor_serial_number",
|
|
193
|
+
"sensor_date",
|
|
194
|
+
"sensor_time",
|
|
195
|
+
]
|
|
196
|
+
df = df.drop(columns=columns_to_drop)
|
|
197
|
+
return df
|
|
@@ -15,21 +15,22 @@
|
|
|
15
15
|
# You should have received a copy of the GNU General Public License
|
|
16
16
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
17
|
# -----------------------------------------------------------------------------.
|
|
18
|
-
from disdrodb.l0 import run_l0b_from_nc
|
|
19
18
|
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
19
|
+
from disdrodb.l0.l0b_nc_processing import open_raw_netcdf_file, standardize_raw_dataset
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
@is_documented_by(reader_generic_docstring)
|
|
23
23
|
def reader(
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
station_name,
|
|
27
|
-
# Processing options
|
|
28
|
-
force=False,
|
|
29
|
-
verbose=False,
|
|
30
|
-
parallel=False,
|
|
31
|
-
debugging_mode=False,
|
|
24
|
+
filepath,
|
|
25
|
+
logger=None,
|
|
32
26
|
):
|
|
27
|
+
"""Reader."""
|
|
28
|
+
##------------------------------------------------------------------------.
|
|
29
|
+
#### Open the netCDF
|
|
30
|
+
ds = open_raw_netcdf_file(filepath=filepath, logger=logger)
|
|
31
|
+
|
|
32
|
+
##------------------------------------------------------------------------.
|
|
33
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
33
34
|
# Define dictionary mapping dataset variables to select and rename
|
|
34
35
|
dict_names = {
|
|
35
36
|
### Dimensions
|
|
@@ -61,30 +62,8 @@ def reader(
|
|
|
61
62
|
# 'hydrometeor_type_5m' # Pickering et al., 2019
|
|
62
63
|
}
|
|
63
64
|
|
|
64
|
-
#
|
|
65
|
-
|
|
66
|
-
# Return dataset
|
|
67
|
-
return ds
|
|
68
|
-
|
|
69
|
-
# Define glob pattern to search data files in <raw_dir>/data/<station_name>
|
|
70
|
-
glob_patterns = "*.nc*"
|
|
71
|
-
|
|
72
|
-
####----------------------------------------------------------------------.
|
|
73
|
-
#### - Create L0A products
|
|
74
|
-
run_l0b_from_nc(
|
|
75
|
-
raw_dir=raw_dir,
|
|
76
|
-
processed_dir=processed_dir,
|
|
77
|
-
station_name=station_name,
|
|
78
|
-
# Custom arguments of the reader
|
|
79
|
-
glob_patterns=glob_patterns,
|
|
80
|
-
dict_names=dict_names,
|
|
81
|
-
ds_sanitizer_fun=ds_sanitizer_fun,
|
|
82
|
-
# Processing options
|
|
83
|
-
force=force,
|
|
84
|
-
verbose=verbose,
|
|
85
|
-
parallel=parallel,
|
|
86
|
-
debugging_mode=debugging_mode,
|
|
87
|
-
)
|
|
88
|
-
|
|
65
|
+
# Rename dataset variables and columns and infill missing variables
|
|
66
|
+
ds = standardize_raw_dataset(ds=ds, dict_names=dict_names, sensor_name="LPM")
|
|
89
67
|
|
|
90
|
-
#
|
|
68
|
+
# Return the dataset adhering to DISDRODB L0B standards
|
|
69
|
+
return ds
|