disdrodb 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +68 -34
- disdrodb/_config.py +5 -4
- disdrodb/_version.py +16 -3
- disdrodb/accessor/__init__.py +20 -0
- disdrodb/accessor/methods.py +125 -0
- disdrodb/api/checks.py +177 -24
- disdrodb/api/configs.py +3 -3
- disdrodb/api/info.py +13 -13
- disdrodb/api/io.py +281 -22
- disdrodb/api/path.py +184 -195
- disdrodb/api/search.py +18 -9
- disdrodb/cli/disdrodb_create_summary.py +103 -0
- disdrodb/cli/disdrodb_create_summary_station.py +91 -0
- disdrodb/cli/disdrodb_run_l0.py +1 -1
- disdrodb/cli/disdrodb_run_l0_station.py +1 -1
- disdrodb/cli/disdrodb_run_l0a_station.py +1 -1
- disdrodb/cli/disdrodb_run_l0b.py +1 -1
- disdrodb/cli/disdrodb_run_l0b_station.py +3 -3
- disdrodb/cli/disdrodb_run_l0c.py +1 -1
- disdrodb/cli/disdrodb_run_l0c_station.py +3 -3
- disdrodb/cli/disdrodb_run_l1_station.py +2 -2
- disdrodb/cli/disdrodb_run_l2e_station.py +2 -2
- disdrodb/cli/disdrodb_run_l2m_station.py +2 -2
- disdrodb/configs.py +149 -4
- disdrodb/constants.py +61 -0
- disdrodb/data_transfer/download_data.py +127 -11
- disdrodb/etc/configs/attributes.yaml +339 -0
- disdrodb/etc/configs/encodings.yaml +473 -0
- disdrodb/etc/products/L1/global.yaml +13 -0
- disdrodb/etc/products/L2E/10MIN.yaml +12 -0
- disdrodb/etc/products/L2E/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/global.yaml +22 -0
- disdrodb/etc/products/L2M/10MIN.yaml +12 -0
- disdrodb/etc/products/L2M/GAMMA_ML.yaml +8 -0
- disdrodb/etc/products/L2M/NGAMMA_GS_LOG_ND_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/NGAMMA_GS_ND_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/NGAMMA_GS_Z_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/global.yaml +26 -0
- disdrodb/issue/writer.py +2 -0
- disdrodb/l0/__init__.py +13 -0
- disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +4 -4
- disdrodb/l0/configs/PARSIVEL/l0b_cf_attrs.yml +1 -1
- disdrodb/l0/configs/PARSIVEL/l0b_encodings.yml +3 -3
- disdrodb/l0/configs/PARSIVEL/raw_data_format.yml +1 -1
- disdrodb/l0/configs/PARSIVEL2/l0b_cf_attrs.yml +5 -5
- disdrodb/l0/configs/PARSIVEL2/l0b_encodings.yml +3 -3
- disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml +1 -1
- disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +4 -4
- disdrodb/l0/configs/PWS100/raw_data_format.yml +1 -1
- disdrodb/l0/l0a_processing.py +37 -32
- disdrodb/l0/l0b_nc_processing.py +118 -8
- disdrodb/l0/l0b_processing.py +30 -65
- disdrodb/l0/l0c_processing.py +369 -259
- disdrodb/l0/readers/LPM/ARM/ARM_LPM.py +7 -0
- disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_LPM_NC.py +66 -0
- disdrodb/l0/readers/LPM/SLOVENIA/{CRNI_VRH.py → UL.py} +3 -0
- disdrodb/l0/readers/LPM/SWITZERLAND/INNERERIZ_LPM.py +195 -0
- disdrodb/l0/readers/PARSIVEL/GPM/PIERS.py +0 -2
- disdrodb/l0/readers/PARSIVEL/JAPAN/JMA.py +4 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/ARM/ARM_PARSIVEL2.py +4 -0
- disdrodb/l0/readers/PARSIVEL2/BELGIUM/ILVO.py +168 -0
- disdrodb/l0/readers/PARSIVEL2/CANADA/UQAM_NC.py +69 -0
- disdrodb/l0/readers/PARSIVEL2/DENMARK/DTU.py +165 -0
- disdrodb/l0/readers/PARSIVEL2/FINLAND/FMI_PARSIVEL2.py +69 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +255 -134
- disdrodb/l0/readers/PARSIVEL2/FRANCE/OSUG.py +525 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +9 -7
- disdrodb/l0/readers/PARSIVEL2/KIT/BURKINA_FASO.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/KIT/TEAMX.py +123 -0
- disdrodb/l0/readers/PARSIVEL2/{NETHERLANDS/DELFT.py → MPI/BCO_PARSIVEL2.py} +41 -71
- disdrodb/l0/readers/PARSIVEL2/MPI/BOWTIE.py +220 -0
- disdrodb/l0/readers/PARSIVEL2/NASA/APU.py +120 -0
- disdrodb/l0/readers/PARSIVEL2/NASA/LPVEX.py +109 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/FARM_PARSIVEL2.py +1 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_MIPS.py +126 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_PIPS.py +165 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +20 -12
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT_NC.py +5 -0
- disdrodb/l0/readers/PARSIVEL2/SPAIN/CENER.py +144 -0
- disdrodb/l0/readers/PARSIVEL2/SPAIN/CR1000DL.py +201 -0
- disdrodb/l0/readers/PARSIVEL2/SPAIN/LIAISE.py +137 -0
- disdrodb/l0/readers/PARSIVEL2/USA/C3WE.py +146 -0
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +105 -99
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100_SIRTA.py +151 -0
- disdrodb/l1/__init__.py +5 -0
- disdrodb/l1/fall_velocity.py +46 -0
- disdrodb/l1/filters.py +34 -20
- disdrodb/l1/processing.py +46 -45
- disdrodb/l1/resampling.py +77 -66
- disdrodb/l1_env/routines.py +18 -3
- disdrodb/l2/__init__.py +7 -0
- disdrodb/l2/empirical_dsd.py +58 -10
- disdrodb/l2/processing.py +268 -117
- disdrodb/metadata/checks.py +132 -125
- disdrodb/metadata/standards.py +3 -1
- disdrodb/psd/fitting.py +631 -345
- disdrodb/psd/models.py +9 -6
- disdrodb/routines/__init__.py +54 -0
- disdrodb/{l0/routines.py → routines/l0.py} +316 -355
- disdrodb/{l1/routines.py → routines/l1.py} +76 -116
- disdrodb/routines/l2.py +1019 -0
- disdrodb/{routines.py → routines/wrappers.py} +98 -10
- disdrodb/scattering/__init__.py +16 -4
- disdrodb/scattering/axis_ratio.py +61 -37
- disdrodb/scattering/permittivity.py +504 -0
- disdrodb/scattering/routines.py +746 -184
- disdrodb/summary/__init__.py +17 -0
- disdrodb/summary/routines.py +4196 -0
- disdrodb/utils/archiving.py +434 -0
- disdrodb/utils/attrs.py +68 -125
- disdrodb/utils/cli.py +5 -5
- disdrodb/utils/compression.py +30 -1
- disdrodb/utils/dask.py +121 -9
- disdrodb/utils/dataframe.py +61 -7
- disdrodb/utils/decorators.py +31 -0
- disdrodb/utils/directories.py +35 -15
- disdrodb/utils/encoding.py +37 -19
- disdrodb/{l2 → utils}/event.py +15 -173
- disdrodb/utils/logger.py +14 -7
- disdrodb/utils/manipulations.py +81 -0
- disdrodb/utils/routines.py +166 -0
- disdrodb/utils/subsetting.py +214 -0
- disdrodb/utils/time.py +35 -177
- disdrodb/utils/writer.py +20 -7
- disdrodb/utils/xarray.py +5 -4
- disdrodb/viz/__init__.py +13 -0
- disdrodb/viz/plots.py +398 -0
- {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/METADATA +4 -3
- {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/RECORD +139 -98
- {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/entry_points.txt +2 -0
- disdrodb/l1/encoding_attrs.py +0 -642
- disdrodb/l2/processing_options.py +0 -213
- disdrodb/l2/routines.py +0 -868
- /disdrodb/l0/readers/PARSIVEL/SLOVENIA/{UL_FGG.py → UL.py} +0 -0
- {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/WHEEL +0 -0
- {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/licenses/LICENSE +0 -0
- {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
"""This reader allows to read raw data from NASA APU stations.
|
|
19
|
+
|
|
20
|
+
The reader allows to read raw APU data from the following NASA campaigns:
|
|
21
|
+
|
|
22
|
+
- HYMEX
|
|
23
|
+
- IFLOODS
|
|
24
|
+
- IPHEX
|
|
25
|
+
- OLYMPEX
|
|
26
|
+
- ICEPOP
|
|
27
|
+
- IMPACTS
|
|
28
|
+
- GCPEX
|
|
29
|
+
- WFF
|
|
30
|
+
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
import pandas as pd
|
|
34
|
+
|
|
35
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
36
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@is_documented_by(reader_generic_docstring)
|
|
40
|
+
def reader(
|
|
41
|
+
filepath,
|
|
42
|
+
logger=None,
|
|
43
|
+
):
|
|
44
|
+
"""Reader."""
|
|
45
|
+
##------------------------------------------------------------------------.
|
|
46
|
+
#### Define column names
|
|
47
|
+
column_names = ["time", "TO_BE_SPLITTED"]
|
|
48
|
+
|
|
49
|
+
##------------------------------------------------------------------------.
|
|
50
|
+
#### Define reader options
|
|
51
|
+
reader_kwargs = {}
|
|
52
|
+
# - Define delimiter
|
|
53
|
+
reader_kwargs["delimiter"] = ";"
|
|
54
|
+
# - Skip first row as columns names
|
|
55
|
+
reader_kwargs["header"] = None
|
|
56
|
+
reader_kwargs["skiprows"] = 0
|
|
57
|
+
# - Skip file with encoding errors
|
|
58
|
+
reader_kwargs["encoding_errors"] = "ignore"
|
|
59
|
+
# - Avoid first column to become df index !!!
|
|
60
|
+
reader_kwargs["index_col"] = False
|
|
61
|
+
# - Define behaviour when encountering bad lines
|
|
62
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
63
|
+
# - Define reader engine
|
|
64
|
+
# - C engine is faster
|
|
65
|
+
# - Python engine is more feature-complete
|
|
66
|
+
reader_kwargs["engine"] = "python"
|
|
67
|
+
# - Define on-the-fly decompression of on-disk data
|
|
68
|
+
# - Available: gzip, bz2, zip
|
|
69
|
+
reader_kwargs["compression"] = "infer"
|
|
70
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
71
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
72
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
73
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
74
|
+
reader_kwargs["na_values"] = ["na", "", "error", "NA", "-.-"]
|
|
75
|
+
|
|
76
|
+
##------------------------------------------------------------------------.
|
|
77
|
+
#### Read the data
|
|
78
|
+
df = read_raw_text_file(
|
|
79
|
+
filepath=filepath,
|
|
80
|
+
column_names=column_names,
|
|
81
|
+
reader_kwargs=reader_kwargs,
|
|
82
|
+
logger=logger,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
##------------------------------------------------------------------------.
|
|
86
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
87
|
+
# Convert time column to datetime
|
|
88
|
+
df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
|
|
89
|
+
|
|
90
|
+
# Split the 'TO_BE_SPLITTED' column
|
|
91
|
+
df = df["TO_BE_SPLITTED"].str.split(",", n=3, expand=True)
|
|
92
|
+
|
|
93
|
+
# Assign column names
|
|
94
|
+
names = [
|
|
95
|
+
"station_name",
|
|
96
|
+
"unknown",
|
|
97
|
+
"unknown2",
|
|
98
|
+
"raw_drop_number",
|
|
99
|
+
]
|
|
100
|
+
df.columns = names
|
|
101
|
+
|
|
102
|
+
# Add the time column
|
|
103
|
+
df["time"] = df_time
|
|
104
|
+
|
|
105
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
106
|
+
df = df.drop(columns=["station_name", "unknown", "unknown2"])
|
|
107
|
+
|
|
108
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
109
|
+
return df
|
|
@@ -108,7 +108,7 @@ def reader(
|
|
|
108
108
|
# --> "" generates an array of zeros in L0B processing
|
|
109
109
|
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
|
|
110
110
|
|
|
111
|
-
# Remove <SPECTRUM> and </SPECTRUM>
|
|
111
|
+
# Remove <SPECTRUM> and </SPECTRUM> prefix and suffix from the raw_drop_number field
|
|
112
112
|
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
|
|
113
113
|
df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
|
|
114
114
|
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
"""Reader for the PERILS 2022 and 2023 MIPS and RADAPS files."""
|
|
19
|
+
import pandas as pd
|
|
20
|
+
|
|
21
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
22
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@is_documented_by(reader_generic_docstring)
|
|
26
|
+
def reader(
|
|
27
|
+
filepath,
|
|
28
|
+
logger=None,
|
|
29
|
+
):
|
|
30
|
+
"""Reader."""
|
|
31
|
+
##------------------------------------------------------------------------.
|
|
32
|
+
#### Define column names
|
|
33
|
+
column_names = ["TO_PARSE"]
|
|
34
|
+
|
|
35
|
+
##------------------------------------------------------------------------.
|
|
36
|
+
#### Define reader options
|
|
37
|
+
reader_kwargs = {}
|
|
38
|
+
# - Define delimiter
|
|
39
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
40
|
+
# - Avoid first column to become df index !!!
|
|
41
|
+
reader_kwargs["index_col"] = False
|
|
42
|
+
|
|
43
|
+
# - Define behaviour when encountering bad lines
|
|
44
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
45
|
+
|
|
46
|
+
# Skip first row as columns names
|
|
47
|
+
reader_kwargs["header"] = None
|
|
48
|
+
|
|
49
|
+
# - Define encoding
|
|
50
|
+
reader_kwargs["encoding"] = "ISO-8859-1"
|
|
51
|
+
|
|
52
|
+
# - Define reader engine
|
|
53
|
+
# - C engine is faster
|
|
54
|
+
# - Python engine is more feature-complete
|
|
55
|
+
reader_kwargs["engine"] = "python"
|
|
56
|
+
|
|
57
|
+
# - Define on-the-fly decompression of on-disk data
|
|
58
|
+
# - Available: gzip, bz2, zip
|
|
59
|
+
reader_kwargs["compression"] = "infer"
|
|
60
|
+
|
|
61
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
62
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
63
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
64
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
65
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
66
|
+
|
|
67
|
+
##------------------------------------------------------------------------.
|
|
68
|
+
#### Read the data
|
|
69
|
+
df = read_raw_text_file(
|
|
70
|
+
filepath=filepath,
|
|
71
|
+
column_names=column_names,
|
|
72
|
+
reader_kwargs=reader_kwargs,
|
|
73
|
+
logger=logger,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
##------------------------------------------------------------------------.
|
|
77
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
78
|
+
# Split and assign integrated variables names
|
|
79
|
+
df = df["TO_PARSE"].str.split(",", expand=True, n=6)
|
|
80
|
+
names = [
|
|
81
|
+
"year",
|
|
82
|
+
"doy",
|
|
83
|
+
"time",
|
|
84
|
+
"sensor_temperature",
|
|
85
|
+
"number_particles",
|
|
86
|
+
"rainfall_rate_32bit",
|
|
87
|
+
"TO_SPLIT",
|
|
88
|
+
]
|
|
89
|
+
df.columns = names
|
|
90
|
+
|
|
91
|
+
# Derive raw drop arrays
|
|
92
|
+
def split_string(s):
|
|
93
|
+
vals = [v.strip() for v in s.split(",")]
|
|
94
|
+
c1 = ", ".join(vals[:32])
|
|
95
|
+
c2 = ", ".join(vals[32:64])
|
|
96
|
+
# c3 = ", ".join(vals[64:])
|
|
97
|
+
series = pd.Series(
|
|
98
|
+
{
|
|
99
|
+
"raw_drop_concentration": c1,
|
|
100
|
+
"raw_drop_average_velocity": c2,
|
|
101
|
+
# "raw_drop_number": c3,
|
|
102
|
+
},
|
|
103
|
+
)
|
|
104
|
+
return series
|
|
105
|
+
|
|
106
|
+
splitted_string = df["TO_SPLIT"].apply(split_string)
|
|
107
|
+
df["raw_drop_concentration"] = splitted_string["raw_drop_concentration"]
|
|
108
|
+
df["raw_drop_average_velocity"] = splitted_string["raw_drop_average_velocity"]
|
|
109
|
+
|
|
110
|
+
# Define datetime time column
|
|
111
|
+
df["year"] = df["year"].str.replace(".0", "")
|
|
112
|
+
df["doy"] = df["doy"].str.replace(".0", "")
|
|
113
|
+
df["time"] = df["time"].str.replace(".0", "")
|
|
114
|
+
df["time"] = df["year"].astype(str) + "-" + df["doy"].astype(str) + " " + df["time"].astype(str)
|
|
115
|
+
df["time"] = pd.to_datetime(df["time"], format="%Y-%j %H%M%S", errors="coerce")
|
|
116
|
+
|
|
117
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
118
|
+
columns_to_drop = [
|
|
119
|
+
"doy",
|
|
120
|
+
"year",
|
|
121
|
+
"TO_SPLIT",
|
|
122
|
+
]
|
|
123
|
+
df = df.drop(columns=columns_to_drop)
|
|
124
|
+
|
|
125
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
126
|
+
return df
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
"""Reader for the PERILS 2022 and 2023 PIPS files."""
|
|
19
|
+
import pandas as pd
|
|
20
|
+
|
|
21
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
22
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@is_documented_by(reader_generic_docstring)
|
|
26
|
+
def reader(
|
|
27
|
+
filepath,
|
|
28
|
+
logger=None,
|
|
29
|
+
):
|
|
30
|
+
"""Reader."""
|
|
31
|
+
##------------------------------------------------------------------------.
|
|
32
|
+
#### Define column names
|
|
33
|
+
column_names = [
|
|
34
|
+
"logger_time",
|
|
35
|
+
"record_number",
|
|
36
|
+
"battery_voltage",
|
|
37
|
+
"logger_temperature",
|
|
38
|
+
"wind_direction",
|
|
39
|
+
"wind_speed",
|
|
40
|
+
"wind_flag",
|
|
41
|
+
"fast_temperature",
|
|
42
|
+
"slow_temperature",
|
|
43
|
+
"relative_humidity",
|
|
44
|
+
"pressure",
|
|
45
|
+
"compass_direction",
|
|
46
|
+
"gps_time",
|
|
47
|
+
"gps_status",
|
|
48
|
+
"gps_latitude", # DD.DM, DM = decimal minutes/100, DD = degrees
|
|
49
|
+
"gps_latitude_hemisphere", # N/S
|
|
50
|
+
"gps_longitude",
|
|
51
|
+
"gps_longitude_hemisphere", # W/E
|
|
52
|
+
"gps_speed",
|
|
53
|
+
"gps_direction",
|
|
54
|
+
"gps_date",
|
|
55
|
+
"gps_magnetic_version",
|
|
56
|
+
"gps_altitude",
|
|
57
|
+
"relative_wind_direction",
|
|
58
|
+
"dew_point_temperature",
|
|
59
|
+
"rh",
|
|
60
|
+
"disdrometer_data",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
##------------------------------------------------------------------------.
|
|
64
|
+
#### Define reader options
|
|
65
|
+
reader_kwargs = {}
|
|
66
|
+
|
|
67
|
+
# - Define delimiter
|
|
68
|
+
reader_kwargs["delimiter"] = ","
|
|
69
|
+
|
|
70
|
+
# - Avoid first column to become df index !!!
|
|
71
|
+
reader_kwargs["index_col"] = False
|
|
72
|
+
|
|
73
|
+
# - Define behaviour when encountering bad lines
|
|
74
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
75
|
+
|
|
76
|
+
# - Define encoding
|
|
77
|
+
reader_kwargs["encoding"] = "ISO-8859-1"
|
|
78
|
+
|
|
79
|
+
# - Define reader engine
|
|
80
|
+
# - C engine is faster
|
|
81
|
+
# - Python engine is more feature-complete
|
|
82
|
+
reader_kwargs["engine"] = "python"
|
|
83
|
+
|
|
84
|
+
# - Define on-the-fly decompression of on-disk data
|
|
85
|
+
# - Available: gzip, bz2, zip
|
|
86
|
+
reader_kwargs["compression"] = "infer"
|
|
87
|
+
|
|
88
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
89
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
90
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
91
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
92
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
93
|
+
|
|
94
|
+
# Skip first row as columns names
|
|
95
|
+
reader_kwargs["header"] = None
|
|
96
|
+
|
|
97
|
+
##------------------------------------------------------------------------.
|
|
98
|
+
#### Read the data
|
|
99
|
+
df = read_raw_text_file(
|
|
100
|
+
filepath=filepath,
|
|
101
|
+
column_names=column_names,
|
|
102
|
+
reader_kwargs=reader_kwargs,
|
|
103
|
+
logger=logger,
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
##------------------------------------------------------------------------.
|
|
107
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
108
|
+
# Drop timesteps without disdrometer data
|
|
109
|
+
df = df[~df["disdrometer_data"].isna()]
|
|
110
|
+
|
|
111
|
+
# Retrieve disdrometer data
|
|
112
|
+
df_data = df["disdrometer_data"].str.split(";", expand=True, n=11)
|
|
113
|
+
|
|
114
|
+
# Assign column names
|
|
115
|
+
column_names = [
|
|
116
|
+
"serial_number",
|
|
117
|
+
"rainfall_rate_32bit",
|
|
118
|
+
"rainfall_accumulated_32bit",
|
|
119
|
+
"reflectivity_32bit",
|
|
120
|
+
"sample_interval",
|
|
121
|
+
"laser_amplitude",
|
|
122
|
+
"number_particles",
|
|
123
|
+
"sensor_temperature",
|
|
124
|
+
"sensor_battery_voltage",
|
|
125
|
+
"sensor_time", # Note: logger_time is currently used !
|
|
126
|
+
"sensor_date",
|
|
127
|
+
"raw_drop_number",
|
|
128
|
+
]
|
|
129
|
+
df_data.columns = column_names
|
|
130
|
+
|
|
131
|
+
# Add weather information
|
|
132
|
+
df_data["air_temperature"] = df["fast_temperature"]
|
|
133
|
+
df_data["relative_humidity"] = df["relative_humidity"]
|
|
134
|
+
df_data["wind_direction"] = df["wind_direction"]
|
|
135
|
+
df_data["wind_speed"] = df["wind_speed"]
|
|
136
|
+
|
|
137
|
+
# df_data["dew_point"] = df["dew_point"]
|
|
138
|
+
# df_data["air_pressure"] = df["pressure"]
|
|
139
|
+
|
|
140
|
+
# Retrieve time
|
|
141
|
+
df_time = pd.to_datetime(df["logger_time"], errors="coerce")
|
|
142
|
+
|
|
143
|
+
# Retrieve coordinates information
|
|
144
|
+
# --> Latitude in degrees_north
|
|
145
|
+
# --> Longitude in degrees_east
|
|
146
|
+
# df_lat_sign = df["gps_latitude_hemisphere"].str.replace("N", "1").str.replace("S", "-1")
|
|
147
|
+
# df_lon_sign = df["gps_longitude_hemisphere"].str.replace("E", "1").str.replace("W", "-1")
|
|
148
|
+
# df_lat_sign = df_lat_sign.astype(float)
|
|
149
|
+
# df_lon_sign = df_lon_sign.astype(float)
|
|
150
|
+
# df_lon = df["gps_longitude"].astype(float)
|
|
151
|
+
# df_lat = df["gps_latitude"].astype(float)
|
|
152
|
+
# df_lon = df_lon * df_lon_sign
|
|
153
|
+
# df_lat = df_lat * df_lat_sign
|
|
154
|
+
|
|
155
|
+
# Create dataframe
|
|
156
|
+
df = df_data
|
|
157
|
+
df["time"] = df_time
|
|
158
|
+
# df["latitude"] = df_lat
|
|
159
|
+
# df["longitude"] = df_lon
|
|
160
|
+
|
|
161
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
162
|
+
df = df.drop(columns=["serial_number", "sensor_time", "sensor_date", "serial_number"])
|
|
163
|
+
|
|
164
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
165
|
+
return df
|
|
@@ -102,7 +102,7 @@ def reader(
|
|
|
102
102
|
df["raw_drop_number"] = df["raw_drop_number"].str.strip()
|
|
103
103
|
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
|
|
104
104
|
|
|
105
|
-
# Remove <SPECTRUM> and </SPECTRUM>
|
|
105
|
+
# Remove <SPECTRUM> and </SPECTRUM> prefix and suffix from the raw_drop_number field
|
|
106
106
|
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
|
|
107
107
|
df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
|
|
108
108
|
|
|
@@ -15,6 +15,9 @@
|
|
|
15
15
|
# You should have received a copy of the GNU General Public License
|
|
16
16
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
17
|
# -----------------------------------------------------------------------------.
|
|
18
|
+
"""Reader for the PERILS 2022 and 2023 PIPS files."""
|
|
19
|
+
import os
|
|
20
|
+
|
|
18
21
|
import pandas as pd
|
|
19
22
|
|
|
20
23
|
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
@@ -62,6 +65,7 @@ def reader(
|
|
|
62
65
|
##------------------------------------------------------------------------.
|
|
63
66
|
#### Define reader options
|
|
64
67
|
reader_kwargs = {}
|
|
68
|
+
|
|
65
69
|
# - Define delimiter
|
|
66
70
|
reader_kwargs["delimiter"] = ","
|
|
67
71
|
|
|
@@ -91,6 +95,8 @@ def reader(
|
|
|
91
95
|
|
|
92
96
|
# Skip first row as columns names
|
|
93
97
|
reader_kwargs["header"] = None
|
|
98
|
+
if "FMCW" in os.path.basename(filepath):
|
|
99
|
+
reader_kwargs["skiprows"] = 1
|
|
94
100
|
|
|
95
101
|
##------------------------------------------------------------------------.
|
|
96
102
|
#### Read the data
|
|
@@ -135,24 +141,26 @@ def reader(
|
|
|
135
141
|
# df_data["dew_point"] = df["dew_point"]
|
|
136
142
|
# df_data["air_pressure"] = df["pressure"]
|
|
137
143
|
|
|
138
|
-
# Retrieve time
|
|
144
|
+
# Retrieve time
|
|
145
|
+
df_time = pd.to_datetime(df["logger_time"], errors="coerce")
|
|
146
|
+
|
|
147
|
+
# Retrieve coordinates information
|
|
139
148
|
# --> Latitude in degrees_north
|
|
140
149
|
# --> Longitude in degrees_east
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
df_lat = df_lat * df_lat_sign
|
|
150
|
+
# df_lat_sign = df["gps_latitude_hemisphere"].str.replace("N", "1").str.replace("S", "-1")
|
|
151
|
+
# df_lon_sign = df["gps_longitude_hemisphere"].str.replace("E", "1").str.replace("W", "-1")
|
|
152
|
+
# df_lat_sign = df_lat_sign.astype(float)
|
|
153
|
+
# df_lon_sign = df_lon_sign.astype(float)
|
|
154
|
+
# df_lon = df["gps_longitude"].astype(float)
|
|
155
|
+
# df_lat = df["gps_latitude"].astype(float)
|
|
156
|
+
# df_lon = df_lon * df_lon_sign
|
|
157
|
+
# df_lat = df_lat * df_lat_sign
|
|
150
158
|
|
|
151
159
|
# Create dataframe
|
|
152
160
|
df = df_data
|
|
153
161
|
df["time"] = df_time
|
|
154
|
-
df["latitude"] = df_lat
|
|
155
|
-
df["longitude"] = df_lon
|
|
162
|
+
# df["latitude"] = df_lat
|
|
163
|
+
# df["longitude"] = df_lon
|
|
156
164
|
|
|
157
165
|
# Drop columns not agreeing with DISDRODB L0 standards
|
|
158
166
|
df = df.drop(columns=["serial_number", "sensor_time", "sensor_date", "serial_number"])
|
|
@@ -35,6 +35,8 @@ def reader(
|
|
|
35
35
|
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
36
36
|
# Add time coordinate
|
|
37
37
|
ds["time"] = ds["time_as_string"].astype("M8[s]")
|
|
38
|
+
ds["time"].attrs.pop("comment", None)
|
|
39
|
+
ds["time"].attrs.pop("units", None)
|
|
38
40
|
ds = ds.set_coords("time")
|
|
39
41
|
|
|
40
42
|
# Define dictionary mapping dataset variables to select and rename
|
|
@@ -63,5 +65,8 @@ def reader(
|
|
|
63
65
|
# Rename dataset variables and columns and infill missing variables
|
|
64
66
|
ds = standardize_raw_dataset(ds=ds, dict_names=dict_names, sensor_name="PARSIVEL2")
|
|
65
67
|
|
|
68
|
+
# Replace NaN flags
|
|
69
|
+
ds["raw_drop_number"] = ds["raw_drop_number"].where(ds["raw_drop_number"] < 65437) # dummy flag 65437.0
|
|
70
|
+
|
|
66
71
|
# Return the dataset adhering to DISDRODB L0B standards
|
|
67
72
|
return ds
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
20
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
21
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@is_documented_by(reader_generic_docstring)
|
|
25
|
+
def reader(
|
|
26
|
+
filepath,
|
|
27
|
+
logger=None,
|
|
28
|
+
):
|
|
29
|
+
"""Reader."""
|
|
30
|
+
##------------------------------------------------------------------------.
|
|
31
|
+
#### Define column names
|
|
32
|
+
column_names = ["TO_PARSE"]
|
|
33
|
+
|
|
34
|
+
##------------------------------------------------------------------------.
|
|
35
|
+
#### Define reader options
|
|
36
|
+
reader_kwargs = {}
|
|
37
|
+
# - Define delimiter
|
|
38
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
39
|
+
# - Skip first row as columns names
|
|
40
|
+
# - Define encoding
|
|
41
|
+
reader_kwargs["encoding"] = "latin" # "ISO-8859-1"
|
|
42
|
+
# - Avoid first column to become df index !!!
|
|
43
|
+
reader_kwargs["index_col"] = False
|
|
44
|
+
# - Define behaviour when encountering bad lines
|
|
45
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
46
|
+
# - Define reader engine
|
|
47
|
+
# - C engine is faster
|
|
48
|
+
# - Python engine is more feature-complete
|
|
49
|
+
reader_kwargs["engine"] = "python"
|
|
50
|
+
# - Define on-the-fly decompression of on-disk data
|
|
51
|
+
# - Available: gzip, bz2, zip
|
|
52
|
+
reader_kwargs["compression"] = "infer"
|
|
53
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
54
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
55
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
56
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
57
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
58
|
+
|
|
59
|
+
##------------------------------------------------------------------------.
|
|
60
|
+
#### Read the data
|
|
61
|
+
df = read_raw_text_file(
|
|
62
|
+
filepath=filepath,
|
|
63
|
+
column_names=column_names,
|
|
64
|
+
reader_kwargs=reader_kwargs,
|
|
65
|
+
logger=logger,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
##------------------------------------------------------------------------.
|
|
69
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
70
|
+
# Define time
|
|
71
|
+
df = df["TO_PARSE"].str.split(",", n=2, expand=True)
|
|
72
|
+
df.columns = ["date", "time", "TO_PARSE"]
|
|
73
|
+
datetime_str = df["date"] + " " + df["time"]
|
|
74
|
+
df["time"] = pd.to_datetime(datetime_str, format="%d.%m.%Y %H:%M:%S", errors="coerce")
|
|
75
|
+
|
|
76
|
+
# Identify rows with integral variables
|
|
77
|
+
df_vars = df[df["TO_PARSE"].str.len() == 94]
|
|
78
|
+
|
|
79
|
+
# Split and assign column names
|
|
80
|
+
df_data = df_vars["TO_PARSE"].str.split(",", expand=True)
|
|
81
|
+
var_names = [
|
|
82
|
+
"rainfall_rate_32bit",
|
|
83
|
+
"rainfall_accumulated_32bit",
|
|
84
|
+
"weather_code_synop_4680",
|
|
85
|
+
"weather_code_synop_4677",
|
|
86
|
+
"reflectivity_32bit",
|
|
87
|
+
"mor_visibility",
|
|
88
|
+
"laser_amplitude",
|
|
89
|
+
"number_particles",
|
|
90
|
+
"sensor_temperature",
|
|
91
|
+
"sensor_heating_current",
|
|
92
|
+
"sensor_battery_voltage",
|
|
93
|
+
"sensor_status",
|
|
94
|
+
"sensor_serial_number",
|
|
95
|
+
"sensor_temperature_receiver",
|
|
96
|
+
"sensor_temperature_trasmitter",
|
|
97
|
+
"snowfall_rate",
|
|
98
|
+
"rain_kinetic_energy",
|
|
99
|
+
]
|
|
100
|
+
df_data.columns = var_names
|
|
101
|
+
df_data["time"] = df_vars["time"]
|
|
102
|
+
df_data = df_data.drop(columns="sensor_serial_number")
|
|
103
|
+
|
|
104
|
+
# Initialize empty arrays
|
|
105
|
+
# --> 0 values array produced in L0B
|
|
106
|
+
df_data["raw_drop_concentration"] = ""
|
|
107
|
+
df_data["raw_drop_average_velocity"] = ""
|
|
108
|
+
df_data["raw_drop_number"] = ""
|
|
109
|
+
|
|
110
|
+
# Identify raw spectrum
|
|
111
|
+
df_raw_spectrum = df[df["TO_PARSE"].str.len() == 4545]
|
|
112
|
+
|
|
113
|
+
# Derive raw drop arrays
|
|
114
|
+
def split_string(s):
|
|
115
|
+
vals = [v.strip() for v in s.split(",")]
|
|
116
|
+
c1 = ",".join(vals[:32])
|
|
117
|
+
c2 = ",".join(vals[32:64])
|
|
118
|
+
c3 = ",".join(vals[64].replace("r", "").split("/"))
|
|
119
|
+
series = pd.Series(
|
|
120
|
+
{
|
|
121
|
+
"raw_drop_concentration": c1,
|
|
122
|
+
"raw_drop_average_velocity": c2,
|
|
123
|
+
"raw_drop_number": c3,
|
|
124
|
+
},
|
|
125
|
+
)
|
|
126
|
+
return series
|
|
127
|
+
|
|
128
|
+
splitted_string = df_raw_spectrum["TO_PARSE"].apply(split_string)
|
|
129
|
+
df_raw_spectrum["raw_drop_concentration"] = splitted_string["raw_drop_concentration"]
|
|
130
|
+
df_raw_spectrum["raw_drop_average_velocity"] = splitted_string["raw_drop_average_velocity"]
|
|
131
|
+
df_raw_spectrum["raw_drop_number"] = splitted_string["raw_drop_number"]
|
|
132
|
+
df_raw_spectrum = df_raw_spectrum.drop(columns=["date", "TO_PARSE"])
|
|
133
|
+
|
|
134
|
+
# Add raw array
|
|
135
|
+
df = df_data.set_index("time")
|
|
136
|
+
df_raw_spectrum = df_raw_spectrum.set_index("time")
|
|
137
|
+
|
|
138
|
+
df.update(df_raw_spectrum)
|
|
139
|
+
|
|
140
|
+
# Set back time as column
|
|
141
|
+
df = df.reset_index()
|
|
142
|
+
|
|
143
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
144
|
+
return df
|