disdrodb 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/_version.py +2 -2
- disdrodb/accessor/methods.py +10 -3
- disdrodb/api/checks.py +1 -1
- disdrodb/api/io.py +6 -1
- disdrodb/constants.py +1 -1
- disdrodb/etc/products/L1/LPM_V0/1MIN.yaml +13 -0
- disdrodb/etc/products/L1/global.yaml +1 -1
- disdrodb/etc/products/L2E/global.yaml +1 -1
- disdrodb/etc/products/L2M/global.yaml +1 -1
- disdrodb/issue/checks.py +2 -2
- disdrodb/l0/check_configs.py +1 -1
- disdrodb/l0/configs/LPM/l0a_encodings.yml +0 -1
- disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +0 -4
- disdrodb/l0/configs/LPM/l0b_encodings.yml +9 -9
- disdrodb/l0/configs/LPM/raw_data_format.yml +11 -11
- disdrodb/l0/configs/LPM_V0/bins_diameter.yml +103 -0
- disdrodb/l0/configs/LPM_V0/bins_velocity.yml +103 -0
- disdrodb/l0/configs/LPM_V0/l0a_encodings.yml +45 -0
- disdrodb/l0/configs/LPM_V0/l0b_cf_attrs.yml +180 -0
- disdrodb/l0/configs/LPM_V0/l0b_encodings.yml +410 -0
- disdrodb/l0/configs/LPM_V0/raw_data_format.yml +474 -0
- disdrodb/l0/configs/PARSIVEL/raw_data_format.yml +8 -8
- disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml +9 -9
- disdrodb/l0/l0a_processing.py +6 -2
- disdrodb/l0/l0b_processing.py +26 -19
- disdrodb/l0/l0c_processing.py +10 -0
- disdrodb/l0/manuals/LPM_V0.pdf +0 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +15 -7
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_PI.py +279 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_T.py +276 -0
- disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +2 -2
- disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_RWANDA_LPM_NC.py +103 -0
- disdrodb/l0/readers/LPM/NORWAY/HAUKELISETER_LPM.py +216 -0
- disdrodb/l0/readers/LPM/NORWAY/NMBU_LPM.py +208 -0
- disdrodb/l0/readers/LPM/UK/WITHWORTH_LPM.py +219 -0
- disdrodb/l0/readers/LPM/USA/CHARLESTON.py +229 -0
- disdrodb/l0/readers/{LPM → LPM_V0}/BELGIUM/ULIEGE.py +33 -49
- disdrodb/l0/readers/LPM_V0/ITALY/GID_LPM_V0.py +240 -0
- disdrodb/l0/readers/PARSIVEL/NASA/LPVEX.py +25 -13
- disdrodb/l0/readers/PARSIVEL/NASA/MC3E.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/BASQUECOUNTRY/EUSKALMET_OTT2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/JAPAN/PRECIP.py +155 -0
- disdrodb/l0/readers/PARSIVEL2/MPI/BCO_PARSIVEL2.py +14 -7
- disdrodb/l0/readers/PARSIVEL2/MPI/BOWTIE.py +8 -3
- disdrodb/l0/readers/PARSIVEL2/NASA/APU.py +28 -5
- disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/{NASA/GCPEX.py → NORWAY/UIB.py} +54 -29
- disdrodb/l0/readers/PARSIVEL2/PHILIPPINES/PAGASA.py +6 -3
- disdrodb/l0/readers/{PARSIVEL/NASA/PIERS.py → PARSIVEL2/USA/CSU.py} +62 -29
- disdrodb/l0/readers/PARSIVEL2/USA/CW3E.py +48 -21
- disdrodb/l0/readers/{PARSIVEL/NASA/IFLOODS.py → RD80/BRAZIL/ATTO_RD80.py} +50 -34
- disdrodb/l0/readers/{SW250 → SWS250}/BELGIUM/KMI.py +1 -1
- disdrodb/l1/beard_model.py +45 -1
- disdrodb/l1/fall_velocity.py +1 -6
- disdrodb/l1/filters.py +2 -0
- disdrodb/l2/empirical_dsd.py +12 -8
- disdrodb/routines/l0.py +2 -2
- disdrodb/routines/options.py +2 -0
- disdrodb/scattering/axis_ratio.py +3 -0
- disdrodb/scattering/routines.py +1 -1
- disdrodb/summary/routines.py +63 -61
- disdrodb/utils/compression.py +4 -2
- disdrodb/utils/dask.py +31 -11
- disdrodb/utils/manipulations.py +7 -1
- disdrodb/viz/plots.py +5 -3
- {disdrodb-0.2.0.dist-info → disdrodb-0.2.1.dist-info}/METADATA +1 -1
- {disdrodb-0.2.0.dist-info → disdrodb-0.2.1.dist-info}/RECORD +71 -54
- {disdrodb-0.2.0.dist-info → disdrodb-0.2.1.dist-info}/WHEEL +0 -0
- {disdrodb-0.2.0.dist-info → disdrodb-0.2.1.dist-info}/entry_points.txt +0 -0
- {disdrodb-0.2.0.dist-info → disdrodb-0.2.1.dist-info}/licenses/LICENSE +0 -0
- {disdrodb-0.2.0.dist-info → disdrodb-0.2.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
# -----------------------------------------------------------------------------.
|
|
2
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
3
|
+
#
|
|
4
|
+
# This program is free software: you can redistribute it and/or modify
|
|
5
|
+
# it under the terms of the GNU General Public License as published by
|
|
6
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
7
|
+
# (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This program is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
12
|
+
# GNU General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU General Public License
|
|
15
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
16
|
+
# -----------------------------------------------------------------------------.
|
|
17
|
+
"""DISDRODB reader for GID LPM V0 sensor (TC-TO) with incorrect reported time."""
|
|
18
|
+
import numpy as np
|
|
19
|
+
import pandas as pd
|
|
20
|
+
|
|
21
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
22
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
23
|
+
from disdrodb.utils.logger import log_error, log_warning
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def read_txt_file(file, filename, logger):
|
|
27
|
+
"""Parse for TC-TO LPM hourly file."""
|
|
28
|
+
#### - Define raw data headers
|
|
29
|
+
column_names = ["TO_PARSE"]
|
|
30
|
+
|
|
31
|
+
##------------------------------------------------------------------------.
|
|
32
|
+
#### Define reader options
|
|
33
|
+
# - For more info: https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
|
|
34
|
+
reader_kwargs = {}
|
|
35
|
+
|
|
36
|
+
# - Define delimiter
|
|
37
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
38
|
+
|
|
39
|
+
# - Avoid first column to become df index !!!
|
|
40
|
+
reader_kwargs["index_col"] = False
|
|
41
|
+
|
|
42
|
+
# Since column names are expected to be passed explicitly, header is set to None
|
|
43
|
+
reader_kwargs["header"] = None
|
|
44
|
+
|
|
45
|
+
# - Number of rows to be skipped at the beginning of the file
|
|
46
|
+
reader_kwargs["skiprows"] = None
|
|
47
|
+
|
|
48
|
+
# - Define behaviour when encountering bad lines
|
|
49
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
50
|
+
|
|
51
|
+
# - Define reader engine
|
|
52
|
+
# - C engine is faster
|
|
53
|
+
# - Python engine is more feature-complete
|
|
54
|
+
reader_kwargs["engine"] = "python"
|
|
55
|
+
|
|
56
|
+
# - Define on-the-fly decompression of on-disk data
|
|
57
|
+
# - Available: gzip, bz2, zip
|
|
58
|
+
reader_kwargs["compression"] = "infer"
|
|
59
|
+
|
|
60
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
61
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
62
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
63
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
64
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
65
|
+
|
|
66
|
+
##------------------------------------------------------------------------.
|
|
67
|
+
#### Read the data
|
|
68
|
+
df = read_raw_text_file(
|
|
69
|
+
filepath=file,
|
|
70
|
+
column_names=column_names,
|
|
71
|
+
reader_kwargs=reader_kwargs,
|
|
72
|
+
logger=logger,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
##------------------------------------------------------------------------.
|
|
76
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
77
|
+
# Raise error if empty file
|
|
78
|
+
if len(df) == 0:
|
|
79
|
+
raise ValueError(f"{filename} is empty.")
|
|
80
|
+
|
|
81
|
+
# Select only rows with expected number of delimiters
|
|
82
|
+
df = df[df["TO_PARSE"].str.count(";") == 442]
|
|
83
|
+
|
|
84
|
+
# Check there are still valid rows
|
|
85
|
+
if len(df) == 0:
|
|
86
|
+
raise ValueError(f"No valid rows in {filename}.")
|
|
87
|
+
|
|
88
|
+
# Split by ; delimiter (before raw drop number)
|
|
89
|
+
df = df["TO_PARSE"].str.split(";", expand=True, n=43)
|
|
90
|
+
|
|
91
|
+
# Assign column names
|
|
92
|
+
names = [
|
|
93
|
+
"start_identifier",
|
|
94
|
+
"sensor_serial_number",
|
|
95
|
+
"weather_code_synop_4680_5min",
|
|
96
|
+
"weather_code_metar_4678_5min",
|
|
97
|
+
"precipitation_rate_5min",
|
|
98
|
+
"weather_code_synop_4680",
|
|
99
|
+
"weather_code_metar_4678",
|
|
100
|
+
"precipitation_rate",
|
|
101
|
+
"precipitation_accumulated",
|
|
102
|
+
"sensor_time",
|
|
103
|
+
"temperature_interior",
|
|
104
|
+
"laser_temperature",
|
|
105
|
+
"laser_current_average",
|
|
106
|
+
"control_voltage",
|
|
107
|
+
"optical_control_voltage_output",
|
|
108
|
+
"number_particles",
|
|
109
|
+
"number_particles_internal_data",
|
|
110
|
+
"number_particles_min_speed",
|
|
111
|
+
"number_particles_min_speed_internal_data",
|
|
112
|
+
"number_particles_max_speed",
|
|
113
|
+
"number_particles_max_speed_internal_data",
|
|
114
|
+
"number_particles_min_diameter",
|
|
115
|
+
"number_particles_min_diameter_internal_data",
|
|
116
|
+
"number_particles_no_hydrometeor",
|
|
117
|
+
"number_particles_no_hydrometeor_internal_data",
|
|
118
|
+
"number_particles_unknown_classification",
|
|
119
|
+
"total_gross_volume_unknown_classification",
|
|
120
|
+
"number_particles_hail",
|
|
121
|
+
"total_gross_volume_hail",
|
|
122
|
+
"number_particles_solid_precipitation",
|
|
123
|
+
"total_gross_volume_solid_precipitation",
|
|
124
|
+
"number_particles_great_pellet",
|
|
125
|
+
"total_gross_volume_great_pellet",
|
|
126
|
+
"number_particles_small_pellet",
|
|
127
|
+
"total_gross_volume_small_pellet",
|
|
128
|
+
"number_particles_snowgrain",
|
|
129
|
+
"total_gross_volume_snowgrain",
|
|
130
|
+
"number_particles_rain",
|
|
131
|
+
"total_gross_volume_rain",
|
|
132
|
+
"number_particles_small_rain",
|
|
133
|
+
"total_gross_volume_small_rain",
|
|
134
|
+
"number_particles_drizzle",
|
|
135
|
+
"total_gross_volume_drizzle",
|
|
136
|
+
"raw_drop_number",
|
|
137
|
+
]
|
|
138
|
+
df.columns = names
|
|
139
|
+
|
|
140
|
+
# Deal with case if there are 61 timesteps
|
|
141
|
+
# - Occurs sometimes when previous hourly file miss timesteps
|
|
142
|
+
if len(df) == 61:
|
|
143
|
+
log_warning(logger=logger, msg=f"{filename} contains 61 timesteps. Dropping the first.")
|
|
144
|
+
df = df.iloc[1:]
|
|
145
|
+
|
|
146
|
+
# Raise error if more than 60 timesteps/rows
|
|
147
|
+
n_rows = len(df)
|
|
148
|
+
if n_rows > 60:
|
|
149
|
+
raise ValueError(f"The hourly file contains {n_rows} timesteps.")
|
|
150
|
+
|
|
151
|
+
# Infer and define "time" column
|
|
152
|
+
start_time_str = filename.split(".")[0] # '2024020200.txt'
|
|
153
|
+
start_time = pd.to_datetime(start_time_str, format="%Y%m%d%H")
|
|
154
|
+
|
|
155
|
+
# - Define timedelta based on sensor_time
|
|
156
|
+
dt = pd.to_timedelta(df["sensor_time"] + ":00").to_numpy().astype("m8[s]")
|
|
157
|
+
rollover_indices = np.where(np.diff(dt) < np.timedelta64(0, "s"))[0]
|
|
158
|
+
if rollover_indices.size > 0:
|
|
159
|
+
for idx in rollover_indices:
|
|
160
|
+
dt[idx + 1 :] += np.timedelta64(24, "h")
|
|
161
|
+
dt = dt - dt[0]
|
|
162
|
+
|
|
163
|
+
# - Define approximate time
|
|
164
|
+
df["time"] = start_time + dt
|
|
165
|
+
|
|
166
|
+
# - Keep rows where time increment is between 00 and 59 minutes
|
|
167
|
+
valid_rows = dt <= np.timedelta64(3540, "s")
|
|
168
|
+
df = df[valid_rows]
|
|
169
|
+
|
|
170
|
+
# Drop rows with invalid raw_drop_number
|
|
171
|
+
# --> 440 value # 22x20
|
|
172
|
+
df = df[df["raw_drop_number"].astype(str).str.len() == 1599]
|
|
173
|
+
|
|
174
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
175
|
+
columns_to_drop = [
|
|
176
|
+
"sensor_time",
|
|
177
|
+
"start_identifier",
|
|
178
|
+
"sensor_serial_number",
|
|
179
|
+
]
|
|
180
|
+
df = df.drop(columns=columns_to_drop)
|
|
181
|
+
return df
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@is_documented_by(reader_generic_docstring)
|
|
185
|
+
def reader(
|
|
186
|
+
filepath,
|
|
187
|
+
logger=None,
|
|
188
|
+
):
|
|
189
|
+
"""Reader."""
|
|
190
|
+
import zipfile
|
|
191
|
+
|
|
192
|
+
##------------------------------------------------------------------------.
|
|
193
|
+
# filename = os.path.basename(filepath)
|
|
194
|
+
# return read_txt_file(file=filepath, filename=filename, logger=logger)
|
|
195
|
+
|
|
196
|
+
# ---------------------------------------------------------------------.
|
|
197
|
+
#### Iterate over all files (aka timesteps) in the daily zip archive
|
|
198
|
+
# - Each file contain a single timestep !
|
|
199
|
+
# list_df = []
|
|
200
|
+
# with tempfile.TemporaryDirectory() as temp_dir:
|
|
201
|
+
# # Extract all files
|
|
202
|
+
# unzip_file_on_terminal(filepath, temp_dir)
|
|
203
|
+
|
|
204
|
+
# # Walk through extracted files
|
|
205
|
+
# for root, _, files in os.walk(temp_dir):
|
|
206
|
+
# for filename in sorted(files):
|
|
207
|
+
# if filename.endswith(".txt"):
|
|
208
|
+
# full_path = os.path.join(root, filename)
|
|
209
|
+
# try:
|
|
210
|
+
# df = read_txt_file(file=full_path, filename=filename, logger=logger)
|
|
211
|
+
# if df is not None:
|
|
212
|
+
# list_df.append(df)
|
|
213
|
+
# except Exception as e:
|
|
214
|
+
# msg = f"An error occurred while reading {filename}: {e}"
|
|
215
|
+
# log_error(logger=logger, msg=msg, verbose=True)
|
|
216
|
+
|
|
217
|
+
list_df = []
|
|
218
|
+
with zipfile.ZipFile(filepath, "r") as zip_ref:
|
|
219
|
+
filenames = sorted(zip_ref.namelist())
|
|
220
|
+
for filename in filenames:
|
|
221
|
+
if filename.endswith(".txt"):
|
|
222
|
+
# Open file
|
|
223
|
+
with zip_ref.open(filename) as file:
|
|
224
|
+
try:
|
|
225
|
+
df = read_txt_file(file=file, filename=filename, logger=logger)
|
|
226
|
+
if df is not None:
|
|
227
|
+
list_df.append(df)
|
|
228
|
+
except Exception as e:
|
|
229
|
+
msg = f"An error occurred while reading {filename}. The error is: {e}"
|
|
230
|
+
log_error(logger=logger, msg=msg, verbose=True)
|
|
231
|
+
|
|
232
|
+
# Check the zip file contains at least some non.empty files
|
|
233
|
+
if len(list_df) == 0:
|
|
234
|
+
raise ValueError(f"{filepath} contains only empty files!")
|
|
235
|
+
|
|
236
|
+
# Concatenate all dataframes into a single one
|
|
237
|
+
df = pd.concat(list_df)
|
|
238
|
+
|
|
239
|
+
# ---------------------------------------------------------------------.
|
|
240
|
+
return df
|
|
@@ -29,22 +29,19 @@ def reader(
|
|
|
29
29
|
"""Reader."""
|
|
30
30
|
##------------------------------------------------------------------------.
|
|
31
31
|
#### Define column names
|
|
32
|
-
column_names = ["
|
|
32
|
+
column_names = ["TO_PARSE"]
|
|
33
33
|
|
|
34
34
|
##------------------------------------------------------------------------.
|
|
35
35
|
#### Define reader options
|
|
36
36
|
reader_kwargs = {}
|
|
37
37
|
# - Define delimiter
|
|
38
|
-
reader_kwargs["delimiter"] = "
|
|
38
|
+
reader_kwargs["delimiter"] = "//n"
|
|
39
39
|
# - Skip first row as columns names
|
|
40
40
|
reader_kwargs["header"] = None
|
|
41
|
+
reader_kwargs["skiprows"] = 0
|
|
41
42
|
# - Skip file with encoding errors
|
|
42
43
|
reader_kwargs["encoding_errors"] = "ignore"
|
|
43
|
-
# -
|
|
44
|
-
reader_kwargs["zipped"] = True
|
|
45
|
-
# - Searched file into tar files
|
|
46
|
-
reader_kwargs["filename_to_read_zipped"] = "spectrum.txt"
|
|
47
|
-
# - Avoid first column to become df index
|
|
44
|
+
# - Avoid first column to become df index !!!
|
|
48
45
|
reader_kwargs["index_col"] = False
|
|
49
46
|
# - Define behaviour when encountering bad lines
|
|
50
47
|
reader_kwargs["on_bad_lines"] = "skip"
|
|
@@ -59,7 +56,7 @@ def reader(
|
|
|
59
56
|
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
60
57
|
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
61
58
|
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
62
|
-
reader_kwargs["na_values"] = ["na", "", "error", "-.-"]
|
|
59
|
+
reader_kwargs["na_values"] = ["na", "", "error", "NA", "-.-"]
|
|
63
60
|
|
|
64
61
|
##------------------------------------------------------------------------.
|
|
65
62
|
#### Read the data
|
|
@@ -72,14 +69,29 @@ def reader(
|
|
|
72
69
|
|
|
73
70
|
##------------------------------------------------------------------------.
|
|
74
71
|
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
72
|
+
# Remove rows with invalid number of separators
|
|
73
|
+
df = df[df["TO_PARSE"].str.count(";") == 1]
|
|
74
|
+
if len(df) == 0:
|
|
75
|
+
raise ValueError(f"No valid data in {filepath}")
|
|
76
|
+
|
|
77
|
+
# Split the columns
|
|
78
|
+
df = df["TO_PARSE"].str.split(";", n=2, expand=True)
|
|
79
|
+
|
|
80
|
+
# Assign column names
|
|
81
|
+
df.columns = ["time", "TO_BE_SPLITTED"]
|
|
82
|
+
|
|
75
83
|
# Convert time column to datetime
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
84
|
+
df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
|
|
85
|
+
|
|
86
|
+
# Split the 'TO_BE_SPLITTED' column
|
|
87
|
+
df = df["TO_BE_SPLITTED"].str.split(",", n=3, expand=True)
|
|
88
|
+
df.columns = ["station_id", "sensor_status", "sensor_temperature", "raw_drop_number"]
|
|
89
|
+
|
|
90
|
+
# Add time
|
|
91
|
+
df["time"] = df_time
|
|
80
92
|
|
|
81
93
|
# Drop columns not agreeing with DISDRODB L0 standards
|
|
82
|
-
df = df.drop(columns=["
|
|
94
|
+
df = df.drop(columns=["station_id"])
|
|
83
95
|
|
|
84
96
|
# Return the dataframe adhering to DISDRODB L0 standards
|
|
85
97
|
return df
|
|
@@ -116,7 +116,7 @@ def reader(
|
|
|
116
116
|
return df
|
|
117
117
|
# ---------------------------------------------------------
|
|
118
118
|
#### Case of 1032 delimiters
|
|
119
|
-
if n_delimiters == 1033: # (most of the files)
|
|
119
|
+
if n_delimiters == 1033: # (most of the files ... PIERS FORMAT)
|
|
120
120
|
# Select valid rows
|
|
121
121
|
df = df.loc[df["TO_BE_SPLITTED"].str.count(",") == 1033]
|
|
122
122
|
# Get time column
|
|
@@ -124,7 +124,7 @@ def read_txt_file(file, filename, logger):
|
|
|
124
124
|
# Select rows with valid spectrum
|
|
125
125
|
# df = df[df["TO_PARSE"].str.count(";") == 1191] # 1112
|
|
126
126
|
|
|
127
|
-
# Raise
|
|
127
|
+
# Raise error if corrupted file
|
|
128
128
|
if len(df) == 4:
|
|
129
129
|
raise ValueError(f"{filename} is corrupted.")
|
|
130
130
|
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# -----------------------------------------------------------------------------.
|
|
2
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
3
|
+
#
|
|
4
|
+
# This program is free software: you can redistribute it and/or modify
|
|
5
|
+
# it under the terms of the GNU General Public License as published by
|
|
6
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
7
|
+
# (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This program is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
12
|
+
# GNU General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU General Public License
|
|
15
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
16
|
+
# -----------------------------------------------------------------------------.
|
|
17
|
+
"""DISDRODB reader for Colorado State University PRECIP OTT Parsivel 2 raw data."""
|
|
18
|
+
|
|
19
|
+
import pandas as pd
|
|
20
|
+
|
|
21
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
22
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@is_documented_by(reader_generic_docstring)
|
|
26
|
+
def reader(
|
|
27
|
+
filepath,
|
|
28
|
+
logger=None,
|
|
29
|
+
):
|
|
30
|
+
"""Reader."""
|
|
31
|
+
##------------------------------------------------------------------------.
|
|
32
|
+
#### Define column names
|
|
33
|
+
column_names = ["TO_PARSE"]
|
|
34
|
+
|
|
35
|
+
##------------------------------------------------------------------------.
|
|
36
|
+
#### Define reader options
|
|
37
|
+
reader_kwargs = {}
|
|
38
|
+
|
|
39
|
+
# - Define delimiter
|
|
40
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
41
|
+
|
|
42
|
+
# - Skip first row as columns names
|
|
43
|
+
reader_kwargs["header"] = None
|
|
44
|
+
|
|
45
|
+
# - Skip header
|
|
46
|
+
reader_kwargs["skiprows"] = 0
|
|
47
|
+
|
|
48
|
+
# - Define encoding
|
|
49
|
+
reader_kwargs["encoding"] = "ISO-8859-1"
|
|
50
|
+
|
|
51
|
+
# - Avoid first column to become df index !!!
|
|
52
|
+
reader_kwargs["index_col"] = False
|
|
53
|
+
|
|
54
|
+
# - Define behaviour when encountering bad lines
|
|
55
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
56
|
+
|
|
57
|
+
# - Define reader engine
|
|
58
|
+
# - C engine is faster
|
|
59
|
+
# - Python engine is more feature-complete
|
|
60
|
+
reader_kwargs["engine"] = "python"
|
|
61
|
+
|
|
62
|
+
# - Define on-the-fly decompression of on-disk data
|
|
63
|
+
# - Available: gzip, bz2, zip
|
|
64
|
+
# reader_kwargs['compression'] = 'xz'
|
|
65
|
+
|
|
66
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
67
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
68
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
69
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
70
|
+
reader_kwargs["na_values"] = ["na", "error", "-.-", " NA"]
|
|
71
|
+
|
|
72
|
+
##------------------------------------------------------------------------.
|
|
73
|
+
#### Read the data
|
|
74
|
+
df = read_raw_text_file(
|
|
75
|
+
filepath=filepath,
|
|
76
|
+
column_names=column_names,
|
|
77
|
+
reader_kwargs=reader_kwargs,
|
|
78
|
+
logger=logger,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
##------------------------------------------------------------------------.
|
|
82
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
83
|
+
# Raise error if empty file
|
|
84
|
+
if len(df) == 0:
|
|
85
|
+
raise ValueError(f"{filepath} is empty.")
|
|
86
|
+
|
|
87
|
+
# Select only rows with expected number of delimiters
|
|
88
|
+
df = df[df["TO_PARSE"].str.count(",") == 1041]
|
|
89
|
+
|
|
90
|
+
# Raise error if no data left
|
|
91
|
+
if len(df) == 0:
|
|
92
|
+
raise ValueError(f"No valid data in {filepath}.")
|
|
93
|
+
|
|
94
|
+
# Split into columns
|
|
95
|
+
df = df["TO_PARSE"].str.split(",", expand=True, n=17)
|
|
96
|
+
|
|
97
|
+
# Assign columns names
|
|
98
|
+
names = [
|
|
99
|
+
"date",
|
|
100
|
+
"time",
|
|
101
|
+
"rainfall_rate_32bit",
|
|
102
|
+
"rainfall_accumulated_32bit",
|
|
103
|
+
"weather_code_synop_4680",
|
|
104
|
+
"weather_code_metar_4678",
|
|
105
|
+
"weather_code_nws",
|
|
106
|
+
"reflectivity_32bit",
|
|
107
|
+
"mor_visibility",
|
|
108
|
+
"laser_amplitude",
|
|
109
|
+
"number_particles",
|
|
110
|
+
"sensor_temperature",
|
|
111
|
+
"sensor_heating_current",
|
|
112
|
+
"sensor_battery_voltage",
|
|
113
|
+
"sensor_status",
|
|
114
|
+
"rain_kinetic_energy",
|
|
115
|
+
"snowfall_rate",
|
|
116
|
+
"raw_drop_number",
|
|
117
|
+
]
|
|
118
|
+
df.columns = names
|
|
119
|
+
|
|
120
|
+
# Add datetime time column
|
|
121
|
+
time_str = df["date"] + "-" + df["time"]
|
|
122
|
+
df["time"] = pd.to_datetime(time_str, format="%d.%m.%Y-%H:%M:%S", errors="coerce")
|
|
123
|
+
|
|
124
|
+
# Derive the raw spectrum
|
|
125
|
+
# Remove <SPECTRUM> and </SPECTRUM> prefix and suffix from the raw_drop_number field
|
|
126
|
+
df["raw_drop_number"] = df["raw_drop_number"].astype("string")
|
|
127
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
|
|
128
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
|
|
129
|
+
|
|
130
|
+
# Preprocess the raw spectrum and raw_drop_average_velocity
|
|
131
|
+
# - Add 0 before every ; if ; not preceded by a digit
|
|
132
|
+
# - Example: ';;1;;' --> '0;0;1;0;'
|
|
133
|
+
df["raw_drop_number"] = df["raw_drop_number"].str.replace(r"(?<!\d),", "0,", regex=True)
|
|
134
|
+
|
|
135
|
+
# Infill missing timesteps with raw_drop_number = 0 spectrum
|
|
136
|
+
# - Define the full time range with 30-second frequency
|
|
137
|
+
full_time_index = pd.date_range(start=df["time"].iloc[0], end=df["time"].iloc[-1], freq="30s")
|
|
138
|
+
|
|
139
|
+
# - Reindex the DataFrame to include all 30-second timesteps
|
|
140
|
+
df = df.set_index("time").reindex(full_time_index)
|
|
141
|
+
|
|
142
|
+
# - Fill missing raw_drop_number with 0
|
|
143
|
+
df["raw_drop_number"] = df["raw_drop_number"].fillna(0)
|
|
144
|
+
|
|
145
|
+
# - Restore 'time' as a column
|
|
146
|
+
df = df.rename_axis("time").reset_index()
|
|
147
|
+
|
|
148
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
149
|
+
columns_to_drop = [
|
|
150
|
+
"date",
|
|
151
|
+
]
|
|
152
|
+
df = df.drop(columns=columns_to_drop)
|
|
153
|
+
|
|
154
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
155
|
+
return df
|
|
@@ -38,11 +38,14 @@ def reader(
|
|
|
38
38
|
# - Define delimiter
|
|
39
39
|
reader_kwargs["delimiter"] = "/\n"
|
|
40
40
|
|
|
41
|
-
# Skip first row as columns names
|
|
41
|
+
# - Skip first row as columns names
|
|
42
42
|
reader_kwargs["header"] = None
|
|
43
43
|
|
|
44
|
-
# Skip first 2 rows
|
|
45
|
-
reader_kwargs["skiprows"] =
|
|
44
|
+
# - Skip first 2 rows
|
|
45
|
+
reader_kwargs["skiprows"] = 0
|
|
46
|
+
|
|
47
|
+
# - Define encoding
|
|
48
|
+
reader_kwargs["encoding"] = "ISO-8859-1"
|
|
46
49
|
|
|
47
50
|
# - Avoid first column to become df index !!!
|
|
48
51
|
reader_kwargs["index_col"] = False
|
|
@@ -76,12 +79,17 @@ def reader(
|
|
|
76
79
|
|
|
77
80
|
##------------------------------------------------------------------------.
|
|
78
81
|
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
79
|
-
#
|
|
80
|
-
|
|
82
|
+
# Raise error if empty file
|
|
83
|
+
if len(df) == 0:
|
|
84
|
+
raise ValueError(f"{filepath} is empty.")
|
|
81
85
|
|
|
82
|
-
#
|
|
86
|
+
# Select only rows with expected number of delimiters
|
|
83
87
|
df = df[df["TO_BE_PARSED"].str.count(";") == 1107]
|
|
84
88
|
|
|
89
|
+
# Raise error if no data left
|
|
90
|
+
if len(df) == 0:
|
|
91
|
+
raise ValueError(f"No valid data in {filepath}.")
|
|
92
|
+
|
|
85
93
|
# Split by ; delimiter
|
|
86
94
|
df = df["TO_BE_PARSED"].str.split(";", expand=True, n=19)
|
|
87
95
|
|
|
@@ -132,5 +140,4 @@ def reader(
|
|
|
132
140
|
"sample_interval",
|
|
133
141
|
]
|
|
134
142
|
df = df.drop(columns=columns_to_drop)
|
|
135
|
-
|
|
136
143
|
return df
|
|
@@ -157,12 +157,17 @@ def reader(
|
|
|
157
157
|
|
|
158
158
|
##------------------------------------------------------------------------.
|
|
159
159
|
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
160
|
-
#
|
|
161
|
-
|
|
160
|
+
# Raise error if empty file
|
|
161
|
+
if len(df) == 0:
|
|
162
|
+
raise ValueError(f"{filepath} is empty.")
|
|
162
163
|
|
|
163
|
-
#
|
|
164
|
+
# Select only rows with expected number of delimiters
|
|
164
165
|
df = df[df["TO_BE_PARSED"].str.count(";") == 1107]
|
|
165
166
|
|
|
167
|
+
# Raise error if no data left
|
|
168
|
+
if len(df) == 0:
|
|
169
|
+
raise ValueError(f"No valid data in {filepath}.")
|
|
170
|
+
|
|
166
171
|
# Split by ; delimiter
|
|
167
172
|
df = df["TO_BE_PARSED"].str.split(";", expand=True, n=19)
|
|
168
173
|
|
|
@@ -44,13 +44,13 @@ def reader(
|
|
|
44
44
|
"""Reader."""
|
|
45
45
|
##------------------------------------------------------------------------.
|
|
46
46
|
#### Define column names
|
|
47
|
-
column_names = ["
|
|
47
|
+
column_names = ["TO_PARSE"]
|
|
48
48
|
|
|
49
49
|
##------------------------------------------------------------------------.
|
|
50
50
|
#### Define reader options
|
|
51
51
|
reader_kwargs = {}
|
|
52
52
|
# - Define delimiter
|
|
53
|
-
reader_kwargs["delimiter"] = "
|
|
53
|
+
reader_kwargs["delimiter"] = "//n"
|
|
54
54
|
# - Skip first row as columns names
|
|
55
55
|
reader_kwargs["header"] = None
|
|
56
56
|
reader_kwargs["skiprows"] = 0
|
|
@@ -84,6 +84,20 @@ def reader(
|
|
|
84
84
|
|
|
85
85
|
##------------------------------------------------------------------------.
|
|
86
86
|
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
87
|
+
# Remove rows with invalid number of separators
|
|
88
|
+
df = df[df["TO_PARSE"].str.count(";") == 1]
|
|
89
|
+
if len(df) == 0:
|
|
90
|
+
raise ValueError(f"No valid data in {filepath}")
|
|
91
|
+
|
|
92
|
+
# Retrieve time and telegram field
|
|
93
|
+
df = df["TO_PARSE"].str.split(";", expand=True)
|
|
94
|
+
df.columns = ["time", "TO_BE_SPLITTED"]
|
|
95
|
+
|
|
96
|
+
# Remove rows with invalid number of separators
|
|
97
|
+
df = df[df["TO_BE_SPLITTED"].str.count(",") == 1033]
|
|
98
|
+
if len(df) == 0:
|
|
99
|
+
raise ValueError(f"No valid data in {filepath}")
|
|
100
|
+
|
|
87
101
|
# Convert time column to datetime
|
|
88
102
|
df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
|
|
89
103
|
|
|
@@ -91,7 +105,7 @@ def reader(
|
|
|
91
105
|
df = df["TO_BE_SPLITTED"].str.split(",", n=9, expand=True)
|
|
92
106
|
|
|
93
107
|
# Assign column names
|
|
94
|
-
|
|
108
|
+
columns_names = [
|
|
95
109
|
"station_name",
|
|
96
110
|
"sensor_status",
|
|
97
111
|
"sensor_temperature",
|
|
@@ -103,7 +117,7 @@ def reader(
|
|
|
103
117
|
"weather_code_synop_4677",
|
|
104
118
|
"raw_drop_number",
|
|
105
119
|
]
|
|
106
|
-
df.columns =
|
|
120
|
+
df.columns = columns_names
|
|
107
121
|
|
|
108
122
|
# Add the time column
|
|
109
123
|
df["time"] = df_time
|
|
@@ -111,10 +125,19 @@ def reader(
|
|
|
111
125
|
# Drop columns not agreeing with DISDRODB L0 standards
|
|
112
126
|
df = df.drop(columns=["station_name"])
|
|
113
127
|
|
|
128
|
+
# Remove rows with invalid raw drop number
|
|
129
|
+
# --> Occurs e.g. in UCONN apu28
|
|
130
|
+
# def mask_invalid_raw_drop_number(df)
|
|
131
|
+
# df_split = df["raw_drop_number"].str.split(",", expand=True)
|
|
132
|
+
# idx = np.where(np.any(df_split.astype(float) > 998, axis=1))[0]
|
|
133
|
+
# df.loc[idx, "raw_drop_number"] = "NaN"
|
|
134
|
+
# return df
|
|
135
|
+
df = df[df["raw_drop_number"].str.len() == 4096]
|
|
136
|
+
|
|
114
137
|
# Drop rows with invalid values
|
|
115
138
|
# --> Ensure that weather_code_synop_4677 has length 2
|
|
116
139
|
# --> If a previous column is missing it will have 000
|
|
117
|
-
df = df[df["weather_code_synop_4677"].str.len() == 2]
|
|
140
|
+
# df = df[df["weather_code_synop_4677"].str.len() == 2]
|
|
118
141
|
|
|
119
142
|
# Return the dataframe adhering to DISDRODB L0 standards
|
|
120
143
|
return df
|