disdrodb 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. disdrodb/_version.py +2 -2
  2. disdrodb/accessor/methods.py +10 -3
  3. disdrodb/api/checks.py +1 -1
  4. disdrodb/api/io.py +6 -1
  5. disdrodb/constants.py +1 -1
  6. disdrodb/etc/products/L1/LPM_V0/1MIN.yaml +13 -0
  7. disdrodb/etc/products/L1/global.yaml +1 -1
  8. disdrodb/etc/products/L2E/global.yaml +1 -1
  9. disdrodb/etc/products/L2M/global.yaml +1 -1
  10. disdrodb/issue/checks.py +2 -2
  11. disdrodb/l0/check_configs.py +1 -1
  12. disdrodb/l0/configs/LPM/l0a_encodings.yml +0 -1
  13. disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +0 -4
  14. disdrodb/l0/configs/LPM/l0b_encodings.yml +9 -9
  15. disdrodb/l0/configs/LPM/raw_data_format.yml +11 -11
  16. disdrodb/l0/configs/LPM_V0/bins_diameter.yml +103 -0
  17. disdrodb/l0/configs/LPM_V0/bins_velocity.yml +103 -0
  18. disdrodb/l0/configs/LPM_V0/l0a_encodings.yml +45 -0
  19. disdrodb/l0/configs/LPM_V0/l0b_cf_attrs.yml +180 -0
  20. disdrodb/l0/configs/LPM_V0/l0b_encodings.yml +410 -0
  21. disdrodb/l0/configs/LPM_V0/raw_data_format.yml +474 -0
  22. disdrodb/l0/configs/PARSIVEL/raw_data_format.yml +8 -8
  23. disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml +9 -9
  24. disdrodb/l0/l0a_processing.py +6 -2
  25. disdrodb/l0/l0b_processing.py +26 -19
  26. disdrodb/l0/l0c_processing.py +10 -0
  27. disdrodb/l0/manuals/LPM_V0.pdf +0 -0
  28. disdrodb/l0/readers/LPM/ITALY/GID_LPM.py +15 -7
  29. disdrodb/l0/readers/LPM/ITALY/GID_LPM_PI.py +279 -0
  30. disdrodb/l0/readers/LPM/ITALY/GID_LPM_T.py +276 -0
  31. disdrodb/l0/readers/LPM/ITALY/GID_LPM_W.py +2 -2
  32. disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_RWANDA_LPM_NC.py +103 -0
  33. disdrodb/l0/readers/LPM/NORWAY/HAUKELISETER_LPM.py +216 -0
  34. disdrodb/l0/readers/LPM/NORWAY/NMBU_LPM.py +208 -0
  35. disdrodb/l0/readers/LPM/UK/WITHWORTH_LPM.py +219 -0
  36. disdrodb/l0/readers/LPM/USA/CHARLESTON.py +229 -0
  37. disdrodb/l0/readers/{LPM → LPM_V0}/BELGIUM/ULIEGE.py +33 -49
  38. disdrodb/l0/readers/LPM_V0/ITALY/GID_LPM_V0.py +240 -0
  39. disdrodb/l0/readers/PARSIVEL/NASA/LPVEX.py +25 -13
  40. disdrodb/l0/readers/PARSIVEL/NASA/MC3E.py +1 -1
  41. disdrodb/l0/readers/PARSIVEL2/BASQUECOUNTRY/EUSKALMET_OTT2.py +1 -1
  42. disdrodb/l0/readers/PARSIVEL2/JAPAN/PRECIP.py +155 -0
  43. disdrodb/l0/readers/PARSIVEL2/MPI/BCO_PARSIVEL2.py +14 -7
  44. disdrodb/l0/readers/PARSIVEL2/MPI/BOWTIE.py +8 -3
  45. disdrodb/l0/readers/PARSIVEL2/NASA/APU.py +28 -5
  46. disdrodb/l0/readers/PARSIVEL2/NCAR/RELAMPAGO_PARSIVEL2.py +1 -1
  47. disdrodb/l0/readers/PARSIVEL2/{NASA/GCPEX.py → NORWAY/UIB.py} +54 -29
  48. disdrodb/l0/readers/PARSIVEL2/PHILIPPINES/PAGASA.py +6 -3
  49. disdrodb/l0/readers/{PARSIVEL/NASA/PIERS.py → PARSIVEL2/USA/CSU.py} +62 -29
  50. disdrodb/l0/readers/PARSIVEL2/USA/CW3E.py +48 -21
  51. disdrodb/l0/readers/{PARSIVEL/NASA/IFLOODS.py → RD80/BRAZIL/ATTO_RD80.py} +50 -34
  52. disdrodb/l0/readers/{SW250 → SWS250}/BELGIUM/KMI.py +1 -1
  53. disdrodb/l1/beard_model.py +45 -1
  54. disdrodb/l1/fall_velocity.py +1 -6
  55. disdrodb/l1/filters.py +2 -0
  56. disdrodb/l2/empirical_dsd.py +12 -8
  57. disdrodb/routines/l0.py +2 -2
  58. disdrodb/routines/options.py +2 -0
  59. disdrodb/scattering/axis_ratio.py +3 -0
  60. disdrodb/scattering/routines.py +1 -1
  61. disdrodb/summary/routines.py +63 -61
  62. disdrodb/utils/compression.py +4 -2
  63. disdrodb/utils/dask.py +31 -11
  64. disdrodb/utils/manipulations.py +7 -1
  65. disdrodb/viz/plots.py +5 -3
  66. {disdrodb-0.2.0.dist-info → disdrodb-0.2.1.dist-info}/METADATA +1 -1
  67. {disdrodb-0.2.0.dist-info → disdrodb-0.2.1.dist-info}/RECORD +71 -54
  68. {disdrodb-0.2.0.dist-info → disdrodb-0.2.1.dist-info}/WHEEL +0 -0
  69. {disdrodb-0.2.0.dist-info → disdrodb-0.2.1.dist-info}/entry_points.txt +0 -0
  70. {disdrodb-0.2.0.dist-info → disdrodb-0.2.1.dist-info}/licenses/LICENSE +0 -0
  71. {disdrodb-0.2.0.dist-info → disdrodb-0.2.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,240 @@
1
+ # -----------------------------------------------------------------------------.
2
+ # Copyright (c) 2021-2023 DISDRODB developers
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU General Public License
15
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
16
+ # -----------------------------------------------------------------------------.
17
+ """DISDRODB reader for GID LPM V0 sensor (TC-TO) with incorrect reported time."""
18
+ import numpy as np
19
+ import pandas as pd
20
+
21
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
22
+ from disdrodb.l0.l0a_processing import read_raw_text_file
23
+ from disdrodb.utils.logger import log_error, log_warning
24
+
25
+
26
+ def read_txt_file(file, filename, logger):
27
+ """Parse for TC-TO LPM hourly file."""
28
+ #### - Define raw data headers
29
+ column_names = ["TO_PARSE"]
30
+
31
+ ##------------------------------------------------------------------------.
32
+ #### Define reader options
33
+ # - For more info: https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
34
+ reader_kwargs = {}
35
+
36
+ # - Define delimiter
37
+ reader_kwargs["delimiter"] = "\\n"
38
+
39
+ # - Avoid first column to become df index !!!
40
+ reader_kwargs["index_col"] = False
41
+
42
+ # Since column names are expected to be passed explicitly, header is set to None
43
+ reader_kwargs["header"] = None
44
+
45
+ # - Number of rows to be skipped at the beginning of the file
46
+ reader_kwargs["skiprows"] = None
47
+
48
+ # - Define behaviour when encountering bad lines
49
+ reader_kwargs["on_bad_lines"] = "skip"
50
+
51
+ # - Define reader engine
52
+ # - C engine is faster
53
+ # - Python engine is more feature-complete
54
+ reader_kwargs["engine"] = "python"
55
+
56
+ # - Define on-the-fly decompression of on-disk data
57
+ # - Available: gzip, bz2, zip
58
+ reader_kwargs["compression"] = "infer"
59
+
60
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
61
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
62
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
63
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
64
+ reader_kwargs["na_values"] = ["na", "", "error"]
65
+
66
+ ##------------------------------------------------------------------------.
67
+ #### Read the data
68
+ df = read_raw_text_file(
69
+ filepath=file,
70
+ column_names=column_names,
71
+ reader_kwargs=reader_kwargs,
72
+ logger=logger,
73
+ )
74
+
75
+ ##------------------------------------------------------------------------.
76
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
77
+ # Raise error if empty file
78
+ if len(df) == 0:
79
+ raise ValueError(f"{filename} is empty.")
80
+
81
+ # Select only rows with expected number of delimiters
82
+ df = df[df["TO_PARSE"].str.count(";") == 442]
83
+
84
+ # Check there are still valid rows
85
+ if len(df) == 0:
86
+ raise ValueError(f"No valid rows in {filename}.")
87
+
88
+ # Split by ; delimiter (before raw drop number)
89
+ df = df["TO_PARSE"].str.split(";", expand=True, n=43)
90
+
91
+ # Assign column names
92
+ names = [
93
+ "start_identifier",
94
+ "sensor_serial_number",
95
+ "weather_code_synop_4680_5min",
96
+ "weather_code_metar_4678_5min",
97
+ "precipitation_rate_5min",
98
+ "weather_code_synop_4680",
99
+ "weather_code_metar_4678",
100
+ "precipitation_rate",
101
+ "precipitation_accumulated",
102
+ "sensor_time",
103
+ "temperature_interior",
104
+ "laser_temperature",
105
+ "laser_current_average",
106
+ "control_voltage",
107
+ "optical_control_voltage_output",
108
+ "number_particles",
109
+ "number_particles_internal_data",
110
+ "number_particles_min_speed",
111
+ "number_particles_min_speed_internal_data",
112
+ "number_particles_max_speed",
113
+ "number_particles_max_speed_internal_data",
114
+ "number_particles_min_diameter",
115
+ "number_particles_min_diameter_internal_data",
116
+ "number_particles_no_hydrometeor",
117
+ "number_particles_no_hydrometeor_internal_data",
118
+ "number_particles_unknown_classification",
119
+ "total_gross_volume_unknown_classification",
120
+ "number_particles_hail",
121
+ "total_gross_volume_hail",
122
+ "number_particles_solid_precipitation",
123
+ "total_gross_volume_solid_precipitation",
124
+ "number_particles_great_pellet",
125
+ "total_gross_volume_great_pellet",
126
+ "number_particles_small_pellet",
127
+ "total_gross_volume_small_pellet",
128
+ "number_particles_snowgrain",
129
+ "total_gross_volume_snowgrain",
130
+ "number_particles_rain",
131
+ "total_gross_volume_rain",
132
+ "number_particles_small_rain",
133
+ "total_gross_volume_small_rain",
134
+ "number_particles_drizzle",
135
+ "total_gross_volume_drizzle",
136
+ "raw_drop_number",
137
+ ]
138
+ df.columns = names
139
+
140
+ # Deal with case if there are 61 timesteps
141
+ # - Occurs sometimes when previous hourly file miss timesteps
142
+ if len(df) == 61:
143
+ log_warning(logger=logger, msg=f"{filename} contains 61 timesteps. Dropping the first.")
144
+ df = df.iloc[1:]
145
+
146
+ # Raise error if more than 60 timesteps/rows
147
+ n_rows = len(df)
148
+ if n_rows > 60:
149
+ raise ValueError(f"The hourly file contains {n_rows} timesteps.")
150
+
151
+ # Infer and define "time" column
152
+ start_time_str = filename.split(".")[0] # '2024020200.txt'
153
+ start_time = pd.to_datetime(start_time_str, format="%Y%m%d%H")
154
+
155
+ # - Define timedelta based on sensor_time
156
+ dt = pd.to_timedelta(df["sensor_time"] + ":00").to_numpy().astype("m8[s]")
157
+ rollover_indices = np.where(np.diff(dt) < np.timedelta64(0, "s"))[0]
158
+ if rollover_indices.size > 0:
159
+ for idx in rollover_indices:
160
+ dt[idx + 1 :] += np.timedelta64(24, "h")
161
+ dt = dt - dt[0]
162
+
163
+ # - Define approximate time
164
+ df["time"] = start_time + dt
165
+
166
+ # - Keep rows where time increment is between 00 and 59 minutes
167
+ valid_rows = dt <= np.timedelta64(3540, "s")
168
+ df = df[valid_rows]
169
+
170
+ # Drop rows with invalid raw_drop_number
171
+ # --> 440 value # 22x20
172
+ df = df[df["raw_drop_number"].astype(str).str.len() == 1599]
173
+
174
+ # Drop columns not agreeing with DISDRODB L0 standards
175
+ columns_to_drop = [
176
+ "sensor_time",
177
+ "start_identifier",
178
+ "sensor_serial_number",
179
+ ]
180
+ df = df.drop(columns=columns_to_drop)
181
+ return df
182
+
183
+
184
+ @is_documented_by(reader_generic_docstring)
185
+ def reader(
186
+ filepath,
187
+ logger=None,
188
+ ):
189
+ """Reader."""
190
+ import zipfile
191
+
192
+ ##------------------------------------------------------------------------.
193
+ # filename = os.path.basename(filepath)
194
+ # return read_txt_file(file=filepath, filename=filename, logger=logger)
195
+
196
+ # ---------------------------------------------------------------------.
197
+ #### Iterate over all files (aka timesteps) in the daily zip archive
198
+ # - Each file contain a single timestep !
199
+ # list_df = []
200
+ # with tempfile.TemporaryDirectory() as temp_dir:
201
+ # # Extract all files
202
+ # unzip_file_on_terminal(filepath, temp_dir)
203
+
204
+ # # Walk through extracted files
205
+ # for root, _, files in os.walk(temp_dir):
206
+ # for filename in sorted(files):
207
+ # if filename.endswith(".txt"):
208
+ # full_path = os.path.join(root, filename)
209
+ # try:
210
+ # df = read_txt_file(file=full_path, filename=filename, logger=logger)
211
+ # if df is not None:
212
+ # list_df.append(df)
213
+ # except Exception as e:
214
+ # msg = f"An error occurred while reading {filename}: {e}"
215
+ # log_error(logger=logger, msg=msg, verbose=True)
216
+
217
+ list_df = []
218
+ with zipfile.ZipFile(filepath, "r") as zip_ref:
219
+ filenames = sorted(zip_ref.namelist())
220
+ for filename in filenames:
221
+ if filename.endswith(".txt"):
222
+ # Open file
223
+ with zip_ref.open(filename) as file:
224
+ try:
225
+ df = read_txt_file(file=file, filename=filename, logger=logger)
226
+ if df is not None:
227
+ list_df.append(df)
228
+ except Exception as e:
229
+ msg = f"An error occurred while reading {filename}. The error is: {e}"
230
+ log_error(logger=logger, msg=msg, verbose=True)
231
+
232
+ # Check the zip file contains at least some non.empty files
233
+ if len(list_df) == 0:
234
+ raise ValueError(f"{filepath} contains only empty files!")
235
+
236
+ # Concatenate all dataframes into a single one
237
+ df = pd.concat(list_df)
238
+
239
+ # ---------------------------------------------------------------------.
240
+ return df
@@ -29,22 +29,19 @@ def reader(
29
29
  """Reader."""
30
30
  ##------------------------------------------------------------------------.
31
31
  #### Define column names
32
- column_names = ["time", "unknown", "raw_drop_number"]
32
+ column_names = ["TO_PARSE"]
33
33
 
34
34
  ##------------------------------------------------------------------------.
35
35
  #### Define reader options
36
36
  reader_kwargs = {}
37
37
  # - Define delimiter
38
- reader_kwargs["delimiter"] = " "
38
+ reader_kwargs["delimiter"] = "//n"
39
39
  # - Skip first row as columns names
40
40
  reader_kwargs["header"] = None
41
+ reader_kwargs["skiprows"] = 0
41
42
  # - Skip file with encoding errors
42
43
  reader_kwargs["encoding_errors"] = "ignore"
43
- # - Need for zipped raw file (NASA files)
44
- reader_kwargs["zipped"] = True
45
- # - Searched file into tar files
46
- reader_kwargs["filename_to_read_zipped"] = "spectrum.txt"
47
- # - Avoid first column to become df index
44
+ # - Avoid first column to become df index !!!
48
45
  reader_kwargs["index_col"] = False
49
46
  # - Define behaviour when encountering bad lines
50
47
  reader_kwargs["on_bad_lines"] = "skip"
@@ -59,7 +56,7 @@ def reader(
59
56
  # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
60
57
  # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
61
58
  # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
62
- reader_kwargs["na_values"] = ["na", "", "error", "-.-"]
59
+ reader_kwargs["na_values"] = ["na", "", "error", "NA", "-.-"]
63
60
 
64
61
  ##------------------------------------------------------------------------.
65
62
  #### Read the data
@@ -72,14 +69,29 @@ def reader(
72
69
 
73
70
  ##------------------------------------------------------------------------.
74
71
  #### Adapt the dataframe to adhere to DISDRODB L0 standards
72
+ # Remove rows with invalid number of separators
73
+ df = df[df["TO_PARSE"].str.count(";") == 1]
74
+ if len(df) == 0:
75
+ raise ValueError(f"No valid data in {filepath}")
76
+
77
+ # Split the columns
78
+ df = df["TO_PARSE"].str.split(";", n=2, expand=True)
79
+
80
+ # Assign column names
81
+ df.columns = ["time", "TO_BE_SPLITTED"]
82
+
75
83
  # Convert time column to datetime
76
- try:
77
- df["time"] = pd.to_datetime(df["time"], format="%Y %m %d %H %M %S", errors="coerce")
78
- except ValueError:
79
- df["time"] = pd.to_datetime(df["time"], format="%Y-%m-%d %H:%M:%S", errors="coerce")
84
+ df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
85
+
86
+ # Split the 'TO_BE_SPLITTED' column
87
+ df = df["TO_BE_SPLITTED"].str.split(",", n=3, expand=True)
88
+ df.columns = ["station_id", "sensor_status", "sensor_temperature", "raw_drop_number"]
89
+
90
+ # Add time
91
+ df["time"] = df_time
80
92
 
81
93
  # Drop columns not agreeing with DISDRODB L0 standards
82
- df = df.drop(columns=["unknown"])
94
+ df = df.drop(columns=["station_id"])
83
95
 
84
96
  # Return the dataframe adhering to DISDRODB L0 standards
85
97
  return df
@@ -116,7 +116,7 @@ def reader(
116
116
  return df
117
117
  # ---------------------------------------------------------
118
118
  #### Case of 1032 delimiters
119
- if n_delimiters == 1033: # (most of the files)
119
+ if n_delimiters == 1033: # (most of the files ... PIERS FORMAT)
120
120
  # Select valid rows
121
121
  df = df.loc[df["TO_BE_SPLITTED"].str.count(",") == 1033]
122
122
  # Get time column
@@ -124,7 +124,7 @@ def read_txt_file(file, filename, logger):
124
124
  # Select rows with valid spectrum
125
125
  # df = df[df["TO_PARSE"].str.count(";") == 1191] # 1112
126
126
 
127
- # Raise errof if corrupted file
127
+ # Raise error if corrupted file
128
128
  if len(df) == 4:
129
129
  raise ValueError(f"{filename} is corrupted.")
130
130
 
@@ -0,0 +1,155 @@
1
+ # -----------------------------------------------------------------------------.
2
+ # Copyright (c) 2021-2023 DISDRODB developers
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU General Public License
15
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
16
+ # -----------------------------------------------------------------------------.
17
+ """DISDRODB reader for Colorado State University PRECIP OTT Parsivel 2 raw data."""
18
+
19
+ import pandas as pd
20
+
21
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
22
+ from disdrodb.l0.l0a_processing import read_raw_text_file
23
+
24
+
25
+ @is_documented_by(reader_generic_docstring)
26
+ def reader(
27
+ filepath,
28
+ logger=None,
29
+ ):
30
+ """Reader."""
31
+ ##------------------------------------------------------------------------.
32
+ #### Define column names
33
+ column_names = ["TO_PARSE"]
34
+
35
+ ##------------------------------------------------------------------------.
36
+ #### Define reader options
37
+ reader_kwargs = {}
38
+
39
+ # - Define delimiter
40
+ reader_kwargs["delimiter"] = "\\n"
41
+
42
+ # - Skip first row as columns names
43
+ reader_kwargs["header"] = None
44
+
45
+ # - Skip header
46
+ reader_kwargs["skiprows"] = 0
47
+
48
+ # - Define encoding
49
+ reader_kwargs["encoding"] = "ISO-8859-1"
50
+
51
+ # - Avoid first column to become df index !!!
52
+ reader_kwargs["index_col"] = False
53
+
54
+ # - Define behaviour when encountering bad lines
55
+ reader_kwargs["on_bad_lines"] = "skip"
56
+
57
+ # - Define reader engine
58
+ # - C engine is faster
59
+ # - Python engine is more feature-complete
60
+ reader_kwargs["engine"] = "python"
61
+
62
+ # - Define on-the-fly decompression of on-disk data
63
+ # - Available: gzip, bz2, zip
64
+ # reader_kwargs['compression'] = 'xz'
65
+
66
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
67
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
68
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
69
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
70
+ reader_kwargs["na_values"] = ["na", "error", "-.-", " NA"]
71
+
72
+ ##------------------------------------------------------------------------.
73
+ #### Read the data
74
+ df = read_raw_text_file(
75
+ filepath=filepath,
76
+ column_names=column_names,
77
+ reader_kwargs=reader_kwargs,
78
+ logger=logger,
79
+ )
80
+
81
+ ##------------------------------------------------------------------------.
82
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
83
+ # Raise error if empty file
84
+ if len(df) == 0:
85
+ raise ValueError(f"{filepath} is empty.")
86
+
87
+ # Select only rows with expected number of delimiters
88
+ df = df[df["TO_PARSE"].str.count(",") == 1041]
89
+
90
+ # Raise error if no data left
91
+ if len(df) == 0:
92
+ raise ValueError(f"No valid data in {filepath}.")
93
+
94
+ # Split into columns
95
+ df = df["TO_PARSE"].str.split(",", expand=True, n=17)
96
+
97
+ # Assign columns names
98
+ names = [
99
+ "date",
100
+ "time",
101
+ "rainfall_rate_32bit",
102
+ "rainfall_accumulated_32bit",
103
+ "weather_code_synop_4680",
104
+ "weather_code_metar_4678",
105
+ "weather_code_nws",
106
+ "reflectivity_32bit",
107
+ "mor_visibility",
108
+ "laser_amplitude",
109
+ "number_particles",
110
+ "sensor_temperature",
111
+ "sensor_heating_current",
112
+ "sensor_battery_voltage",
113
+ "sensor_status",
114
+ "rain_kinetic_energy",
115
+ "snowfall_rate",
116
+ "raw_drop_number",
117
+ ]
118
+ df.columns = names
119
+
120
+ # Add datetime time column
121
+ time_str = df["date"] + "-" + df["time"]
122
+ df["time"] = pd.to_datetime(time_str, format="%d.%m.%Y-%H:%M:%S", errors="coerce")
123
+
124
+ # Derive the raw spectrum
125
+ # Remove <SPECTRUM> and </SPECTRUM> prefix and suffix from the raw_drop_number field
126
+ df["raw_drop_number"] = df["raw_drop_number"].astype("string")
127
+ df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
128
+ df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
129
+
130
+ # Preprocess the raw spectrum and raw_drop_average_velocity
131
+ # - Add 0 before every ; if ; not preceded by a digit
132
+ # - Example: ';;1;;' --> '0;0;1;0;'
133
+ df["raw_drop_number"] = df["raw_drop_number"].str.replace(r"(?<!\d),", "0,", regex=True)
134
+
135
+ # Infill missing timesteps with raw_drop_number = 0 spectrum
136
+ # - Define the full time range with 30-second frequency
137
+ full_time_index = pd.date_range(start=df["time"].iloc[0], end=df["time"].iloc[-1], freq="30s")
138
+
139
+ # - Reindex the DataFrame to include all 30-second timesteps
140
+ df = df.set_index("time").reindex(full_time_index)
141
+
142
+ # - Fill missing raw_drop_number with 0
143
+ df["raw_drop_number"] = df["raw_drop_number"].fillna(0)
144
+
145
+ # - Restore 'time' as a column
146
+ df = df.rename_axis("time").reset_index()
147
+
148
+ # Drop columns not agreeing with DISDRODB L0 standards
149
+ columns_to_drop = [
150
+ "date",
151
+ ]
152
+ df = df.drop(columns=columns_to_drop)
153
+
154
+ # Return the dataframe adhering to DISDRODB L0 standards
155
+ return df
@@ -38,11 +38,14 @@ def reader(
38
38
  # - Define delimiter
39
39
  reader_kwargs["delimiter"] = "/\n"
40
40
 
41
- # Skip first row as columns names
41
+ # - Skip first row as columns names
42
42
  reader_kwargs["header"] = None
43
43
 
44
- # Skip first 2 rows
45
- reader_kwargs["skiprows"] = 1
44
+ # - Skip first 2 rows
45
+ reader_kwargs["skiprows"] = 0
46
+
47
+ # - Define encoding
48
+ reader_kwargs["encoding"] = "ISO-8859-1"
46
49
 
47
50
  # - Avoid first column to become df index !!!
48
51
  reader_kwargs["index_col"] = False
@@ -76,12 +79,17 @@ def reader(
76
79
 
77
80
  ##------------------------------------------------------------------------.
78
81
  #### Adapt the dataframe to adhere to DISDRODB L0 standards
79
- # Remove rows with invalid length
80
- # df = df[df["TO_BE_PARSED"].str.len().isin([4664])]
82
+ # Raise error if empty file
83
+ if len(df) == 0:
84
+ raise ValueError(f"{filepath} is empty.")
81
85
 
82
- # Count number of delimiters to select valid rows
86
+ # Select only rows with expected number of delimiters
83
87
  df = df[df["TO_BE_PARSED"].str.count(";") == 1107]
84
88
 
89
+ # Raise error if no data left
90
+ if len(df) == 0:
91
+ raise ValueError(f"No valid data in {filepath}.")
92
+
85
93
  # Split by ; delimiter
86
94
  df = df["TO_BE_PARSED"].str.split(";", expand=True, n=19)
87
95
 
@@ -132,5 +140,4 @@ def reader(
132
140
  "sample_interval",
133
141
  ]
134
142
  df = df.drop(columns=columns_to_drop)
135
-
136
143
  return df
@@ -157,12 +157,17 @@ def reader(
157
157
 
158
158
  ##------------------------------------------------------------------------.
159
159
  #### Adapt the dataframe to adhere to DISDRODB L0 standards
160
- # Remove rows with invalid length
161
- # df = df[df["TO_BE_PARSED"].str.len().isin([4664])]
160
+ # Raise error if empty file
161
+ if len(df) == 0:
162
+ raise ValueError(f"{filepath} is empty.")
162
163
 
163
- # Count number of delimiters to select valid rows
164
+ # Select only rows with expected number of delimiters
164
165
  df = df[df["TO_BE_PARSED"].str.count(";") == 1107]
165
166
 
167
+ # Raise error if no data left
168
+ if len(df) == 0:
169
+ raise ValueError(f"No valid data in {filepath}.")
170
+
166
171
  # Split by ; delimiter
167
172
  df = df["TO_BE_PARSED"].str.split(";", expand=True, n=19)
168
173
 
@@ -44,13 +44,13 @@ def reader(
44
44
  """Reader."""
45
45
  ##------------------------------------------------------------------------.
46
46
  #### Define column names
47
- column_names = ["time", "TO_BE_SPLITTED"]
47
+ column_names = ["TO_PARSE"]
48
48
 
49
49
  ##------------------------------------------------------------------------.
50
50
  #### Define reader options
51
51
  reader_kwargs = {}
52
52
  # - Define delimiter
53
- reader_kwargs["delimiter"] = ";"
53
+ reader_kwargs["delimiter"] = "//n"
54
54
  # - Skip first row as columns names
55
55
  reader_kwargs["header"] = None
56
56
  reader_kwargs["skiprows"] = 0
@@ -84,6 +84,20 @@ def reader(
84
84
 
85
85
  ##------------------------------------------------------------------------.
86
86
  #### Adapt the dataframe to adhere to DISDRODB L0 standards
87
+ # Remove rows with invalid number of separators
88
+ df = df[df["TO_PARSE"].str.count(";") == 1]
89
+ if len(df) == 0:
90
+ raise ValueError(f"No valid data in {filepath}")
91
+
92
+ # Retrieve time and telegram field
93
+ df = df["TO_PARSE"].str.split(";", expand=True)
94
+ df.columns = ["time", "TO_BE_SPLITTED"]
95
+
96
+ # Remove rows with invalid number of separators
97
+ df = df[df["TO_BE_SPLITTED"].str.count(",") == 1033]
98
+ if len(df) == 0:
99
+ raise ValueError(f"No valid data in {filepath}")
100
+
87
101
  # Convert time column to datetime
88
102
  df_time = pd.to_datetime(df["time"], format="%Y%m%d%H%M%S", errors="coerce")
89
103
 
@@ -91,7 +105,7 @@ def reader(
91
105
  df = df["TO_BE_SPLITTED"].str.split(",", n=9, expand=True)
92
106
 
93
107
  # Assign column names
94
- names = [
108
+ columns_names = [
95
109
  "station_name",
96
110
  "sensor_status",
97
111
  "sensor_temperature",
@@ -103,7 +117,7 @@ def reader(
103
117
  "weather_code_synop_4677",
104
118
  "raw_drop_number",
105
119
  ]
106
- df.columns = names
120
+ df.columns = columns_names
107
121
 
108
122
  # Add the time column
109
123
  df["time"] = df_time
@@ -111,10 +125,19 @@ def reader(
111
125
  # Drop columns not agreeing with DISDRODB L0 standards
112
126
  df = df.drop(columns=["station_name"])
113
127
 
128
+ # Remove rows with invalid raw drop number
129
+ # --> Occurs e.g. in UCONN apu28
130
+ # def mask_invalid_raw_drop_number(df)
131
+ # df_split = df["raw_drop_number"].str.split(",", expand=True)
132
+ # idx = np.where(np.any(df_split.astype(float) > 998, axis=1))[0]
133
+ # df.loc[idx, "raw_drop_number"] = "NaN"
134
+ # return df
135
+ df = df[df["raw_drop_number"].str.len() == 4096]
136
+
114
137
  # Drop rows with invalid values
115
138
  # --> Ensure that weather_code_synop_4677 has length 2
116
139
  # --> If a previous column is missing it will have 000
117
- df = df[df["weather_code_synop_4677"].str.len() == 2]
140
+ # df = df[df["weather_code_synop_4677"].str.len() == 2]
118
141
 
119
142
  # Return the dataframe adhering to DISDRODB L0 standards
120
143
  return df
@@ -42,7 +42,7 @@ def reader(
42
42
  # - Define encoding
43
43
  reader_kwargs["encoding"] = "ISO-8859-1"
44
44
 
45
- # Skip first row as columns names
45
+ # - Skip first row as columns names
46
46
  reader_kwargs["header"] = None
47
47
 
48
48
  # - Avoid first column to become df index !!!