disdrodb 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- disdrodb/__init__.py +64 -34
- disdrodb/_config.py +5 -4
- disdrodb/_version.py +16 -3
- disdrodb/accessor/__init__.py +20 -0
- disdrodb/accessor/methods.py +125 -0
- disdrodb/api/checks.py +139 -9
- disdrodb/api/configs.py +4 -2
- disdrodb/api/info.py +10 -10
- disdrodb/api/io.py +237 -18
- disdrodb/api/path.py +81 -75
- disdrodb/api/search.py +6 -6
- disdrodb/cli/disdrodb_create_summary_station.py +91 -0
- disdrodb/cli/disdrodb_run_l0.py +1 -1
- disdrodb/cli/disdrodb_run_l0_station.py +1 -1
- disdrodb/cli/disdrodb_run_l0b.py +1 -1
- disdrodb/cli/disdrodb_run_l0b_station.py +1 -1
- disdrodb/cli/disdrodb_run_l0c.py +1 -1
- disdrodb/cli/disdrodb_run_l0c_station.py +1 -1
- disdrodb/cli/disdrodb_run_l2e_station.py +1 -1
- disdrodb/configs.py +149 -4
- disdrodb/constants.py +61 -0
- disdrodb/data_transfer/download_data.py +145 -14
- disdrodb/etc/configs/attributes.yaml +339 -0
- disdrodb/etc/configs/encodings.yaml +473 -0
- disdrodb/etc/products/L1/global.yaml +13 -0
- disdrodb/etc/products/L2E/10MIN.yaml +12 -0
- disdrodb/etc/products/L2E/1MIN.yaml +1 -0
- disdrodb/etc/products/L2E/global.yaml +22 -0
- disdrodb/etc/products/L2M/10MIN.yaml +12 -0
- disdrodb/etc/products/L2M/GAMMA_ML.yaml +8 -0
- disdrodb/etc/products/L2M/NGAMMA_GS_LOG_ND_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/NGAMMA_GS_ND_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/NGAMMA_GS_Z_MAE.yaml +6 -0
- disdrodb/etc/products/L2M/global.yaml +26 -0
- disdrodb/l0/__init__.py +13 -0
- disdrodb/l0/configs/LPM/bins_diameter.yml +3 -3
- disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +4 -4
- disdrodb/l0/configs/PARSIVEL/l0b_cf_attrs.yml +1 -1
- disdrodb/l0/configs/PARSIVEL/l0b_encodings.yml +3 -3
- disdrodb/l0/configs/PARSIVEL/raw_data_format.yml +1 -1
- disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +4 -0
- disdrodb/l0/configs/PARSIVEL2/l0b_cf_attrs.yml +20 -4
- disdrodb/l0/configs/PARSIVEL2/l0b_encodings.yml +44 -3
- disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml +41 -1
- disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +4 -4
- disdrodb/l0/configs/PWS100/raw_data_format.yml +1 -1
- disdrodb/l0/l0a_processing.py +30 -30
- disdrodb/l0/l0b_nc_processing.py +108 -2
- disdrodb/l0/l0b_processing.py +4 -4
- disdrodb/l0/l0c_processing.py +5 -13
- disdrodb/l0/manuals/SWS250.pdf +0 -0
- disdrodb/l0/manuals/VPF730.pdf +0 -0
- disdrodb/l0/manuals/VPF750.pdf +0 -0
- disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_LPM_NC.py +66 -0
- disdrodb/l0/readers/LPM/SLOVENIA/{CRNI_VRH.py → UL.py} +3 -0
- disdrodb/l0/readers/LPM/SWITZERLAND/INNERERIZ_LPM.py +195 -0
- disdrodb/l0/readers/PARSIVEL/GPM/PIERS.py +105 -0
- disdrodb/l0/readers/PARSIVEL/JAPAN/JMA.py +128 -0
- disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +1 -1
- disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/BELGIUM/ILVO.py +168 -0
- disdrodb/l0/readers/PARSIVEL2/DENMARK/DTU.py +165 -0
- disdrodb/l0/readers/PARSIVEL2/FINLAND/FMI_PARSIVEL2.py +69 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +255 -134
- disdrodb/l0/readers/PARSIVEL2/FRANCE/OSUG.py +525 -0
- disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +9 -7
- disdrodb/l0/readers/{PARSIVEL → PARSIVEL2}/KIT/BURKINA_FASO.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/KIT/TEAMX.py +123 -0
- disdrodb/l0/readers/PARSIVEL2/NASA/APU.py +120 -0
- disdrodb/l0/readers/PARSIVEL2/{NETHERLANDS/DELFT.py → NCAR/FARM_PARSIVEL2.py} +43 -70
- disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_MIPS.py +126 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_PIPS.py +165 -0
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +1 -1
- disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +29 -12
- disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT_NC.py +69 -0
- disdrodb/l0/readers/PARSIVEL2/SPAIN/CENER.py +144 -0
- disdrodb/l0/readers/PARSIVEL2/SPAIN/CR1000DL.py +201 -0
- disdrodb/l0/readers/PARSIVEL2/SPAIN/LIAISE.py +137 -0
- disdrodb/l0/readers/PARSIVEL2/USA/C3WE.py +146 -0
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +105 -99
- disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100_SIRTA.py +151 -0
- disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py +31 -14
- disdrodb/l0/routines.py +105 -14
- disdrodb/l1/__init__.py +5 -0
- disdrodb/l1/filters.py +34 -20
- disdrodb/l1/processing.py +45 -44
- disdrodb/l1/resampling.py +77 -66
- disdrodb/l1/routines.py +35 -42
- disdrodb/l1_env/routines.py +18 -3
- disdrodb/l2/__init__.py +7 -0
- disdrodb/l2/empirical_dsd.py +58 -10
- disdrodb/l2/event.py +27 -120
- disdrodb/l2/processing.py +267 -116
- disdrodb/l2/routines.py +618 -254
- disdrodb/metadata/standards.py +3 -1
- disdrodb/psd/fitting.py +463 -144
- disdrodb/psd/models.py +8 -5
- disdrodb/routines.py +3 -3
- disdrodb/scattering/__init__.py +16 -4
- disdrodb/scattering/axis_ratio.py +56 -36
- disdrodb/scattering/permittivity.py +486 -0
- disdrodb/scattering/routines.py +701 -159
- disdrodb/summary/__init__.py +17 -0
- disdrodb/summary/routines.py +4120 -0
- disdrodb/utils/attrs.py +68 -125
- disdrodb/utils/compression.py +30 -1
- disdrodb/utils/dask.py +59 -8
- disdrodb/utils/dataframe.py +63 -9
- disdrodb/utils/directories.py +49 -17
- disdrodb/utils/encoding.py +33 -19
- disdrodb/utils/logger.py +13 -6
- disdrodb/utils/manipulations.py +71 -0
- disdrodb/utils/subsetting.py +214 -0
- disdrodb/utils/time.py +165 -19
- disdrodb/utils/writer.py +20 -7
- disdrodb/utils/xarray.py +85 -4
- disdrodb/viz/__init__.py +13 -0
- disdrodb/viz/plots.py +327 -0
- {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/METADATA +3 -2
- {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/RECORD +127 -87
- {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/entry_points.txt +1 -0
- disdrodb/l1/encoding_attrs.py +0 -635
- disdrodb/l2/processing_options.py +0 -213
- /disdrodb/l0/readers/PARSIVEL/SLOVENIA/{UL_FGG.py → UL.py} +0 -0
- {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/WHEEL +0 -0
- {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/licenses/LICENSE +0 -0
- {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
"""Reader for the PERILS 2022 and 2023 PIPS files."""
|
|
19
|
+
import pandas as pd
|
|
20
|
+
|
|
21
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
22
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@is_documented_by(reader_generic_docstring)
|
|
26
|
+
def reader(
|
|
27
|
+
filepath,
|
|
28
|
+
logger=None,
|
|
29
|
+
):
|
|
30
|
+
"""Reader."""
|
|
31
|
+
##------------------------------------------------------------------------.
|
|
32
|
+
#### Define column names
|
|
33
|
+
column_names = [
|
|
34
|
+
"logger_time",
|
|
35
|
+
"record_number",
|
|
36
|
+
"battery_voltage",
|
|
37
|
+
"logger_temperature",
|
|
38
|
+
"wind_direction",
|
|
39
|
+
"wind_speed",
|
|
40
|
+
"wind_flag",
|
|
41
|
+
"fast_temperature",
|
|
42
|
+
"slow_temperature",
|
|
43
|
+
"relative_humidity",
|
|
44
|
+
"pressure",
|
|
45
|
+
"compass_direction",
|
|
46
|
+
"gps_time",
|
|
47
|
+
"gps_status",
|
|
48
|
+
"gps_latitude", # DD.DM, DM = decimal minutes/100, DD = degrees
|
|
49
|
+
"gps_latitude_hemisphere", # N/S
|
|
50
|
+
"gps_longitude",
|
|
51
|
+
"gps_longitude_hemisphere", # W/E
|
|
52
|
+
"gps_speed",
|
|
53
|
+
"gps_direction",
|
|
54
|
+
"gps_date",
|
|
55
|
+
"gps_magnetic_version",
|
|
56
|
+
"gps_altitude",
|
|
57
|
+
"relative_wind_direction",
|
|
58
|
+
"dew_point_temperature",
|
|
59
|
+
"rh",
|
|
60
|
+
"disdrometer_data",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
##------------------------------------------------------------------------.
|
|
64
|
+
#### Define reader options
|
|
65
|
+
reader_kwargs = {}
|
|
66
|
+
|
|
67
|
+
# - Define delimiter
|
|
68
|
+
reader_kwargs["delimiter"] = ","
|
|
69
|
+
|
|
70
|
+
# - Avoid first column to become df index !!!
|
|
71
|
+
reader_kwargs["index_col"] = False
|
|
72
|
+
|
|
73
|
+
# - Define behaviour when encountering bad lines
|
|
74
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
75
|
+
|
|
76
|
+
# - Define encoding
|
|
77
|
+
reader_kwargs["encoding"] = "ISO-8859-1"
|
|
78
|
+
|
|
79
|
+
# - Define reader engine
|
|
80
|
+
# - C engine is faster
|
|
81
|
+
# - Python engine is more feature-complete
|
|
82
|
+
reader_kwargs["engine"] = "python"
|
|
83
|
+
|
|
84
|
+
# - Define on-the-fly decompression of on-disk data
|
|
85
|
+
# - Available: gzip, bz2, zip
|
|
86
|
+
reader_kwargs["compression"] = "infer"
|
|
87
|
+
|
|
88
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
89
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
90
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
91
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
92
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
93
|
+
|
|
94
|
+
# Skip first row as columns names
|
|
95
|
+
reader_kwargs["header"] = None
|
|
96
|
+
|
|
97
|
+
##------------------------------------------------------------------------.
|
|
98
|
+
#### Read the data
|
|
99
|
+
df = read_raw_text_file(
|
|
100
|
+
filepath=filepath,
|
|
101
|
+
column_names=column_names,
|
|
102
|
+
reader_kwargs=reader_kwargs,
|
|
103
|
+
logger=logger,
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
##------------------------------------------------------------------------.
|
|
107
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
108
|
+
# Drop timesteps without disdrometer data
|
|
109
|
+
df = df[~df["disdrometer_data"].isna()]
|
|
110
|
+
|
|
111
|
+
# Retrieve disdrometer data
|
|
112
|
+
df_data = df["disdrometer_data"].str.split(";", expand=True, n=11)
|
|
113
|
+
|
|
114
|
+
# Assign column names
|
|
115
|
+
column_names = [
|
|
116
|
+
"serial_number",
|
|
117
|
+
"rainfall_rate_32bit",
|
|
118
|
+
"rainfall_accumulated_32bit",
|
|
119
|
+
"reflectivity_32bit",
|
|
120
|
+
"sample_interval",
|
|
121
|
+
"laser_amplitude",
|
|
122
|
+
"number_particles",
|
|
123
|
+
"sensor_temperature",
|
|
124
|
+
"sensor_battery_voltage",
|
|
125
|
+
"sensor_time", # Note: logger_time is currently used !
|
|
126
|
+
"sensor_date",
|
|
127
|
+
"raw_drop_number",
|
|
128
|
+
]
|
|
129
|
+
df_data.columns = column_names
|
|
130
|
+
|
|
131
|
+
# Add weather information
|
|
132
|
+
df_data["air_temperature"] = df["fast_temperature"]
|
|
133
|
+
df_data["relative_humidity"] = df["relative_humidity"]
|
|
134
|
+
df_data["wind_direction"] = df["wind_direction"]
|
|
135
|
+
df_data["wind_speed"] = df["wind_speed"]
|
|
136
|
+
|
|
137
|
+
# df_data["dew_point"] = df["dew_point"]
|
|
138
|
+
# df_data["air_pressure"] = df["pressure"]
|
|
139
|
+
|
|
140
|
+
# Retrieve time
|
|
141
|
+
df_time = pd.to_datetime(df["logger_time"], errors="coerce")
|
|
142
|
+
|
|
143
|
+
# Retrieve coordinates information
|
|
144
|
+
# --> Latitude in degrees_north
|
|
145
|
+
# --> Longitude in degrees_east
|
|
146
|
+
# df_lat_sign = df["gps_latitude_hemisphere"].str.replace("N", "1").str.replace("S", "-1")
|
|
147
|
+
# df_lon_sign = df["gps_longitude_hemisphere"].str.replace("E", "1").str.replace("W", "-1")
|
|
148
|
+
# df_lat_sign = df_lat_sign.astype(float)
|
|
149
|
+
# df_lon_sign = df_lon_sign.astype(float)
|
|
150
|
+
# df_lon = df["gps_longitude"].astype(float)
|
|
151
|
+
# df_lat = df["gps_latitude"].astype(float)
|
|
152
|
+
# df_lon = df_lon * df_lon_sign
|
|
153
|
+
# df_lat = df_lat * df_lat_sign
|
|
154
|
+
|
|
155
|
+
# Create dataframe
|
|
156
|
+
df = df_data
|
|
157
|
+
df["time"] = df_time
|
|
158
|
+
# df["latitude"] = df_lat
|
|
159
|
+
# df["longitude"] = df_lon
|
|
160
|
+
|
|
161
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
162
|
+
df = df.drop(columns=["serial_number", "sensor_time", "sensor_date", "serial_number"])
|
|
163
|
+
|
|
164
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
165
|
+
return df
|
|
@@ -102,7 +102,7 @@ def reader(
|
|
|
102
102
|
df["raw_drop_number"] = df["raw_drop_number"].str.strip()
|
|
103
103
|
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
|
|
104
104
|
|
|
105
|
-
# Remove <SPECTRUM> and </SPECTRUM>
|
|
105
|
+
# Remove <SPECTRUM> and </SPECTRUM> prefix and suffix from the raw_drop_number field
|
|
106
106
|
df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
|
|
107
107
|
df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
|
|
108
108
|
|
|
@@ -15,6 +15,9 @@
|
|
|
15
15
|
# You should have received a copy of the GNU General Public License
|
|
16
16
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
17
|
# -----------------------------------------------------------------------------.
|
|
18
|
+
"""Reader for the PERILS 2022 and 2023 PIPS files."""
|
|
19
|
+
import os
|
|
20
|
+
|
|
18
21
|
import pandas as pd
|
|
19
22
|
|
|
20
23
|
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
@@ -62,6 +65,7 @@ def reader(
|
|
|
62
65
|
##------------------------------------------------------------------------.
|
|
63
66
|
#### Define reader options
|
|
64
67
|
reader_kwargs = {}
|
|
68
|
+
|
|
65
69
|
# - Define delimiter
|
|
66
70
|
reader_kwargs["delimiter"] = ","
|
|
67
71
|
|
|
@@ -91,6 +95,8 @@ def reader(
|
|
|
91
95
|
|
|
92
96
|
# Skip first row as columns names
|
|
93
97
|
reader_kwargs["header"] = None
|
|
98
|
+
if "FMCW" in os.path.basename(filepath):
|
|
99
|
+
reader_kwargs["skiprows"] = 1
|
|
94
100
|
|
|
95
101
|
##------------------------------------------------------------------------.
|
|
96
102
|
#### Read the data
|
|
@@ -126,24 +132,35 @@ def reader(
|
|
|
126
132
|
]
|
|
127
133
|
df_data.columns = column_names
|
|
128
134
|
|
|
129
|
-
#
|
|
135
|
+
# Add weather information
|
|
136
|
+
df_data["air_temperature"] = df["fast_temperature"]
|
|
137
|
+
df_data["relative_humidity"] = df["relative_humidity"]
|
|
138
|
+
df_data["wind_direction"] = df["wind_direction"]
|
|
139
|
+
df_data["wind_speed"] = df["wind_speed"]
|
|
140
|
+
|
|
141
|
+
# df_data["dew_point"] = df["dew_point"]
|
|
142
|
+
# df_data["air_pressure"] = df["pressure"]
|
|
143
|
+
|
|
144
|
+
# Retrieve time
|
|
145
|
+
df_time = pd.to_datetime(df["logger_time"], errors="coerce")
|
|
146
|
+
|
|
147
|
+
# Retrieve coordinates information
|
|
130
148
|
# --> Latitude in degrees_north
|
|
131
149
|
# --> Longitude in degrees_east
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
df_lat = df_lat * df_lat_sign
|
|
150
|
+
# df_lat_sign = df["gps_latitude_hemisphere"].str.replace("N", "1").str.replace("S", "-1")
|
|
151
|
+
# df_lon_sign = df["gps_longitude_hemisphere"].str.replace("E", "1").str.replace("W", "-1")
|
|
152
|
+
# df_lat_sign = df_lat_sign.astype(float)
|
|
153
|
+
# df_lon_sign = df_lon_sign.astype(float)
|
|
154
|
+
# df_lon = df["gps_longitude"].astype(float)
|
|
155
|
+
# df_lat = df["gps_latitude"].astype(float)
|
|
156
|
+
# df_lon = df_lon * df_lon_sign
|
|
157
|
+
# df_lat = df_lat * df_lat_sign
|
|
141
158
|
|
|
142
159
|
# Create dataframe
|
|
143
160
|
df = df_data
|
|
144
161
|
df["time"] = df_time
|
|
145
|
-
df["latitude"] = df_lat
|
|
146
|
-
df["longitude"] = df_lon
|
|
162
|
+
# df["latitude"] = df_lat
|
|
163
|
+
# df["longitude"] = df_lon
|
|
147
164
|
|
|
148
165
|
# Drop columns not agreeing with DISDRODB L0 standards
|
|
149
166
|
df = df.drop(columns=["serial_number", "sensor_time", "sensor_date", "serial_number"])
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
"""Reader for DELFT OTT PARSIVEL2 sensor in netCDF format."""
|
|
19
|
+
|
|
20
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
21
|
+
from disdrodb.l0.l0b_nc_processing import open_raw_netcdf_file, standardize_raw_dataset
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@is_documented_by(reader_generic_docstring)
|
|
25
|
+
def reader(
|
|
26
|
+
filepath,
|
|
27
|
+
logger=None,
|
|
28
|
+
):
|
|
29
|
+
"""Reader."""
|
|
30
|
+
##------------------------------------------------------------------------.
|
|
31
|
+
#### Open the netCDF
|
|
32
|
+
ds = open_raw_netcdf_file(filepath=filepath, logger=logger)
|
|
33
|
+
|
|
34
|
+
##------------------------------------------------------------------------.
|
|
35
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
36
|
+
# Add time coordinate
|
|
37
|
+
ds["time"] = ds["time_as_string"].astype("M8[s]")
|
|
38
|
+
ds["time"].attrs.pop("comment", None)
|
|
39
|
+
ds["time"].attrs.pop("units", None)
|
|
40
|
+
ds = ds.set_coords("time")
|
|
41
|
+
|
|
42
|
+
# Define dictionary mapping dataset variables to select and rename
|
|
43
|
+
dict_names = {
|
|
44
|
+
### Dimensions
|
|
45
|
+
"diameter_classes": "diameter_bin_center",
|
|
46
|
+
"velocity_classes": "velocity_bin_center",
|
|
47
|
+
### Variables
|
|
48
|
+
"rainfall_rate_32bit": "rainfall_rate_32bit",
|
|
49
|
+
"weather_code_synop_4680": "weather_code_synop_4680",
|
|
50
|
+
"weather_code_synop_4677": "weather_code_synop_4677",
|
|
51
|
+
"weather_code_metar_4678": "weather_code_metar_4678",
|
|
52
|
+
"weather_code_nws": "weather_code_nws",
|
|
53
|
+
"reflectivity_32bit": "reflectivity_32bit",
|
|
54
|
+
"mor_visibility": "mor_visibility",
|
|
55
|
+
"laser_amplitude": "laser_amplitude",
|
|
56
|
+
"number_particles_validated": "number_particles",
|
|
57
|
+
"sensor_temperature": "sensor_temperature",
|
|
58
|
+
"error_code": "error_code",
|
|
59
|
+
"kinetic_energy": "rain_kinetic_energy",
|
|
60
|
+
"fieldV": "raw_drop_average_velocity",
|
|
61
|
+
"fieldN": "raw_drop_concentration",
|
|
62
|
+
"raw_data": "raw_drop_number",
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
# Rename dataset variables and columns and infill missing variables
|
|
66
|
+
ds = standardize_raw_dataset(ds=ds, dict_names=dict_names, sensor_name="PARSIVEL2")
|
|
67
|
+
|
|
68
|
+
# Return the dataset adhering to DISDRODB L0B standards
|
|
69
|
+
return ds
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
20
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
21
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@is_documented_by(reader_generic_docstring)
|
|
25
|
+
def reader(
|
|
26
|
+
filepath,
|
|
27
|
+
logger=None,
|
|
28
|
+
):
|
|
29
|
+
"""Reader."""
|
|
30
|
+
##------------------------------------------------------------------------.
|
|
31
|
+
#### Define column names
|
|
32
|
+
column_names = ["TO_PARSE"]
|
|
33
|
+
|
|
34
|
+
##------------------------------------------------------------------------.
|
|
35
|
+
#### Define reader options
|
|
36
|
+
reader_kwargs = {}
|
|
37
|
+
# - Define delimiter
|
|
38
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
39
|
+
# - Skip first row as columns names
|
|
40
|
+
# - Define encoding
|
|
41
|
+
reader_kwargs["encoding"] = "latin" # "ISO-8859-1"
|
|
42
|
+
# - Avoid first column to become df index !!!
|
|
43
|
+
reader_kwargs["index_col"] = False
|
|
44
|
+
# - Define behaviour when encountering bad lines
|
|
45
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
46
|
+
# - Define reader engine
|
|
47
|
+
# - C engine is faster
|
|
48
|
+
# - Python engine is more feature-complete
|
|
49
|
+
reader_kwargs["engine"] = "python"
|
|
50
|
+
# - Define on-the-fly decompression of on-disk data
|
|
51
|
+
# - Available: gzip, bz2, zip
|
|
52
|
+
reader_kwargs["compression"] = "infer"
|
|
53
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
54
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
55
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
56
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
57
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
58
|
+
|
|
59
|
+
##------------------------------------------------------------------------.
|
|
60
|
+
#### Read the data
|
|
61
|
+
df = read_raw_text_file(
|
|
62
|
+
filepath=filepath,
|
|
63
|
+
column_names=column_names,
|
|
64
|
+
reader_kwargs=reader_kwargs,
|
|
65
|
+
logger=logger,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
##------------------------------------------------------------------------.
|
|
69
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
70
|
+
# Define time
|
|
71
|
+
df = df["TO_PARSE"].str.split(",", n=2, expand=True)
|
|
72
|
+
df.columns = ["date", "time", "TO_PARSE"]
|
|
73
|
+
datetime_str = df["date"] + " " + df["time"]
|
|
74
|
+
df["time"] = pd.to_datetime(datetime_str, format="%d.%m.%Y %H:%M:%S", errors="coerce")
|
|
75
|
+
|
|
76
|
+
# Identify rows with integral variables
|
|
77
|
+
df_vars = df[df["TO_PARSE"].str.len() == 94]
|
|
78
|
+
|
|
79
|
+
# Split and assign column names
|
|
80
|
+
df_data = df_vars["TO_PARSE"].str.split(",", expand=True)
|
|
81
|
+
var_names = [
|
|
82
|
+
"rainfall_rate_32bit",
|
|
83
|
+
"rainfall_accumulated_32bit",
|
|
84
|
+
"weather_code_synop_4680",
|
|
85
|
+
"weather_code_synop_4677",
|
|
86
|
+
"reflectivity_32bit",
|
|
87
|
+
"mor_visibility",
|
|
88
|
+
"laser_amplitude",
|
|
89
|
+
"number_particles",
|
|
90
|
+
"sensor_temperature",
|
|
91
|
+
"sensor_heating_current",
|
|
92
|
+
"sensor_battery_voltage",
|
|
93
|
+
"sensor_status",
|
|
94
|
+
"sensor_serial_number",
|
|
95
|
+
"sensor_temperature_receiver",
|
|
96
|
+
"sensor_temperature_trasmitter",
|
|
97
|
+
"snowfall_rate",
|
|
98
|
+
"rain_kinetic_energy",
|
|
99
|
+
]
|
|
100
|
+
df_data.columns = var_names
|
|
101
|
+
df_data["time"] = df_vars["time"]
|
|
102
|
+
df_data = df_data.drop(columns="sensor_serial_number")
|
|
103
|
+
|
|
104
|
+
# Initialize empty arrays
|
|
105
|
+
# --> 0 values array produced in L0B
|
|
106
|
+
df_data["raw_drop_concentration"] = ""
|
|
107
|
+
df_data["raw_drop_average_velocity"] = ""
|
|
108
|
+
df_data["raw_drop_number"] = ""
|
|
109
|
+
|
|
110
|
+
# Identify raw spectrum
|
|
111
|
+
df_raw_spectrum = df[df["TO_PARSE"].str.len() == 4545]
|
|
112
|
+
|
|
113
|
+
# Derive raw drop arrays
|
|
114
|
+
def split_string(s):
|
|
115
|
+
vals = [v.strip() for v in s.split(",")]
|
|
116
|
+
c1 = ",".join(vals[:32])
|
|
117
|
+
c2 = ",".join(vals[32:64])
|
|
118
|
+
c3 = ",".join(vals[64].replace("r", "").split("/"))
|
|
119
|
+
series = pd.Series(
|
|
120
|
+
{
|
|
121
|
+
"raw_drop_concentration": c1,
|
|
122
|
+
"raw_drop_average_velocity": c2,
|
|
123
|
+
"raw_drop_number": c3,
|
|
124
|
+
},
|
|
125
|
+
)
|
|
126
|
+
return series
|
|
127
|
+
|
|
128
|
+
splitted_string = df_raw_spectrum["TO_PARSE"].apply(split_string)
|
|
129
|
+
df_raw_spectrum["raw_drop_concentration"] = splitted_string["raw_drop_concentration"]
|
|
130
|
+
df_raw_spectrum["raw_drop_average_velocity"] = splitted_string["raw_drop_average_velocity"]
|
|
131
|
+
df_raw_spectrum["raw_drop_number"] = splitted_string["raw_drop_number"]
|
|
132
|
+
df_raw_spectrum = df_raw_spectrum.drop(columns=["date", "TO_PARSE"])
|
|
133
|
+
|
|
134
|
+
# Add raw array
|
|
135
|
+
df = df_data.set_index("time")
|
|
136
|
+
df_raw_spectrum = df_raw_spectrum.set_index("time")
|
|
137
|
+
|
|
138
|
+
df.update(df_raw_spectrum)
|
|
139
|
+
|
|
140
|
+
# Set back time as column
|
|
141
|
+
df = df.reset_index()
|
|
142
|
+
|
|
143
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
144
|
+
return df
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -----------------------------------------------------------------------------.
|
|
3
|
+
# Copyright (c) 2021-2023 DISDRODB developers
|
|
4
|
+
#
|
|
5
|
+
# This program is free software: you can redistribute it and/or modify
|
|
6
|
+
# it under the terms of the GNU General Public License as published by
|
|
7
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
8
|
+
# (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This program is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
+
# GNU General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU General Public License
|
|
16
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
# -----------------------------------------------------------------------------.
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
20
|
+
from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
|
|
21
|
+
from disdrodb.l0.l0a_processing import read_raw_text_file
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def reader_parsivel(filepath, logger):
|
|
25
|
+
"""Reader for Parsivel CR1000 Data Logger file."""
|
|
26
|
+
##------------------------------------------------------------------------.
|
|
27
|
+
#### Define column names
|
|
28
|
+
column_names = [
|
|
29
|
+
"time",
|
|
30
|
+
"RECORD",
|
|
31
|
+
"rainfall_rate_32bit",
|
|
32
|
+
"rainfall_accumulated_32bit",
|
|
33
|
+
"weather_code_synop_4680",
|
|
34
|
+
"weather_code_synop_4677",
|
|
35
|
+
"reflectivity_32bit",
|
|
36
|
+
"mor_visibility",
|
|
37
|
+
"laser_amplitude",
|
|
38
|
+
"number_particles",
|
|
39
|
+
"sensor_temperature",
|
|
40
|
+
"sensor_heating_current",
|
|
41
|
+
"sensor_battery_voltage",
|
|
42
|
+
"sensor_status",
|
|
43
|
+
"rain_kinetic_energy",
|
|
44
|
+
"V_Batt_Min",
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
##------------------------------------------------------------------------.
|
|
48
|
+
#### Define reader options
|
|
49
|
+
reader_kwargs = {}
|
|
50
|
+
# - Define delimiter
|
|
51
|
+
reader_kwargs["delimiter"] = ","
|
|
52
|
+
# - Skip first row as columns names
|
|
53
|
+
reader_kwargs["header"] = None
|
|
54
|
+
# - Skip first 3 rows
|
|
55
|
+
reader_kwargs["skiprows"] = 4
|
|
56
|
+
# - Define encoding
|
|
57
|
+
reader_kwargs["encoding"] = "latin" # "ISO-8859-1"
|
|
58
|
+
# - Avoid first column to become df index !!!
|
|
59
|
+
reader_kwargs["index_col"] = False
|
|
60
|
+
# - Define behaviour when encountering bad lines
|
|
61
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
62
|
+
# - Define reader engine
|
|
63
|
+
# - C engine is faster
|
|
64
|
+
# - Python engine is more feature-complete
|
|
65
|
+
reader_kwargs["engine"] = "python"
|
|
66
|
+
# - Define on-the-fly decompression of on-disk data
|
|
67
|
+
# - Available: gzip, bz2, zip
|
|
68
|
+
reader_kwargs["compression"] = "infer"
|
|
69
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
70
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
71
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
72
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
73
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
74
|
+
|
|
75
|
+
##------------------------------------------------------------------------.
|
|
76
|
+
#### Read the data
|
|
77
|
+
df = read_raw_text_file(
|
|
78
|
+
filepath=filepath,
|
|
79
|
+
column_names=column_names,
|
|
80
|
+
reader_kwargs=reader_kwargs,
|
|
81
|
+
logger=logger,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
##------------------------------------------------------------------------.
|
|
85
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
86
|
+
# Define time
|
|
87
|
+
df["time"] = pd.to_datetime(df["time"], format="%Y-%m-%d %H:%M:%S", errors="coerce")
|
|
88
|
+
|
|
89
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
90
|
+
df = df.drop(columns=["RECORD", "V_Batt_Min"])
|
|
91
|
+
return df
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def reader_spectrum(filepath, logger):
|
|
95
|
+
"""Reader for Spectrum CR1000 Data Logger file."""
|
|
96
|
+
##------------------------------------------------------------------------.
|
|
97
|
+
#### Define column names
|
|
98
|
+
column_names = ["TO_PARSE"]
|
|
99
|
+
|
|
100
|
+
##------------------------------------------------------------------------.
|
|
101
|
+
#### Define reader options
|
|
102
|
+
reader_kwargs = {}
|
|
103
|
+
# - Define delimiter
|
|
104
|
+
reader_kwargs["delimiter"] = "\\n"
|
|
105
|
+
# - Skip first row as columns names
|
|
106
|
+
reader_kwargs["header"] = None
|
|
107
|
+
# - Skip first 3 rows
|
|
108
|
+
reader_kwargs["skiprows"] = 4
|
|
109
|
+
# - Define encoding
|
|
110
|
+
reader_kwargs["encoding"] = "latin" # "ISO-8859-1"
|
|
111
|
+
# - Avoid first column to become df index !!!
|
|
112
|
+
reader_kwargs["index_col"] = False
|
|
113
|
+
# - Define behaviour when encountering bad lines
|
|
114
|
+
reader_kwargs["on_bad_lines"] = "skip"
|
|
115
|
+
# - Define reader engine
|
|
116
|
+
# - C engine is faster
|
|
117
|
+
# - Python engine is more feature-complete
|
|
118
|
+
reader_kwargs["engine"] = "python"
|
|
119
|
+
# - Define on-the-fly decompression of on-disk data
|
|
120
|
+
# - Available: gzip, bz2, zip
|
|
121
|
+
reader_kwargs["compression"] = "infer"
|
|
122
|
+
# - Strings to recognize as NA/NaN and replace with standard NA flags
|
|
123
|
+
# - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
|
|
124
|
+
# '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
|
|
125
|
+
# 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
|
|
126
|
+
reader_kwargs["na_values"] = ["na", "", "error"]
|
|
127
|
+
|
|
128
|
+
##------------------------------------------------------------------------.
|
|
129
|
+
#### Read the data
|
|
130
|
+
df = read_raw_text_file(
|
|
131
|
+
filepath=filepath,
|
|
132
|
+
column_names=column_names,
|
|
133
|
+
reader_kwargs=reader_kwargs,
|
|
134
|
+
logger=logger,
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
##------------------------------------------------------------------------.
|
|
138
|
+
#### Adapt the dataframe to adhere to DISDRODB L0 standards
|
|
139
|
+
# Define time
|
|
140
|
+
df = df["TO_PARSE"].str.split(",", n=2, expand=True)
|
|
141
|
+
df.columns = ["time", "RECORD", "TO_PARSE"]
|
|
142
|
+
|
|
143
|
+
df["time"] = pd.to_datetime(df["time"].str.strip('"'), format="%Y-%m-%d %H:%M:%S", errors="coerce")
|
|
144
|
+
|
|
145
|
+
# Derive raw drop arrays
|
|
146
|
+
def split_string(s):
|
|
147
|
+
vals = [v.strip() for v in s.split(",")]
|
|
148
|
+
c1 = ",".join(vals[:32]) # -10
|
|
149
|
+
c1 = c1.replace("-10", "0")
|
|
150
|
+
c2 = "0,0," + ",".join(vals[32:62])
|
|
151
|
+
c3 = ",".join(vals[62:])
|
|
152
|
+
series = pd.Series(
|
|
153
|
+
{
|
|
154
|
+
"raw_drop_concentration": c1,
|
|
155
|
+
"raw_drop_average_velocity": c2,
|
|
156
|
+
"raw_drop_number": c3,
|
|
157
|
+
},
|
|
158
|
+
)
|
|
159
|
+
return series
|
|
160
|
+
|
|
161
|
+
splitted_string = df["TO_PARSE"].apply(split_string)
|
|
162
|
+
df["raw_drop_concentration"] = splitted_string["raw_drop_concentration"]
|
|
163
|
+
df["raw_drop_average_velocity"] = splitted_string["raw_drop_average_velocity"]
|
|
164
|
+
df["raw_drop_number"] = splitted_string["raw_drop_number"]
|
|
165
|
+
|
|
166
|
+
# Drop columns not agreeing with DISDRODB L0 standards
|
|
167
|
+
df = df.drop(columns=["TO_PARSE", "RECORD"])
|
|
168
|
+
return df
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
@is_documented_by(reader_generic_docstring)
|
|
172
|
+
def reader(
|
|
173
|
+
filepath,
|
|
174
|
+
logger=None,
|
|
175
|
+
):
|
|
176
|
+
"""Reader."""
|
|
177
|
+
# Retrieve Spectrum filepath
|
|
178
|
+
spectrum_filepath = filepath.replace("Parsivel", "Spectre")
|
|
179
|
+
|
|
180
|
+
# Read integral variables
|
|
181
|
+
df = reader_parsivel(filepath, logger=logger)
|
|
182
|
+
|
|
183
|
+
# Initialize empty arrays
|
|
184
|
+
# --> 0 values array produced in L0B
|
|
185
|
+
df["raw_drop_concentration"] = ""
|
|
186
|
+
df["raw_drop_average_velocity"] = ""
|
|
187
|
+
df["raw_drop_number"] = ""
|
|
188
|
+
|
|
189
|
+
# Read raw spectrum for corresponding timesteps
|
|
190
|
+
df_raw_spectrum = reader_spectrum(spectrum_filepath, logger=logger)
|
|
191
|
+
|
|
192
|
+
# Add raw array to df
|
|
193
|
+
df = df.set_index("time")
|
|
194
|
+
df_raw_spectrum = df_raw_spectrum.set_index("time")
|
|
195
|
+
df.update(df_raw_spectrum)
|
|
196
|
+
|
|
197
|
+
# Set back time as column
|
|
198
|
+
df = df.reset_index()
|
|
199
|
+
|
|
200
|
+
# Return the dataframe adhering to DISDRODB L0 standards
|
|
201
|
+
return df
|