disdrodb 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. disdrodb/__init__.py +64 -34
  2. disdrodb/_config.py +5 -4
  3. disdrodb/_version.py +16 -3
  4. disdrodb/accessor/__init__.py +20 -0
  5. disdrodb/accessor/methods.py +125 -0
  6. disdrodb/api/checks.py +139 -9
  7. disdrodb/api/configs.py +4 -2
  8. disdrodb/api/info.py +10 -10
  9. disdrodb/api/io.py +237 -18
  10. disdrodb/api/path.py +81 -75
  11. disdrodb/api/search.py +6 -6
  12. disdrodb/cli/disdrodb_create_summary_station.py +91 -0
  13. disdrodb/cli/disdrodb_run_l0.py +1 -1
  14. disdrodb/cli/disdrodb_run_l0_station.py +1 -1
  15. disdrodb/cli/disdrodb_run_l0b.py +1 -1
  16. disdrodb/cli/disdrodb_run_l0b_station.py +1 -1
  17. disdrodb/cli/disdrodb_run_l0c.py +1 -1
  18. disdrodb/cli/disdrodb_run_l0c_station.py +1 -1
  19. disdrodb/cli/disdrodb_run_l2e_station.py +1 -1
  20. disdrodb/configs.py +149 -4
  21. disdrodb/constants.py +61 -0
  22. disdrodb/data_transfer/download_data.py +145 -14
  23. disdrodb/etc/configs/attributes.yaml +339 -0
  24. disdrodb/etc/configs/encodings.yaml +473 -0
  25. disdrodb/etc/products/L1/global.yaml +13 -0
  26. disdrodb/etc/products/L2E/10MIN.yaml +12 -0
  27. disdrodb/etc/products/L2E/1MIN.yaml +1 -0
  28. disdrodb/etc/products/L2E/global.yaml +22 -0
  29. disdrodb/etc/products/L2M/10MIN.yaml +12 -0
  30. disdrodb/etc/products/L2M/GAMMA_ML.yaml +8 -0
  31. disdrodb/etc/products/L2M/NGAMMA_GS_LOG_ND_MAE.yaml +6 -0
  32. disdrodb/etc/products/L2M/NGAMMA_GS_ND_MAE.yaml +6 -0
  33. disdrodb/etc/products/L2M/NGAMMA_GS_Z_MAE.yaml +6 -0
  34. disdrodb/etc/products/L2M/global.yaml +26 -0
  35. disdrodb/l0/__init__.py +13 -0
  36. disdrodb/l0/configs/LPM/bins_diameter.yml +3 -3
  37. disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +4 -4
  38. disdrodb/l0/configs/PARSIVEL/l0b_cf_attrs.yml +1 -1
  39. disdrodb/l0/configs/PARSIVEL/l0b_encodings.yml +3 -3
  40. disdrodb/l0/configs/PARSIVEL/raw_data_format.yml +1 -1
  41. disdrodb/l0/configs/PARSIVEL2/l0a_encodings.yml +4 -0
  42. disdrodb/l0/configs/PARSIVEL2/l0b_cf_attrs.yml +20 -4
  43. disdrodb/l0/configs/PARSIVEL2/l0b_encodings.yml +44 -3
  44. disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml +41 -1
  45. disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +4 -4
  46. disdrodb/l0/configs/PWS100/raw_data_format.yml +1 -1
  47. disdrodb/l0/l0a_processing.py +30 -30
  48. disdrodb/l0/l0b_nc_processing.py +108 -2
  49. disdrodb/l0/l0b_processing.py +4 -4
  50. disdrodb/l0/l0c_processing.py +5 -13
  51. disdrodb/l0/manuals/SWS250.pdf +0 -0
  52. disdrodb/l0/manuals/VPF730.pdf +0 -0
  53. disdrodb/l0/manuals/VPF750.pdf +0 -0
  54. disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_LPM_NC.py +66 -0
  55. disdrodb/l0/readers/LPM/SLOVENIA/{CRNI_VRH.py → UL.py} +3 -0
  56. disdrodb/l0/readers/LPM/SWITZERLAND/INNERERIZ_LPM.py +195 -0
  57. disdrodb/l0/readers/PARSIVEL/GPM/PIERS.py +105 -0
  58. disdrodb/l0/readers/PARSIVEL/JAPAN/JMA.py +128 -0
  59. disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +1 -1
  60. disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +1 -1
  61. disdrodb/l0/readers/PARSIVEL2/BELGIUM/ILVO.py +168 -0
  62. disdrodb/l0/readers/PARSIVEL2/DENMARK/DTU.py +165 -0
  63. disdrodb/l0/readers/PARSIVEL2/FINLAND/FMI_PARSIVEL2.py +69 -0
  64. disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +255 -134
  65. disdrodb/l0/readers/PARSIVEL2/FRANCE/OSUG.py +525 -0
  66. disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +1 -1
  67. disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +9 -7
  68. disdrodb/l0/readers/{PARSIVEL → PARSIVEL2}/KIT/BURKINA_FASO.py +1 -1
  69. disdrodb/l0/readers/PARSIVEL2/KIT/TEAMX.py +123 -0
  70. disdrodb/l0/readers/PARSIVEL2/NASA/APU.py +120 -0
  71. disdrodb/l0/readers/PARSIVEL2/{NETHERLANDS/DELFT.py → NCAR/FARM_PARSIVEL2.py} +43 -70
  72. disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +1 -1
  73. disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_MIPS.py +126 -0
  74. disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_PIPS.py +165 -0
  75. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +1 -1
  76. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +29 -12
  77. disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT_NC.py +69 -0
  78. disdrodb/l0/readers/PARSIVEL2/SPAIN/CENER.py +144 -0
  79. disdrodb/l0/readers/PARSIVEL2/SPAIN/CR1000DL.py +201 -0
  80. disdrodb/l0/readers/PARSIVEL2/SPAIN/LIAISE.py +137 -0
  81. disdrodb/l0/readers/PARSIVEL2/USA/C3WE.py +146 -0
  82. disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +105 -99
  83. disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100_SIRTA.py +151 -0
  84. disdrodb/l0/readers/RD80/NOAA/PSL_RD80.py +31 -14
  85. disdrodb/l0/routines.py +105 -14
  86. disdrodb/l1/__init__.py +5 -0
  87. disdrodb/l1/filters.py +34 -20
  88. disdrodb/l1/processing.py +45 -44
  89. disdrodb/l1/resampling.py +77 -66
  90. disdrodb/l1/routines.py +35 -42
  91. disdrodb/l1_env/routines.py +18 -3
  92. disdrodb/l2/__init__.py +7 -0
  93. disdrodb/l2/empirical_dsd.py +58 -10
  94. disdrodb/l2/event.py +27 -120
  95. disdrodb/l2/processing.py +267 -116
  96. disdrodb/l2/routines.py +618 -254
  97. disdrodb/metadata/standards.py +3 -1
  98. disdrodb/psd/fitting.py +463 -144
  99. disdrodb/psd/models.py +8 -5
  100. disdrodb/routines.py +3 -3
  101. disdrodb/scattering/__init__.py +16 -4
  102. disdrodb/scattering/axis_ratio.py +56 -36
  103. disdrodb/scattering/permittivity.py +486 -0
  104. disdrodb/scattering/routines.py +701 -159
  105. disdrodb/summary/__init__.py +17 -0
  106. disdrodb/summary/routines.py +4120 -0
  107. disdrodb/utils/attrs.py +68 -125
  108. disdrodb/utils/compression.py +30 -1
  109. disdrodb/utils/dask.py +59 -8
  110. disdrodb/utils/dataframe.py +63 -9
  111. disdrodb/utils/directories.py +49 -17
  112. disdrodb/utils/encoding.py +33 -19
  113. disdrodb/utils/logger.py +13 -6
  114. disdrodb/utils/manipulations.py +71 -0
  115. disdrodb/utils/subsetting.py +214 -0
  116. disdrodb/utils/time.py +165 -19
  117. disdrodb/utils/writer.py +20 -7
  118. disdrodb/utils/xarray.py +85 -4
  119. disdrodb/viz/__init__.py +13 -0
  120. disdrodb/viz/plots.py +327 -0
  121. {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/METADATA +3 -2
  122. {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/RECORD +127 -87
  123. {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/entry_points.txt +1 -0
  124. disdrodb/l1/encoding_attrs.py +0 -635
  125. disdrodb/l2/processing_options.py +0 -213
  126. /disdrodb/l0/readers/PARSIVEL/SLOVENIA/{UL_FGG.py → UL.py} +0 -0
  127. {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/WHEEL +0 -0
  128. {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/licenses/LICENSE +0 -0
  129. {disdrodb-0.1.1.dist-info → disdrodb-0.1.3.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,165 @@
1
+ #!/usr/bin/env python3
2
+ # -----------------------------------------------------------------------------.
3
+ # Copyright (c) 2021-2023 DISDRODB developers
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ # -----------------------------------------------------------------------------.
18
+ """Reader for the PERILS 2022 and 2023 PIPS files."""
19
+ import pandas as pd
20
+
21
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
22
+ from disdrodb.l0.l0a_processing import read_raw_text_file
23
+
24
+
25
+ @is_documented_by(reader_generic_docstring)
26
+ def reader(
27
+ filepath,
28
+ logger=None,
29
+ ):
30
+ """Reader."""
31
+ ##------------------------------------------------------------------------.
32
+ #### Define column names
33
+ column_names = [
34
+ "logger_time",
35
+ "record_number",
36
+ "battery_voltage",
37
+ "logger_temperature",
38
+ "wind_direction",
39
+ "wind_speed",
40
+ "wind_flag",
41
+ "fast_temperature",
42
+ "slow_temperature",
43
+ "relative_humidity",
44
+ "pressure",
45
+ "compass_direction",
46
+ "gps_time",
47
+ "gps_status",
48
+ "gps_latitude", # DD.DM, DM = decimal minutes/100, DD = degrees
49
+ "gps_latitude_hemisphere", # N/S
50
+ "gps_longitude",
51
+ "gps_longitude_hemisphere", # W/E
52
+ "gps_speed",
53
+ "gps_direction",
54
+ "gps_date",
55
+ "gps_magnetic_version",
56
+ "gps_altitude",
57
+ "relative_wind_direction",
58
+ "dew_point_temperature",
59
+ "rh",
60
+ "disdrometer_data",
61
+ ]
62
+
63
+ ##------------------------------------------------------------------------.
64
+ #### Define reader options
65
+ reader_kwargs = {}
66
+
67
+ # - Define delimiter
68
+ reader_kwargs["delimiter"] = ","
69
+
70
+ # - Avoid first column to become df index !!!
71
+ reader_kwargs["index_col"] = False
72
+
73
+ # - Define behaviour when encountering bad lines
74
+ reader_kwargs["on_bad_lines"] = "skip"
75
+
76
+ # - Define encoding
77
+ reader_kwargs["encoding"] = "ISO-8859-1"
78
+
79
+ # - Define reader engine
80
+ # - C engine is faster
81
+ # - Python engine is more feature-complete
82
+ reader_kwargs["engine"] = "python"
83
+
84
+ # - Define on-the-fly decompression of on-disk data
85
+ # - Available: gzip, bz2, zip
86
+ reader_kwargs["compression"] = "infer"
87
+
88
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
89
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
90
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
91
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
92
+ reader_kwargs["na_values"] = ["na", "", "error"]
93
+
94
+ # Skip first row as columns names
95
+ reader_kwargs["header"] = None
96
+
97
+ ##------------------------------------------------------------------------.
98
+ #### Read the data
99
+ df = read_raw_text_file(
100
+ filepath=filepath,
101
+ column_names=column_names,
102
+ reader_kwargs=reader_kwargs,
103
+ logger=logger,
104
+ )
105
+
106
+ ##------------------------------------------------------------------------.
107
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
108
+ # Drop timesteps without disdrometer data
109
+ df = df[~df["disdrometer_data"].isna()]
110
+
111
+ # Retrieve disdrometer data
112
+ df_data = df["disdrometer_data"].str.split(";", expand=True, n=11)
113
+
114
+ # Assign column names
115
+ column_names = [
116
+ "serial_number",
117
+ "rainfall_rate_32bit",
118
+ "rainfall_accumulated_32bit",
119
+ "reflectivity_32bit",
120
+ "sample_interval",
121
+ "laser_amplitude",
122
+ "number_particles",
123
+ "sensor_temperature",
124
+ "sensor_battery_voltage",
125
+ "sensor_time", # Note: logger_time is currently used !
126
+ "sensor_date",
127
+ "raw_drop_number",
128
+ ]
129
+ df_data.columns = column_names
130
+
131
+ # Add weather information
132
+ df_data["air_temperature"] = df["fast_temperature"]
133
+ df_data["relative_humidity"] = df["relative_humidity"]
134
+ df_data["wind_direction"] = df["wind_direction"]
135
+ df_data["wind_speed"] = df["wind_speed"]
136
+
137
+ # df_data["dew_point"] = df["dew_point"]
138
+ # df_data["air_pressure"] = df["pressure"]
139
+
140
+ # Retrieve time
141
+ df_time = pd.to_datetime(df["logger_time"], errors="coerce")
142
+
143
+ # Retrieve coordinates information
144
+ # --> Latitude in degrees_north
145
+ # --> Longitude in degrees_east
146
+ # df_lat_sign = df["gps_latitude_hemisphere"].str.replace("N", "1").str.replace("S", "-1")
147
+ # df_lon_sign = df["gps_longitude_hemisphere"].str.replace("E", "1").str.replace("W", "-1")
148
+ # df_lat_sign = df_lat_sign.astype(float)
149
+ # df_lon_sign = df_lon_sign.astype(float)
150
+ # df_lon = df["gps_longitude"].astype(float)
151
+ # df_lat = df["gps_latitude"].astype(float)
152
+ # df_lon = df_lon * df_lon_sign
153
+ # df_lat = df_lat * df_lat_sign
154
+
155
+ # Create dataframe
156
+ df = df_data
157
+ df["time"] = df_time
158
+ # df["latitude"] = df_lat
159
+ # df["longitude"] = df_lon
160
+
161
+ # Drop columns not agreeing with DISDRODB L0 standards
162
+ df = df.drop(columns=["serial_number", "sensor_time", "sensor_date", "serial_number"])
163
+
164
+ # Return the dataframe adhering to DISDRODB L0 standards
165
+ return df
@@ -102,7 +102,7 @@ def reader(
102
102
  df["raw_drop_number"] = df["raw_drop_number"].str.strip()
103
103
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
104
104
 
105
- # Remove <SPECTRUM> and </SPECTRUM>" acronyms from the raw_drop_number field
105
+ # Remove <SPECTRUM> and </SPECTRUM> prefix and suffix from the raw_drop_number field
106
106
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
107
107
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
108
108
 
@@ -15,6 +15,9 @@
15
15
  # You should have received a copy of the GNU General Public License
16
16
  # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
17
  # -----------------------------------------------------------------------------.
18
+ """Reader for the PERILS 2022 and 2023 PIPS files."""
19
+ import os
20
+
18
21
  import pandas as pd
19
22
 
20
23
  from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
@@ -62,6 +65,7 @@ def reader(
62
65
  ##------------------------------------------------------------------------.
63
66
  #### Define reader options
64
67
  reader_kwargs = {}
68
+
65
69
  # - Define delimiter
66
70
  reader_kwargs["delimiter"] = ","
67
71
 
@@ -91,6 +95,8 @@ def reader(
91
95
 
92
96
  # Skip first row as columns names
93
97
  reader_kwargs["header"] = None
98
+ if "FMCW" in os.path.basename(filepath):
99
+ reader_kwargs["skiprows"] = 1
94
100
 
95
101
  ##------------------------------------------------------------------------.
96
102
  #### Read the data
@@ -126,24 +132,35 @@ def reader(
126
132
  ]
127
133
  df_data.columns = column_names
128
134
 
129
- # Retrieve time and coordinates information
135
+ # Add weather information
136
+ df_data["air_temperature"] = df["fast_temperature"]
137
+ df_data["relative_humidity"] = df["relative_humidity"]
138
+ df_data["wind_direction"] = df["wind_direction"]
139
+ df_data["wind_speed"] = df["wind_speed"]
140
+
141
+ # df_data["dew_point"] = df["dew_point"]
142
+ # df_data["air_pressure"] = df["pressure"]
143
+
144
+ # Retrieve time
145
+ df_time = pd.to_datetime(df["logger_time"], errors="coerce")
146
+
147
+ # Retrieve coordinates information
130
148
  # --> Latitude in degrees_north
131
149
  # --> Longitude in degrees_east
132
- df_time = pd.to_datetime(df["logger_time"], errors="coerce")
133
- df_lat_sign = df["gps_latitude_hemisphere"].str.replace("N", "1").str.replace("S", "-1")
134
- df_lon_sign = df["gps_longitude_hemisphere"].str.replace("E", "1").str.replace("W", "-1")
135
- df_lat_sign = df_lat_sign.astype(float)
136
- df_lon_sign = df_lon_sign.astype(float)
137
- df_lon = df["gps_longitude"].astype(float)
138
- df_lat = df["gps_latitude"].astype(float)
139
- df_lon = df_lon * df_lon_sign
140
- df_lat = df_lat * df_lat_sign
150
+ # df_lat_sign = df["gps_latitude_hemisphere"].str.replace("N", "1").str.replace("S", "-1")
151
+ # df_lon_sign = df["gps_longitude_hemisphere"].str.replace("E", "1").str.replace("W", "-1")
152
+ # df_lat_sign = df_lat_sign.astype(float)
153
+ # df_lon_sign = df_lon_sign.astype(float)
154
+ # df_lon = df["gps_longitude"].astype(float)
155
+ # df_lat = df["gps_latitude"].astype(float)
156
+ # df_lon = df_lon * df_lon_sign
157
+ # df_lat = df_lat * df_lat_sign
141
158
 
142
159
  # Create dataframe
143
160
  df = df_data
144
161
  df["time"] = df_time
145
- df["latitude"] = df_lat
146
- df["longitude"] = df_lon
162
+ # df["latitude"] = df_lat
163
+ # df["longitude"] = df_lon
147
164
 
148
165
  # Drop columns not agreeing with DISDRODB L0 standards
149
166
  df = df.drop(columns=["serial_number", "sensor_time", "sensor_date", "serial_number"])
@@ -0,0 +1,69 @@
1
+ #!/usr/bin/env python3
2
+ # -----------------------------------------------------------------------------.
3
+ # Copyright (c) 2021-2023 DISDRODB developers
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ # -----------------------------------------------------------------------------.
18
+ """Reader for DELFT OTT PARSIVEL2 sensor in netCDF format."""
19
+
20
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
21
+ from disdrodb.l0.l0b_nc_processing import open_raw_netcdf_file, standardize_raw_dataset
22
+
23
+
24
+ @is_documented_by(reader_generic_docstring)
25
+ def reader(
26
+ filepath,
27
+ logger=None,
28
+ ):
29
+ """Reader."""
30
+ ##------------------------------------------------------------------------.
31
+ #### Open the netCDF
32
+ ds = open_raw_netcdf_file(filepath=filepath, logger=logger)
33
+
34
+ ##------------------------------------------------------------------------.
35
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
36
+ # Add time coordinate
37
+ ds["time"] = ds["time_as_string"].astype("M8[s]")
38
+ ds["time"].attrs.pop("comment", None)
39
+ ds["time"].attrs.pop("units", None)
40
+ ds = ds.set_coords("time")
41
+
42
+ # Define dictionary mapping dataset variables to select and rename
43
+ dict_names = {
44
+ ### Dimensions
45
+ "diameter_classes": "diameter_bin_center",
46
+ "velocity_classes": "velocity_bin_center",
47
+ ### Variables
48
+ "rainfall_rate_32bit": "rainfall_rate_32bit",
49
+ "weather_code_synop_4680": "weather_code_synop_4680",
50
+ "weather_code_synop_4677": "weather_code_synop_4677",
51
+ "weather_code_metar_4678": "weather_code_metar_4678",
52
+ "weather_code_nws": "weather_code_nws",
53
+ "reflectivity_32bit": "reflectivity_32bit",
54
+ "mor_visibility": "mor_visibility",
55
+ "laser_amplitude": "laser_amplitude",
56
+ "number_particles_validated": "number_particles",
57
+ "sensor_temperature": "sensor_temperature",
58
+ "error_code": "error_code",
59
+ "kinetic_energy": "rain_kinetic_energy",
60
+ "fieldV": "raw_drop_average_velocity",
61
+ "fieldN": "raw_drop_concentration",
62
+ "raw_data": "raw_drop_number",
63
+ }
64
+
65
+ # Rename dataset variables and columns and infill missing variables
66
+ ds = standardize_raw_dataset(ds=ds, dict_names=dict_names, sensor_name="PARSIVEL2")
67
+
68
+ # Return the dataset adhering to DISDRODB L0B standards
69
+ return ds
@@ -0,0 +1,144 @@
1
+ #!/usr/bin/env python3
2
+ # -----------------------------------------------------------------------------.
3
+ # Copyright (c) 2021-2023 DISDRODB developers
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ # -----------------------------------------------------------------------------.
18
+ import pandas as pd
19
+
20
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
21
+ from disdrodb.l0.l0a_processing import read_raw_text_file
22
+
23
+
24
+ @is_documented_by(reader_generic_docstring)
25
+ def reader(
26
+ filepath,
27
+ logger=None,
28
+ ):
29
+ """Reader."""
30
+ ##------------------------------------------------------------------------.
31
+ #### Define column names
32
+ column_names = ["TO_PARSE"]
33
+
34
+ ##------------------------------------------------------------------------.
35
+ #### Define reader options
36
+ reader_kwargs = {}
37
+ # - Define delimiter
38
+ reader_kwargs["delimiter"] = "\\n"
39
+ # - Skip first row as columns names
40
+ # - Define encoding
41
+ reader_kwargs["encoding"] = "latin" # "ISO-8859-1"
42
+ # - Avoid first column to become df index !!!
43
+ reader_kwargs["index_col"] = False
44
+ # - Define behaviour when encountering bad lines
45
+ reader_kwargs["on_bad_lines"] = "skip"
46
+ # - Define reader engine
47
+ # - C engine is faster
48
+ # - Python engine is more feature-complete
49
+ reader_kwargs["engine"] = "python"
50
+ # - Define on-the-fly decompression of on-disk data
51
+ # - Available: gzip, bz2, zip
52
+ reader_kwargs["compression"] = "infer"
53
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
54
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
55
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
56
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
57
+ reader_kwargs["na_values"] = ["na", "", "error"]
58
+
59
+ ##------------------------------------------------------------------------.
60
+ #### Read the data
61
+ df = read_raw_text_file(
62
+ filepath=filepath,
63
+ column_names=column_names,
64
+ reader_kwargs=reader_kwargs,
65
+ logger=logger,
66
+ )
67
+
68
+ ##------------------------------------------------------------------------.
69
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
70
+ # Define time
71
+ df = df["TO_PARSE"].str.split(",", n=2, expand=True)
72
+ df.columns = ["date", "time", "TO_PARSE"]
73
+ datetime_str = df["date"] + " " + df["time"]
74
+ df["time"] = pd.to_datetime(datetime_str, format="%d.%m.%Y %H:%M:%S", errors="coerce")
75
+
76
+ # Identify rows with integral variables
77
+ df_vars = df[df["TO_PARSE"].str.len() == 94]
78
+
79
+ # Split and assign column names
80
+ df_data = df_vars["TO_PARSE"].str.split(",", expand=True)
81
+ var_names = [
82
+ "rainfall_rate_32bit",
83
+ "rainfall_accumulated_32bit",
84
+ "weather_code_synop_4680",
85
+ "weather_code_synop_4677",
86
+ "reflectivity_32bit",
87
+ "mor_visibility",
88
+ "laser_amplitude",
89
+ "number_particles",
90
+ "sensor_temperature",
91
+ "sensor_heating_current",
92
+ "sensor_battery_voltage",
93
+ "sensor_status",
94
+ "sensor_serial_number",
95
+ "sensor_temperature_receiver",
96
+ "sensor_temperature_trasmitter",
97
+ "snowfall_rate",
98
+ "rain_kinetic_energy",
99
+ ]
100
+ df_data.columns = var_names
101
+ df_data["time"] = df_vars["time"]
102
+ df_data = df_data.drop(columns="sensor_serial_number")
103
+
104
+ # Initialize empty arrays
105
+ # --> 0 values array produced in L0B
106
+ df_data["raw_drop_concentration"] = ""
107
+ df_data["raw_drop_average_velocity"] = ""
108
+ df_data["raw_drop_number"] = ""
109
+
110
+ # Identify raw spectrum
111
+ df_raw_spectrum = df[df["TO_PARSE"].str.len() == 4545]
112
+
113
+ # Derive raw drop arrays
114
+ def split_string(s):
115
+ vals = [v.strip() for v in s.split(",")]
116
+ c1 = ",".join(vals[:32])
117
+ c2 = ",".join(vals[32:64])
118
+ c3 = ",".join(vals[64].replace("r", "").split("/"))
119
+ series = pd.Series(
120
+ {
121
+ "raw_drop_concentration": c1,
122
+ "raw_drop_average_velocity": c2,
123
+ "raw_drop_number": c3,
124
+ },
125
+ )
126
+ return series
127
+
128
+ splitted_string = df_raw_spectrum["TO_PARSE"].apply(split_string)
129
+ df_raw_spectrum["raw_drop_concentration"] = splitted_string["raw_drop_concentration"]
130
+ df_raw_spectrum["raw_drop_average_velocity"] = splitted_string["raw_drop_average_velocity"]
131
+ df_raw_spectrum["raw_drop_number"] = splitted_string["raw_drop_number"]
132
+ df_raw_spectrum = df_raw_spectrum.drop(columns=["date", "TO_PARSE"])
133
+
134
+ # Add raw array
135
+ df = df_data.set_index("time")
136
+ df_raw_spectrum = df_raw_spectrum.set_index("time")
137
+
138
+ df.update(df_raw_spectrum)
139
+
140
+ # Set back time as column
141
+ df = df.reset_index()
142
+
143
+ # Return the dataframe adhering to DISDRODB L0 standards
144
+ return df
@@ -0,0 +1,201 @@
1
+ #!/usr/bin/env python3
2
+ # -----------------------------------------------------------------------------.
3
+ # Copyright (c) 2021-2023 DISDRODB developers
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ # -----------------------------------------------------------------------------.
18
+ import pandas as pd
19
+
20
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
21
+ from disdrodb.l0.l0a_processing import read_raw_text_file
22
+
23
+
24
+ def reader_parsivel(filepath, logger):
25
+ """Reader for Parsivel CR1000 Data Logger file."""
26
+ ##------------------------------------------------------------------------.
27
+ #### Define column names
28
+ column_names = [
29
+ "time",
30
+ "RECORD",
31
+ "rainfall_rate_32bit",
32
+ "rainfall_accumulated_32bit",
33
+ "weather_code_synop_4680",
34
+ "weather_code_synop_4677",
35
+ "reflectivity_32bit",
36
+ "mor_visibility",
37
+ "laser_amplitude",
38
+ "number_particles",
39
+ "sensor_temperature",
40
+ "sensor_heating_current",
41
+ "sensor_battery_voltage",
42
+ "sensor_status",
43
+ "rain_kinetic_energy",
44
+ "V_Batt_Min",
45
+ ]
46
+
47
+ ##------------------------------------------------------------------------.
48
+ #### Define reader options
49
+ reader_kwargs = {}
50
+ # - Define delimiter
51
+ reader_kwargs["delimiter"] = ","
52
+ # - Skip first row as columns names
53
+ reader_kwargs["header"] = None
54
+ # - Skip first 3 rows
55
+ reader_kwargs["skiprows"] = 4
56
+ # - Define encoding
57
+ reader_kwargs["encoding"] = "latin" # "ISO-8859-1"
58
+ # - Avoid first column to become df index !!!
59
+ reader_kwargs["index_col"] = False
60
+ # - Define behaviour when encountering bad lines
61
+ reader_kwargs["on_bad_lines"] = "skip"
62
+ # - Define reader engine
63
+ # - C engine is faster
64
+ # - Python engine is more feature-complete
65
+ reader_kwargs["engine"] = "python"
66
+ # - Define on-the-fly decompression of on-disk data
67
+ # - Available: gzip, bz2, zip
68
+ reader_kwargs["compression"] = "infer"
69
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
70
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
71
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
72
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
73
+ reader_kwargs["na_values"] = ["na", "", "error"]
74
+
75
+ ##------------------------------------------------------------------------.
76
+ #### Read the data
77
+ df = read_raw_text_file(
78
+ filepath=filepath,
79
+ column_names=column_names,
80
+ reader_kwargs=reader_kwargs,
81
+ logger=logger,
82
+ )
83
+
84
+ ##------------------------------------------------------------------------.
85
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
86
+ # Define time
87
+ df["time"] = pd.to_datetime(df["time"], format="%Y-%m-%d %H:%M:%S", errors="coerce")
88
+
89
+ # Drop columns not agreeing with DISDRODB L0 standards
90
+ df = df.drop(columns=["RECORD", "V_Batt_Min"])
91
+ return df
92
+
93
+
94
+ def reader_spectrum(filepath, logger):
95
+ """Reader for Spectrum CR1000 Data Logger file."""
96
+ ##------------------------------------------------------------------------.
97
+ #### Define column names
98
+ column_names = ["TO_PARSE"]
99
+
100
+ ##------------------------------------------------------------------------.
101
+ #### Define reader options
102
+ reader_kwargs = {}
103
+ # - Define delimiter
104
+ reader_kwargs["delimiter"] = "\\n"
105
+ # - Skip first row as columns names
106
+ reader_kwargs["header"] = None
107
+ # - Skip first 3 rows
108
+ reader_kwargs["skiprows"] = 4
109
+ # - Define encoding
110
+ reader_kwargs["encoding"] = "latin" # "ISO-8859-1"
111
+ # - Avoid first column to become df index !!!
112
+ reader_kwargs["index_col"] = False
113
+ # - Define behaviour when encountering bad lines
114
+ reader_kwargs["on_bad_lines"] = "skip"
115
+ # - Define reader engine
116
+ # - C engine is faster
117
+ # - Python engine is more feature-complete
118
+ reader_kwargs["engine"] = "python"
119
+ # - Define on-the-fly decompression of on-disk data
120
+ # - Available: gzip, bz2, zip
121
+ reader_kwargs["compression"] = "infer"
122
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
123
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
124
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
125
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
126
+ reader_kwargs["na_values"] = ["na", "", "error"]
127
+
128
+ ##------------------------------------------------------------------------.
129
+ #### Read the data
130
+ df = read_raw_text_file(
131
+ filepath=filepath,
132
+ column_names=column_names,
133
+ reader_kwargs=reader_kwargs,
134
+ logger=logger,
135
+ )
136
+
137
+ ##------------------------------------------------------------------------.
138
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
139
+ # Define time
140
+ df = df["TO_PARSE"].str.split(",", n=2, expand=True)
141
+ df.columns = ["time", "RECORD", "TO_PARSE"]
142
+
143
+ df["time"] = pd.to_datetime(df["time"].str.strip('"'), format="%Y-%m-%d %H:%M:%S", errors="coerce")
144
+
145
+ # Derive raw drop arrays
146
+ def split_string(s):
147
+ vals = [v.strip() for v in s.split(",")]
148
+ c1 = ",".join(vals[:32]) # -10
149
+ c1 = c1.replace("-10", "0")
150
+ c2 = "0,0," + ",".join(vals[32:62])
151
+ c3 = ",".join(vals[62:])
152
+ series = pd.Series(
153
+ {
154
+ "raw_drop_concentration": c1,
155
+ "raw_drop_average_velocity": c2,
156
+ "raw_drop_number": c3,
157
+ },
158
+ )
159
+ return series
160
+
161
+ splitted_string = df["TO_PARSE"].apply(split_string)
162
+ df["raw_drop_concentration"] = splitted_string["raw_drop_concentration"]
163
+ df["raw_drop_average_velocity"] = splitted_string["raw_drop_average_velocity"]
164
+ df["raw_drop_number"] = splitted_string["raw_drop_number"]
165
+
166
+ # Drop columns not agreeing with DISDRODB L0 standards
167
+ df = df.drop(columns=["TO_PARSE", "RECORD"])
168
+ return df
169
+
170
+
171
+ @is_documented_by(reader_generic_docstring)
172
+ def reader(
173
+ filepath,
174
+ logger=None,
175
+ ):
176
+ """Reader."""
177
+ # Retrieve Spectrum filepath
178
+ spectrum_filepath = filepath.replace("Parsivel", "Spectre")
179
+
180
+ # Read integral variables
181
+ df = reader_parsivel(filepath, logger=logger)
182
+
183
+ # Initialize empty arrays
184
+ # --> 0 values array produced in L0B
185
+ df["raw_drop_concentration"] = ""
186
+ df["raw_drop_average_velocity"] = ""
187
+ df["raw_drop_number"] = ""
188
+
189
+ # Read raw spectrum for corresponding timesteps
190
+ df_raw_spectrum = reader_spectrum(spectrum_filepath, logger=logger)
191
+
192
+ # Add raw array to df
193
+ df = df.set_index("time")
194
+ df_raw_spectrum = df_raw_spectrum.set_index("time")
195
+ df.update(df_raw_spectrum)
196
+
197
+ # Set back time as column
198
+ df = df.reset_index()
199
+
200
+ # Return the dataframe adhering to DISDRODB L0 standards
201
+ return df