disdrodb 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. disdrodb/__init__.py +68 -34
  2. disdrodb/_config.py +5 -4
  3. disdrodb/_version.py +16 -3
  4. disdrodb/accessor/__init__.py +20 -0
  5. disdrodb/accessor/methods.py +125 -0
  6. disdrodb/api/checks.py +177 -24
  7. disdrodb/api/configs.py +3 -3
  8. disdrodb/api/info.py +13 -13
  9. disdrodb/api/io.py +281 -22
  10. disdrodb/api/path.py +184 -195
  11. disdrodb/api/search.py +18 -9
  12. disdrodb/cli/disdrodb_create_summary.py +103 -0
  13. disdrodb/cli/disdrodb_create_summary_station.py +91 -0
  14. disdrodb/cli/disdrodb_run_l0.py +1 -1
  15. disdrodb/cli/disdrodb_run_l0_station.py +1 -1
  16. disdrodb/cli/disdrodb_run_l0a_station.py +1 -1
  17. disdrodb/cli/disdrodb_run_l0b.py +1 -1
  18. disdrodb/cli/disdrodb_run_l0b_station.py +3 -3
  19. disdrodb/cli/disdrodb_run_l0c.py +1 -1
  20. disdrodb/cli/disdrodb_run_l0c_station.py +3 -3
  21. disdrodb/cli/disdrodb_run_l1_station.py +2 -2
  22. disdrodb/cli/disdrodb_run_l2e_station.py +2 -2
  23. disdrodb/cli/disdrodb_run_l2m_station.py +2 -2
  24. disdrodb/configs.py +149 -4
  25. disdrodb/constants.py +61 -0
  26. disdrodb/data_transfer/download_data.py +127 -11
  27. disdrodb/etc/configs/attributes.yaml +339 -0
  28. disdrodb/etc/configs/encodings.yaml +473 -0
  29. disdrodb/etc/products/L1/global.yaml +13 -0
  30. disdrodb/etc/products/L2E/10MIN.yaml +12 -0
  31. disdrodb/etc/products/L2E/1MIN.yaml +1 -0
  32. disdrodb/etc/products/L2E/global.yaml +22 -0
  33. disdrodb/etc/products/L2M/10MIN.yaml +12 -0
  34. disdrodb/etc/products/L2M/GAMMA_ML.yaml +8 -0
  35. disdrodb/etc/products/L2M/NGAMMA_GS_LOG_ND_MAE.yaml +6 -0
  36. disdrodb/etc/products/L2M/NGAMMA_GS_ND_MAE.yaml +6 -0
  37. disdrodb/etc/products/L2M/NGAMMA_GS_Z_MAE.yaml +6 -0
  38. disdrodb/etc/products/L2M/global.yaml +26 -0
  39. disdrodb/issue/writer.py +2 -0
  40. disdrodb/l0/__init__.py +13 -0
  41. disdrodb/l0/configs/LPM/l0b_cf_attrs.yml +4 -4
  42. disdrodb/l0/configs/PARSIVEL/l0b_cf_attrs.yml +1 -1
  43. disdrodb/l0/configs/PARSIVEL/l0b_encodings.yml +3 -3
  44. disdrodb/l0/configs/PARSIVEL/raw_data_format.yml +1 -1
  45. disdrodb/l0/configs/PARSIVEL2/l0b_cf_attrs.yml +5 -5
  46. disdrodb/l0/configs/PARSIVEL2/l0b_encodings.yml +3 -3
  47. disdrodb/l0/configs/PARSIVEL2/raw_data_format.yml +1 -1
  48. disdrodb/l0/configs/PWS100/l0b_cf_attrs.yml +4 -4
  49. disdrodb/l0/configs/PWS100/raw_data_format.yml +1 -1
  50. disdrodb/l0/l0a_processing.py +37 -32
  51. disdrodb/l0/l0b_nc_processing.py +118 -8
  52. disdrodb/l0/l0b_processing.py +30 -65
  53. disdrodb/l0/l0c_processing.py +369 -259
  54. disdrodb/l0/readers/LPM/ARM/ARM_LPM.py +7 -0
  55. disdrodb/l0/readers/LPM/NETHERLANDS/DELFT_LPM_NC.py +66 -0
  56. disdrodb/l0/readers/LPM/SLOVENIA/{CRNI_VRH.py → UL.py} +3 -0
  57. disdrodb/l0/readers/LPM/SWITZERLAND/INNERERIZ_LPM.py +195 -0
  58. disdrodb/l0/readers/PARSIVEL/GPM/PIERS.py +0 -2
  59. disdrodb/l0/readers/PARSIVEL/JAPAN/JMA.py +4 -1
  60. disdrodb/l0/readers/PARSIVEL/NCAR/PECAN_MOBILE.py +1 -1
  61. disdrodb/l0/readers/PARSIVEL/NCAR/VORTEX2_2009.py +1 -1
  62. disdrodb/l0/readers/PARSIVEL2/ARM/ARM_PARSIVEL2.py +4 -0
  63. disdrodb/l0/readers/PARSIVEL2/BELGIUM/ILVO.py +168 -0
  64. disdrodb/l0/readers/PARSIVEL2/CANADA/UQAM_NC.py +69 -0
  65. disdrodb/l0/readers/PARSIVEL2/DENMARK/DTU.py +165 -0
  66. disdrodb/l0/readers/PARSIVEL2/FINLAND/FMI_PARSIVEL2.py +69 -0
  67. disdrodb/l0/readers/PARSIVEL2/FRANCE/ENPC_PARSIVEL2.py +255 -134
  68. disdrodb/l0/readers/PARSIVEL2/FRANCE/OSUG.py +525 -0
  69. disdrodb/l0/readers/PARSIVEL2/FRANCE/SIRTA_PARSIVEL2.py +1 -1
  70. disdrodb/l0/readers/PARSIVEL2/GPM/GCPEX.py +9 -7
  71. disdrodb/l0/readers/PARSIVEL2/KIT/BURKINA_FASO.py +1 -1
  72. disdrodb/l0/readers/PARSIVEL2/KIT/TEAMX.py +123 -0
  73. disdrodb/l0/readers/PARSIVEL2/{NETHERLANDS/DELFT.py → MPI/BCO_PARSIVEL2.py} +41 -71
  74. disdrodb/l0/readers/PARSIVEL2/MPI/BOWTIE.py +220 -0
  75. disdrodb/l0/readers/PARSIVEL2/NASA/APU.py +120 -0
  76. disdrodb/l0/readers/PARSIVEL2/NASA/LPVEX.py +109 -0
  77. disdrodb/l0/readers/PARSIVEL2/NCAR/FARM_PARSIVEL2.py +1 -0
  78. disdrodb/l0/readers/PARSIVEL2/NCAR/PECAN_FP3.py +1 -1
  79. disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_MIPS.py +126 -0
  80. disdrodb/l0/readers/PARSIVEL2/NCAR/PERILS_PIPS.py +165 -0
  81. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_P2.py +1 -1
  82. disdrodb/l0/readers/PARSIVEL2/NCAR/VORTEX_SE_2016_PIPS.py +20 -12
  83. disdrodb/l0/readers/PARSIVEL2/NETHERLANDS/DELFT_NC.py +5 -0
  84. disdrodb/l0/readers/PARSIVEL2/SPAIN/CENER.py +144 -0
  85. disdrodb/l0/readers/PARSIVEL2/SPAIN/CR1000DL.py +201 -0
  86. disdrodb/l0/readers/PARSIVEL2/SPAIN/LIAISE.py +137 -0
  87. disdrodb/l0/readers/PARSIVEL2/USA/C3WE.py +146 -0
  88. disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100.py +105 -99
  89. disdrodb/l0/readers/PWS100/FRANCE/ENPC_PWS100_SIRTA.py +151 -0
  90. disdrodb/l1/__init__.py +5 -0
  91. disdrodb/l1/fall_velocity.py +46 -0
  92. disdrodb/l1/filters.py +34 -20
  93. disdrodb/l1/processing.py +46 -45
  94. disdrodb/l1/resampling.py +77 -66
  95. disdrodb/l1_env/routines.py +18 -3
  96. disdrodb/l2/__init__.py +7 -0
  97. disdrodb/l2/empirical_dsd.py +58 -10
  98. disdrodb/l2/processing.py +268 -117
  99. disdrodb/metadata/checks.py +132 -125
  100. disdrodb/metadata/standards.py +3 -1
  101. disdrodb/psd/fitting.py +631 -345
  102. disdrodb/psd/models.py +9 -6
  103. disdrodb/routines/__init__.py +54 -0
  104. disdrodb/{l0/routines.py → routines/l0.py} +316 -355
  105. disdrodb/{l1/routines.py → routines/l1.py} +76 -116
  106. disdrodb/routines/l2.py +1019 -0
  107. disdrodb/{routines.py → routines/wrappers.py} +98 -10
  108. disdrodb/scattering/__init__.py +16 -4
  109. disdrodb/scattering/axis_ratio.py +61 -37
  110. disdrodb/scattering/permittivity.py +504 -0
  111. disdrodb/scattering/routines.py +746 -184
  112. disdrodb/summary/__init__.py +17 -0
  113. disdrodb/summary/routines.py +4196 -0
  114. disdrodb/utils/archiving.py +434 -0
  115. disdrodb/utils/attrs.py +68 -125
  116. disdrodb/utils/cli.py +5 -5
  117. disdrodb/utils/compression.py +30 -1
  118. disdrodb/utils/dask.py +121 -9
  119. disdrodb/utils/dataframe.py +61 -7
  120. disdrodb/utils/decorators.py +31 -0
  121. disdrodb/utils/directories.py +35 -15
  122. disdrodb/utils/encoding.py +37 -19
  123. disdrodb/{l2 → utils}/event.py +15 -173
  124. disdrodb/utils/logger.py +14 -7
  125. disdrodb/utils/manipulations.py +81 -0
  126. disdrodb/utils/routines.py +166 -0
  127. disdrodb/utils/subsetting.py +214 -0
  128. disdrodb/utils/time.py +35 -177
  129. disdrodb/utils/writer.py +20 -7
  130. disdrodb/utils/xarray.py +5 -4
  131. disdrodb/viz/__init__.py +13 -0
  132. disdrodb/viz/plots.py +398 -0
  133. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/METADATA +4 -3
  134. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/RECORD +139 -98
  135. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/entry_points.txt +2 -0
  136. disdrodb/l1/encoding_attrs.py +0 -642
  137. disdrodb/l2/processing_options.py +0 -213
  138. disdrodb/l2/routines.py +0 -868
  139. /disdrodb/l0/readers/PARSIVEL/SLOVENIA/{UL_FGG.py → UL.py} +0 -0
  140. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/WHEEL +0 -0
  141. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/licenses/LICENSE +0 -0
  142. {disdrodb-0.1.2.dist-info → disdrodb-0.1.4.dist-info}/top_level.txt +0 -0
@@ -30,6 +30,13 @@ def reader(
30
30
  #### Open the netCDF
31
31
  ds = open_raw_netcdf_file(filepath=filepath, logger=logger)
32
32
 
33
+ ##------------------------------------------------------------------------.
34
+ # Check correct dimensions
35
+ if ds.sizes["particle_diameter"] != 22 or ds.sizes["particle_fall_velocity"] != 20:
36
+ raise ValueError(
37
+ f"Dimensions of {filepath} {ds.sizes} do not match the expected dimensions for LPM sensor.",
38
+ )
39
+
33
40
  ##------------------------------------------------------------------------.
34
41
  #### Adapt the dataframe to adhere to DISDRODB L0 standards
35
42
  # Define dictionary mapping dataset variables to select and rename
@@ -0,0 +1,66 @@
1
+ #!/usr/bin/env python3
2
+ # -----------------------------------------------------------------------------.
3
+ # Copyright (c) 2021-2023 DISDRODB developers
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ # -----------------------------------------------------------------------------.
18
+ """Reader for DELFT Thies LPM sensor in netCDF format."""
19
+
20
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
21
+ from disdrodb.l0.l0b_nc_processing import open_raw_netcdf_file, standardize_raw_dataset
22
+
23
+
24
+ @is_documented_by(reader_generic_docstring)
25
+ def reader(
26
+ filepath,
27
+ logger=None,
28
+ ):
29
+ """Reader."""
30
+ ##------------------------------------------------------------------------.
31
+ #### Open the netCDF
32
+ ds = open_raw_netcdf_file(filepath=filepath, logger=logger)
33
+
34
+ ##------------------------------------------------------------------------.
35
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
36
+ # Add time coordinate
37
+ ds["time"] = ds["time_as_string"].astype("M8[s]")
38
+ ds["time"].attrs.pop("comment", None)
39
+ ds["time"].attrs.pop("units", None)
40
+ ds = ds.set_coords("time")
41
+
42
+ # Define dictionary mapping dataset variables to select and rename
43
+ dict_names = {
44
+ ### Dimensions
45
+ "diameter_classes": "diameter_bin_center",
46
+ "velocity_classes": "velocity_bin_center",
47
+ ### Variables
48
+ "liquid_precip_intensity": "rainfall_rate",
49
+ "solid_precip_intensity": "snowfall_rate",
50
+ "all_precip_intensity": "precipitation_rate",
51
+ "weather_code_synop_4680": "weather_code_synop_4680",
52
+ "weather_code_synop_4677": "weather_code_synop_4677",
53
+ "reflectivity": "reflectivity",
54
+ "visibility": "mor_visibility",
55
+ "total_number_particles": "number_particles",
56
+ "ambient_temperature": "temperature_ambient",
57
+ "status_laser": "laser_status",
58
+ "measurement_quality": "quality_index",
59
+ "raw_spectrum": "raw_drop_number",
60
+ }
61
+
62
+ # Rename dataset variables and columns and infill missing variables
63
+ ds = standardize_raw_dataset(ds=ds, dict_names=dict_names, sensor_name="LPM")
64
+
65
+ # Return the dataset adhering to DISDRODB L0B standards
66
+ return ds
@@ -62,6 +62,9 @@ def reader(
62
62
  # - Available: gzip, bz2, zip
63
63
  reader_kwargs["compression"] = "infer"
64
64
 
65
+ # - Skip rows with badly encoded data
66
+ reader_kwargs["encoding_errors"] = "replace"
67
+
65
68
  # - Strings to recognize as NA/NaN and replace with standard NA flags
66
69
  # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
67
70
  # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
@@ -0,0 +1,195 @@
1
+ #!/usr/bin/env python3
2
+
3
+ # -----------------------------------------------------------------------------.
4
+ # Copyright (c) 2021-2023 DISDRODB developers
5
+ #
6
+ # This program is free software: you can redistribute it and/or modify
7
+ # it under the terms of the GNU General Public License as published by
8
+ # the Free Software Foundation, either version 3 of the License, or
9
+ # (at your option) any later version.
10
+ #
11
+ # This program is distributed in the hope that it will be useful,
12
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+ # GNU General Public License for more details.
15
+ #
16
+ # You should have received a copy of the GNU General Public License
17
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
18
+ # -----------------------------------------------------------------------------.
19
+ """DISDRODB reader for GID LPM sensors not measuring wind."""
20
+ import pandas as pd
21
+
22
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
23
+ from disdrodb.l0.l0a_processing import read_raw_text_file
24
+
25
+
26
+ @is_documented_by(reader_generic_docstring)
27
+ def reader(
28
+ filepath,
29
+ logger=None,
30
+ ):
31
+ """Reader."""
32
+ ##------------------------------------------------------------------------.
33
+ #### - Define raw data headers
34
+ column_names = ["TO_BE_SPLITTED"]
35
+
36
+ ##------------------------------------------------------------------------.
37
+ #### Define reader options
38
+ # - For more info: https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
39
+ reader_kwargs = {}
40
+
41
+ # - Define delimiter
42
+ reader_kwargs["delimiter"] = "\\n"
43
+
44
+ # - Avoid first column to become df index !!!
45
+ reader_kwargs["index_col"] = False
46
+
47
+ # Since column names are expected to be passed explicitly, header is set to None
48
+ reader_kwargs["header"] = None
49
+
50
+ # - Number of rows to be skipped at the beginning of the file
51
+ reader_kwargs["skiprows"] = None
52
+
53
+ # - Define behaviour when encountering bad lines
54
+ reader_kwargs["on_bad_lines"] = "skip"
55
+
56
+ # - Define reader engine
57
+ # - C engine is faster
58
+ # - Python engine is more feature-complete
59
+ reader_kwargs["engine"] = "python"
60
+
61
+ # - Define on-the-fly decompression of on-disk data
62
+ # - Available: gzip, bz2, zip
63
+ reader_kwargs["compression"] = "infer"
64
+
65
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
66
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
67
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
68
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
69
+ reader_kwargs["na_values"] = ["na", "", "error"]
70
+
71
+ ##------------------------------------------------------------------------.
72
+ #### Read the data
73
+ df = read_raw_text_file(
74
+ filepath=filepath,
75
+ column_names=column_names,
76
+ reader_kwargs=reader_kwargs,
77
+ logger=logger,
78
+ )
79
+
80
+ ##------------------------------------------------------------------------.
81
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
82
+ # Count number of delimiters to identify valid rows
83
+ df = df[df["TO_BE_SPLITTED"].str.count(";") == 520]
84
+
85
+ # Split by ; delimiter (before raw drop number)
86
+ df = df["TO_BE_SPLITTED"].str.split(";", expand=True, n=79)
87
+
88
+ # Assign column names
89
+ column_names = [
90
+ "start_identifier",
91
+ "device_address",
92
+ "sensor_serial_number",
93
+ "sensor_date",
94
+ "sensor_time",
95
+ "weather_code_synop_4677_5min",
96
+ "weather_code_synop_4680_5min",
97
+ "weather_code_metar_4678_5min",
98
+ "precipitation_rate_5min",
99
+ "weather_code_synop_4677",
100
+ "weather_code_synop_4680",
101
+ "weather_code_metar_4678",
102
+ "precipitation_rate",
103
+ "rainfall_rate",
104
+ "snowfall_rate",
105
+ "precipitation_accumulated",
106
+ "mor_visibility",
107
+ "reflectivity",
108
+ "quality_index",
109
+ "max_hail_diameter",
110
+ "laser_status",
111
+ "static_signal",
112
+ "laser_temperature_analog_status",
113
+ "laser_temperature_digital_status",
114
+ "laser_current_analog_status",
115
+ "laser_current_digital_status",
116
+ "sensor_voltage_supply_status",
117
+ "current_heating_pane_transmitter_head_status",
118
+ "current_heating_pane_receiver_head_status",
119
+ "temperature_sensor_status",
120
+ "current_heating_voltage_supply_status",
121
+ "current_heating_house_status",
122
+ "current_heating_heads_status",
123
+ "current_heating_carriers_status",
124
+ "control_output_laser_power_status",
125
+ "reserve_status",
126
+ "temperature_interior",
127
+ "laser_temperature",
128
+ "laser_current_average",
129
+ "control_voltage",
130
+ "optical_control_voltage_output",
131
+ "sensor_voltage_supply",
132
+ "current_heating_pane_transmitter_head",
133
+ "current_heating_pane_receiver_head",
134
+ "temperature_ambient",
135
+ "current_heating_voltage_supply",
136
+ "current_heating_house",
137
+ "current_heating_heads",
138
+ "current_heating_carriers",
139
+ "number_particles",
140
+ "number_particles_internal_data",
141
+ "number_particles_min_speed",
142
+ "number_particles_min_speed_internal_data",
143
+ "number_particles_max_speed",
144
+ "number_particles_max_speed_internal_data",
145
+ "number_particles_min_diameter",
146
+ "number_particles_min_diameter_internal_data",
147
+ "number_particles_no_hydrometeor",
148
+ "number_particles_no_hydrometeor_internal_data",
149
+ "number_particles_unknown_classification",
150
+ "number_particles_unknown_classification_internal_data",
151
+ "number_particles_class_1",
152
+ "number_particles_class_1_internal_data",
153
+ "number_particles_class_2",
154
+ "number_particles_class_2_internal_data",
155
+ "number_particles_class_3",
156
+ "number_particles_class_3_internal_data",
157
+ "number_particles_class_4",
158
+ "number_particles_class_4_internal_data",
159
+ "number_particles_class_5",
160
+ "number_particles_class_5_internal_data",
161
+ "number_particles_class_6",
162
+ "number_particles_class_6_internal_data",
163
+ "number_particles_class_7",
164
+ "number_particles_class_7_internal_data",
165
+ "number_particles_class_8",
166
+ "number_particles_class_8_internal_data",
167
+ "number_particles_class_9",
168
+ "number_particles_class_9_internal_data",
169
+ "raw_drop_number",
170
+ ]
171
+ df.columns = column_names
172
+
173
+ # Remove checksum from raw_drop_number
174
+ df["raw_drop_number"] = df["raw_drop_number"].str.rsplit(";", n=2, expand=True)[0]
175
+
176
+ # Define datetime "time" column
177
+ df["time"] = df["sensor_date"] + "-" + df["sensor_time"]
178
+ df["time"] = pd.to_datetime(df["time"], format="%d.%m.%y-%H:%M:%S", errors="coerce")
179
+
180
+ # Drop row if start_identifier different than 00
181
+ df = df[df["start_identifier"].astype(str) == "00"]
182
+
183
+ # Drop rows with invalid raw_drop_number
184
+ df = df[df["raw_drop_number"].astype(str).str.len() == 1759]
185
+
186
+ # Drop columns not agreeing with DISDRODB L0 standards
187
+ columns_to_drop = [
188
+ "start_identifier",
189
+ "device_address",
190
+ "sensor_serial_number",
191
+ "sensor_date",
192
+ "sensor_time",
193
+ ]
194
+ df = df.drop(columns=columns_to_drop)
195
+ return df
@@ -38,8 +38,6 @@ def reader(
38
38
  reader_kwargs["delimiter"] = ";"
39
39
  # - Skip first row as columns names
40
40
  reader_kwargs["header"] = None
41
- # Skip first row as columns names
42
- reader_kwargs["header"] = None
43
41
  reader_kwargs["skiprows"] = 0
44
42
  # - Skip file with encoding errors
45
43
  reader_kwargs["encoding_errors"] = "ignore"
@@ -108,12 +108,15 @@ def reader(
108
108
  df["time"] = pd.to_datetime(df["time"], format="%Y/%m/%d-%H:%M:%S", errors="coerce")
109
109
  df = df.drop(columns=["date"])
110
110
 
111
+ # Convert timezone from JST to UTC
112
+ df = df.set_index("time").tz_localize("Asia/Tokyo").tz_convert(None).reset_index()
113
+
111
114
  # Preprocess the raw spectrum
112
115
  # - The '<SPECTRUM>ZERO</SPECTRUM>' indicates no drops detected
113
116
  # --> "" generates an array of zeros in L0B processing
114
117
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
115
118
 
116
- # Remove <SPECTRUM> and </SPECTRUM>" acronyms from the raw_drop_number field
119
+ # Remove <SPECTRUM> and </SPECTRUM> prefix and suffix from the raw_drop_number field
117
120
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
118
121
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
119
122
 
@@ -113,7 +113,7 @@ def reader(
113
113
  # --> "" generates an array of zeros in L0B processing
114
114
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
115
115
 
116
- # Remove <SPECTRUM> and </SPECTRUM>" acronyms from the raw_drop_number field
116
+ # Remove <SPECTRUM> and </SPECTRUM> prefix and suffix from the raw_drop_number field
117
117
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
118
118
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
119
119
 
@@ -102,7 +102,7 @@ def reader(
102
102
  # --> "" generates an array of zeros in L0B processing
103
103
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>ZERO</SPECTRUM>", "")
104
104
 
105
- # Remove <SPECTRUM> and </SPECTRUM>" acronyms from the raw_drop_number field
105
+ # Remove <SPECTRUM> and </SPECTRUM> prefix and suffix from the raw_drop_number field
106
106
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("<SPECTRUM>", "")
107
107
  df["raw_drop_number"] = df["raw_drop_number"].str.replace("</SPECTRUM>", "")
108
108
 
@@ -32,6 +32,10 @@ def reader(
32
32
 
33
33
  ##------------------------------------------------------------------------.
34
34
  #### Adapt the dataframe to adhere to DISDRODB L0 standards
35
+ # Correct for inverted raw_spectrum axis
36
+ ds["raw_spectrum"] = ds["raw_spectrum"].transpose("time", "particle_size", "raw_fall_velocity")
37
+ ds["raw_spectrum"].data = ds["raw_spectrum"].data.swapaxes(1, 2)
38
+
35
39
  # Define dictionary mapping dataset variables to select and rename
36
40
  dict_names = {
37
41
  ### Dimensions
@@ -0,0 +1,168 @@
1
+ #!/usr/bin/env python3
2
+
3
+ # -----------------------------------------------------------------------------.
4
+ # Copyright (c) 2021-2023 DISDRODB developers
5
+ #
6
+ # This program is free software: you can redistribute it and/or modify
7
+ # it under the terms of the GNU General Public License as published by
8
+ # the Free Software Foundation, either version 3 of the License, or
9
+ # (at your option) any later version.
10
+ #
11
+ # This program is distributed in the hope that it will be useful,
12
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+ # GNU General Public License for more details.
15
+ #
16
+ # You should have received a copy of the GNU General Public License
17
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
18
+ # -----------------------------------------------------------------------------.
19
+ """DISDRODB reader for UGENT ILVO Parsivel2 raw text data."""
20
+ import tarfile
21
+
22
+ import numpy as np
23
+ import pandas as pd
24
+
25
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
26
+ from disdrodb.l0.l0a_processing import read_raw_text_file
27
+ from disdrodb.utils.logger import log_error
28
+
29
+
30
+ @is_documented_by(reader_generic_docstring)
31
+ def reader(
32
+ filepath,
33
+ logger=None,
34
+ ):
35
+ """Reader."""
36
+
37
+ ##------------------------------------------------------------------------.
38
+ #### Define function to read each txt file inside each daily zip file
39
+ def read_txt_file(file, filename):
40
+ """Parse a single txt file within the daily zip file."""
41
+ ##------------------------------------------------------------------------.
42
+ #### Define column names
43
+ column_names = ["TO_PARSE"]
44
+
45
+ ##------------------------------------------------------------------------.
46
+ #### Define reader options
47
+ reader_kwargs = {}
48
+ # - Define delimiter
49
+ reader_kwargs["delimiter"] = "\\n"
50
+ # - Skip first row as columns names
51
+ # - Define encoding
52
+ reader_kwargs["encoding"] = "latin" # "ISO-8859-1"
53
+ # - Avoid first column to become df index !!!
54
+ reader_kwargs["index_col"] = False
55
+ # - Define behaviour when encountering bad lines
56
+ reader_kwargs["on_bad_lines"] = "skip"
57
+ # - Define reader engine
58
+ # - C engine is faster
59
+ # - Python engine is more feature-complete
60
+ reader_kwargs["engine"] = "python"
61
+ # - Define on-the-fly decompression of on-disk data
62
+ # - Available: gzip, bz2, zip
63
+ reader_kwargs["compression"] = "infer"
64
+ # - Strings to recognize as NA/NaN and replace with standard NA flags
65
+ # - Already included: '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN',
66
+ # '-NaN', '-nan', '1.#IND', '1.#QNAN', '<NA>', 'N/A',
67
+ # 'NA', 'NULL', 'NaN', 'n/a', 'nan', 'null'
68
+ reader_kwargs["na_values"] = ["na", "", "error"]
69
+
70
+ ##------------------------------------------------------------------------.
71
+ #### Read the data
72
+ df = read_raw_text_file(
73
+ filepath=file,
74
+ column_names=column_names,
75
+ reader_kwargs=reader_kwargs,
76
+ logger=logger,
77
+ )
78
+
79
+ ##------------------------------------------------------------------------.
80
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
81
+ # Create ID and Value columns
82
+ df = df["TO_PARSE"].str.split(":", expand=True, n=1)
83
+ df.columns = ["ID", "Value"]
84
+
85
+ # Select only rows with values
86
+ df = df[df["Value"].apply(lambda x: x is not None)]
87
+
88
+ # Drop rows with invalid IDs
89
+ valid_id_str = np.char.rjust(np.arange(0, 94).astype(str), width=2, fillchar="0")
90
+ df = df[df["ID"].astype(str).isin(valid_id_str)]
91
+
92
+ # Reshape dataframe
93
+ df = df.set_index("ID").T
94
+
95
+ # Assign column names
96
+ column_dict = {
97
+ "01": "rainfall_rate_32bit",
98
+ "02": "rainfall_accumulated_32bit",
99
+ "03": "weather_code_synop_4680",
100
+ "04": "weather_code_synop_4677",
101
+ "05": "weather_code_metar_4678",
102
+ "06": "weather_code_nws",
103
+ "07": "reflectivity_32bit",
104
+ "08": "mor_visibility",
105
+ "09": "sample_interval",
106
+ "10": "laser_amplitude",
107
+ "11": "number_particles",
108
+ "12": "sensor_temperature",
109
+ # "13": "sensor_serial_number",
110
+ # "14": "firmware_iop",
111
+ # "15": "firmware_dsp",
112
+ "16": "sensor_heating_current",
113
+ "17": "sensor_battery_voltage",
114
+ "18": "sensor_status",
115
+ # "19": "start_time",
116
+ # "20": "sensor_time",
117
+ # "21": "sensor_date",
118
+ # "22": "station_name",
119
+ # "23": "station_number",
120
+ "24": "rainfall_amount_absolute_32bit",
121
+ "25": "error_code",
122
+ "26": "sensor_temperature_pcb",
123
+ "27": "sensor_temperature_receiver",
124
+ "28": "sensor_temperature_trasmitter",
125
+ "30": "rainfall_rate_16_bit_30",
126
+ "31": "rainfall_rate_16_bit_1200",
127
+ "32": "rainfall_accumulated_16bit",
128
+ "34": "rain_kinetic_energy",
129
+ "35": "snowfall_rate",
130
+ "90": "raw_drop_concentration",
131
+ "91": "raw_drop_average_velocity",
132
+ "93": "raw_drop_number",
133
+ }
134
+
135
+ df = df.rename(column_dict, axis=1)
136
+
137
+ # Keep only columns defined in the dictionary
138
+ df = df[list(column_dict.values())]
139
+
140
+ # Define datetime "time" column from filename
141
+ datetime_str = filename.replace(".txt", "").split("_")[1]
142
+ df["time"] = pd.to_datetime(datetime_str, format="%Y%m%d%H%M%S")
143
+
144
+ return df
145
+
146
+ # ---------------------------------------------------------------------.
147
+ #### Iterate over all files (aka timesteps) in the daily zip archive
148
+ # - Each file contain a single timestep !
149
+ list_df = []
150
+ with tarfile.open(filepath, "r:gz") as tar:
151
+ members = sorted(tar.getmembers(), key=lambda m: m.name)
152
+ for member in members:
153
+ filename = member.name
154
+ if member.isfile() and filename.endswith(".txt"):
155
+ # Open file
156
+ with tar.extractfile(member) as file:
157
+ try:
158
+ df = read_txt_file(file=file, filename=filename)
159
+ list_df.append(df)
160
+ except Exception as e:
161
+ msg = f"An error occurred while reading {filename}. The error is: {e}."
162
+ log_error(logger=logger, msg=msg, verbose=True)
163
+
164
+ # Concatenate all dataframes into a single one
165
+ df = pd.concat(list_df)
166
+
167
+ # ---------------------------------------------------------------------.
168
+ return df
@@ -0,0 +1,69 @@
1
+ #!/usr/bin/env python3
2
+ # -----------------------------------------------------------------------------.
3
+ # Copyright (c) 2021-2023 DISDRODB developers
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ # -----------------------------------------------------------------------------.
18
+ """Reader for UQAM OTT PARSIVEL2 sensor in netCDF format."""
19
+
20
+ from disdrodb.l0.l0_reader import is_documented_by, reader_generic_docstring
21
+ from disdrodb.l0.l0b_nc_processing import open_raw_netcdf_file, standardize_raw_dataset
22
+
23
+
24
+ @is_documented_by(reader_generic_docstring)
25
+ def reader(
26
+ filepath,
27
+ logger=None,
28
+ ):
29
+ """Reader."""
30
+ ##------------------------------------------------------------------------.
31
+ #### Open the netCDF
32
+ ds = open_raw_netcdf_file(filepath=filepath, logger=logger)
33
+
34
+ ##------------------------------------------------------------------------.
35
+ #### Adapt the dataframe to adhere to DISDRODB L0 standards
36
+ # Add time coordinate
37
+ ds["time"] = ds["time_as_string"].astype("M8[s]")
38
+ ds["time"].attrs.pop("comment", None)
39
+ ds["time"].attrs.pop("units", None)
40
+ ds = ds.set_coords("time")
41
+
42
+ # Define dictionary mapping dataset variables to select and rename
43
+ dict_names = {
44
+ ### Dimensions
45
+ "diameter_classes": "diameter_bin_center",
46
+ "velocity_classes": "velocity_bin_center",
47
+ ### Variables
48
+ "rainfall_rate_32bit": "rainfall_rate_32bit",
49
+ "weather_code_synop_4680": "weather_code_synop_4680",
50
+ "weather_code_synop_4677": "weather_code_synop_4677",
51
+ "weather_code_metar_4678": "weather_code_metar_4678",
52
+ "weather_code_nws": "weather_code_nws",
53
+ "reflectivity_32bit": "reflectivity_32bit",
54
+ "mor_visibility": "mor_visibility",
55
+ "laser_amplitude": "laser_amplitude",
56
+ "number_particles_validated": "number_particles",
57
+ "sensor_temperature": "sensor_temperature",
58
+ "error_code": "error_code",
59
+ "kinetic_energy": "rain_kinetic_energy",
60
+ "fieldV": "raw_drop_average_velocity",
61
+ "fieldN": "raw_drop_concentration",
62
+ "raw_data": "raw_drop_number",
63
+ }
64
+
65
+ # Rename dataset variables and columns and infill missing variables
66
+ ds = standardize_raw_dataset(ds=ds, dict_names=dict_names, sensor_name="PARSIVEL2")
67
+
68
+ # Return the dataset adhering to DISDRODB L0B standards
69
+ return ds