cloudnetpy 1.55.20__py3-none-any.whl → 1.55.22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudnetpy/categorize/atmos.py +46 -14
- cloudnetpy/categorize/atmos_utils.py +11 -1
- cloudnetpy/categorize/categorize.py +38 -21
- cloudnetpy/categorize/classify.py +31 -9
- cloudnetpy/categorize/containers.py +19 -7
- cloudnetpy/categorize/droplet.py +24 -8
- cloudnetpy/categorize/falling.py +17 -7
- cloudnetpy/categorize/freezing.py +19 -5
- cloudnetpy/categorize/insects.py +27 -14
- cloudnetpy/categorize/lidar.py +38 -36
- cloudnetpy/categorize/melting.py +19 -9
- cloudnetpy/categorize/model.py +28 -9
- cloudnetpy/categorize/mwr.py +4 -2
- cloudnetpy/categorize/radar.py +58 -22
- cloudnetpy/cloudnetarray.py +15 -6
- cloudnetpy/concat_lib.py +39 -16
- cloudnetpy/constants.py +7 -0
- cloudnetpy/datasource.py +39 -19
- cloudnetpy/instruments/basta.py +6 -2
- cloudnetpy/instruments/campbell_scientific.py +33 -16
- cloudnetpy/instruments/ceilo.py +30 -13
- cloudnetpy/instruments/ceilometer.py +76 -37
- cloudnetpy/instruments/cl61d.py +8 -3
- cloudnetpy/instruments/cloudnet_instrument.py +2 -1
- cloudnetpy/instruments/copernicus.py +27 -14
- cloudnetpy/instruments/disdrometer/common.py +51 -32
- cloudnetpy/instruments/disdrometer/parsivel.py +79 -48
- cloudnetpy/instruments/disdrometer/thies.py +10 -6
- cloudnetpy/instruments/galileo.py +23 -12
- cloudnetpy/instruments/hatpro.py +27 -11
- cloudnetpy/instruments/instruments.py +4 -1
- cloudnetpy/instruments/lufft.py +20 -11
- cloudnetpy/instruments/mira.py +60 -49
- cloudnetpy/instruments/mrr.py +31 -20
- cloudnetpy/instruments/nc_lidar.py +15 -6
- cloudnetpy/instruments/nc_radar.py +31 -22
- cloudnetpy/instruments/pollyxt.py +36 -21
- cloudnetpy/instruments/radiometrics.py +32 -18
- cloudnetpy/instruments/rpg.py +48 -22
- cloudnetpy/instruments/rpg_reader.py +39 -30
- cloudnetpy/instruments/vaisala.py +39 -27
- cloudnetpy/instruments/weather_station.py +15 -11
- cloudnetpy/metadata.py +3 -1
- cloudnetpy/model_evaluation/file_handler.py +31 -21
- cloudnetpy/model_evaluation/metadata.py +3 -1
- cloudnetpy/model_evaluation/model_metadata.py +1 -1
- cloudnetpy/model_evaluation/plotting/plot_tools.py +20 -15
- cloudnetpy/model_evaluation/plotting/plotting.py +114 -64
- cloudnetpy/model_evaluation/products/advance_methods.py +48 -28
- cloudnetpy/model_evaluation/products/grid_methods.py +44 -19
- cloudnetpy/model_evaluation/products/model_products.py +22 -18
- cloudnetpy/model_evaluation/products/observation_products.py +15 -9
- cloudnetpy/model_evaluation/products/product_resampling.py +14 -4
- cloudnetpy/model_evaluation/products/tools.py +16 -7
- cloudnetpy/model_evaluation/statistics/statistical_methods.py +28 -15
- cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
- cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +14 -13
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +14 -13
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +14 -13
- cloudnetpy/model_evaluation/tests/unit/conftest.py +11 -11
- cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +33 -27
- cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +83 -83
- cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
- cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +24 -25
- cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +40 -39
- cloudnetpy/model_evaluation/tests/unit/test_plotting.py +12 -11
- cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +30 -30
- cloudnetpy/model_evaluation/tests/unit/test_tools.py +18 -17
- cloudnetpy/model_evaluation/utils.py +3 -2
- cloudnetpy/output.py +45 -19
- cloudnetpy/plotting/plot_meta.py +35 -11
- cloudnetpy/plotting/plotting.py +172 -104
- cloudnetpy/products/classification.py +20 -8
- cloudnetpy/products/der.py +25 -10
- cloudnetpy/products/drizzle.py +41 -26
- cloudnetpy/products/drizzle_error.py +10 -5
- cloudnetpy/products/drizzle_tools.py +43 -24
- cloudnetpy/products/ier.py +10 -5
- cloudnetpy/products/iwc.py +16 -9
- cloudnetpy/products/lwc.py +34 -12
- cloudnetpy/products/mwr_multi.py +4 -1
- cloudnetpy/products/mwr_single.py +4 -1
- cloudnetpy/products/product_tools.py +33 -10
- cloudnetpy/utils.py +175 -74
- cloudnetpy/version.py +1 -1
- {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/METADATA +11 -10
- cloudnetpy-1.55.22.dist-info/RECORD +114 -0
- docs/source/conf.py +2 -2
- cloudnetpy-1.55.20.dist-info/RECORD +0 -114
- {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/LICENSE +0 -0
- {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/WHEEL +0 -0
- {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/top_level.txt +0 -0
@@ -6,6 +6,7 @@ from numpy import ma
|
|
6
6
|
from numpy.lib import recfunctions as rfn
|
7
7
|
from rpgpy import read_rpg
|
8
8
|
|
9
|
+
from cloudnetpy.constants import G_TO_KG
|
9
10
|
from cloudnetpy.exceptions import ValidTimeStampError
|
10
11
|
|
11
12
|
|
@@ -108,7 +109,9 @@ class Fmcw94Bin:
|
|
108
109
|
|
109
110
|
|
110
111
|
def _read_from_file(
|
111
|
-
file: BinaryIO,
|
112
|
+
file: BinaryIO,
|
113
|
+
fields: list[tuple[str, str]],
|
114
|
+
count: int | None = None,
|
112
115
|
) -> ma.MaskedArray:
|
113
116
|
arr = np.fromfile(file, np.dtype(fields), 1 if count is None else count)
|
114
117
|
masked_arr = ma.array(arr)
|
@@ -118,10 +121,10 @@ def _read_from_file(
|
|
118
121
|
|
119
122
|
|
120
123
|
def _decode_angles(
|
121
|
-
x: np.ndarray,
|
124
|
+
x: np.ndarray,
|
125
|
+
version: Literal[1, 2],
|
122
126
|
) -> tuple[np.ndarray, np.ndarray]:
|
123
|
-
"""
|
124
|
-
Decode elevation and azimuth angles.
|
127
|
+
"""Decode elevation and azimuth angles.
|
125
128
|
|
126
129
|
>>> _decode_angles(np.array([1267438.5]), version=1)
|
127
130
|
(array([138.5]), array([267.4]))
|
@@ -131,7 +134,6 @@ def _decode_angles(
|
|
131
134
|
Based on `interpret_angle` from mwr_raw2l1 licensed under BSD 3-Clause:
|
132
135
|
https://github.com/MeteoSwiss/mwr_raw2l1/blob/0738490d22f77138cdf9329bf102f319c78be584/mwr_raw2l1/readers/reader_rpg_helpers.py#L30
|
133
136
|
"""
|
134
|
-
|
135
137
|
if version == 1:
|
136
138
|
# Description in the manual is quite unclear so here's an improved one:
|
137
139
|
# Ang=sign(El)*(|El|+1000*Az), -90°<=El<100°, 0°<=Az<360°. If El>=100°
|
@@ -153,9 +155,8 @@ def _decode_angles(
|
|
153
155
|
ele = np.sign(x) * (np.abs(x) // 1e5) / 100
|
154
156
|
azi = (np.abs(x) - np.abs(ele) * 1e7) / 100
|
155
157
|
else:
|
156
|
-
|
157
|
-
|
158
|
-
)
|
158
|
+
msg = f"Known versions for angle encoding are 1 and 2, but received {version}"
|
159
|
+
raise NotImplementedError(msg)
|
159
160
|
|
160
161
|
return ele, azi
|
161
162
|
|
@@ -163,7 +164,8 @@ def _decode_angles(
|
|
163
164
|
class HatproBin:
|
164
165
|
"""HATPRO binary file reader. Byte order is assumed to be little endian.
|
165
166
|
|
166
|
-
References
|
167
|
+
References
|
168
|
+
----------
|
167
169
|
Radiometer Physics (2014): Instrument Operation and Software Guide
|
168
170
|
Operation Principles and Software Description for RPG standard single
|
169
171
|
polarization radiometers (G5 series).
|
@@ -188,30 +190,33 @@ class HatproBin:
|
|
188
190
|
self._remove_duplicate_timestamps()
|
189
191
|
self._add_zenith_angle()
|
190
192
|
|
191
|
-
def screen_bad_profiles(self):
|
193
|
+
def screen_bad_profiles(self) -> None:
|
192
194
|
is_bad = self.data["_quality_flag"] & 0b110 == self.QUALITY_LOW << 1
|
193
195
|
n_bad = np.count_nonzero(is_bad)
|
194
196
|
if n_bad == len(is_bad):
|
195
|
-
|
197
|
+
msg = "All data are low quality"
|
198
|
+
raise ValidTimeStampError(msg)
|
196
199
|
if n_bad:
|
197
200
|
percentage = round(100 * n_bad / len(is_bad))
|
198
201
|
logging.info(
|
199
|
-
|
200
|
-
|
202
|
+
"Screening %s %% (%s/%s) data points with low quality",
|
203
|
+
percentage,
|
204
|
+
n_bad,
|
205
|
+
len(is_bad),
|
201
206
|
)
|
202
207
|
self.data[self.variable][is_bad] = ma.masked
|
203
208
|
|
204
|
-
def _remove_duplicate_timestamps(self):
|
209
|
+
def _remove_duplicate_timestamps(self) -> None:
|
205
210
|
_, ind = np.unique(self.data["time"], return_index=True)
|
206
211
|
self.data = self.data[ind]
|
207
212
|
|
208
|
-
def _read_header(self, file: BinaryIO):
|
209
|
-
raise NotImplementedError
|
213
|
+
def _read_header(self, file: BinaryIO) -> None:
|
214
|
+
raise NotImplementedError
|
210
215
|
|
211
|
-
def _read_data(self, file: BinaryIO):
|
212
|
-
raise NotImplementedError
|
216
|
+
def _read_data(self, file: BinaryIO) -> None:
|
217
|
+
raise NotImplementedError
|
213
218
|
|
214
|
-
def _add_zenith_angle(self):
|
219
|
+
def _add_zenith_angle(self) -> None:
|
215
220
|
ele, _azi = _decode_angles(self.data["_instrument_angles"], self.version)
|
216
221
|
self.data = rfn.append_fields(self.data, "zenith_angle", 90 - ele)
|
217
222
|
|
@@ -221,7 +226,7 @@ class HatproBinLwp(HatproBin):
|
|
221
226
|
|
222
227
|
variable = "lwp"
|
223
228
|
|
224
|
-
def _read_header(self, file):
|
229
|
+
def _read_header(self, file) -> None:
|
225
230
|
self.header = _read_from_file(
|
226
231
|
file,
|
227
232
|
[
|
@@ -238,9 +243,10 @@ class HatproBinLwp(HatproBin):
|
|
238
243
|
elif self.header["file_code"] == 934501000:
|
239
244
|
self.version = 2
|
240
245
|
else:
|
241
|
-
|
246
|
+
msg = f'Unknown HATPRO version. {self.header["file_code"]}'
|
247
|
+
raise ValueError(msg)
|
242
248
|
|
243
|
-
def _read_data(self, file):
|
249
|
+
def _read_data(self, file) -> None:
|
244
250
|
self.data = _read_from_file(
|
245
251
|
file,
|
246
252
|
[
|
@@ -251,7 +257,7 @@ class HatproBinLwp(HatproBin):
|
|
251
257
|
],
|
252
258
|
self.header["_n_samples"],
|
253
259
|
)
|
254
|
-
self.data["lwp"] *=
|
260
|
+
self.data["lwp"] *= G_TO_KG
|
255
261
|
|
256
262
|
|
257
263
|
class HatproBinIwv(HatproBin):
|
@@ -259,7 +265,7 @@ class HatproBinIwv(HatproBin):
|
|
259
265
|
|
260
266
|
variable = "iwv"
|
261
267
|
|
262
|
-
def _read_header(self, file):
|
268
|
+
def _read_header(self, file) -> None:
|
263
269
|
self.header = _read_from_file(
|
264
270
|
file,
|
265
271
|
[
|
@@ -276,9 +282,10 @@ class HatproBinIwv(HatproBin):
|
|
276
282
|
elif self.header["file_code"] == 594811000:
|
277
283
|
self.version = 2
|
278
284
|
else:
|
279
|
-
|
285
|
+
msg = f'Unknown HATPRO version. {self.header["file_code"]}'
|
286
|
+
raise ValueError(msg)
|
280
287
|
|
281
|
-
def _read_data(self, file):
|
288
|
+
def _read_data(self, file) -> None:
|
282
289
|
self.data = _read_from_file(
|
283
290
|
file,
|
284
291
|
[
|
@@ -310,16 +317,18 @@ class HatproBinCombined:
|
|
310
317
|
_combine_values(arr["zenith_angle1"], arr["zenith_angle2"]),
|
311
318
|
)
|
312
319
|
# Workaround because rfn.drop_fields seems to incorrectly drop mask...
|
313
|
-
# arr = rfn.drop_fields(arr, ["zenith_angle1", "zenith_angle2"])
|
314
320
|
arr = rfn.rename_fields(
|
315
|
-
arr,
|
321
|
+
arr,
|
322
|
+
{"zenith_angle1": "_tmp1", "zenith_angle2": "_tmp2"},
|
316
323
|
)
|
317
324
|
else:
|
318
|
-
|
325
|
+
msg = "Only implemented up to 2 files"
|
326
|
+
raise NotImplementedError(msg)
|
319
327
|
self.data = {field: arr[field] for field in arr.dtype.fields}
|
320
328
|
|
321
329
|
|
322
330
|
def _combine_values(arr1: ma.MaskedArray, arr2: ma.MaskedArray) -> ma.MaskedArray:
|
323
331
|
if not ma.allequal(arr1, arr2):
|
324
|
-
|
332
|
+
msg = "Inconsistent values"
|
333
|
+
raise ValueError(msg)
|
325
334
|
return ma.where(~arr1.mask, arr1, arr2)
|
@@ -1,23 +1,26 @@
|
|
1
1
|
"""Module with classes for Vaisala ceilometers."""
|
2
|
+
import itertools
|
2
3
|
import logging
|
3
4
|
|
4
5
|
import numpy as np
|
5
6
|
|
6
7
|
from cloudnetpy import utils
|
8
|
+
from cloudnetpy.constants import SEC_IN_HOUR, SEC_IN_MINUTE
|
7
9
|
from cloudnetpy.exceptions import ValidTimeStampError
|
8
10
|
from cloudnetpy.instruments import instruments
|
9
11
|
from cloudnetpy.instruments.ceilometer import Ceilometer, NoiseParam
|
10
12
|
|
11
13
|
M2KM = 0.001
|
12
|
-
SECONDS_IN_MINUTE = 60
|
13
|
-
SECONDS_IN_HOUR = 3600
|
14
14
|
|
15
15
|
|
16
16
|
class VaisalaCeilo(Ceilometer):
|
17
17
|
"""Base class for Vaisala ceilometers."""
|
18
18
|
|
19
19
|
def __init__(
|
20
|
-
self,
|
20
|
+
self,
|
21
|
+
full_path: str,
|
22
|
+
site_meta: dict,
|
23
|
+
expected_date: str | None = None,
|
21
24
|
):
|
22
25
|
super().__init__(self.noise_param)
|
23
26
|
self.full_path = full_path
|
@@ -87,7 +90,8 @@ class VaisalaCeilo(Ceilometer):
|
|
87
90
|
for i, line in enumerate(data[timestamp_line_number:]):
|
88
91
|
if utils.is_empty_line(line):
|
89
92
|
return i
|
90
|
-
|
93
|
+
msg = "Can not parse number of data lines"
|
94
|
+
raise RuntimeError(msg)
|
91
95
|
|
92
96
|
def _parse_data_lines(data: list, starting_indices: list) -> list:
|
93
97
|
return [
|
@@ -103,20 +107,23 @@ class VaisalaCeilo(Ceilometer):
|
|
103
107
|
timestamp_line_numbers = _find_timestamp_line_numbers(valid_lines)
|
104
108
|
if self.expected_date is not None:
|
105
109
|
timestamp_line_numbers = _find_correct_dates(
|
106
|
-
valid_lines,
|
110
|
+
valid_lines,
|
111
|
+
timestamp_line_numbers,
|
107
112
|
)
|
108
113
|
if not timestamp_line_numbers:
|
109
114
|
raise ValidTimeStampError
|
110
115
|
number_of_data_lines = _find_number_of_data_lines(
|
111
|
-
valid_lines,
|
116
|
+
valid_lines,
|
117
|
+
timestamp_line_numbers[0],
|
112
118
|
)
|
113
|
-
|
114
|
-
return data_lines
|
119
|
+
return _parse_data_lines(valid_lines, timestamp_line_numbers)
|
115
120
|
|
116
121
|
@staticmethod
|
117
122
|
def _get_message_number(header_line_1: dict) -> int:
|
118
123
|
msg_no = header_line_1["message_number"]
|
119
|
-
|
124
|
+
if len(np.unique(msg_no)) != 1:
|
125
|
+
msg = "Error: inconsistent message numbers."
|
126
|
+
raise RuntimeError(msg)
|
120
127
|
return int(msg_no[0])
|
121
128
|
|
122
129
|
@staticmethod
|
@@ -134,8 +141,7 @@ class VaisalaCeilo(Ceilometer):
|
|
134
141
|
def _handle_metadata(cls, header: list) -> dict:
|
135
142
|
meta = cls._concatenate_meta(header)
|
136
143
|
meta = cls._remove_meta_duplicates(meta)
|
137
|
-
|
138
|
-
return meta
|
144
|
+
return cls._convert_meta_strings(meta)
|
139
145
|
|
140
146
|
@staticmethod
|
141
147
|
def _concatenate_meta(header: list) -> dict:
|
@@ -196,10 +202,7 @@ class VaisalaCeilo(Ceilometer):
|
|
196
202
|
"message_number",
|
197
203
|
"message_subclass",
|
198
204
|
)
|
199
|
-
if self._is_ct25k()
|
200
|
-
indices = [1, 3, 4, 6, 7, 8]
|
201
|
-
else:
|
202
|
-
indices = [1, 3, 4, 7, 8, 9]
|
205
|
+
indices = [1, 3, 4, 6, 7, 8] if self._is_ct25k() else [1, 3, 4, 7, 8, 9]
|
203
206
|
values = [split_string(line, indices) for line in lines]
|
204
207
|
return values_to_dict(fields, values)
|
205
208
|
|
@@ -222,7 +225,10 @@ class ClCeilo(VaisalaCeilo):
|
|
222
225
|
noise_param = NoiseParam(noise_min=3.1e-8, noise_smooth_min=1.1e-8)
|
223
226
|
|
224
227
|
def __init__(
|
225
|
-
self,
|
228
|
+
self,
|
229
|
+
full_path: str,
|
230
|
+
site_meta: dict,
|
231
|
+
expected_date: str | None = None,
|
226
232
|
):
|
227
233
|
super().__init__(full_path, site_meta, expected_date)
|
228
234
|
self._hex_conversion_params = (5, 524288, 1048576)
|
@@ -241,7 +247,7 @@ class ClCeilo(VaisalaCeilo):
|
|
241
247
|
self._store_ceilometer_info()
|
242
248
|
self._sort_time()
|
243
249
|
|
244
|
-
def _sort_time(self):
|
250
|
+
def _sort_time(self) -> None:
|
245
251
|
"""Sorts timestamps and removes duplicates."""
|
246
252
|
time = np.copy(self.data["time"][:])
|
247
253
|
ind_sorted = np.argsort(time)
|
@@ -258,7 +264,7 @@ class ClCeilo(VaisalaCeilo):
|
|
258
264
|
if array.ndim == 2 and array.shape[0] == n_time:
|
259
265
|
self.data[key] = self.data[key][ind_valid, :]
|
260
266
|
|
261
|
-
def _store_ceilometer_info(self):
|
267
|
+
def _store_ceilometer_info(self) -> None:
|
262
268
|
n_gates = self.data["beta_raw"].shape[1]
|
263
269
|
if n_gates < 1540:
|
264
270
|
self.instrument = instruments.CL31
|
@@ -267,7 +273,8 @@ class ClCeilo(VaisalaCeilo):
|
|
267
273
|
|
268
274
|
def _read_header_line_3(self, lines: list) -> dict:
|
269
275
|
if self._message_number != 2:
|
270
|
-
|
276
|
+
msg = f"Unsupported message number: {self._message_number}"
|
277
|
+
raise RuntimeError(msg)
|
271
278
|
keys = ("cloud_detection_status", "cloud_amount_data")
|
272
279
|
values = [[line[0:3], line[3:].strip()] for line in lines]
|
273
280
|
return values_to_dict(keys, values)
|
@@ -293,7 +300,8 @@ class ClCeilo(VaisalaCeilo):
|
|
293
300
|
class Ct25k(VaisalaCeilo):
|
294
301
|
"""Class for Vaisala CT25k ceilometer.
|
295
302
|
|
296
|
-
References
|
303
|
+
References
|
304
|
+
----------
|
297
305
|
https://www.manualslib.com/manual/1414094/Vaisala-Ct25k.html
|
298
306
|
|
299
307
|
"""
|
@@ -334,7 +342,8 @@ class Ct25k(VaisalaCeilo):
|
|
334
342
|
|
335
343
|
def _read_header_line_3(self, lines: list) -> dict:
|
336
344
|
if self._message_number in (1, 3, 6):
|
337
|
-
|
345
|
+
msg = f"Unsupported message number: {self._message_number}"
|
346
|
+
raise RuntimeError(msg)
|
338
347
|
keys = (
|
339
348
|
"measurement_mode",
|
340
349
|
"laser_energy",
|
@@ -347,31 +356,34 @@ class Ct25k(VaisalaCeilo):
|
|
347
356
|
"backscatter_sum",
|
348
357
|
)
|
349
358
|
values = [line.split() for line in lines]
|
350
|
-
keys_out = ("scale",
|
359
|
+
keys_out = ("scale", *keys) if len(values[0]) == 10 else keys
|
351
360
|
return values_to_dict(keys_out, values)
|
352
361
|
|
353
362
|
|
354
363
|
def split_string(string: str, indices: list) -> list:
|
355
364
|
"""Splits string between indices.
|
356
365
|
|
357
|
-
Notes
|
366
|
+
Notes
|
367
|
+
-----
|
358
368
|
It is possible to skip characters from the beginning and end of the
|
359
369
|
string but not from the middle.
|
360
370
|
|
361
|
-
Examples
|
371
|
+
Examples
|
372
|
+
--------
|
362
373
|
>>> s = 'abcde'
|
363
374
|
>>> indices = [1, 2, 4]
|
364
375
|
>>> split_string(s, indices)
|
365
376
|
['b', 'cd']
|
366
377
|
|
367
378
|
"""
|
368
|
-
return [string[n:m] for n, m in
|
379
|
+
return [string[n:m] for n, m in itertools.pairwise(indices)]
|
369
380
|
|
370
381
|
|
371
382
|
def values_to_dict(keys: tuple, values: list) -> dict:
|
372
383
|
"""Converts list elements to dictionary.
|
373
384
|
|
374
|
-
Examples
|
385
|
+
Examples
|
386
|
+
--------
|
375
387
|
>>> keys = ('a', 'b')
|
376
388
|
>>> values = [[1, 2], [1, 2], [1, 2], [1, 2]]
|
377
389
|
>>> values_to_dict(keys, values)
|
@@ -387,4 +399,4 @@ def values_to_dict(keys: tuple, values: list) -> dict:
|
|
387
399
|
def time_to_fraction_hour(time: str) -> float:
|
388
400
|
"""Returns time (hh:mm:ss) as fraction hour"""
|
389
401
|
hour, minute, sec = time.split(":")
|
390
|
-
return int(hour) + (int(minute) *
|
402
|
+
return int(hour) + (int(minute) * SEC_IN_MINUTE + int(sec)) / SEC_IN_HOUR
|
@@ -22,6 +22,7 @@ def ws2nc(
|
|
22
22
|
"""Converts weather-station data into Cloudnet Level 1b netCDF file.
|
23
23
|
|
24
24
|
Args:
|
25
|
+
----
|
25
26
|
weather_station_file: Filename of weather-station ASCII file.
|
26
27
|
output_file: Output filename.
|
27
28
|
site_meta: Dictionary containing information about the site. Required key
|
@@ -30,13 +31,14 @@ def ws2nc(
|
|
30
31
|
date: Expected date of the measurements as YYYY-MM-DD.
|
31
32
|
|
32
33
|
Returns:
|
34
|
+
-------
|
33
35
|
UUID of the generated file.
|
34
36
|
|
35
37
|
Raises:
|
38
|
+
------
|
36
39
|
WeatherStationDataError : Unable to read the file.
|
37
40
|
ValidTimeStampError: No valid timestamps found.
|
38
41
|
"""
|
39
|
-
|
40
42
|
try:
|
41
43
|
ws = WS(weather_station_file, site_meta)
|
42
44
|
if date is not None:
|
@@ -50,8 +52,7 @@ def ws2nc(
|
|
50
52
|
output.update_attributes(ws.data, attributes)
|
51
53
|
except ValueError as err:
|
52
54
|
raise WeatherStationDataError from err
|
53
|
-
|
54
|
-
return uuid
|
55
|
+
return output.save_level1b(ws, output_file, uuid)
|
55
56
|
|
56
57
|
|
57
58
|
class WS(CloudnetInstrument):
|
@@ -63,14 +64,17 @@ class WS(CloudnetInstrument):
|
|
63
64
|
self.instrument = instruments.GENERIC_WEATHER_STATION
|
64
65
|
self._data = self._read_data()
|
65
66
|
|
66
|
-
def _read_data(self):
|
67
|
+
def _read_data(self) -> dict:
|
67
68
|
timestamps, values, header = [], [], []
|
68
69
|
with open(self.filename, encoding="latin-1") as f:
|
69
70
|
data = f.readlines()
|
70
71
|
for row in data:
|
71
72
|
splat = row.split()
|
72
73
|
try:
|
73
|
-
timestamp = datetime.datetime.strptime(
|
74
|
+
timestamp = datetime.datetime.strptime(
|
75
|
+
splat[0],
|
76
|
+
"%Y-%m-%dT%H:%M:%SZ",
|
77
|
+
).replace(tzinfo=datetime.timezone.utc)
|
74
78
|
temp: list[str | float] = list(splat)
|
75
79
|
temp[1:] = [float(x) for x in temp[1:]]
|
76
80
|
values.append(temp)
|
@@ -93,16 +97,16 @@ class WS(CloudnetInstrument):
|
|
93
97
|
error_msg = "Unexpected weather station file format"
|
94
98
|
if len(column_titles) != len(expected_identifiers):
|
95
99
|
raise ValueError(error_msg)
|
96
|
-
for title, identifier in zip(column_titles, expected_identifiers):
|
100
|
+
for title, identifier in zip(column_titles, expected_identifiers, strict=True):
|
97
101
|
if identifier not in title:
|
98
102
|
raise ValueError(error_msg)
|
99
103
|
return {"timestamps": timestamps, "values": values}
|
100
104
|
|
101
|
-
def convert_time(self):
|
105
|
+
def convert_time(self) -> None:
|
102
106
|
decimal_hours = datetime2decimal_hours(self._data["timestamps"])
|
103
107
|
self.data["time"] = CloudnetArray(decimal_hours, "time")
|
104
108
|
|
105
|
-
def screen_timestamps(self, date: str):
|
109
|
+
def screen_timestamps(self, date: str) -> None:
|
106
110
|
dates = [str(d.date()) for d in self._data["timestamps"]]
|
107
111
|
valid_ind = [ind for ind, d in enumerate(dates) if d == date]
|
108
112
|
if not valid_ind:
|
@@ -112,7 +116,7 @@ class WS(CloudnetInstrument):
|
|
112
116
|
x for ind, x in enumerate(self._data[key]) if ind in valid_ind
|
113
117
|
]
|
114
118
|
|
115
|
-
def add_date(self):
|
119
|
+
def add_date(self) -> None:
|
116
120
|
first_date = self._data["timestamps"][0].date()
|
117
121
|
self.date = [
|
118
122
|
str(first_date.year),
|
@@ -120,7 +124,7 @@ class WS(CloudnetInstrument):
|
|
120
124
|
str(first_date.day).zfill(2),
|
121
125
|
]
|
122
126
|
|
123
|
-
def add_data(self):
|
127
|
+
def add_data(self) -> None:
|
124
128
|
keys = (
|
125
129
|
"wind_speed",
|
126
130
|
"wind_direction",
|
@@ -135,7 +139,7 @@ class WS(CloudnetInstrument):
|
|
135
139
|
array_masked = ma.masked_invalid(array)
|
136
140
|
self.data[key] = CloudnetArray(array_masked, key)
|
137
141
|
|
138
|
-
def convert_units(self):
|
142
|
+
def convert_units(self) -> None:
|
139
143
|
temperature_kelvins = atmos_utils.c2k(self.data["air_temperature"][:])
|
140
144
|
self.data["air_temperature"].data = temperature_kelvins
|
141
145
|
self.data["relative_humidity"].data = self.data["relative_humidity"][:] / 100
|
cloudnetpy/metadata.py
CHANGED
@@ -46,7 +46,9 @@ COMMON_ATTRIBUTES = {
|
|
46
46
|
standard_name="longitude",
|
47
47
|
),
|
48
48
|
"altitude": MetaData(
|
49
|
-
long_name="Altitude of site",
|
49
|
+
long_name="Altitude of site",
|
50
|
+
standard_name="altitude",
|
51
|
+
units="m",
|
50
52
|
),
|
51
53
|
"Zh": MetaData(
|
52
54
|
long_name="Radar reflectivity factor",
|
@@ -14,12 +14,13 @@ from .metadata import (
|
|
14
14
|
from .products.model_products import ModelManager
|
15
15
|
|
16
16
|
|
17
|
-
def update_attributes(model_downsample_variables: dict, attributes: dict):
|
17
|
+
def update_attributes(model_downsample_variables: dict, attributes: dict) -> None:
|
18
18
|
"""Overrides existing Cloudnet-ME Array-attributes.
|
19
19
|
Overrides existing attributes using hard-coded values.
|
20
20
|
New attributes are added.
|
21
21
|
|
22
22
|
Args:
|
23
|
+
----
|
23
24
|
model_downsample_variables (dict): Array instances.
|
24
25
|
attributes (dict): Product-specific attributes.
|
25
26
|
"""
|
@@ -32,11 +33,11 @@ def update_attributes(model_downsample_variables: dict, attributes: dict):
|
|
32
33
|
model_downsample_variables[key].set_attributes(MODEL_ATTRIBUTES[key])
|
33
34
|
elif "_".join(key_parts[0:-1]) in REGRID_PRODUCT_ATTRIBUTES:
|
34
35
|
model_downsample_variables[key].set_attributes(
|
35
|
-
REGRID_PRODUCT_ATTRIBUTES["_".join(key_parts[0:-1])]
|
36
|
+
REGRID_PRODUCT_ATTRIBUTES["_".join(key_parts[0:-1])],
|
36
37
|
)
|
37
38
|
elif "_".join(key_parts[0:-2]) in REGRID_PRODUCT_ATTRIBUTES:
|
38
39
|
model_downsample_variables[key].set_attributes(
|
39
|
-
REGRID_PRODUCT_ATTRIBUTES["_".join(key_parts[0:-2])]
|
40
|
+
REGRID_PRODUCT_ATTRIBUTES["_".join(key_parts[0:-2])],
|
40
41
|
)
|
41
42
|
elif (
|
42
43
|
"_".join(key_parts[1:]) in MODEL_L3_ATTRIBUTES
|
@@ -44,19 +45,19 @@ def update_attributes(model_downsample_variables: dict, attributes: dict):
|
|
44
45
|
):
|
45
46
|
try:
|
46
47
|
model_downsample_variables[key].set_attributes(
|
47
|
-
MODEL_L3_ATTRIBUTES["_".join(key_parts[1:])]
|
48
|
+
MODEL_L3_ATTRIBUTES["_".join(key_parts[1:])],
|
48
49
|
)
|
49
50
|
except KeyError:
|
50
51
|
model_downsample_variables[key].set_attributes(
|
51
|
-
MODEL_L3_ATTRIBUTES["_".join(key_parts[2:])]
|
52
|
+
MODEL_L3_ATTRIBUTES["_".join(key_parts[2:])],
|
52
53
|
)
|
53
54
|
elif "_".join(key_parts[1:]) in CYCLE_ATTRIBUTES:
|
54
55
|
model_downsample_variables[key].set_attributes(
|
55
|
-
CYCLE_ATTRIBUTES["_".join(key_parts[1:])]
|
56
|
+
CYCLE_ATTRIBUTES["_".join(key_parts[1:])],
|
56
57
|
)
|
57
58
|
elif "_".join(key_parts[2:]) in CYCLE_ATTRIBUTES:
|
58
59
|
model_downsample_variables[key].set_attributes(
|
59
|
-
CYCLE_ATTRIBUTES["_".join(key_parts[2:])]
|
60
|
+
CYCLE_ATTRIBUTES["_".join(key_parts[2:])],
|
60
61
|
)
|
61
62
|
|
62
63
|
|
@@ -66,10 +67,11 @@ def save_downsampled_file(
|
|
66
67
|
objects: tuple,
|
67
68
|
files: tuple,
|
68
69
|
uuid: str | None,
|
69
|
-
):
|
70
|
+
) -> str:
|
70
71
|
"""Saves a standard downsampled day product file.
|
71
72
|
|
72
73
|
Args:
|
74
|
+
----
|
73
75
|
id_mark (str): File identifier, format "(product name)_(model name)"
|
74
76
|
file_name (str): Name of the output file to be generated
|
75
77
|
objects (tuple): Include two objects: The :class:'ModelManager' and
|
@@ -92,25 +94,26 @@ def save_downsampled_file(
|
|
92
94
|
)
|
93
95
|
_add_source(root_group, objects, files)
|
94
96
|
output.copy_global(obj.dataset, root_group, ("location", "day", "month", "year"))
|
95
|
-
|
96
|
-
obj.dataset.day
|
97
|
-
except AttributeError:
|
97
|
+
if not hasattr(obj.dataset, "day"):
|
98
98
|
root_group.year, root_group.month, root_group.day = obj.date
|
99
99
|
output.merge_history(root_group, id_mark, {"l3": obj})
|
100
100
|
root_group.close()
|
101
|
+
if not isinstance(uuid, str):
|
102
|
+
msg = "UUID is not a string."
|
103
|
+
raise TypeError(msg)
|
101
104
|
return uuid
|
102
105
|
|
103
106
|
|
104
|
-
def add_var2ncfile(obj: ModelManager, file_name: str):
|
107
|
+
def add_var2ncfile(obj: ModelManager, file_name: str) -> None:
|
105
108
|
nc_file = netCDF4.Dataset(file_name, "r+", format="NETCDF4_CLASSIC")
|
106
109
|
_write_vars2nc(nc_file, obj.data)
|
107
110
|
nc_file.close()
|
108
111
|
|
109
112
|
|
110
|
-
def _write_vars2nc(rootgrp: netCDF4.Dataset, cloudnet_variables: dict):
|
113
|
+
def _write_vars2nc(rootgrp: netCDF4.Dataset, cloudnet_variables: dict) -> None:
|
111
114
|
"""Iterates over Cloudnet-ME instances and write to given rootgrp."""
|
112
115
|
|
113
|
-
def _get_dimensions(array):
|
116
|
+
def _get_dimensions(array) -> tuple:
|
114
117
|
"""Finds correct dimensions for a variable."""
|
115
118
|
if utils.isscalar(array):
|
116
119
|
return ()
|
@@ -118,8 +121,8 @@ def _write_vars2nc(rootgrp: netCDF4.Dataset, cloudnet_variables: dict):
|
|
118
121
|
file_dims = rootgrp.dimensions
|
119
122
|
array_dims = array.shape
|
120
123
|
for length in array_dims:
|
121
|
-
dim = [key for key in file_dims
|
122
|
-
variable_size = variable_size
|
124
|
+
dim = [key for key in file_dims if file_dims[key].size == length][0] # noqa: RUF015
|
125
|
+
variable_size = (*variable_size, dim)
|
123
126
|
return variable_size
|
124
127
|
|
125
128
|
for key in cloudnet_variables:
|
@@ -127,7 +130,10 @@ def _write_vars2nc(rootgrp: netCDF4.Dataset, cloudnet_variables: dict):
|
|
127
130
|
size = _get_dimensions(obj.data)
|
128
131
|
try:
|
129
132
|
nc_variable = rootgrp.createVariable(
|
130
|
-
obj.name,
|
133
|
+
obj.name,
|
134
|
+
obj.data_type,
|
135
|
+
size,
|
136
|
+
zlib=True,
|
131
137
|
)
|
132
138
|
nc_variable[:] = obj.data
|
133
139
|
for attr in obj.fetch_attributes():
|
@@ -136,11 +142,11 @@ def _write_vars2nc(rootgrp: netCDF4.Dataset, cloudnet_variables: dict):
|
|
136
142
|
continue
|
137
143
|
|
138
144
|
|
139
|
-
def _augment_global_attributes(root_group: netCDF4.Dataset):
|
145
|
+
def _augment_global_attributes(root_group: netCDF4.Dataset) -> None:
|
140
146
|
root_group.Conventions = "CF-1.8"
|
141
147
|
|
142
148
|
|
143
|
-
def _add_source(root_ground: netCDF4.Dataset, objects: tuple, files: tuple):
|
149
|
+
def _add_source(root_ground: netCDF4.Dataset, objects: tuple, files: tuple) -> None:
|
144
150
|
"""Generates source info for multiple files"""
|
145
151
|
model, obs = objects
|
146
152
|
model_files, obs_file = files
|
@@ -157,14 +163,18 @@ def _add_source(root_ground: netCDF4.Dataset, objects: tuple, files: tuple):
|
|
157
163
|
|
158
164
|
def add_time_attribute(date: datetime) -> dict:
|
159
165
|
""" "Adds time attribute with correct units.
|
166
|
+
|
160
167
|
Args:
|
168
|
+
----
|
161
169
|
attributes: Attributes of variables.
|
162
170
|
date: Date as Y M D 0 0 0.
|
171
|
+
|
163
172
|
Returns:
|
173
|
+
-------
|
164
174
|
dict: Same attributes with 'time' attribute added.
|
165
175
|
"""
|
166
176
|
return {
|
167
177
|
"time": MODEL_ATTRIBUTES["time"]._replace(
|
168
|
-
units=f"hours since {date:%Y-%m-%d} 00:00:00 +00:00"
|
169
|
-
)
|
178
|
+
units=f"hours since {date:%Y-%m-%d} 00:00:00 +00:00",
|
179
|
+
),
|
170
180
|
}
|
@@ -51,7 +51,9 @@ CYCLE_ATTRIBUTES = {
|
|
51
51
|
"pressure": MetaData(long_name="Pressure", units="Pa"),
|
52
52
|
"temperature": MetaData(long_name="Temperature", units="K"),
|
53
53
|
"uwind": MetaData(
|
54
|
-
long_name="Zonal wind",
|
54
|
+
long_name="Zonal wind",
|
55
|
+
units="m s-1",
|
56
|
+
standard_name="eastward_wind",
|
55
57
|
),
|
56
58
|
"vwind": MetaData(
|
57
59
|
long_name="Meridional wind",
|
@@ -30,7 +30,7 @@ MODELS = {
|
|
30
30
|
),
|
31
31
|
"harmonie": ModelMetaData(
|
32
32
|
model_name="HARMONIE-AROME",
|
33
|
-
long_name="the HIRLAM–ALADIN Research on Mesoscale Operational NWP in Euromed",
|
33
|
+
long_name="the HIRLAM–ALADIN Research on Mesoscale Operational NWP in Euromed", # noqa: RUF001
|
34
34
|
level=65,
|
35
35
|
cycle="6-11",
|
36
36
|
),
|