cloudnetpy 1.80.8__py3-none-any.whl → 1.81.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudnetpy/categorize/__init__.py +1 -1
- cloudnetpy/categorize/atmos_utils.py +31 -27
- cloudnetpy/categorize/attenuations/__init__.py +4 -4
- cloudnetpy/categorize/attenuations/liquid_attenuation.py +7 -5
- cloudnetpy/categorize/attenuations/melting_attenuation.py +3 -3
- cloudnetpy/categorize/attenuations/rain_attenuation.py +4 -4
- cloudnetpy/categorize/categorize.py +25 -11
- cloudnetpy/categorize/classify.py +9 -8
- cloudnetpy/categorize/containers.py +13 -10
- cloudnetpy/categorize/disdrometer.py +5 -3
- cloudnetpy/categorize/droplet.py +12 -9
- cloudnetpy/categorize/falling.py +9 -8
- cloudnetpy/categorize/freezing.py +10 -7
- cloudnetpy/categorize/insects.py +18 -17
- cloudnetpy/categorize/lidar.py +7 -3
- cloudnetpy/categorize/melting.py +16 -15
- cloudnetpy/categorize/model.py +17 -10
- cloudnetpy/categorize/mwr.py +5 -3
- cloudnetpy/categorize/radar.py +15 -13
- cloudnetpy/cli.py +10 -8
- cloudnetpy/cloudnetarray.py +8 -7
- cloudnetpy/concat_lib.py +29 -20
- cloudnetpy/datasource.py +26 -21
- cloudnetpy/exceptions.py +12 -10
- cloudnetpy/instruments/basta.py +19 -9
- cloudnetpy/instruments/bowtie.py +18 -11
- cloudnetpy/instruments/ceilo.py +22 -10
- cloudnetpy/instruments/ceilometer.py +33 -34
- cloudnetpy/instruments/cl61d.py +5 -3
- cloudnetpy/instruments/cloudnet_instrument.py +7 -7
- cloudnetpy/instruments/copernicus.py +16 -7
- cloudnetpy/instruments/disdrometer/common.py +5 -4
- cloudnetpy/instruments/disdrometer/parsivel.py +14 -9
- cloudnetpy/instruments/disdrometer/thies.py +11 -7
- cloudnetpy/instruments/fd12p.py +7 -6
- cloudnetpy/instruments/galileo.py +16 -7
- cloudnetpy/instruments/hatpro.py +33 -24
- cloudnetpy/instruments/lufft.py +6 -4
- cloudnetpy/instruments/mira.py +33 -19
- cloudnetpy/instruments/mrr.py +12 -12
- cloudnetpy/instruments/nc_lidar.py +1 -1
- cloudnetpy/instruments/nc_radar.py +8 -8
- cloudnetpy/instruments/pollyxt.py +19 -12
- cloudnetpy/instruments/radiometrics.py +17 -10
- cloudnetpy/instruments/rain_e_h3.py +9 -5
- cloudnetpy/instruments/rpg.py +32 -21
- cloudnetpy/instruments/rpg_reader.py +15 -12
- cloudnetpy/instruments/vaisala.py +32 -24
- cloudnetpy/instruments/weather_station.py +28 -21
- cloudnetpy/model_evaluation/file_handler.py +27 -29
- cloudnetpy/model_evaluation/plotting/plot_tools.py +7 -5
- cloudnetpy/model_evaluation/plotting/plotting.py +41 -32
- cloudnetpy/model_evaluation/products/advance_methods.py +38 -34
- cloudnetpy/model_evaluation/products/grid_methods.py +10 -9
- cloudnetpy/model_evaluation/products/model_products.py +15 -9
- cloudnetpy/model_evaluation/products/observation_products.py +12 -10
- cloudnetpy/model_evaluation/products/product_resampling.py +11 -7
- cloudnetpy/model_evaluation/products/tools.py +18 -14
- cloudnetpy/model_evaluation/statistics/statistical_methods.py +6 -5
- cloudnetpy/model_evaluation/tests/unit/test_plotting.py +18 -25
- cloudnetpy/model_evaluation/utils.py +3 -3
- cloudnetpy/output.py +15 -32
- cloudnetpy/plotting/plotting.py +22 -12
- cloudnetpy/products/classification.py +15 -9
- cloudnetpy/products/der.py +24 -19
- cloudnetpy/products/drizzle.py +21 -13
- cloudnetpy/products/drizzle_error.py +8 -7
- cloudnetpy/products/drizzle_tools.py +27 -23
- cloudnetpy/products/epsilon.py +6 -5
- cloudnetpy/products/ier.py +11 -5
- cloudnetpy/products/iwc.py +18 -9
- cloudnetpy/products/lwc.py +41 -31
- cloudnetpy/products/mwr_tools.py +30 -19
- cloudnetpy/products/product_tools.py +23 -19
- cloudnetpy/utils.py +84 -98
- cloudnetpy/version.py +2 -2
- {cloudnetpy-1.80.8.dist-info → cloudnetpy-1.81.1.dist-info}/METADATA +3 -2
- cloudnetpy-1.81.1.dist-info/RECORD +126 -0
- cloudnetpy-1.80.8.dist-info/RECORD +0 -126
- {cloudnetpy-1.80.8.dist-info → cloudnetpy-1.81.1.dist-info}/WHEEL +0 -0
- {cloudnetpy-1.80.8.dist-info → cloudnetpy-1.81.1.dist-info}/entry_points.txt +0 -0
- {cloudnetpy-1.80.8.dist-info → cloudnetpy-1.81.1.dist-info}/licenses/LICENSE +0 -0
- {cloudnetpy-1.80.8.dist-info → cloudnetpy-1.81.1.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,9 @@
|
|
1
|
+
import datetime
|
1
2
|
import logging
|
2
3
|
from typing import NamedTuple
|
3
4
|
|
4
5
|
import numpy as np
|
6
|
+
import numpy.typing as npt
|
5
7
|
from numpy import ma
|
6
8
|
from scipy.ndimage import gaussian_filter
|
7
9
|
|
@@ -9,7 +11,6 @@ from cloudnetpy import utils
|
|
9
11
|
from cloudnetpy.cloudnetarray import CloudnetArray
|
10
12
|
from cloudnetpy.exceptions import ValidTimeStampError
|
11
13
|
from cloudnetpy.instruments.instruments import Instrument
|
12
|
-
from cloudnetpy.utils import Epoch
|
13
14
|
|
14
15
|
|
15
16
|
class NoiseParam(NamedTuple):
|
@@ -22,24 +23,24 @@ class NoiseParam(NamedTuple):
|
|
22
23
|
class Ceilometer:
|
23
24
|
"""Base class for all types of ceilometers and pollyxt."""
|
24
25
|
|
25
|
-
def __init__(self, noise_param: NoiseParam | None = None):
|
26
|
+
def __init__(self, noise_param: NoiseParam | None = None) -> None:
|
26
27
|
self.noise_param = noise_param or NoiseParam()
|
27
28
|
self.data: dict = {} # Need to contain 'beta_raw', 'range' and 'time'
|
28
29
|
self.metadata: dict = {} # Need to contain 'date' as ('yyyy', 'mm', 'dd')
|
29
|
-
self.expected_date:
|
30
|
+
self.expected_date: datetime.date | None = None
|
30
31
|
self.site_meta: dict = {}
|
31
|
-
self.date:
|
32
|
+
self.date: datetime.date
|
32
33
|
self.instrument: Instrument | None = None
|
33
34
|
self.serial_number: str | None = None
|
34
35
|
|
35
36
|
def calc_screened_product(
|
36
37
|
self,
|
37
|
-
array:
|
38
|
+
array: npt.NDArray,
|
38
39
|
snr_limit: int = 5,
|
39
40
|
n_negatives: int = 5,
|
40
41
|
*,
|
41
42
|
range_corrected: bool = True,
|
42
|
-
) ->
|
43
|
+
) -> npt.NDArray:
|
43
44
|
"""Screens noise from lidar variable."""
|
44
45
|
noisy_data = NoisyData(
|
45
46
|
self.data,
|
@@ -55,12 +56,12 @@ class Ceilometer:
|
|
55
56
|
|
56
57
|
def calc_beta_smooth(
|
57
58
|
self,
|
58
|
-
beta:
|
59
|
+
beta: npt.NDArray,
|
59
60
|
snr_limit: int = 5,
|
60
61
|
n_negatives: int = 5,
|
61
62
|
*,
|
62
63
|
range_corrected: bool = True,
|
63
|
-
) ->
|
64
|
+
) -> npt.NDArray:
|
64
65
|
noisy_data = NoisyData(
|
65
66
|
self.data,
|
66
67
|
self.noise_param,
|
@@ -94,23 +95,23 @@ class Ceilometer:
|
|
94
95
|
raise RuntimeError(msg)
|
95
96
|
self.data["wavelength"] = float(self.instrument.wavelength)
|
96
97
|
|
97
|
-
def get_date_and_time(self, epoch:
|
98
|
+
def get_date_and_time(self, epoch: datetime.datetime) -> None:
|
98
99
|
if "time" not in self.data:
|
99
100
|
msg = "Time array missing from data"
|
100
101
|
raise ValidTimeStampError(msg)
|
101
102
|
if self.expected_date is not None:
|
102
103
|
self.data = utils.screen_by_time(self.data, epoch, self.expected_date)
|
103
|
-
self.date = utils.seconds2date(self.data["time"][0], epoch=epoch)
|
104
|
+
self.date = utils.seconds2date(self.data["time"][0], epoch=epoch).date()
|
104
105
|
self.data["time"] = utils.seconds2hours(self.data["time"])
|
105
106
|
|
106
|
-
def data_to_cloudnet_arrays(self, time_dtype="f4") -> None:
|
107
|
+
def data_to_cloudnet_arrays(self, time_dtype: str = "f4") -> None:
|
107
108
|
for key, array in self.data.items():
|
108
109
|
if key == "time":
|
109
110
|
self.data[key] = CloudnetArray(array, key, data_type=time_dtype)
|
110
111
|
else:
|
111
112
|
self.data[key] = CloudnetArray(array, key)
|
112
113
|
|
113
|
-
def add_site_geolocation(self):
|
114
|
+
def add_site_geolocation(self) -> None:
|
114
115
|
utils.add_site_geolocation(self.data, gps=False, site_meta=self.site_meta)
|
115
116
|
|
116
117
|
def screen_depol(self) -> None:
|
@@ -164,7 +165,7 @@ class NoisyData:
|
|
164
165
|
*,
|
165
166
|
range_corrected: bool = True,
|
166
167
|
instrument: Instrument | None = None,
|
167
|
-
):
|
168
|
+
) -> None:
|
168
169
|
self.data = data
|
169
170
|
self.noise_param = noise_param
|
170
171
|
self.range_corrected = range_corrected
|
@@ -172,7 +173,7 @@ class NoisyData:
|
|
172
173
|
|
173
174
|
def screen_data(
|
174
175
|
self,
|
175
|
-
data_in:
|
176
|
+
data_in: npt.NDArray,
|
176
177
|
snr_limit: float = 5,
|
177
178
|
n_negatives: int = 5,
|
178
179
|
*,
|
@@ -181,7 +182,7 @@ class NoisyData:
|
|
181
182
|
filter_fog: bool = True,
|
182
183
|
filter_negatives: bool = True,
|
183
184
|
filter_snr: bool = True,
|
184
|
-
) ->
|
185
|
+
) -> npt.NDArray:
|
185
186
|
data = ma.copy(data_in)
|
186
187
|
self._calc_range_uncorrected(data)
|
187
188
|
noise = _estimate_background_noise(data)
|
@@ -206,7 +207,7 @@ class NoisyData:
|
|
206
207
|
self._calc_range_corrected(data)
|
207
208
|
return data
|
208
209
|
|
209
|
-
def _adjust_noise(self, noise:
|
210
|
+
def _adjust_noise(self, noise: npt.NDArray, *, is_smoothed: bool) -> npt.NDArray:
|
210
211
|
noise_min = (
|
211
212
|
self.noise_param.noise_smooth_min
|
212
213
|
if is_smoothed is True
|
@@ -222,12 +223,12 @@ class NoisyData:
|
|
222
223
|
|
223
224
|
@staticmethod
|
224
225
|
def _mask_low_values_above_consequent_negatives(
|
225
|
-
data:
|
226
|
+
data: npt.NDArray,
|
226
227
|
n_negatives: int = 5,
|
227
228
|
threshold: float = 8e-6,
|
228
229
|
n_gates: int = 95,
|
229
230
|
n_skip_lowest: int = 5,
|
230
|
-
) ->
|
231
|
+
) -> npt.NDArray:
|
231
232
|
negative_data = data[:, n_skip_lowest : n_gates + n_skip_lowest] < 0
|
232
233
|
n_consequent_negatives = utils.cumsumr(negative_data, axis=1)
|
233
234
|
time_indices, alt_indices = np.where(n_consequent_negatives > n_negatives)
|
@@ -247,7 +248,7 @@ class NoisyData:
|
|
247
248
|
n_gates_for_signal_sum: int = 20,
|
248
249
|
signal_sum_threshold: float = 1e-3,
|
249
250
|
variance_threshold: float = 1e-15,
|
250
|
-
) ->
|
251
|
+
) -> npt.NDArray:
|
251
252
|
"""Finds saturated (usually fog) profiles from beta_raw."""
|
252
253
|
signal_sum = ma.sum(
|
253
254
|
ma.abs(self.data["beta_raw"][:, :n_gates_for_signal_sum]),
|
@@ -260,12 +261,12 @@ class NoisyData:
|
|
260
261
|
|
261
262
|
def _remove_noise(
|
262
263
|
self,
|
263
|
-
array:
|
264
|
-
noise:
|
264
|
+
array: npt.NDArray,
|
265
|
+
noise: npt.NDArray,
|
265
266
|
*,
|
266
267
|
keep_negative: bool,
|
267
268
|
snr_limit: float,
|
268
|
-
) ->
|
269
|
+
) -> npt.NDArray:
|
269
270
|
snr = array / utils.transpose(noise)
|
270
271
|
if self.range_corrected is False:
|
271
272
|
snr_scale_factor = 6
|
@@ -279,11 +280,11 @@ class NoisyData:
|
|
279
280
|
array[snr < snr_limit] = ma.masked
|
280
281
|
return array
|
281
282
|
|
282
|
-
def _calc_range_uncorrected(self, data:
|
283
|
+
def _calc_range_uncorrected(self, data: npt.NDArray) -> None:
|
283
284
|
ind = self._get_altitude_ind()
|
284
285
|
data[:, ind] = data[:, ind] / self._get_range_squared()[ind]
|
285
286
|
|
286
|
-
def _calc_range_corrected(self, data:
|
287
|
+
def _calc_range_corrected(self, data: npt.NDArray) -> None:
|
287
288
|
ind = self._get_altitude_ind()
|
288
289
|
data[:, ind] = data[:, ind] * self._get_range_squared()[ind]
|
289
290
|
|
@@ -301,15 +302,15 @@ class NoisyData:
|
|
301
302
|
alt_limit = 2400.0
|
302
303
|
return np.where(self.data["range"] < alt_limit)
|
303
304
|
|
304
|
-
def _get_range_squared(self) ->
|
305
|
+
def _get_range_squared(self) -> npt.NDArray:
|
305
306
|
"""Returns range (m), squared and converted to km."""
|
306
307
|
m2km = 0.001
|
307
308
|
return (self.data["range"] * m2km) ** 2
|
308
309
|
|
309
310
|
@staticmethod
|
310
311
|
def _clean_fog_profiles(
|
311
|
-
data:
|
312
|
-
is_fog:
|
312
|
+
data: npt.NDArray,
|
313
|
+
is_fog: npt.NDArray,
|
313
314
|
threshold: float = 2e-6,
|
314
315
|
) -> None:
|
315
316
|
"""Removes values in saturated (e.g. fog) profiles above peak."""
|
@@ -319,20 +320,20 @@ class NoisyData:
|
|
319
320
|
profile[peak_ind:][profile[peak_ind:] < threshold] = ma.masked
|
320
321
|
|
321
322
|
|
322
|
-
def _estimate_background_noise(data:
|
323
|
+
def _estimate_background_noise(data: npt.NDArray) -> npt.NDArray:
|
323
324
|
var = _calc_var_from_top_gates(data)
|
324
325
|
return np.sqrt(var)
|
325
326
|
|
326
327
|
|
327
|
-
def _calc_var_from_top_gates(data:
|
328
|
+
def _calc_var_from_top_gates(data: npt.NDArray) -> npt.NDArray:
|
328
329
|
fraction = 0.1
|
329
330
|
n_gates = round(data.shape[1] * fraction)
|
330
331
|
return ma.var(data[:, -n_gates:], axis=1)
|
331
332
|
|
332
333
|
|
333
334
|
def calc_sigma_units(
|
334
|
-
time_vector:
|
335
|
-
range_los:
|
335
|
+
time_vector: npt.NDArray,
|
336
|
+
range_los: npt.NDArray,
|
336
337
|
sigma_minutes: float = 1,
|
337
338
|
sigma_metres: float = 10,
|
338
339
|
) -> tuple[float, float]:
|
@@ -363,9 +364,7 @@ def calc_sigma_units(
|
|
363
364
|
return x_std, y_std
|
364
365
|
|
365
366
|
|
366
|
-
def _estimate_clouds_from_beta(
|
367
|
-
beta: np.ndarray,
|
368
|
-
) -> tuple[tuple, np.ndarray, float]:
|
367
|
+
def _estimate_clouds_from_beta(beta: npt.NDArray) -> tuple[tuple, npt.NDArray, float]:
|
369
368
|
"""Naively finds strong clouds from ceilometer backscatter."""
|
370
369
|
cloud_limit = 1e-6
|
371
370
|
cloud_ind = np.where(beta > cloud_limit)
|
cloudnetpy/instruments/cl61d.py
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
"""Module with a class for Lufft chm15k ceilometer."""
|
2
2
|
|
3
|
+
import datetime
|
3
4
|
import logging
|
5
|
+
from os import PathLike
|
4
6
|
|
5
7
|
import netCDF4
|
6
8
|
|
@@ -15,10 +17,10 @@ class Cl61d(NcLidar):
|
|
15
17
|
|
16
18
|
def __init__(
|
17
19
|
self,
|
18
|
-
file_name: str,
|
20
|
+
file_name: str | PathLike,
|
19
21
|
site_meta: dict,
|
20
|
-
expected_date:
|
21
|
-
):
|
22
|
+
expected_date: datetime.date | None = None,
|
23
|
+
) -> None:
|
22
24
|
super().__init__()
|
23
25
|
self.file_name = file_name
|
24
26
|
self.site_meta = site_meta
|
@@ -2,6 +2,7 @@ import logging
|
|
2
2
|
from typing import TYPE_CHECKING
|
3
3
|
|
4
4
|
import numpy as np
|
5
|
+
import numpy.typing as npt
|
5
6
|
from numpy import ma
|
6
7
|
|
7
8
|
from cloudnetpy import utils
|
@@ -14,9 +15,9 @@ if TYPE_CHECKING:
|
|
14
15
|
|
15
16
|
|
16
17
|
class CloudnetInstrument:
|
17
|
-
def __init__(self):
|
18
|
+
def __init__(self) -> None:
|
18
19
|
self.dataset: netCDF4.Dataset
|
19
|
-
self.time:
|
20
|
+
self.time: npt.NDArray = np.array([])
|
20
21
|
self.site_meta: dict = {}
|
21
22
|
self.data: dict = {}
|
22
23
|
self.serial_number: str | None = None
|
@@ -55,7 +56,7 @@ class CloudnetInstrument:
|
|
55
56
|
ind = time.argsort()
|
56
57
|
self.screen_time_indices(ind)
|
57
58
|
|
58
|
-
def screen_time_indices(self, valid_indices: list |
|
59
|
+
def screen_time_indices(self, valid_indices: list | npt.NDArray) -> None:
|
59
60
|
time = self._get_time()
|
60
61
|
n_time = len(time)
|
61
62
|
if len(valid_indices) == 0 or (
|
@@ -79,7 +80,7 @@ class CloudnetInstrument:
|
|
79
80
|
if self.time.size > 0:
|
80
81
|
self.time = self.time[valid_indices]
|
81
82
|
|
82
|
-
def _get_time(self) ->
|
83
|
+
def _get_time(self) -> npt.NDArray:
|
83
84
|
try:
|
84
85
|
return self.data["time"].data[:]
|
85
86
|
except KeyError:
|
@@ -95,14 +96,13 @@ class CloudnetInstrument:
|
|
95
96
|
|
96
97
|
|
97
98
|
class CSVFile(CloudnetInstrument):
|
98
|
-
def __init__(self, site_meta: dict):
|
99
|
+
def __init__(self, site_meta: dict) -> None:
|
99
100
|
super().__init__()
|
100
101
|
self.site_meta = site_meta
|
101
102
|
self._data: dict = {}
|
102
103
|
|
103
104
|
def add_date(self) -> None:
|
104
|
-
|
105
|
-
self.date = dt.strftime("%Y %m %d").split()
|
105
|
+
self.date = self._data["time"][0].date()
|
106
106
|
|
107
107
|
def add_data(self) -> None:
|
108
108
|
for key, value in self._data.items():
|
@@ -1,8 +1,11 @@
|
|
1
1
|
"""Module for reading raw cloud radar data."""
|
2
2
|
|
3
|
+
import datetime
|
3
4
|
import os
|
4
5
|
import tempfile
|
6
|
+
from os import PathLike
|
5
7
|
from tempfile import TemporaryDirectory
|
8
|
+
from uuid import UUID
|
6
9
|
|
7
10
|
import numpy as np
|
8
11
|
|
@@ -13,12 +16,12 @@ from cloudnetpy.metadata import MetaData
|
|
13
16
|
|
14
17
|
|
15
18
|
def copernicus2nc(
|
16
|
-
raw_files: str,
|
17
|
-
output_file: str,
|
19
|
+
raw_files: str | PathLike,
|
20
|
+
output_file: str | PathLike,
|
18
21
|
site_meta: dict,
|
19
|
-
uuid: str | None = None,
|
20
|
-
date: str | None = None,
|
21
|
-
) ->
|
22
|
+
uuid: str | UUID | None = None,
|
23
|
+
date: str | datetime.date | None = None,
|
24
|
+
) -> UUID:
|
22
25
|
"""Converts 'Copernicus' cloud radar data into Cloudnet Level 1b netCDF file.
|
23
26
|
|
24
27
|
Args:
|
@@ -43,6 +46,10 @@ def copernicus2nc(
|
|
43
46
|
>>> copernicus2nc('/one/day/of/copernicus/files/', 'radar.nc', site_meta)
|
44
47
|
|
45
48
|
"""
|
49
|
+
if isinstance(date, str):
|
50
|
+
date = datetime.date.fromisoformat(date)
|
51
|
+
uuid = utils.get_uuid(uuid)
|
52
|
+
|
46
53
|
keymap = {
|
47
54
|
"ZED_HC": "Zh",
|
48
55
|
"VEL_HC": "v",
|
@@ -57,6 +64,7 @@ def copernicus2nc(
|
|
57
64
|
"beamwidthH": "beamwidthH",
|
58
65
|
}
|
59
66
|
|
67
|
+
nc_filename: str | PathLike
|
60
68
|
with TemporaryDirectory() as temp_dir:
|
61
69
|
if os.path.isdir(raw_files):
|
62
70
|
with tempfile.NamedTemporaryFile(
|
@@ -105,7 +113,8 @@ def copernicus2nc(
|
|
105
113
|
copernicus.test_if_all_masked()
|
106
114
|
attributes = output.add_time_attribute(ATTRIBUTES, copernicus.date)
|
107
115
|
output.update_attributes(copernicus.data, attributes)
|
108
|
-
|
116
|
+
output.save_level1b(copernicus, output_file, uuid)
|
117
|
+
return uuid
|
109
118
|
|
110
119
|
|
111
120
|
class Copernicus(ChilboltonRadar):
|
@@ -117,7 +126,7 @@ class Copernicus(ChilboltonRadar):
|
|
117
126
|
|
118
127
|
"""
|
119
128
|
|
120
|
-
def __init__(self, full_path: str, site_meta: dict):
|
129
|
+
def __init__(self, full_path: str | PathLike, site_meta: dict) -> None:
|
121
130
|
super().__init__(full_path, site_meta)
|
122
131
|
self.instrument = COPERNICUS
|
123
132
|
|
@@ -3,6 +3,7 @@
|
|
3
3
|
from typing import Literal
|
4
4
|
|
5
5
|
import numpy as np
|
6
|
+
import numpy.typing as npt
|
6
7
|
|
7
8
|
from cloudnetpy.cloudnetarray import CloudnetArray
|
8
9
|
from cloudnetpy.instruments.cloudnet_instrument import CloudnetInstrument
|
@@ -42,7 +43,7 @@ class Disdrometer(CloudnetInstrument):
|
|
42
43
|
spreads: list,
|
43
44
|
name: str,
|
44
45
|
start: float = 0.0,
|
45
|
-
):
|
46
|
+
) -> None:
|
46
47
|
mid, bounds, spread = self._create_vectors(n_values, spreads, start)
|
47
48
|
self.data[name] = CloudnetArray(mid, name, dimensions=(name,))
|
48
49
|
key = f"{name}_spread"
|
@@ -56,9 +57,9 @@ class Disdrometer(CloudnetInstrument):
|
|
56
57
|
spreads: list[float],
|
57
58
|
start: float,
|
58
59
|
) -> tuple:
|
59
|
-
mid_value:
|
60
|
-
lower_limit:
|
61
|
-
upper_limit:
|
60
|
+
mid_value: npt.NDArray = np.array([])
|
61
|
+
lower_limit: npt.NDArray = np.array([])
|
62
|
+
upper_limit: npt.NDArray = np.array([])
|
62
63
|
for spread, n in zip(spreads, n_values, strict=True):
|
63
64
|
lower = np.linspace(start, start + (n - 1) * spread, n)
|
64
65
|
upper = lower + spread
|
@@ -7,8 +7,10 @@ from collections.abc import Callable, Iterable, Iterator, Sequence
|
|
7
7
|
from itertools import islice
|
8
8
|
from os import PathLike
|
9
9
|
from typing import Any
|
10
|
+
from uuid import UUID
|
10
11
|
|
11
12
|
import numpy as np
|
13
|
+
import numpy.typing as npt
|
12
14
|
from numpy import ma
|
13
15
|
|
14
16
|
from cloudnetpy import output
|
@@ -16,19 +18,20 @@ from cloudnetpy.cloudnetarray import CloudnetArray
|
|
16
18
|
from cloudnetpy.constants import MM_TO_M, SEC_IN_HOUR
|
17
19
|
from cloudnetpy.exceptions import DisdrometerDataError
|
18
20
|
from cloudnetpy.instruments import instruments
|
21
|
+
from cloudnetpy.utils import get_uuid
|
19
22
|
|
20
23
|
from .common import ATTRIBUTES, Disdrometer
|
21
24
|
|
22
25
|
|
23
26
|
def parsivel2nc(
|
24
27
|
disdrometer_file: str | PathLike | Iterable[str | PathLike],
|
25
|
-
output_file: str,
|
28
|
+
output_file: str | PathLike,
|
26
29
|
site_meta: dict,
|
27
|
-
uuid: str | None = None,
|
30
|
+
uuid: str | UUID | None = None,
|
28
31
|
date: str | datetime.date | None = None,
|
29
32
|
telegram: Sequence[int | None] | None = None,
|
30
33
|
timestamps: Sequence[datetime.datetime] | None = None,
|
31
|
-
) ->
|
34
|
+
) -> UUID:
|
32
35
|
"""Converts OTT Parsivel-2 disdrometer data into Cloudnet Level 1b netCDF
|
33
36
|
file.
|
34
37
|
|
@@ -61,6 +64,7 @@ def parsivel2nc(
|
|
61
64
|
"""
|
62
65
|
if isinstance(date, str):
|
63
66
|
date = datetime.date.fromisoformat(date)
|
67
|
+
uuid = get_uuid(uuid)
|
64
68
|
if isinstance(disdrometer_file, str | PathLike):
|
65
69
|
disdrometer_file = [disdrometer_file]
|
66
70
|
disdrometer = Parsivel(disdrometer_file, site_meta, telegram, date, timestamps)
|
@@ -74,7 +78,8 @@ def parsivel2nc(
|
|
74
78
|
disdrometer.add_meta()
|
75
79
|
attributes = output.add_time_attribute(ATTRIBUTES, disdrometer.date)
|
76
80
|
output.update_attributes(disdrometer.data, attributes)
|
77
|
-
|
81
|
+
output.save_level1b(disdrometer, output_file, uuid)
|
82
|
+
return uuid
|
78
83
|
|
79
84
|
|
80
85
|
class Parsivel(Disdrometer):
|
@@ -85,7 +90,7 @@ class Parsivel(Disdrometer):
|
|
85
90
|
telegram: Sequence[int | None] | None = None,
|
86
91
|
expected_date: datetime.date | None = None,
|
87
92
|
timestamps: Sequence[datetime.datetime] | None = None,
|
88
|
-
):
|
93
|
+
) -> None:
|
89
94
|
super().__init__()
|
90
95
|
self.site_meta = site_meta
|
91
96
|
self.raw_data = _read_parsivel(filenames, telegram, timestamps)
|
@@ -337,11 +342,11 @@ def _parse_datetime(tokens: Iterator[str]) -> datetime.datetime:
|
|
337
342
|
)
|
338
343
|
|
339
344
|
|
340
|
-
def _parse_vector(tokens: Iterator[str]) ->
|
345
|
+
def _parse_vector(tokens: Iterator[str]) -> npt.NDArray:
|
341
346
|
return np.array([_parse_float(tokens) for _i in range(32)])
|
342
347
|
|
343
348
|
|
344
|
-
def _parse_spectrum(tokens: Iterator[str]) ->
|
349
|
+
def _parse_spectrum(tokens: Iterator[str]) -> npt.NDArray:
|
345
350
|
first = next(tokens)
|
346
351
|
if first == "<SPECTRUM>ZERO</SPECTRUM>":
|
347
352
|
return np.zeros((32, 32), dtype="i2")
|
@@ -615,7 +620,7 @@ def _read_typ_op4a(lines: list[str]) -> dict[str, Any]:
|
|
615
620
|
return data
|
616
621
|
|
617
622
|
|
618
|
-
def _read_fmi(content: str):
|
623
|
+
def _read_fmi(content: str) -> dict[str, list]:
|
619
624
|
r"""Read format used by Finnish Meteorological Institute and University of
|
620
625
|
Helsinki.
|
621
626
|
|
@@ -661,7 +666,7 @@ def _read_parsivel(
|
|
661
666
|
filenames: Iterable[str | PathLike],
|
662
667
|
telegram: Sequence[int | None] | None = None,
|
663
668
|
timestamps: Sequence[datetime.datetime] | None = None,
|
664
|
-
) -> dict[str,
|
669
|
+
) -> dict[str, npt.NDArray]:
|
665
670
|
combined_data = defaultdict(list)
|
666
671
|
for filename in filenames:
|
667
672
|
with open(filename, encoding="latin1", errors="ignore") as file:
|
@@ -2,6 +2,7 @@ import datetime
|
|
2
2
|
from collections import defaultdict
|
3
3
|
from os import PathLike
|
4
4
|
from typing import Any
|
5
|
+
from uuid import UUID
|
5
6
|
|
6
7
|
import numpy as np
|
7
8
|
from numpy import ma
|
@@ -12,6 +13,7 @@ from cloudnetpy.constants import MM_TO_M, SEC_IN_HOUR
|
|
12
13
|
from cloudnetpy.exceptions import DisdrometerDataError, ValidTimeStampError
|
13
14
|
from cloudnetpy.instruments import instruments
|
14
15
|
from cloudnetpy.instruments.toa5 import read_toa5
|
16
|
+
from cloudnetpy.utils import get_uuid
|
15
17
|
|
16
18
|
from .common import ATTRIBUTES, Disdrometer
|
17
19
|
|
@@ -69,12 +71,12 @@ TELEGRAM4 = [
|
|
69
71
|
|
70
72
|
|
71
73
|
def thies2nc(
|
72
|
-
disdrometer_file: str,
|
73
|
-
output_file: str,
|
74
|
+
disdrometer_file: str | PathLike,
|
75
|
+
output_file: str | PathLike,
|
74
76
|
site_meta: dict,
|
75
|
-
uuid: str | None = None,
|
77
|
+
uuid: str | UUID | None = None,
|
76
78
|
date: str | datetime.date | None = None,
|
77
|
-
) ->
|
79
|
+
) -> UUID:
|
78
80
|
"""Converts Thies-LNM disdrometer data into Cloudnet Level 1b netCDF file.
|
79
81
|
|
80
82
|
Args:
|
@@ -101,6 +103,7 @@ def thies2nc(
|
|
101
103
|
"""
|
102
104
|
if isinstance(date, str):
|
103
105
|
date = datetime.date.fromisoformat(date)
|
106
|
+
uuid = get_uuid(uuid)
|
104
107
|
try:
|
105
108
|
disdrometer = Thies(disdrometer_file, site_meta, date)
|
106
109
|
except (ValueError, IndexError) as err:
|
@@ -113,7 +116,8 @@ def thies2nc(
|
|
113
116
|
disdrometer.convert_units()
|
114
117
|
attributes = output.add_time_attribute(ATTRIBUTES, disdrometer.date)
|
115
118
|
output.update_attributes(disdrometer.data, attributes)
|
116
|
-
|
119
|
+
output.save_level1b(disdrometer, output_file, uuid)
|
120
|
+
return uuid
|
117
121
|
|
118
122
|
|
119
123
|
class Thies(Disdrometer):
|
@@ -122,7 +126,7 @@ class Thies(Disdrometer):
|
|
122
126
|
filename: str | PathLike,
|
123
127
|
site_meta: dict,
|
124
128
|
expected_date: datetime.date | None = None,
|
125
|
-
):
|
129
|
+
) -> None:
|
126
130
|
super().__init__()
|
127
131
|
self.instrument = instruments.THIES
|
128
132
|
self.n_velocity = 20
|
@@ -205,7 +209,7 @@ class Thies(Disdrometer):
|
|
205
209
|
raise DisdrometerDataError(msg)
|
206
210
|
self.serial_number = first_id
|
207
211
|
|
208
|
-
def _read_line(self, line: str, timestamp: datetime.datetime | None = None):
|
212
|
+
def _read_line(self, line: str, timestamp: datetime.datetime | None = None) -> None:
|
209
213
|
raw_values = line.strip().strip(";").split(";")
|
210
214
|
# Support custom truncated format used in Leipzig LIM.
|
211
215
|
expected_columns = self.site_meta.get("truncate_columns", 521) - 1
|
cloudnetpy/instruments/fd12p.py
CHANGED
@@ -13,6 +13,7 @@ from cloudnetpy.exceptions import ValidTimeStampError
|
|
13
13
|
from cloudnetpy.instruments import instruments
|
14
14
|
from cloudnetpy.instruments.cloudnet_instrument import CSVFile
|
15
15
|
from cloudnetpy.metadata import MetaData
|
16
|
+
from cloudnetpy.utils import get_uuid
|
16
17
|
|
17
18
|
|
18
19
|
def fd12p2nc(
|
@@ -21,7 +22,7 @@ def fd12p2nc(
|
|
21
22
|
site_meta: dict,
|
22
23
|
uuid: str | UUID | None = None,
|
23
24
|
date: str | datetime.date | None = None,
|
24
|
-
):
|
25
|
+
) -> UUID:
|
25
26
|
"""Converts Vaisala FD12P into Cloudnet Level 1b netCDF file.
|
26
27
|
|
27
28
|
Args:
|
@@ -40,8 +41,7 @@ def fd12p2nc(
|
|
40
41
|
"""
|
41
42
|
if isinstance(date, str):
|
42
43
|
date = datetime.date.fromisoformat(date)
|
43
|
-
|
44
|
-
uuid = UUID(uuid)
|
44
|
+
uuid = get_uuid(uuid)
|
45
45
|
fd12p = FD12P(site_meta)
|
46
46
|
fd12p.parse_input_file(input_file, date)
|
47
47
|
fd12p.add_data()
|
@@ -55,11 +55,12 @@ def fd12p2nc(
|
|
55
55
|
fd12p.add_site_geolocation()
|
56
56
|
attributes = output.add_time_attribute(ATTRIBUTES, fd12p.date)
|
57
57
|
output.update_attributes(fd12p.data, attributes)
|
58
|
-
|
58
|
+
output.save_level1b(fd12p, output_file, uuid)
|
59
|
+
return uuid
|
59
60
|
|
60
61
|
|
61
62
|
class FD12P(CSVFile):
|
62
|
-
def __init__(self, site_meta: dict):
|
63
|
+
def __init__(self, site_meta: dict) -> None:
|
63
64
|
super().__init__(site_meta)
|
64
65
|
self.instrument = instruments.FD12P
|
65
66
|
self._data = {
|
@@ -76,7 +77,7 @@ class FD12P(CSVFile):
|
|
76
77
|
|
77
78
|
def parse_input_file(
|
78
79
|
self, filename: str | PathLike, expected_date: datetime.date | None = None
|
79
|
-
):
|
80
|
+
) -> None:
|
80
81
|
# In Lindenberg, format is date and time followed by Message 2 without
|
81
82
|
# non-printable characters.
|
82
83
|
with open(filename) as file:
|
@@ -1,7 +1,10 @@
|
|
1
1
|
"""Module for reading raw Galileo cloud radar data."""
|
2
2
|
|
3
|
+
import datetime
|
3
4
|
import os
|
5
|
+
from os import PathLike
|
4
6
|
from tempfile import NamedTemporaryFile, TemporaryDirectory
|
7
|
+
from uuid import UUID
|
5
8
|
|
6
9
|
import numpy as np
|
7
10
|
|
@@ -12,12 +15,12 @@ from cloudnetpy.metadata import MetaData
|
|
12
15
|
|
13
16
|
|
14
17
|
def galileo2nc(
|
15
|
-
raw_files: str,
|
16
|
-
output_file: str,
|
18
|
+
raw_files: str | PathLike,
|
19
|
+
output_file: str | PathLike,
|
17
20
|
site_meta: dict,
|
18
|
-
uuid: str | None = None,
|
19
|
-
date: str | None = None,
|
20
|
-
) ->
|
21
|
+
uuid: str | UUID | None = None,
|
22
|
+
date: str | datetime.date | None = None,
|
23
|
+
) -> UUID:
|
21
24
|
"""Converts 'Galileo' cloud radar data into Cloudnet Level 1b netCDF file.
|
22
25
|
|
23
26
|
Args:
|
@@ -42,6 +45,10 @@ def galileo2nc(
|
|
42
45
|
>>> galileo2nc('/one/day/of/galileo/files/', 'radar.nc', site_meta)
|
43
46
|
|
44
47
|
"""
|
48
|
+
if isinstance(date, str):
|
49
|
+
date = datetime.date.fromisoformat(date)
|
50
|
+
uuid = utils.get_uuid(uuid)
|
51
|
+
|
45
52
|
keymap = {
|
46
53
|
"ZED_HC": "Zh",
|
47
54
|
"VEL_HC": "v",
|
@@ -56,6 +63,7 @@ def galileo2nc(
|
|
56
63
|
"beamwidthH": "beamwidthH",
|
57
64
|
}
|
58
65
|
|
66
|
+
nc_filename: str | PathLike
|
59
67
|
with TemporaryDirectory() as temp_dir:
|
60
68
|
if os.path.isdir(raw_files):
|
61
69
|
with NamedTemporaryFile(
|
@@ -101,7 +109,8 @@ def galileo2nc(
|
|
101
109
|
galileo.test_if_all_masked()
|
102
110
|
attributes = output.add_time_attribute(ATTRIBUTES, galileo.date)
|
103
111
|
output.update_attributes(galileo.data, attributes)
|
104
|
-
|
112
|
+
output.save_level1b(galileo, output_file, uuid)
|
113
|
+
return uuid
|
105
114
|
|
106
115
|
|
107
116
|
class Galileo(ChilboltonRadar):
|
@@ -113,7 +122,7 @@ class Galileo(ChilboltonRadar):
|
|
113
122
|
|
114
123
|
"""
|
115
124
|
|
116
|
-
def __init__(self, full_path: str, site_meta: dict):
|
125
|
+
def __init__(self, full_path: str | PathLike, site_meta: dict) -> None:
|
117
126
|
super().__init__(full_path, site_meta)
|
118
127
|
self.date = self._init_date()
|
119
128
|
self.instrument = GALILEO
|