cloudnetpy 1.55.20__py3-none-any.whl → 1.55.22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudnetpy/categorize/atmos.py +46 -14
- cloudnetpy/categorize/atmos_utils.py +11 -1
- cloudnetpy/categorize/categorize.py +38 -21
- cloudnetpy/categorize/classify.py +31 -9
- cloudnetpy/categorize/containers.py +19 -7
- cloudnetpy/categorize/droplet.py +24 -8
- cloudnetpy/categorize/falling.py +17 -7
- cloudnetpy/categorize/freezing.py +19 -5
- cloudnetpy/categorize/insects.py +27 -14
- cloudnetpy/categorize/lidar.py +38 -36
- cloudnetpy/categorize/melting.py +19 -9
- cloudnetpy/categorize/model.py +28 -9
- cloudnetpy/categorize/mwr.py +4 -2
- cloudnetpy/categorize/radar.py +58 -22
- cloudnetpy/cloudnetarray.py +15 -6
- cloudnetpy/concat_lib.py +39 -16
- cloudnetpy/constants.py +7 -0
- cloudnetpy/datasource.py +39 -19
- cloudnetpy/instruments/basta.py +6 -2
- cloudnetpy/instruments/campbell_scientific.py +33 -16
- cloudnetpy/instruments/ceilo.py +30 -13
- cloudnetpy/instruments/ceilometer.py +76 -37
- cloudnetpy/instruments/cl61d.py +8 -3
- cloudnetpy/instruments/cloudnet_instrument.py +2 -1
- cloudnetpy/instruments/copernicus.py +27 -14
- cloudnetpy/instruments/disdrometer/common.py +51 -32
- cloudnetpy/instruments/disdrometer/parsivel.py +79 -48
- cloudnetpy/instruments/disdrometer/thies.py +10 -6
- cloudnetpy/instruments/galileo.py +23 -12
- cloudnetpy/instruments/hatpro.py +27 -11
- cloudnetpy/instruments/instruments.py +4 -1
- cloudnetpy/instruments/lufft.py +20 -11
- cloudnetpy/instruments/mira.py +60 -49
- cloudnetpy/instruments/mrr.py +31 -20
- cloudnetpy/instruments/nc_lidar.py +15 -6
- cloudnetpy/instruments/nc_radar.py +31 -22
- cloudnetpy/instruments/pollyxt.py +36 -21
- cloudnetpy/instruments/radiometrics.py +32 -18
- cloudnetpy/instruments/rpg.py +48 -22
- cloudnetpy/instruments/rpg_reader.py +39 -30
- cloudnetpy/instruments/vaisala.py +39 -27
- cloudnetpy/instruments/weather_station.py +15 -11
- cloudnetpy/metadata.py +3 -1
- cloudnetpy/model_evaluation/file_handler.py +31 -21
- cloudnetpy/model_evaluation/metadata.py +3 -1
- cloudnetpy/model_evaluation/model_metadata.py +1 -1
- cloudnetpy/model_evaluation/plotting/plot_tools.py +20 -15
- cloudnetpy/model_evaluation/plotting/plotting.py +114 -64
- cloudnetpy/model_evaluation/products/advance_methods.py +48 -28
- cloudnetpy/model_evaluation/products/grid_methods.py +44 -19
- cloudnetpy/model_evaluation/products/model_products.py +22 -18
- cloudnetpy/model_evaluation/products/observation_products.py +15 -9
- cloudnetpy/model_evaluation/products/product_resampling.py +14 -4
- cloudnetpy/model_evaluation/products/tools.py +16 -7
- cloudnetpy/model_evaluation/statistics/statistical_methods.py +28 -15
- cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
- cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +14 -13
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +14 -13
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +14 -13
- cloudnetpy/model_evaluation/tests/unit/conftest.py +11 -11
- cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +33 -27
- cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +83 -83
- cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
- cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +24 -25
- cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +40 -39
- cloudnetpy/model_evaluation/tests/unit/test_plotting.py +12 -11
- cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +30 -30
- cloudnetpy/model_evaluation/tests/unit/test_tools.py +18 -17
- cloudnetpy/model_evaluation/utils.py +3 -2
- cloudnetpy/output.py +45 -19
- cloudnetpy/plotting/plot_meta.py +35 -11
- cloudnetpy/plotting/plotting.py +172 -104
- cloudnetpy/products/classification.py +20 -8
- cloudnetpy/products/der.py +25 -10
- cloudnetpy/products/drizzle.py +41 -26
- cloudnetpy/products/drizzle_error.py +10 -5
- cloudnetpy/products/drizzle_tools.py +43 -24
- cloudnetpy/products/ier.py +10 -5
- cloudnetpy/products/iwc.py +16 -9
- cloudnetpy/products/lwc.py +34 -12
- cloudnetpy/products/mwr_multi.py +4 -1
- cloudnetpy/products/mwr_single.py +4 -1
- cloudnetpy/products/product_tools.py +33 -10
- cloudnetpy/utils.py +175 -74
- cloudnetpy/version.py +1 -1
- {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/METADATA +11 -10
- cloudnetpy-1.55.22.dist-info/RECORD +114 -0
- docs/source/conf.py +2 -2
- cloudnetpy-1.55.20.dist-info/RECORD +0 -114
- {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/LICENSE +0 -0
- {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/WHEEL +0 -0
- {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/top_level.txt +0 -0
@@ -1,5 +1,5 @@
|
|
1
1
|
import logging
|
2
|
-
from typing import NamedTuple
|
2
|
+
from typing import TYPE_CHECKING, NamedTuple
|
3
3
|
|
4
4
|
import numpy as np
|
5
5
|
from numpy import ma
|
@@ -8,9 +8,11 @@ from scipy.ndimage import gaussian_filter
|
|
8
8
|
from cloudnetpy import utils
|
9
9
|
from cloudnetpy.cloudnetarray import CloudnetArray
|
10
10
|
from cloudnetpy.exceptions import ValidTimeStampError
|
11
|
-
from cloudnetpy.instruments.instruments import Instrument
|
12
11
|
from cloudnetpy.utils import Epoch
|
13
12
|
|
13
|
+
if TYPE_CHECKING:
|
14
|
+
from cloudnetpy.instruments.instruments import Instrument
|
15
|
+
|
14
16
|
|
15
17
|
class NoiseParam(NamedTuple):
|
16
18
|
"""Noise parameters. Values are weakly instrument-dependent."""
|
@@ -22,8 +24,8 @@ class NoiseParam(NamedTuple):
|
|
22
24
|
class Ceilometer:
|
23
25
|
"""Base class for all types of ceilometers and pollyxt."""
|
24
26
|
|
25
|
-
def __init__(self, noise_param: NoiseParam =
|
26
|
-
self.noise_param = noise_param
|
27
|
+
def __init__(self, noise_param: NoiseParam | None = None):
|
28
|
+
self.noise_param = noise_param or NoiseParam()
|
27
29
|
self.data: dict = {} # Need to contain 'beta_raw', 'range' and 'time'
|
28
30
|
self.metadata: dict = {} # Need to contain 'date' as ('yyyy', 'mm', 'dd')
|
29
31
|
self.expected_date: str | None = None
|
@@ -36,10 +38,15 @@ class Ceilometer:
|
|
36
38
|
self,
|
37
39
|
array: np.ndarray,
|
38
40
|
snr_limit: int = 5,
|
41
|
+
*,
|
39
42
|
range_corrected: bool = True,
|
40
43
|
) -> np.ndarray:
|
41
44
|
"""Screens noise from lidar variable."""
|
42
|
-
noisy_data = NoisyData(
|
45
|
+
noisy_data = NoisyData(
|
46
|
+
self.data,
|
47
|
+
self.noise_param,
|
48
|
+
range_corrected=range_corrected,
|
49
|
+
)
|
43
50
|
if (
|
44
51
|
self.instrument is not None
|
45
52
|
and getattr(self.instrument, "model", "").lower() == "ct25k"
|
@@ -47,40 +54,48 @@ class Ceilometer:
|
|
47
54
|
n_negatives = 20
|
48
55
|
else:
|
49
56
|
n_negatives = 5
|
50
|
-
|
51
|
-
array,
|
57
|
+
return noisy_data.screen_data(
|
58
|
+
array,
|
59
|
+
snr_limit=snr_limit,
|
60
|
+
n_negatives=n_negatives,
|
52
61
|
)
|
53
|
-
return array_screened
|
54
62
|
|
55
63
|
def calc_beta_smooth(
|
56
64
|
self,
|
57
65
|
beta: np.ndarray,
|
58
66
|
snr_limit: int = 5,
|
67
|
+
*,
|
59
68
|
range_corrected: bool = True,
|
60
69
|
) -> np.ndarray:
|
61
|
-
noisy_data = NoisyData(
|
70
|
+
noisy_data = NoisyData(
|
71
|
+
self.data,
|
72
|
+
self.noise_param,
|
73
|
+
range_corrected=range_corrected,
|
74
|
+
)
|
62
75
|
beta_raw = ma.copy(self.data["beta_raw"])
|
63
76
|
cloud_ind, cloud_values, cloud_limit = _estimate_clouds_from_beta(beta)
|
64
77
|
beta_raw[cloud_ind] = cloud_limit
|
65
78
|
sigma = calc_sigma_units(self.data["time"], self.data["range"])
|
66
79
|
beta_raw_smooth = gaussian_filter(beta_raw, sigma)
|
67
80
|
beta_raw_smooth[cloud_ind] = cloud_values
|
68
|
-
|
69
|
-
beta_raw_smooth,
|
81
|
+
return noisy_data.screen_data(
|
82
|
+
beta_raw_smooth,
|
83
|
+
is_smoothed=True,
|
84
|
+
snr_limit=snr_limit,
|
70
85
|
)
|
71
|
-
return beta_smooth
|
72
86
|
|
73
|
-
def prepare_data(self):
|
87
|
+
def prepare_data(self) -> None:
|
74
88
|
"""Add common additional data / metadata and convert into CloudnetArrays."""
|
75
89
|
zenith_angle = self.data["zenith_angle"]
|
76
90
|
self.data["height"] = np.array(
|
77
91
|
self.site_meta["altitude"]
|
78
|
-
+ utils.range_to_height(self.data["range"], zenith_angle)
|
92
|
+
+ utils.range_to_height(self.data["range"], zenith_angle),
|
79
93
|
)
|
80
94
|
for key in ("time", "range"):
|
81
95
|
self.data[key] = np.array(self.data[key])
|
82
|
-
|
83
|
-
|
96
|
+
if self.instrument is None or self.instrument.wavelength is None:
|
97
|
+
msg = "Instrument wavelength not defined"
|
98
|
+
raise RuntimeError(msg)
|
84
99
|
self.data["wavelength"] = float(self.instrument.wavelength)
|
85
100
|
for key in ("latitude", "longitude", "altitude"):
|
86
101
|
if key in self.site_meta:
|
@@ -92,37 +107,42 @@ class Ceilometer:
|
|
92
107
|
self.date = utils.seconds2date(self.data["time"][0], epoch=epoch)[:3]
|
93
108
|
self.data["time"] = utils.seconds2hours(self.data["time"])
|
94
109
|
|
95
|
-
def data_to_cloudnet_arrays(self):
|
110
|
+
def data_to_cloudnet_arrays(self) -> None:
|
96
111
|
for key, array in self.data.items():
|
97
112
|
self.data[key] = CloudnetArray(array, key)
|
98
113
|
|
99
|
-
def screen_depol(self):
|
114
|
+
def screen_depol(self) -> None:
|
100
115
|
key = "depolarisation"
|
101
116
|
if key in self.data:
|
102
117
|
self.data[key][self.data[key] <= 0] = ma.masked
|
103
118
|
self.data[key][self.data[key] > 1] = ma.masked
|
104
119
|
|
105
|
-
def screen_invalid_values(self):
|
106
|
-
for key in self.data
|
120
|
+
def screen_invalid_values(self) -> None:
|
121
|
+
for key in self.data:
|
107
122
|
try:
|
108
123
|
if self.data[key][:].ndim == 2:
|
109
124
|
self.data[key] = ma.masked_invalid(self.data[key])
|
110
125
|
except (IndexError, TypeError):
|
111
126
|
continue
|
112
127
|
|
113
|
-
def add_snr_info(self, key: str, snr_limit: float):
|
128
|
+
def add_snr_info(self, key: str, snr_limit: float) -> None:
|
114
129
|
if key in self.data:
|
115
130
|
self.data[key].comment += f" SNR threshold applied: {snr_limit}."
|
116
131
|
|
117
|
-
def check_beta_raw_shape(self):
|
132
|
+
def check_beta_raw_shape(self) -> None:
|
118
133
|
beta_raw = self.data["beta_raw"]
|
119
134
|
if beta_raw.ndim != 2 or (beta_raw.shape[0] == 1 or beta_raw.shape[1] == 1):
|
120
|
-
|
135
|
+
msg = f"Invalid beta_raw shape: {beta_raw.shape}"
|
136
|
+
raise ValidTimeStampError(msg)
|
121
137
|
|
122
138
|
|
123
139
|
class NoisyData:
|
124
140
|
def __init__(
|
125
|
-
self,
|
141
|
+
self,
|
142
|
+
data: dict,
|
143
|
+
noise_param: NoiseParam,
|
144
|
+
*,
|
145
|
+
range_corrected: bool = True,
|
126
146
|
):
|
127
147
|
self.data = data
|
128
148
|
self.noise_param = noise_param
|
@@ -132,20 +152,22 @@ class NoisyData:
|
|
132
152
|
self,
|
133
153
|
data_in: np.ndarray,
|
134
154
|
snr_limit: float = 5,
|
155
|
+
n_negatives: int = 5,
|
156
|
+
*,
|
135
157
|
is_smoothed: bool = False,
|
136
158
|
keep_negative: bool = False,
|
137
159
|
filter_fog: bool = True,
|
138
160
|
filter_negatives: bool = True,
|
139
161
|
filter_snr: bool = True,
|
140
|
-
n_negatives: int = 5,
|
141
162
|
) -> np.ndarray:
|
142
163
|
data = ma.copy(data_in)
|
143
164
|
self._calc_range_uncorrected(data)
|
144
165
|
noise = _estimate_background_noise(data)
|
145
|
-
noise = self._adjust_noise(noise, is_smoothed)
|
166
|
+
noise = self._adjust_noise(noise, is_smoothed=is_smoothed)
|
146
167
|
if filter_negatives is True:
|
147
168
|
is_negative = self._mask_low_values_above_consequent_negatives(
|
148
|
-
data,
|
169
|
+
data,
|
170
|
+
n_negatives=n_negatives,
|
149
171
|
)
|
150
172
|
noise[is_negative] = 1e-12
|
151
173
|
if filter_fog is True:
|
@@ -153,18 +175,26 @@ class NoisyData:
|
|
153
175
|
self._clean_fog_profiles(data, is_fog)
|
154
176
|
noise[is_fog] = 1e-12
|
155
177
|
if filter_snr is True:
|
156
|
-
data = self._remove_noise(
|
178
|
+
data = self._remove_noise(
|
179
|
+
data,
|
180
|
+
noise,
|
181
|
+
keep_negative=keep_negative,
|
182
|
+
snr_limit=snr_limit,
|
183
|
+
)
|
157
184
|
self._calc_range_corrected(data)
|
158
185
|
return data
|
159
186
|
|
160
|
-
def _adjust_noise(self, noise: np.ndarray, is_smoothed: bool) -> np.ndarray:
|
187
|
+
def _adjust_noise(self, noise: np.ndarray, *, is_smoothed: bool) -> np.ndarray:
|
161
188
|
noise_min = (
|
162
189
|
self.noise_param.noise_smooth_min
|
163
190
|
if is_smoothed is True
|
164
191
|
else self.noise_param.noise_min
|
165
192
|
)
|
166
193
|
noise_below_threshold = noise < noise_min
|
167
|
-
logging.debug(
|
194
|
+
logging.debug(
|
195
|
+
"Adjusted noise of %s profiles",
|
196
|
+
sum(np.array(noise_below_threshold)),
|
197
|
+
)
|
168
198
|
noise[noise_below_threshold] = noise_min
|
169
199
|
return noise
|
170
200
|
|
@@ -180,12 +210,13 @@ class NoisyData:
|
|
180
210
|
n_consequent_negatives = utils.cumsumr(negative_data, axis=1)
|
181
211
|
time_indices, alt_indices = np.where(n_consequent_negatives > n_negatives)
|
182
212
|
alt_indices += n_skip_lowest
|
183
|
-
for time_ind, alt_ind in zip(time_indices, alt_indices):
|
213
|
+
for time_ind, alt_ind in zip(time_indices, alt_indices, strict=True):
|
184
214
|
profile = data[time_ind, alt_ind:]
|
185
215
|
profile[profile < threshold] = ma.masked
|
186
216
|
cleaned_time_indices = np.unique(time_indices)
|
187
217
|
logging.debug(
|
188
|
-
|
218
|
+
"Cleaned %s profiles with negative filter",
|
219
|
+
len(cleaned_time_indices),
|
189
220
|
)
|
190
221
|
return cleaned_time_indices
|
191
222
|
|
@@ -197,17 +228,19 @@ class NoisyData:
|
|
197
228
|
) -> np.ndarray:
|
198
229
|
"""Finds saturated (usually fog) profiles from beta_raw."""
|
199
230
|
signal_sum = ma.sum(
|
200
|
-
ma.abs(self.data["beta_raw"][:, :n_gates_for_signal_sum]),
|
231
|
+
ma.abs(self.data["beta_raw"][:, :n_gates_for_signal_sum]),
|
232
|
+
axis=1,
|
201
233
|
)
|
202
234
|
variance = _calc_var_from_top_gates(self.data["beta_raw"])
|
203
235
|
is_fog = (signal_sum > signal_sum_threshold) | (variance < variance_threshold)
|
204
|
-
logging.debug(
|
236
|
+
logging.debug("Cleaned %s profiles with fog filter", sum(is_fog))
|
205
237
|
return is_fog
|
206
238
|
|
207
239
|
def _remove_noise(
|
208
240
|
self,
|
209
241
|
array: np.ndarray,
|
210
242
|
noise: np.ndarray,
|
243
|
+
*,
|
211
244
|
keep_negative: bool,
|
212
245
|
snr_limit: float,
|
213
246
|
) -> np.ndarray:
|
@@ -236,7 +269,8 @@ class NoisyData:
|
|
236
269
|
if self.range_corrected is False:
|
237
270
|
alt_limit = 2400.0
|
238
271
|
logging.warning(
|
239
|
-
|
272
|
+
"Raw data not range-corrected, correcting below %s m",
|
273
|
+
alt_limit,
|
240
274
|
)
|
241
275
|
else:
|
242
276
|
alt_limit = 1e12
|
@@ -249,7 +283,9 @@ class NoisyData:
|
|
249
283
|
|
250
284
|
@staticmethod
|
251
285
|
def _clean_fog_profiles(
|
252
|
-
data: np.ndarray,
|
286
|
+
data: np.ndarray,
|
287
|
+
is_fog: np.ndarray,
|
288
|
+
threshold: float = 2e-6,
|
253
289
|
) -> None:
|
254
290
|
"""Removes values in saturated (e.g. fog) profiles above peak."""
|
255
291
|
for time_ind in np.where(is_fog)[0]:
|
@@ -281,18 +317,21 @@ def calc_sigma_units(
|
|
281
317
|
how many steps in time and height corresponds to this smoothing.
|
282
318
|
|
283
319
|
Args:
|
320
|
+
----
|
284
321
|
time_vector: 1D vector (fraction hour).
|
285
322
|
range_los: 1D vector (m).
|
286
323
|
sigma_minutes: Smoothing in minutes.
|
287
324
|
sigma_metres: Smoothing in metres.
|
288
325
|
|
289
326
|
Returns:
|
327
|
+
-------
|
290
328
|
tuple: Two element tuple containing number of steps in time and height to
|
291
329
|
achieve wanted smoothing.
|
292
330
|
|
293
331
|
"""
|
294
332
|
if len(time_vector) == 0 or np.max(time_vector) > 24:
|
295
|
-
|
333
|
+
msg = "Invalid time vector"
|
334
|
+
raise ValueError(msg)
|
296
335
|
minutes_in_hour = 60
|
297
336
|
time_step = utils.mdiff(time_vector) * minutes_in_hour
|
298
337
|
alt_step = utils.mdiff(range_los)
|
cloudnetpy/instruments/cl61d.py
CHANGED
@@ -11,7 +11,10 @@ class Cl61d(NcLidar):
|
|
11
11
|
"""Class for Vaisala CL61d ceilometer."""
|
12
12
|
|
13
13
|
def __init__(
|
14
|
-
self,
|
14
|
+
self,
|
15
|
+
file_name: str,
|
16
|
+
site_meta: dict,
|
17
|
+
expected_date: str | None = None,
|
15
18
|
):
|
16
19
|
super().__init__()
|
17
20
|
self.file_name = file_name
|
@@ -31,7 +34,9 @@ class Cl61d(NcLidar):
|
|
31
34
|
self.dataset = None
|
32
35
|
|
33
36
|
def _fetch_lidar_variables(self, calibration_factor: float | None = None) -> None:
|
34
|
-
|
37
|
+
if self.dataset is None:
|
38
|
+
msg = "No dataset found"
|
39
|
+
raise RuntimeError(msg)
|
35
40
|
beta_raw = self.dataset.variables["beta_att"][:]
|
36
41
|
if calibration_factor is None:
|
37
42
|
logging.warning("Using default calibration factor")
|
@@ -44,5 +49,5 @@ class Cl61d(NcLidar):
|
|
44
49
|
)
|
45
50
|
self.data["depolarisation_raw"] = self.data["depolarisation"].copy()
|
46
51
|
|
47
|
-
def _fetch_attributes(self):
|
52
|
+
def _fetch_attributes(self) -> None:
|
48
53
|
self.serial_number = getattr(self.dataset, "instrument_serial_number", None)
|
@@ -24,7 +24,8 @@ class CloudnetInstrument:
|
|
24
24
|
value = self.site_meta[key]
|
25
25
|
# From source global attributes (MIRA):
|
26
26
|
elif isinstance(self.dataset, netCDF4.Dataset) and hasattr(
|
27
|
-
self.dataset,
|
27
|
+
self.dataset,
|
28
|
+
key.capitalize(),
|
28
29
|
):
|
29
30
|
value = self.parse_global_attribute_numeral(key.capitalize())
|
30
31
|
# From source data (BASTA / RPG):
|
@@ -1,5 +1,6 @@
|
|
1
1
|
"""Module for reading raw cloud radar data."""
|
2
2
|
import os
|
3
|
+
import tempfile
|
3
4
|
from tempfile import TemporaryDirectory
|
4
5
|
|
5
6
|
import numpy as np
|
@@ -20,6 +21,7 @@ def copernicus2nc(
|
|
20
21
|
"""Converts 'Copernicus' cloud radar data into Cloudnet Level 1b netCDF file.
|
21
22
|
|
22
23
|
Args:
|
24
|
+
----
|
23
25
|
raw_files: Input file name or folder containing multiple input files.
|
24
26
|
output_file: Output filename.
|
25
27
|
site_meta: Dictionary containing information about the site. Required key
|
@@ -29,12 +31,15 @@ def copernicus2nc(
|
|
29
31
|
date: Expected date as YYYY-MM-DD of all profiles in the file.
|
30
32
|
|
31
33
|
Returns:
|
34
|
+
-------
|
32
35
|
UUID of the generated file.
|
33
36
|
|
34
37
|
Raises:
|
38
|
+
------
|
35
39
|
ValidTimeStampError: No valid timestamps found.
|
36
40
|
|
37
41
|
Examples:
|
42
|
+
--------
|
38
43
|
>>> from cloudnetpy.instruments import copernicus2nc
|
39
44
|
>>> site_meta = {'name': 'Chilbolton'}
|
40
45
|
>>> copernicus2nc('raw_radar.nc', 'radar.nc', site_meta)
|
@@ -57,13 +62,20 @@ def copernicus2nc(
|
|
57
62
|
|
58
63
|
with TemporaryDirectory() as temp_dir:
|
59
64
|
if os.path.isdir(raw_files):
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
65
|
+
with tempfile.NamedTemporaryFile(
|
66
|
+
dir=temp_dir,
|
67
|
+
suffix=".nc",
|
68
|
+
delete=False,
|
69
|
+
) as temp_file:
|
70
|
+
nc_filename = temp_file.name
|
71
|
+
valid_filenames = utils.get_sorted_filenames(raw_files, ".nc")
|
72
|
+
valid_filenames = utils.get_files_with_common_range(valid_filenames)
|
73
|
+
variables = list(keymap.keys())
|
74
|
+
concat_lib.concatenate_files(
|
75
|
+
valid_filenames,
|
76
|
+
nc_filename,
|
77
|
+
variables=variables,
|
78
|
+
)
|
67
79
|
else:
|
68
80
|
nc_filename = raw_files
|
69
81
|
|
@@ -90,14 +102,14 @@ def copernicus2nc(
|
|
90
102
|
copernicus.add_height()
|
91
103
|
attributes = output.add_time_attribute(ATTRIBUTES, copernicus.date)
|
92
104
|
output.update_attributes(copernicus.data, attributes)
|
93
|
-
|
94
|
-
return uuid
|
105
|
+
return output.save_level1b(copernicus, output_file, uuid)
|
95
106
|
|
96
107
|
|
97
108
|
class Copernicus(ChilboltonRadar):
|
98
109
|
"""Class for Copernicus raw radar data. Child of ChilboltonRadar().
|
99
110
|
|
100
111
|
Args:
|
112
|
+
----
|
101
113
|
full_path: Filename of a daily Copernicus .nc NetCDF file.
|
102
114
|
site_meta: Site properties in a dictionary. Required keys are: `name`.
|
103
115
|
|
@@ -107,16 +119,17 @@ class Copernicus(ChilboltonRadar):
|
|
107
119
|
super().__init__(full_path, site_meta)
|
108
120
|
self.instrument = COPERNICUS
|
109
121
|
|
110
|
-
def calibrate_reflectivity(self):
|
122
|
+
def calibrate_reflectivity(self) -> None:
|
111
123
|
default_offset = -146.8 # TODO: check this value
|
112
124
|
calibration_factor = self.site_meta.get("calibration_offset", default_offset)
|
113
125
|
self.data["Zh"].data[:] += calibration_factor
|
114
126
|
self.append_data(np.array(calibration_factor), "calibration_offset")
|
115
127
|
|
116
|
-
def mask_corrupted_values(self):
|
128
|
+
def mask_corrupted_values(self) -> None:
|
117
129
|
"""Experimental masking of corrupted Copernicus data.
|
118
130
|
|
119
|
-
Notes
|
131
|
+
Notes
|
132
|
+
-----
|
120
133
|
This method is based on a few days of test data only. Should be improved
|
121
134
|
and tested more carefully in the future.
|
122
135
|
"""
|
@@ -125,13 +138,13 @@ class Copernicus(ChilboltonRadar):
|
|
125
138
|
ind = np.where(np.abs(self.data[key][:]) > value)
|
126
139
|
self.data["v"].mask_indices(ind)
|
127
140
|
|
128
|
-
def fix_range_offset(self, site_meta: dict):
|
141
|
+
def fix_range_offset(self, site_meta: dict) -> None:
|
129
142
|
"""Fixes range offset."""
|
130
143
|
range_offset = site_meta.get("range_offset", 0)
|
131
144
|
self.data["range"].data[:] += range_offset
|
132
145
|
self.append_data(np.array(range_offset, dtype=float), "range_offset")
|
133
146
|
|
134
|
-
def screen_negative_ranges(self):
|
147
|
+
def screen_negative_ranges(self) -> None:
|
135
148
|
"""Screens negative range values."""
|
136
149
|
valid_ind = np.where(self.data["range"][:] >= 0)[0]
|
137
150
|
for key, cloudnet_array in self.data.items():
|
@@ -6,6 +6,7 @@ from numpy import ma
|
|
6
6
|
|
7
7
|
from cloudnetpy import utils
|
8
8
|
from cloudnetpy.cloudnetarray import CloudnetArray
|
9
|
+
from cloudnetpy.constants import MM_TO_M, SEC_IN_HOUR, SEC_IN_MINUTE
|
9
10
|
from cloudnetpy.exceptions import DisdrometerDataError, ValidTimeStampError
|
10
11
|
from cloudnetpy.instruments.cloudnet_instrument import CloudnetInstrument
|
11
12
|
from cloudnetpy.instruments.vaisala import values_to_dict
|
@@ -27,14 +28,13 @@ class Disdrometer(CloudnetInstrument):
|
|
27
28
|
self.n_velocity: int = 0
|
28
29
|
self._file_data = self._read_file()
|
29
30
|
|
30
|
-
def convert_units(self):
|
31
|
-
|
32
|
-
mmh_to_ms = 3600 * mm_to_m
|
31
|
+
def convert_units(self) -> None:
|
32
|
+
mmh_to_ms = SEC_IN_HOUR / MM_TO_M
|
33
33
|
c_to_k = 273.15
|
34
34
|
self._convert_data(("rainfall_rate_1min_total",), mmh_to_ms)
|
35
35
|
self._convert_data(("rainfall_rate",), mmh_to_ms)
|
36
36
|
self._convert_data(("rainfall_rate_1min_solid",), mmh_to_ms)
|
37
|
-
self._convert_data(("diameter", "diameter_spread", "diameter_bnds"),
|
37
|
+
self._convert_data(("diameter", "diameter_spread", "diameter_bnds"), 1e3)
|
38
38
|
self._convert_data(("V_sensor_supply",), 10)
|
39
39
|
self._convert_data(("I_mean_laser",), 100)
|
40
40
|
self._convert_data(("T_sensor",), c_to_k, method="add")
|
@@ -42,12 +42,12 @@ class Disdrometer(CloudnetInstrument):
|
|
42
42
|
self._convert_data(("T_ambient",), c_to_k, method="add")
|
43
43
|
self._convert_data(("T_laser_driver",), c_to_k, method="add")
|
44
44
|
|
45
|
-
def add_meta(self):
|
46
|
-
|
45
|
+
def add_meta(self) -> None:
|
46
|
+
valid_names = ("latitude", "longitude", "altitude")
|
47
47
|
for key, value in self.site_meta.items():
|
48
|
-
|
49
|
-
if
|
50
|
-
self.data[
|
48
|
+
name = key.lower()
|
49
|
+
if name in valid_names:
|
50
|
+
self.data[name] = CloudnetArray(float(value), name)
|
51
51
|
|
52
52
|
def validate_date(self, expected_date: str) -> None:
|
53
53
|
valid_ind = []
|
@@ -95,7 +95,7 @@ class Disdrometer(CloudnetInstrument):
|
|
95
95
|
return data
|
96
96
|
|
97
97
|
def _append_data(self, column_and_key: list) -> None:
|
98
|
-
indices, keys = zip(*column_and_key)
|
98
|
+
indices, keys = zip(*column_and_key, strict=True)
|
99
99
|
data = self._parse_useful_data(indices)
|
100
100
|
data_dict = values_to_dict(keys, data)
|
101
101
|
for key in keys:
|
@@ -108,7 +108,8 @@ class Disdrometer(CloudnetInstrument):
|
|
108
108
|
float_array = ma.append(float_array, float(value_str))
|
109
109
|
except ValueError:
|
110
110
|
logging.warning(
|
111
|
-
|
111
|
+
"Invalid character: %s, masking a data point",
|
112
|
+
value_str,
|
112
113
|
)
|
113
114
|
float_array = ma.append(float_array, invalid_value)
|
114
115
|
float_array[float_array == invalid_value] = ma.masked
|
@@ -131,12 +132,12 @@ class Disdrometer(CloudnetInstrument):
|
|
131
132
|
first_id = data_dict["_serial_number"][0]
|
132
133
|
for sensor_id in data_dict["_serial_number"]:
|
133
134
|
if sensor_id != first_id:
|
134
|
-
|
135
|
-
|
136
|
-
|
135
|
+
msg = "Multiple serial numbers are not supported"
|
136
|
+
raise DisdrometerDataError(msg)
|
137
|
+
|
137
138
|
self.serial_number = first_id
|
138
139
|
|
139
|
-
def _parse_useful_data(self, indices:
|
140
|
+
def _parse_useful_data(self, indices: tuple) -> list:
|
140
141
|
data = []
|
141
142
|
for row in self._file_data["scalars"]:
|
142
143
|
useful_data = [row[ind] for ind in indices]
|
@@ -149,10 +150,12 @@ class Disdrometer(CloudnetInstrument):
|
|
149
150
|
if self.source == PARSIVEL:
|
150
151
|
raise NotImplementedError
|
151
152
|
hour, minute, sec = timestamp.split(":")
|
152
|
-
seconds.append(
|
153
|
+
seconds.append(
|
154
|
+
int(hour) * SEC_IN_HOUR + int(minute) * SEC_IN_MINUTE + int(sec)
|
155
|
+
)
|
153
156
|
return CloudnetArray(utils.seconds2hours(np.array(seconds)), "time")
|
154
157
|
|
155
|
-
def _convert_data(self, keys: tuple, value: float, method: str = "divide"):
|
158
|
+
def _convert_data(self, keys: tuple, value: float, method: str = "divide") -> None:
|
156
159
|
for key in keys:
|
157
160
|
if key in self.data:
|
158
161
|
if method == "divide":
|
@@ -162,14 +165,15 @@ class Disdrometer(CloudnetInstrument):
|
|
162
165
|
else:
|
163
166
|
raise ValueError
|
164
167
|
|
165
|
-
def _append_spectra(self):
|
168
|
+
def _append_spectra(self) -> None:
|
166
169
|
array = ma.masked_all(
|
167
|
-
(len(self._file_data["scalars"]), self.n_diameter, self.n_velocity)
|
170
|
+
(len(self._file_data["scalars"]), self.n_diameter, self.n_velocity),
|
168
171
|
)
|
169
172
|
for time_ind, row in enumerate(self._file_data["spectra"]):
|
170
173
|
values = _parse_int(row)
|
171
174
|
array[time_ind, :, :] = np.reshape(
|
172
|
-
values,
|
175
|
+
values,
|
176
|
+
(self.n_diameter, self.n_velocity),
|
173
177
|
)
|
174
178
|
self.data["data_raw"] = CloudnetArray(
|
175
179
|
array,
|
@@ -180,7 +184,12 @@ class Disdrometer(CloudnetInstrument):
|
|
180
184
|
|
181
185
|
@classmethod
|
182
186
|
def store_vectors(
|
183
|
-
cls,
|
187
|
+
cls,
|
188
|
+
data,
|
189
|
+
n_values: list,
|
190
|
+
spreads: list,
|
191
|
+
name: str,
|
192
|
+
start: float = 0.0,
|
184
193
|
):
|
185
194
|
mid, bounds, spread = cls._create_vectors(n_values, spreads, start)
|
186
195
|
data[name] = CloudnetArray(mid, name, dimensions=(name,))
|
@@ -191,12 +200,14 @@ class Disdrometer(CloudnetInstrument):
|
|
191
200
|
|
192
201
|
@staticmethod
|
193
202
|
def _create_vectors(
|
194
|
-
n_values: list[int],
|
203
|
+
n_values: list[int],
|
204
|
+
spreads: list[float],
|
205
|
+
start: float,
|
195
206
|
) -> tuple:
|
196
207
|
mid_value: np.ndarray = np.array([])
|
197
208
|
lower_limit: np.ndarray = np.array([])
|
198
209
|
upper_limit: np.ndarray = np.array([])
|
199
|
-
for spread, n in zip(spreads, n_values):
|
210
|
+
for spread, n in zip(spreads, n_values, strict=True):
|
200
211
|
lower = np.linspace(start, start + (n - 1) * spread, n)
|
201
212
|
upper = lower + spread
|
202
213
|
lower_limit = np.append(lower_limit, lower)
|
@@ -208,7 +219,7 @@ class Disdrometer(CloudnetInstrument):
|
|
208
219
|
return mid_value, bounds, spread
|
209
220
|
|
210
221
|
|
211
|
-
def _format_thies_date(date: str):
|
222
|
+
def _format_thies_date(date: str) -> str:
|
212
223
|
day, month, year = date.split(".")
|
213
224
|
year = f"20{year}"
|
214
225
|
return f"{year}-{month.zfill(2)}-{day.zfill(2)}"
|
@@ -218,9 +229,9 @@ def _parse_int(row: np.ndarray) -> np.ndarray:
|
|
218
229
|
values = ma.masked_all((len(row),))
|
219
230
|
for ind, value in enumerate(row):
|
220
231
|
try:
|
221
|
-
|
222
|
-
if
|
223
|
-
values[ind] =
|
232
|
+
value_int = int(value)
|
233
|
+
if value_int != 0:
|
234
|
+
values[ind] = value_int
|
224
235
|
except ValueError:
|
225
236
|
pass
|
226
237
|
return values
|
@@ -262,7 +273,9 @@ ATTRIBUTES = {
|
|
262
273
|
units="m s-1",
|
263
274
|
),
|
264
275
|
"rainfall_rate": MetaData(
|
265
|
-
long_name="Rainfall rate",
|
276
|
+
long_name="Rainfall rate",
|
277
|
+
units="m s-1",
|
278
|
+
standard_name="rainfall_rate",
|
266
279
|
),
|
267
280
|
"rainfall_rate_1min_solid": MetaData(
|
268
281
|
long_name="Solid precipitation rate",
|
@@ -288,7 +301,8 @@ ATTRIBUTES = {
|
|
288
301
|
"interval": MetaData(long_name="Length of measurement interval", units="s"),
|
289
302
|
"sig_laser": MetaData(long_name="Signal amplitude of the laser strip", units="1"),
|
290
303
|
"n_particles": MetaData(
|
291
|
-
long_name="Number of particles in time interval",
|
304
|
+
long_name="Number of particles in time interval",
|
305
|
+
units="1",
|
292
306
|
),
|
293
307
|
"T_sensor": MetaData(
|
294
308
|
long_name="Temperature in the sensor housing",
|
@@ -326,7 +340,8 @@ ATTRIBUTES = {
|
|
326
340
|
units="1",
|
327
341
|
),
|
328
342
|
"kinetic_energy": MetaData(
|
329
|
-
long_name="Kinetic energy of the hydrometeors",
|
343
|
+
long_name="Kinetic energy of the hydrometeors",
|
344
|
+
units="J m-2 h-1",
|
330
345
|
),
|
331
346
|
# Thies-specific:
|
332
347
|
"T_ambient": MetaData(long_name="Ambient temperature", units="K"),
|
@@ -397,12 +412,16 @@ ATTRIBUTES = {
|
|
397
412
|
units="1",
|
398
413
|
),
|
399
414
|
"status_laser": MetaData(
|
400
|
-
long_name="Status of laser",
|
415
|
+
long_name="Status of laser",
|
416
|
+
comment="0 = OK/on , 1 = Off",
|
417
|
+
units="1",
|
401
418
|
),
|
402
419
|
"measurement_quality": MetaData(long_name="Measurement quality", units="%"),
|
403
420
|
"maximum_hail_diameter": MetaData(long_name="Maximum hail diameter", units="mm"),
|
404
421
|
"static_signal": MetaData(
|
405
|
-
long_name="Static signal",
|
422
|
+
long_name="Static signal",
|
423
|
+
comment="0 = OK, 1 = ERROR",
|
424
|
+
units="1",
|
406
425
|
),
|
407
426
|
"T_laser_driver": MetaData(long_name="Temperature of laser driver", units="K"),
|
408
427
|
"I_mean_laser": MetaData(long_name="Mean value of laser current", units="mA"),
|