cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudnetpy/categorize/__init__.py +1 -2
- cloudnetpy/categorize/atmos_utils.py +297 -67
- cloudnetpy/categorize/attenuation.py +31 -0
- cloudnetpy/categorize/attenuations/__init__.py +37 -0
- cloudnetpy/categorize/attenuations/gas_attenuation.py +30 -0
- cloudnetpy/categorize/attenuations/liquid_attenuation.py +84 -0
- cloudnetpy/categorize/attenuations/melting_attenuation.py +78 -0
- cloudnetpy/categorize/attenuations/rain_attenuation.py +84 -0
- cloudnetpy/categorize/categorize.py +332 -156
- cloudnetpy/categorize/classify.py +127 -125
- cloudnetpy/categorize/containers.py +107 -76
- cloudnetpy/categorize/disdrometer.py +40 -0
- cloudnetpy/categorize/droplet.py +23 -21
- cloudnetpy/categorize/falling.py +53 -24
- cloudnetpy/categorize/freezing.py +25 -12
- cloudnetpy/categorize/insects.py +35 -23
- cloudnetpy/categorize/itu.py +243 -0
- cloudnetpy/categorize/lidar.py +36 -41
- cloudnetpy/categorize/melting.py +34 -26
- cloudnetpy/categorize/model.py +84 -37
- cloudnetpy/categorize/mwr.py +18 -14
- cloudnetpy/categorize/radar.py +215 -102
- cloudnetpy/cli.py +578 -0
- cloudnetpy/cloudnetarray.py +43 -89
- cloudnetpy/concat_lib.py +218 -78
- cloudnetpy/constants.py +28 -10
- cloudnetpy/datasource.py +61 -86
- cloudnetpy/exceptions.py +49 -20
- cloudnetpy/instruments/__init__.py +5 -0
- cloudnetpy/instruments/basta.py +29 -12
- cloudnetpy/instruments/bowtie.py +135 -0
- cloudnetpy/instruments/ceilo.py +138 -115
- cloudnetpy/instruments/ceilometer.py +164 -80
- cloudnetpy/instruments/cl61d.py +21 -5
- cloudnetpy/instruments/cloudnet_instrument.py +74 -36
- cloudnetpy/instruments/copernicus.py +108 -30
- cloudnetpy/instruments/da10.py +54 -0
- cloudnetpy/instruments/disdrometer/common.py +126 -223
- cloudnetpy/instruments/disdrometer/parsivel.py +453 -94
- cloudnetpy/instruments/disdrometer/thies.py +254 -87
- cloudnetpy/instruments/fd12p.py +201 -0
- cloudnetpy/instruments/galileo.py +65 -23
- cloudnetpy/instruments/hatpro.py +123 -49
- cloudnetpy/instruments/instruments.py +113 -1
- cloudnetpy/instruments/lufft.py +39 -17
- cloudnetpy/instruments/mira.py +268 -61
- cloudnetpy/instruments/mrr.py +187 -0
- cloudnetpy/instruments/nc_lidar.py +19 -8
- cloudnetpy/instruments/nc_radar.py +109 -55
- cloudnetpy/instruments/pollyxt.py +135 -51
- cloudnetpy/instruments/radiometrics.py +313 -59
- cloudnetpy/instruments/rain_e_h3.py +171 -0
- cloudnetpy/instruments/rpg.py +321 -189
- cloudnetpy/instruments/rpg_reader.py +74 -40
- cloudnetpy/instruments/toa5.py +49 -0
- cloudnetpy/instruments/vaisala.py +95 -343
- cloudnetpy/instruments/weather_station.py +774 -105
- cloudnetpy/metadata.py +90 -19
- cloudnetpy/model_evaluation/file_handler.py +55 -52
- cloudnetpy/model_evaluation/metadata.py +46 -20
- cloudnetpy/model_evaluation/model_metadata.py +1 -1
- cloudnetpy/model_evaluation/plotting/plot_tools.py +32 -37
- cloudnetpy/model_evaluation/plotting/plotting.py +327 -117
- cloudnetpy/model_evaluation/products/advance_methods.py +92 -83
- cloudnetpy/model_evaluation/products/grid_methods.py +88 -63
- cloudnetpy/model_evaluation/products/model_products.py +43 -35
- cloudnetpy/model_evaluation/products/observation_products.py +41 -35
- cloudnetpy/model_evaluation/products/product_resampling.py +17 -7
- cloudnetpy/model_evaluation/products/tools.py +29 -20
- cloudnetpy/model_evaluation/statistics/statistical_methods.py +30 -20
- cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
- cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/unit/conftest.py +42 -41
- cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +41 -48
- cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +216 -194
- cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
- cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +37 -38
- cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +43 -40
- cloudnetpy/model_evaluation/tests/unit/test_plotting.py +30 -36
- cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +68 -31
- cloudnetpy/model_evaluation/tests/unit/test_tools.py +33 -26
- cloudnetpy/model_evaluation/utils.py +2 -1
- cloudnetpy/output.py +170 -111
- cloudnetpy/plotting/__init__.py +2 -1
- cloudnetpy/plotting/plot_meta.py +562 -822
- cloudnetpy/plotting/plotting.py +1142 -704
- cloudnetpy/products/__init__.py +1 -0
- cloudnetpy/products/classification.py +370 -88
- cloudnetpy/products/der.py +85 -55
- cloudnetpy/products/drizzle.py +77 -34
- cloudnetpy/products/drizzle_error.py +15 -11
- cloudnetpy/products/drizzle_tools.py +79 -59
- cloudnetpy/products/epsilon.py +211 -0
- cloudnetpy/products/ier.py +27 -50
- cloudnetpy/products/iwc.py +55 -48
- cloudnetpy/products/lwc.py +96 -70
- cloudnetpy/products/mwr_tools.py +186 -0
- cloudnetpy/products/product_tools.py +170 -128
- cloudnetpy/utils.py +455 -240
- cloudnetpy/version.py +2 -2
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/METADATA +44 -40
- cloudnetpy-1.87.3.dist-info/RECORD +127 -0
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/WHEEL +1 -1
- cloudnetpy-1.87.3.dist-info/entry_points.txt +2 -0
- docs/source/conf.py +2 -2
- cloudnetpy/categorize/atmos.py +0 -361
- cloudnetpy/products/mwr_multi.py +0 -68
- cloudnetpy/products/mwr_single.py +0 -75
- cloudnetpy-1.49.9.dist-info/RECORD +0 -112
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info/licenses}/LICENSE +0 -0
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/top_level.txt +0 -0
|
@@ -1,171 +1,245 @@
|
|
|
1
|
+
import datetime
|
|
1
2
|
import logging
|
|
3
|
+
from typing import NamedTuple
|
|
2
4
|
|
|
3
5
|
import numpy as np
|
|
6
|
+
import numpy.typing as npt
|
|
4
7
|
from numpy import ma
|
|
5
8
|
from scipy.ndimage import gaussian_filter
|
|
6
9
|
|
|
7
10
|
from cloudnetpy import utils
|
|
8
11
|
from cloudnetpy.cloudnetarray import CloudnetArray
|
|
12
|
+
from cloudnetpy.exceptions import ValidTimeStampError
|
|
9
13
|
from cloudnetpy.instruments.instruments import Instrument
|
|
10
|
-
from cloudnetpy.utils import Epoch
|
|
11
14
|
|
|
12
15
|
|
|
13
|
-
class NoiseParam:
|
|
16
|
+
class NoiseParam(NamedTuple):
|
|
14
17
|
"""Noise parameters. Values are weakly instrument-dependent."""
|
|
15
18
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
self.noise_smooth_min = noise_smooth_min
|
|
19
|
+
noise_min: float = 1e-9
|
|
20
|
+
noise_smooth_min: float = 4e-9
|
|
19
21
|
|
|
20
22
|
|
|
21
23
|
class Ceilometer:
|
|
22
24
|
"""Base class for all types of ceilometers and pollyxt."""
|
|
23
25
|
|
|
24
|
-
def __init__(self, noise_param: NoiseParam =
|
|
25
|
-
self.noise_param = noise_param
|
|
26
|
+
def __init__(self, noise_param: NoiseParam | None = None) -> None:
|
|
27
|
+
self.noise_param = noise_param or NoiseParam()
|
|
26
28
|
self.data: dict = {} # Need to contain 'beta_raw', 'range' and 'time'
|
|
27
29
|
self.metadata: dict = {} # Need to contain 'date' as ('yyyy', 'mm', 'dd')
|
|
28
|
-
self.expected_date:
|
|
30
|
+
self.expected_date: datetime.date | None = None
|
|
29
31
|
self.site_meta: dict = {}
|
|
30
|
-
self.date:
|
|
32
|
+
self.date: datetime.date
|
|
31
33
|
self.instrument: Instrument | None = None
|
|
34
|
+
self.serial_number: str | None = None
|
|
32
35
|
|
|
33
36
|
def calc_screened_product(
|
|
34
37
|
self,
|
|
35
|
-
array:
|
|
38
|
+
array: npt.NDArray,
|
|
36
39
|
snr_limit: int = 5,
|
|
40
|
+
n_negatives: int = 5,
|
|
41
|
+
*,
|
|
37
42
|
range_corrected: bool = True,
|
|
38
|
-
) ->
|
|
43
|
+
) -> npt.NDArray:
|
|
39
44
|
"""Screens noise from lidar variable."""
|
|
40
|
-
noisy_data = NoisyData(
|
|
41
|
-
|
|
42
|
-
|
|
45
|
+
noisy_data = NoisyData(
|
|
46
|
+
self.data,
|
|
47
|
+
self.noise_param,
|
|
48
|
+
range_corrected=range_corrected,
|
|
49
|
+
instrument=self.instrument,
|
|
50
|
+
)
|
|
51
|
+
return noisy_data.screen_data(
|
|
52
|
+
array,
|
|
53
|
+
snr_limit=snr_limit,
|
|
54
|
+
n_negatives=n_negatives,
|
|
55
|
+
)
|
|
43
56
|
|
|
44
57
|
def calc_beta_smooth(
|
|
45
58
|
self,
|
|
46
|
-
beta:
|
|
59
|
+
beta: npt.NDArray,
|
|
47
60
|
snr_limit: int = 5,
|
|
61
|
+
n_negatives: int = 5,
|
|
62
|
+
*,
|
|
48
63
|
range_corrected: bool = True,
|
|
49
|
-
) ->
|
|
50
|
-
noisy_data = NoisyData(
|
|
64
|
+
) -> npt.NDArray:
|
|
65
|
+
noisy_data = NoisyData(
|
|
66
|
+
self.data,
|
|
67
|
+
self.noise_param,
|
|
68
|
+
range_corrected=range_corrected,
|
|
69
|
+
instrument=self.instrument,
|
|
70
|
+
)
|
|
51
71
|
beta_raw = ma.copy(self.data["beta_raw"])
|
|
52
72
|
cloud_ind, cloud_values, cloud_limit = _estimate_clouds_from_beta(beta)
|
|
53
73
|
beta_raw[cloud_ind] = cloud_limit
|
|
54
74
|
sigma = calc_sigma_units(self.data["time"], self.data["range"])
|
|
55
75
|
beta_raw_smooth = gaussian_filter(beta_raw, sigma)
|
|
56
76
|
beta_raw_smooth[cloud_ind] = cloud_values
|
|
57
|
-
|
|
58
|
-
beta_raw_smooth,
|
|
77
|
+
return noisy_data.screen_data(
|
|
78
|
+
beta_raw_smooth,
|
|
79
|
+
is_smoothed=True,
|
|
80
|
+
snr_limit=snr_limit,
|
|
81
|
+
n_negatives=n_negatives,
|
|
59
82
|
)
|
|
60
|
-
return beta_smooth
|
|
61
83
|
|
|
62
|
-
def prepare_data(self):
|
|
84
|
+
def prepare_data(self) -> None:
|
|
63
85
|
"""Add common additional data / metadata and convert into CloudnetArrays."""
|
|
64
86
|
zenith_angle = self.data["zenith_angle"]
|
|
65
87
|
self.data["height"] = np.array(
|
|
66
88
|
self.site_meta["altitude"]
|
|
67
|
-
+ utils.range_to_height(self.data["range"], zenith_angle)
|
|
89
|
+
+ utils.range_to_height(self.data["range"], zenith_angle),
|
|
68
90
|
)
|
|
69
91
|
for key in ("time", "range"):
|
|
70
92
|
self.data[key] = np.array(self.data[key])
|
|
71
|
-
|
|
72
|
-
|
|
93
|
+
if self.instrument is None or self.instrument.wavelength is None:
|
|
94
|
+
msg = "Instrument wavelength not defined"
|
|
95
|
+
raise RuntimeError(msg)
|
|
73
96
|
self.data["wavelength"] = float(self.instrument.wavelength)
|
|
74
|
-
for key in ("latitude", "longitude", "altitude"):
|
|
75
|
-
if key in self.site_meta:
|
|
76
|
-
self.data[key] = float(self.site_meta[key])
|
|
77
97
|
|
|
78
|
-
def get_date_and_time(self, epoch:
|
|
98
|
+
def get_date_and_time(self, epoch: datetime.datetime) -> None:
|
|
99
|
+
if "time" not in self.data:
|
|
100
|
+
msg = "Time array missing from data"
|
|
101
|
+
raise ValidTimeStampError(msg)
|
|
79
102
|
if self.expected_date is not None:
|
|
80
103
|
self.data = utils.screen_by_time(self.data, epoch, self.expected_date)
|
|
81
|
-
self.date = utils.seconds2date(self.data["time"][0], epoch=epoch)
|
|
104
|
+
self.date = utils.seconds2date(self.data["time"][0], epoch=epoch).date()
|
|
82
105
|
self.data["time"] = utils.seconds2hours(self.data["time"])
|
|
83
106
|
|
|
84
|
-
def
|
|
85
|
-
keys = [key for key in self.data.keys() if "raw" in key]
|
|
86
|
-
for key in keys:
|
|
87
|
-
del self.data[key]
|
|
88
|
-
self.data.pop("x_pol", None)
|
|
89
|
-
self.data.pop("p_pol", None)
|
|
90
|
-
|
|
91
|
-
def data_to_cloudnet_arrays(self):
|
|
107
|
+
def data_to_cloudnet_arrays(self, time_dtype: str = "f4") -> None:
|
|
92
108
|
for key, array in self.data.items():
|
|
93
|
-
|
|
109
|
+
if key == "time":
|
|
110
|
+
self.data[key] = CloudnetArray(array, key, data_type=time_dtype)
|
|
111
|
+
else:
|
|
112
|
+
self.data[key] = CloudnetArray(array, key)
|
|
94
113
|
|
|
95
|
-
def
|
|
114
|
+
def add_site_geolocation(self) -> None:
|
|
115
|
+
utils.add_site_geolocation(self.data, gps=False, site_meta=self.site_meta)
|
|
116
|
+
|
|
117
|
+
def screen_depol(self) -> None:
|
|
96
118
|
key = "depolarisation"
|
|
97
119
|
if key in self.data:
|
|
98
120
|
self.data[key][self.data[key] <= 0] = ma.masked
|
|
99
121
|
self.data[key][self.data[key] > 1] = ma.masked
|
|
100
122
|
|
|
101
|
-
def
|
|
123
|
+
def screen_invalid_values(self) -> None:
|
|
124
|
+
for key in self.data:
|
|
125
|
+
try:
|
|
126
|
+
if self.data[key][:].ndim == 2:
|
|
127
|
+
self.data[key] = ma.masked_invalid(self.data[key])
|
|
128
|
+
except (IndexError, TypeError):
|
|
129
|
+
continue
|
|
130
|
+
|
|
131
|
+
def add_snr_info(self, key: str, snr_limit: float) -> None:
|
|
102
132
|
if key in self.data:
|
|
103
133
|
self.data[key].comment += f" SNR threshold applied: {snr_limit}."
|
|
104
134
|
|
|
135
|
+
def check_beta_raw_shape(self) -> None:
|
|
136
|
+
beta_raw = self.data["beta_raw"]
|
|
137
|
+
if beta_raw.ndim != 2 or (beta_raw.shape[0] == 1 or beta_raw.shape[1] == 1):
|
|
138
|
+
msg = f"Invalid beta_raw shape: {beta_raw.shape}"
|
|
139
|
+
raise ValidTimeStampError(msg)
|
|
140
|
+
|
|
141
|
+
def screen_sunbeam(self) -> None:
|
|
142
|
+
high_alt_mask = self.data["range"] > 10000
|
|
143
|
+
if not np.any(high_alt_mask):
|
|
144
|
+
return
|
|
145
|
+
|
|
146
|
+
is_data = ~self.data["beta"][:, high_alt_mask].mask
|
|
147
|
+
n_bins = 20
|
|
148
|
+
|
|
149
|
+
n_profiles, n_heights = is_data.shape
|
|
150
|
+
bin_size = n_heights // n_bins
|
|
151
|
+
reshaped = is_data[:, : bin_size * n_bins].reshape(n_profiles, n_bins, bin_size)
|
|
152
|
+
|
|
153
|
+
valid_profiles = np.any(reshaped, axis=2).sum(axis=1) < 15
|
|
154
|
+
|
|
155
|
+
for key, value in self.data.items():
|
|
156
|
+
if key == "time" or (isinstance(value, np.ndarray) and value.ndim == 2):
|
|
157
|
+
self.data[key] = value[valid_profiles]
|
|
158
|
+
|
|
105
159
|
|
|
106
160
|
class NoisyData:
|
|
107
161
|
def __init__(
|
|
108
|
-
self,
|
|
109
|
-
|
|
162
|
+
self,
|
|
163
|
+
data: dict,
|
|
164
|
+
noise_param: NoiseParam,
|
|
165
|
+
*,
|
|
166
|
+
range_corrected: bool = True,
|
|
167
|
+
instrument: Instrument | None = None,
|
|
168
|
+
) -> None:
|
|
110
169
|
self.data = data
|
|
111
170
|
self.noise_param = noise_param
|
|
112
171
|
self.range_corrected = range_corrected
|
|
172
|
+
self.instrument = instrument
|
|
113
173
|
|
|
114
174
|
def screen_data(
|
|
115
175
|
self,
|
|
116
|
-
data_in:
|
|
176
|
+
data_in: npt.NDArray,
|
|
117
177
|
snr_limit: float = 5,
|
|
178
|
+
n_negatives: int = 5,
|
|
179
|
+
*,
|
|
118
180
|
is_smoothed: bool = False,
|
|
119
181
|
keep_negative: bool = False,
|
|
120
182
|
filter_fog: bool = True,
|
|
121
183
|
filter_negatives: bool = True,
|
|
122
184
|
filter_snr: bool = True,
|
|
123
|
-
) ->
|
|
124
|
-
data = ma.copy(data_in)
|
|
185
|
+
) -> npt.NDArray:
|
|
186
|
+
data: npt.NDArray = ma.copy(data_in)
|
|
125
187
|
self._calc_range_uncorrected(data)
|
|
126
188
|
noise = _estimate_background_noise(data)
|
|
127
|
-
noise = self._adjust_noise(noise, is_smoothed)
|
|
189
|
+
noise = self._adjust_noise(noise, is_smoothed=is_smoothed)
|
|
128
190
|
if filter_negatives is True:
|
|
129
|
-
is_negative = self._mask_low_values_above_consequent_negatives(
|
|
191
|
+
is_negative = self._mask_low_values_above_consequent_negatives(
|
|
192
|
+
data,
|
|
193
|
+
n_negatives=n_negatives,
|
|
194
|
+
)
|
|
130
195
|
noise[is_negative] = 1e-12
|
|
131
196
|
if filter_fog is True:
|
|
132
197
|
is_fog = self._find_fog_profiles()
|
|
133
198
|
self._clean_fog_profiles(data, is_fog)
|
|
134
199
|
noise[is_fog] = 1e-12
|
|
135
200
|
if filter_snr is True:
|
|
136
|
-
data = self._remove_noise(
|
|
201
|
+
data = self._remove_noise(
|
|
202
|
+
data,
|
|
203
|
+
noise,
|
|
204
|
+
keep_negative=keep_negative,
|
|
205
|
+
snr_limit=snr_limit,
|
|
206
|
+
)
|
|
137
207
|
self._calc_range_corrected(data)
|
|
138
208
|
return data
|
|
139
209
|
|
|
140
|
-
def _adjust_noise(self, noise:
|
|
210
|
+
def _adjust_noise(self, noise: npt.NDArray, *, is_smoothed: bool) -> npt.NDArray:
|
|
141
211
|
noise_min = (
|
|
142
212
|
self.noise_param.noise_smooth_min
|
|
143
213
|
if is_smoothed is True
|
|
144
214
|
else self.noise_param.noise_min
|
|
145
215
|
)
|
|
146
216
|
noise_below_threshold = noise < noise_min
|
|
147
|
-
logging.debug(
|
|
217
|
+
logging.debug(
|
|
218
|
+
"Adjusted noise of %s profiles",
|
|
219
|
+
sum(np.array(noise_below_threshold)),
|
|
220
|
+
)
|
|
148
221
|
noise[noise_below_threshold] = noise_min
|
|
149
222
|
return noise
|
|
150
223
|
|
|
151
224
|
@staticmethod
|
|
152
225
|
def _mask_low_values_above_consequent_negatives(
|
|
153
|
-
data:
|
|
226
|
+
data: npt.NDArray,
|
|
154
227
|
n_negatives: int = 5,
|
|
155
228
|
threshold: float = 8e-6,
|
|
156
229
|
n_gates: int = 95,
|
|
157
230
|
n_skip_lowest: int = 5,
|
|
158
|
-
) ->
|
|
231
|
+
) -> npt.NDArray:
|
|
159
232
|
negative_data = data[:, n_skip_lowest : n_gates + n_skip_lowest] < 0
|
|
160
233
|
n_consequent_negatives = utils.cumsumr(negative_data, axis=1)
|
|
161
234
|
time_indices, alt_indices = np.where(n_consequent_negatives > n_negatives)
|
|
162
235
|
alt_indices += n_skip_lowest
|
|
163
|
-
for time_ind, alt_ind in zip(time_indices, alt_indices):
|
|
236
|
+
for time_ind, alt_ind in zip(time_indices, alt_indices, strict=True):
|
|
164
237
|
profile = data[time_ind, alt_ind:]
|
|
165
238
|
profile[profile < threshold] = ma.masked
|
|
166
239
|
cleaned_time_indices = np.unique(time_indices)
|
|
167
240
|
logging.debug(
|
|
168
|
-
|
|
241
|
+
"Cleaned %s profiles with negative filter",
|
|
242
|
+
len(cleaned_time_indices),
|
|
169
243
|
)
|
|
170
244
|
return cleaned_time_indices
|
|
171
245
|
|
|
@@ -174,23 +248,25 @@ class NoisyData:
|
|
|
174
248
|
n_gates_for_signal_sum: int = 20,
|
|
175
249
|
signal_sum_threshold: float = 1e-3,
|
|
176
250
|
variance_threshold: float = 1e-15,
|
|
177
|
-
) ->
|
|
251
|
+
) -> npt.NDArray:
|
|
178
252
|
"""Finds saturated (usually fog) profiles from beta_raw."""
|
|
179
253
|
signal_sum = ma.sum(
|
|
180
|
-
ma.abs(self.data["beta_raw"][:, :n_gates_for_signal_sum]),
|
|
254
|
+
ma.abs(self.data["beta_raw"][:, :n_gates_for_signal_sum]),
|
|
255
|
+
axis=1,
|
|
181
256
|
)
|
|
182
257
|
variance = _calc_var_from_top_gates(self.data["beta_raw"])
|
|
183
258
|
is_fog = (signal_sum > signal_sum_threshold) | (variance < variance_threshold)
|
|
184
|
-
logging.debug(
|
|
259
|
+
logging.debug("Cleaned %s profiles with fog filter", sum(is_fog))
|
|
185
260
|
return is_fog
|
|
186
261
|
|
|
187
262
|
def _remove_noise(
|
|
188
263
|
self,
|
|
189
|
-
array:
|
|
190
|
-
noise:
|
|
264
|
+
array: npt.NDArray,
|
|
265
|
+
noise: npt.NDArray,
|
|
266
|
+
*,
|
|
191
267
|
keep_negative: bool,
|
|
192
268
|
snr_limit: float,
|
|
193
|
-
) ->
|
|
269
|
+
) -> npt.NDArray:
|
|
194
270
|
snr = array / utils.transpose(noise)
|
|
195
271
|
if self.range_corrected is False:
|
|
196
272
|
snr_scale_factor = 6
|
|
@@ -204,32 +280,38 @@ class NoisyData:
|
|
|
204
280
|
array[snr < snr_limit] = ma.masked
|
|
205
281
|
return array
|
|
206
282
|
|
|
207
|
-
def _calc_range_uncorrected(self, data:
|
|
283
|
+
def _calc_range_uncorrected(self, data: npt.NDArray) -> None:
|
|
208
284
|
ind = self._get_altitude_ind()
|
|
209
285
|
data[:, ind] = data[:, ind] / self._get_range_squared()[ind]
|
|
210
286
|
|
|
211
|
-
def _calc_range_corrected(self, data:
|
|
287
|
+
def _calc_range_corrected(self, data: npt.NDArray) -> None:
|
|
212
288
|
ind = self._get_altitude_ind()
|
|
213
289
|
data[:, ind] = data[:, ind] * self._get_range_squared()[ind]
|
|
214
290
|
|
|
215
291
|
def _get_altitude_ind(self) -> tuple:
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
292
|
+
alt_limit = 1e12 # All altitudes
|
|
293
|
+
if (
|
|
294
|
+
self.range_corrected is False
|
|
295
|
+
and self.instrument is not None
|
|
296
|
+
and self.instrument.model is not None
|
|
297
|
+
):
|
|
298
|
+
model = self.instrument.model.lower()
|
|
299
|
+
if model == "ct25k":
|
|
300
|
+
alt_limit = 0.0
|
|
301
|
+
elif model in ("cl31", "cl51"):
|
|
302
|
+
alt_limit = 2400.0
|
|
223
303
|
return np.where(self.data["range"] < alt_limit)
|
|
224
304
|
|
|
225
|
-
def _get_range_squared(self) ->
|
|
305
|
+
def _get_range_squared(self) -> npt.NDArray:
|
|
226
306
|
"""Returns range (m), squared and converted to km."""
|
|
227
307
|
m2km = 0.001
|
|
228
308
|
return (self.data["range"] * m2km) ** 2
|
|
229
309
|
|
|
230
310
|
@staticmethod
|
|
231
311
|
def _clean_fog_profiles(
|
|
232
|
-
data:
|
|
312
|
+
data: npt.NDArray,
|
|
313
|
+
is_fog: npt.NDArray,
|
|
314
|
+
threshold: float = 2e-6,
|
|
233
315
|
) -> None:
|
|
234
316
|
"""Removes values in saturated (e.g. fog) profiles above peak."""
|
|
235
317
|
for time_ind in np.where(is_fog)[0]:
|
|
@@ -238,19 +320,22 @@ class NoisyData:
|
|
|
238
320
|
profile[peak_ind:][profile[peak_ind:] < threshold] = ma.masked
|
|
239
321
|
|
|
240
322
|
|
|
241
|
-
def _estimate_background_noise(data:
|
|
323
|
+
def _estimate_background_noise(data: npt.NDArray) -> npt.NDArray:
|
|
242
324
|
var = _calc_var_from_top_gates(data)
|
|
243
325
|
return np.sqrt(var)
|
|
244
326
|
|
|
245
327
|
|
|
246
|
-
def _calc_var_from_top_gates(data:
|
|
328
|
+
def _calc_var_from_top_gates(data: npt.NDArray) -> npt.NDArray:
|
|
247
329
|
fraction = 0.1
|
|
248
330
|
n_gates = round(data.shape[1] * fraction)
|
|
249
331
|
return ma.var(data[:, -n_gates:], axis=1)
|
|
250
332
|
|
|
251
333
|
|
|
252
334
|
def calc_sigma_units(
|
|
253
|
-
time_vector:
|
|
335
|
+
time_vector: npt.NDArray,
|
|
336
|
+
range_los: npt.NDArray,
|
|
337
|
+
sigma_minutes: float = 1,
|
|
338
|
+
sigma_metres: float = 10,
|
|
254
339
|
) -> tuple[float, float]:
|
|
255
340
|
"""Calculates Gaussian peak std parameters.
|
|
256
341
|
|
|
@@ -260,6 +345,8 @@ def calc_sigma_units(
|
|
|
260
345
|
Args:
|
|
261
346
|
time_vector: 1D vector (fraction hour).
|
|
262
347
|
range_los: 1D vector (m).
|
|
348
|
+
sigma_minutes: Smoothing in minutes.
|
|
349
|
+
sigma_metres: Smoothing in metres.
|
|
263
350
|
|
|
264
351
|
Returns:
|
|
265
352
|
tuple: Two element tuple containing number of steps in time and height to
|
|
@@ -267,10 +354,9 @@ def calc_sigma_units(
|
|
|
267
354
|
|
|
268
355
|
"""
|
|
269
356
|
if len(time_vector) == 0 or np.max(time_vector) > 24:
|
|
270
|
-
|
|
357
|
+
msg = "Invalid time vector"
|
|
358
|
+
raise ValueError(msg)
|
|
271
359
|
minutes_in_hour = 60
|
|
272
|
-
sigma_minutes = 2
|
|
273
|
-
sigma_metres = 5
|
|
274
360
|
time_step = utils.mdiff(time_vector) * minutes_in_hour
|
|
275
361
|
alt_step = utils.mdiff(range_los)
|
|
276
362
|
x_std = sigma_minutes / time_step
|
|
@@ -278,9 +364,7 @@ def calc_sigma_units(
|
|
|
278
364
|
return x_std, y_std
|
|
279
365
|
|
|
280
366
|
|
|
281
|
-
def _estimate_clouds_from_beta(
|
|
282
|
-
beta: np.ndarray,
|
|
283
|
-
) -> tuple[tuple, np.ndarray, float]:
|
|
367
|
+
def _estimate_clouds_from_beta(beta: npt.NDArray) -> tuple[tuple, npt.NDArray, float]:
|
|
284
368
|
"""Naively finds strong clouds from ceilometer backscatter."""
|
|
285
369
|
cloud_limit = 1e-6
|
|
286
370
|
cloud_ind = np.where(beta > cloud_limit)
|
cloudnetpy/instruments/cl61d.py
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
import datetime
|
|
2
2
|
import logging
|
|
3
|
+
from os import PathLike
|
|
3
4
|
|
|
4
5
|
import netCDF4
|
|
5
6
|
|
|
7
|
+
from cloudnetpy import utils
|
|
8
|
+
from cloudnetpy.exceptions import LidarDataError
|
|
6
9
|
from cloudnetpy.instruments import instruments
|
|
7
10
|
from cloudnetpy.instruments.nc_lidar import NcLidar
|
|
8
11
|
|
|
@@ -11,8 +14,11 @@ class Cl61d(NcLidar):
|
|
|
11
14
|
"""Class for Vaisala CL61d ceilometer."""
|
|
12
15
|
|
|
13
16
|
def __init__(
|
|
14
|
-
self,
|
|
15
|
-
|
|
17
|
+
self,
|
|
18
|
+
file_name: str | PathLike,
|
|
19
|
+
site_meta: dict,
|
|
20
|
+
expected_date: datetime.date | None = None,
|
|
21
|
+
) -> None:
|
|
16
22
|
super().__init__()
|
|
17
23
|
self.file_name = file_name
|
|
18
24
|
self.site_meta = site_meta
|
|
@@ -23,15 +29,21 @@ class Cl61d(NcLidar):
|
|
|
23
29
|
"""Reads data and metadata from concatenated Vaisala CL61d netCDF file."""
|
|
24
30
|
with netCDF4.Dataset(self.file_name) as dataset:
|
|
25
31
|
self.dataset = dataset
|
|
26
|
-
self.
|
|
32
|
+
self._fetch_attributes()
|
|
33
|
+
self._fetch_zenith_angle("tilt_angle", default=3.0)
|
|
27
34
|
self._fetch_range(reference="lower")
|
|
28
35
|
self._fetch_lidar_variables(calibration_factor)
|
|
29
36
|
self._fetch_time_and_date()
|
|
30
37
|
self.dataset = None
|
|
31
38
|
|
|
32
39
|
def _fetch_lidar_variables(self, calibration_factor: float | None = None) -> None:
|
|
33
|
-
|
|
40
|
+
if self.dataset is None:
|
|
41
|
+
msg = "No dataset found"
|
|
42
|
+
raise RuntimeError(msg)
|
|
34
43
|
beta_raw = self.dataset.variables["beta_att"][:]
|
|
44
|
+
if utils.is_all_masked(beta_raw):
|
|
45
|
+
msg = "All beta_raw values are masked. Check the input file(s)."
|
|
46
|
+
raise LidarDataError(msg)
|
|
35
47
|
if calibration_factor is None:
|
|
36
48
|
logging.warning("Using default calibration factor")
|
|
37
49
|
calibration_factor = 1
|
|
@@ -41,3 +53,7 @@ class Cl61d(NcLidar):
|
|
|
41
53
|
self.data["depolarisation"] = (
|
|
42
54
|
self.dataset.variables["x_pol"][:] / self.dataset.variables["p_pol"][:]
|
|
43
55
|
)
|
|
56
|
+
self.data["depolarisation_raw"] = self.data["depolarisation"].copy()
|
|
57
|
+
|
|
58
|
+
def _fetch_attributes(self) -> None:
|
|
59
|
+
self.serial_number = getattr(self.dataset, "instrument_serial_number", None)
|
|
@@ -1,57 +1,44 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
from typing import TYPE_CHECKING
|
|
2
3
|
|
|
3
|
-
import netCDF4
|
|
4
4
|
import numpy as np
|
|
5
|
+
import numpy.typing as npt
|
|
5
6
|
from numpy import ma
|
|
6
7
|
|
|
7
8
|
from cloudnetpy import utils
|
|
8
9
|
from cloudnetpy.cloudnetarray import CloudnetArray
|
|
10
|
+
from cloudnetpy.exceptions import ValidTimeStampError
|
|
11
|
+
from cloudnetpy.instruments.instruments import BASTA, FMCW35, FMCW94, HATPRO, Instrument
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
import netCDF4
|
|
9
15
|
|
|
10
16
|
|
|
11
17
|
class CloudnetInstrument:
|
|
12
|
-
def __init__(self):
|
|
13
|
-
self.dataset: netCDF4.Dataset
|
|
14
|
-
self.time:
|
|
18
|
+
def __init__(self) -> None:
|
|
19
|
+
self.dataset: netCDF4.Dataset
|
|
20
|
+
self.time: npt.NDArray = np.array([])
|
|
15
21
|
self.site_meta: dict = {}
|
|
16
22
|
self.data: dict = {}
|
|
23
|
+
self.serial_number: str | None = None
|
|
24
|
+
self.instrument: Instrument | None = None
|
|
17
25
|
|
|
18
26
|
def add_site_geolocation(self) -> None:
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
self.dataset, key.capitalize()
|
|
27
|
-
):
|
|
28
|
-
value = self.parse_global_attribute_numeral(key.capitalize())
|
|
29
|
-
# From source data (BASTA / RPG):
|
|
30
|
-
elif (
|
|
31
|
-
isinstance(self.dataset, netCDF4.Dataset)
|
|
32
|
-
and key in self.dataset.variables
|
|
33
|
-
):
|
|
34
|
-
value = self.dataset.variables[key][:]
|
|
35
|
-
if value is not None:
|
|
36
|
-
value = float(ma.mean(value))
|
|
37
|
-
self.data[key] = CloudnetArray(value, key)
|
|
38
|
-
|
|
39
|
-
def parse_global_attribute_numeral(self, key: str) -> float:
|
|
40
|
-
new_str = ""
|
|
41
|
-
for char in getattr(self.dataset, key):
|
|
42
|
-
if char.isdigit() or char == ".":
|
|
43
|
-
new_str += char
|
|
44
|
-
return float(new_str)
|
|
27
|
+
has_gps = self.instrument in (BASTA, FMCW94, FMCW35, HATPRO)
|
|
28
|
+
utils.add_site_geolocation(
|
|
29
|
+
self.data,
|
|
30
|
+
gps=has_gps,
|
|
31
|
+
site_meta=self.site_meta,
|
|
32
|
+
dataset=self.dataset if hasattr(self, "dataset") else None,
|
|
33
|
+
)
|
|
45
34
|
|
|
46
35
|
def add_height(self) -> None:
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
except RuntimeError:
|
|
36
|
+
zenith_angle = self._get_zenith_angle()
|
|
37
|
+
if zenith_angle is None:
|
|
50
38
|
logging.warning("Assuming 0 deg zenith_angle")
|
|
51
39
|
zenith_angle = 0
|
|
52
40
|
height = utils.range_to_height(self.data["range"].data, zenith_angle)
|
|
53
41
|
height += self.data["altitude"].data
|
|
54
|
-
height = np.array(height)
|
|
55
42
|
self.data["height"] = CloudnetArray(height, "height")
|
|
56
43
|
|
|
57
44
|
def linear_to_db(self, variables_to_log: tuple) -> None:
|
|
@@ -69,9 +56,17 @@ class CloudnetInstrument:
|
|
|
69
56
|
ind = time.argsort()
|
|
70
57
|
self.screen_time_indices(ind)
|
|
71
58
|
|
|
72
|
-
def screen_time_indices(self, valid_indices: list |
|
|
59
|
+
def screen_time_indices(self, valid_indices: list | npt.NDArray) -> None:
|
|
73
60
|
time = self._get_time()
|
|
74
61
|
n_time = len(time)
|
|
62
|
+
if len(valid_indices) == 0 or (
|
|
63
|
+
isinstance(valid_indices, np.ndarray)
|
|
64
|
+
and valid_indices.dtype == np.bool_
|
|
65
|
+
and valid_indices.shape == time.shape
|
|
66
|
+
and not np.any(valid_indices)
|
|
67
|
+
):
|
|
68
|
+
msg = "All timestamps screened"
|
|
69
|
+
raise ValidTimeStampError(msg)
|
|
75
70
|
for cloudnet_array in self.data.values():
|
|
76
71
|
array = cloudnet_array.data
|
|
77
72
|
if not utils.isscalar(array) and array.shape[0] == n_time:
|
|
@@ -85,8 +80,51 @@ class CloudnetInstrument:
|
|
|
85
80
|
if self.time.size > 0:
|
|
86
81
|
self.time = self.time[valid_indices]
|
|
87
82
|
|
|
88
|
-
def _get_time(self) ->
|
|
83
|
+
def _get_time(self) -> npt.NDArray:
|
|
89
84
|
try:
|
|
90
85
|
return self.data["time"].data[:]
|
|
91
86
|
except KeyError:
|
|
92
87
|
return self.time
|
|
88
|
+
|
|
89
|
+
def _get_zenith_angle(self) -> float | None:
|
|
90
|
+
if "zenith_angle" not in self.data or self.data["zenith_angle"].data.size == 0:
|
|
91
|
+
return None
|
|
92
|
+
zenith_angle = ma.median(self.data["zenith_angle"].data)
|
|
93
|
+
if np.isnan(zenith_angle) or zenith_angle is ma.masked:
|
|
94
|
+
return None
|
|
95
|
+
return zenith_angle
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class CSVFile(CloudnetInstrument):
|
|
99
|
+
def __init__(self, site_meta: dict) -> None:
|
|
100
|
+
super().__init__()
|
|
101
|
+
self.site_meta = site_meta
|
|
102
|
+
self._data: dict = {}
|
|
103
|
+
|
|
104
|
+
def add_date(self) -> None:
|
|
105
|
+
self.date = self._data["time"][0].date()
|
|
106
|
+
|
|
107
|
+
def add_data(self) -> None:
|
|
108
|
+
for key, value in self._data.items():
|
|
109
|
+
parsed = (
|
|
110
|
+
utils.datetime2decimal_hours(value)
|
|
111
|
+
if key == "time"
|
|
112
|
+
else ma.masked_invalid(value)
|
|
113
|
+
)
|
|
114
|
+
self.data[key] = CloudnetArray(parsed, key)
|
|
115
|
+
|
|
116
|
+
def normalize_cumulative_amount(self, key: str) -> None:
|
|
117
|
+
if key not in self.data:
|
|
118
|
+
return
|
|
119
|
+
amount = self.data[key][:]
|
|
120
|
+
offset = 0
|
|
121
|
+
last_valid = 0
|
|
122
|
+
for i in range(1, len(amount)):
|
|
123
|
+
if amount[i] is ma.masked:
|
|
124
|
+
continue
|
|
125
|
+
if amount[i] + offset < amount[last_valid]:
|
|
126
|
+
offset += amount[last_valid]
|
|
127
|
+
amount[i] += offset
|
|
128
|
+
last_valid = i
|
|
129
|
+
amount -= amount[0]
|
|
130
|
+
self.data[key].data = amount
|