cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. cloudnetpy/categorize/__init__.py +1 -2
  2. cloudnetpy/categorize/atmos_utils.py +297 -67
  3. cloudnetpy/categorize/attenuation.py +31 -0
  4. cloudnetpy/categorize/attenuations/__init__.py +37 -0
  5. cloudnetpy/categorize/attenuations/gas_attenuation.py +30 -0
  6. cloudnetpy/categorize/attenuations/liquid_attenuation.py +84 -0
  7. cloudnetpy/categorize/attenuations/melting_attenuation.py +78 -0
  8. cloudnetpy/categorize/attenuations/rain_attenuation.py +84 -0
  9. cloudnetpy/categorize/categorize.py +332 -156
  10. cloudnetpy/categorize/classify.py +127 -125
  11. cloudnetpy/categorize/containers.py +107 -76
  12. cloudnetpy/categorize/disdrometer.py +40 -0
  13. cloudnetpy/categorize/droplet.py +23 -21
  14. cloudnetpy/categorize/falling.py +53 -24
  15. cloudnetpy/categorize/freezing.py +25 -12
  16. cloudnetpy/categorize/insects.py +35 -23
  17. cloudnetpy/categorize/itu.py +243 -0
  18. cloudnetpy/categorize/lidar.py +36 -41
  19. cloudnetpy/categorize/melting.py +34 -26
  20. cloudnetpy/categorize/model.py +84 -37
  21. cloudnetpy/categorize/mwr.py +18 -14
  22. cloudnetpy/categorize/radar.py +215 -102
  23. cloudnetpy/cli.py +578 -0
  24. cloudnetpy/cloudnetarray.py +43 -89
  25. cloudnetpy/concat_lib.py +218 -78
  26. cloudnetpy/constants.py +28 -10
  27. cloudnetpy/datasource.py +61 -86
  28. cloudnetpy/exceptions.py +49 -20
  29. cloudnetpy/instruments/__init__.py +5 -0
  30. cloudnetpy/instruments/basta.py +29 -12
  31. cloudnetpy/instruments/bowtie.py +135 -0
  32. cloudnetpy/instruments/ceilo.py +138 -115
  33. cloudnetpy/instruments/ceilometer.py +164 -80
  34. cloudnetpy/instruments/cl61d.py +21 -5
  35. cloudnetpy/instruments/cloudnet_instrument.py +74 -36
  36. cloudnetpy/instruments/copernicus.py +108 -30
  37. cloudnetpy/instruments/da10.py +54 -0
  38. cloudnetpy/instruments/disdrometer/common.py +126 -223
  39. cloudnetpy/instruments/disdrometer/parsivel.py +453 -94
  40. cloudnetpy/instruments/disdrometer/thies.py +254 -87
  41. cloudnetpy/instruments/fd12p.py +201 -0
  42. cloudnetpy/instruments/galileo.py +65 -23
  43. cloudnetpy/instruments/hatpro.py +123 -49
  44. cloudnetpy/instruments/instruments.py +113 -1
  45. cloudnetpy/instruments/lufft.py +39 -17
  46. cloudnetpy/instruments/mira.py +268 -61
  47. cloudnetpy/instruments/mrr.py +187 -0
  48. cloudnetpy/instruments/nc_lidar.py +19 -8
  49. cloudnetpy/instruments/nc_radar.py +109 -55
  50. cloudnetpy/instruments/pollyxt.py +135 -51
  51. cloudnetpy/instruments/radiometrics.py +313 -59
  52. cloudnetpy/instruments/rain_e_h3.py +171 -0
  53. cloudnetpy/instruments/rpg.py +321 -189
  54. cloudnetpy/instruments/rpg_reader.py +74 -40
  55. cloudnetpy/instruments/toa5.py +49 -0
  56. cloudnetpy/instruments/vaisala.py +95 -343
  57. cloudnetpy/instruments/weather_station.py +774 -105
  58. cloudnetpy/metadata.py +90 -19
  59. cloudnetpy/model_evaluation/file_handler.py +55 -52
  60. cloudnetpy/model_evaluation/metadata.py +46 -20
  61. cloudnetpy/model_evaluation/model_metadata.py +1 -1
  62. cloudnetpy/model_evaluation/plotting/plot_tools.py +32 -37
  63. cloudnetpy/model_evaluation/plotting/plotting.py +327 -117
  64. cloudnetpy/model_evaluation/products/advance_methods.py +92 -83
  65. cloudnetpy/model_evaluation/products/grid_methods.py +88 -63
  66. cloudnetpy/model_evaluation/products/model_products.py +43 -35
  67. cloudnetpy/model_evaluation/products/observation_products.py +41 -35
  68. cloudnetpy/model_evaluation/products/product_resampling.py +17 -7
  69. cloudnetpy/model_evaluation/products/tools.py +29 -20
  70. cloudnetpy/model_evaluation/statistics/statistical_methods.py +30 -20
  71. cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
  72. cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
  73. cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +15 -14
  74. cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
  75. cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +15 -14
  76. cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
  77. cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +15 -14
  78. cloudnetpy/model_evaluation/tests/unit/conftest.py +42 -41
  79. cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +41 -48
  80. cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +216 -194
  81. cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
  82. cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +37 -38
  83. cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +43 -40
  84. cloudnetpy/model_evaluation/tests/unit/test_plotting.py +30 -36
  85. cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +68 -31
  86. cloudnetpy/model_evaluation/tests/unit/test_tools.py +33 -26
  87. cloudnetpy/model_evaluation/utils.py +2 -1
  88. cloudnetpy/output.py +170 -111
  89. cloudnetpy/plotting/__init__.py +2 -1
  90. cloudnetpy/plotting/plot_meta.py +562 -822
  91. cloudnetpy/plotting/plotting.py +1142 -704
  92. cloudnetpy/products/__init__.py +1 -0
  93. cloudnetpy/products/classification.py +370 -88
  94. cloudnetpy/products/der.py +85 -55
  95. cloudnetpy/products/drizzle.py +77 -34
  96. cloudnetpy/products/drizzle_error.py +15 -11
  97. cloudnetpy/products/drizzle_tools.py +79 -59
  98. cloudnetpy/products/epsilon.py +211 -0
  99. cloudnetpy/products/ier.py +27 -50
  100. cloudnetpy/products/iwc.py +55 -48
  101. cloudnetpy/products/lwc.py +96 -70
  102. cloudnetpy/products/mwr_tools.py +186 -0
  103. cloudnetpy/products/product_tools.py +170 -128
  104. cloudnetpy/utils.py +455 -240
  105. cloudnetpy/version.py +2 -2
  106. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/METADATA +44 -40
  107. cloudnetpy-1.87.3.dist-info/RECORD +127 -0
  108. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/WHEEL +1 -1
  109. cloudnetpy-1.87.3.dist-info/entry_points.txt +2 -0
  110. docs/source/conf.py +2 -2
  111. cloudnetpy/categorize/atmos.py +0 -361
  112. cloudnetpy/products/mwr_multi.py +0 -68
  113. cloudnetpy/products/mwr_single.py +0 -75
  114. cloudnetpy-1.49.9.dist-info/RECORD +0 -112
  115. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info/licenses}/LICENSE +0 -0
  116. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/top_level.txt +0 -0
@@ -1,171 +1,245 @@
1
+ import datetime
1
2
  import logging
3
+ from typing import NamedTuple
2
4
 
3
5
  import numpy as np
6
+ import numpy.typing as npt
4
7
  from numpy import ma
5
8
  from scipy.ndimage import gaussian_filter
6
9
 
7
10
  from cloudnetpy import utils
8
11
  from cloudnetpy.cloudnetarray import CloudnetArray
12
+ from cloudnetpy.exceptions import ValidTimeStampError
9
13
  from cloudnetpy.instruments.instruments import Instrument
10
- from cloudnetpy.utils import Epoch
11
14
 
12
15
 
13
- class NoiseParam:
16
+ class NoiseParam(NamedTuple):
14
17
  """Noise parameters. Values are weakly instrument-dependent."""
15
18
 
16
- def __init__(self, noise_min: float = 1e-9, noise_smooth_min: float = 4e-9):
17
- self.noise_min = noise_min
18
- self.noise_smooth_min = noise_smooth_min
19
+ noise_min: float = 1e-9
20
+ noise_smooth_min: float = 4e-9
19
21
 
20
22
 
21
23
  class Ceilometer:
22
24
  """Base class for all types of ceilometers and pollyxt."""
23
25
 
24
- def __init__(self, noise_param: NoiseParam = NoiseParam()):
25
- self.noise_param = noise_param
26
+ def __init__(self, noise_param: NoiseParam | None = None) -> None:
27
+ self.noise_param = noise_param or NoiseParam()
26
28
  self.data: dict = {} # Need to contain 'beta_raw', 'range' and 'time'
27
29
  self.metadata: dict = {} # Need to contain 'date' as ('yyyy', 'mm', 'dd')
28
- self.expected_date: str | None = None
30
+ self.expected_date: datetime.date | None = None
29
31
  self.site_meta: dict = {}
30
- self.date: list[str] = []
32
+ self.date: datetime.date
31
33
  self.instrument: Instrument | None = None
34
+ self.serial_number: str | None = None
32
35
 
33
36
  def calc_screened_product(
34
37
  self,
35
- array: np.ndarray,
38
+ array: npt.NDArray,
36
39
  snr_limit: int = 5,
40
+ n_negatives: int = 5,
41
+ *,
37
42
  range_corrected: bool = True,
38
- ) -> np.ndarray:
43
+ ) -> npt.NDArray:
39
44
  """Screens noise from lidar variable."""
40
- noisy_data = NoisyData(self.data, self.noise_param, range_corrected)
41
- array_screened = noisy_data.screen_data(array, snr_limit=snr_limit)
42
- return array_screened
45
+ noisy_data = NoisyData(
46
+ self.data,
47
+ self.noise_param,
48
+ range_corrected=range_corrected,
49
+ instrument=self.instrument,
50
+ )
51
+ return noisy_data.screen_data(
52
+ array,
53
+ snr_limit=snr_limit,
54
+ n_negatives=n_negatives,
55
+ )
43
56
 
44
57
  def calc_beta_smooth(
45
58
  self,
46
- beta: np.ndarray,
59
+ beta: npt.NDArray,
47
60
  snr_limit: int = 5,
61
+ n_negatives: int = 5,
62
+ *,
48
63
  range_corrected: bool = True,
49
- ) -> np.ndarray:
50
- noisy_data = NoisyData(self.data, self.noise_param, range_corrected)
64
+ ) -> npt.NDArray:
65
+ noisy_data = NoisyData(
66
+ self.data,
67
+ self.noise_param,
68
+ range_corrected=range_corrected,
69
+ instrument=self.instrument,
70
+ )
51
71
  beta_raw = ma.copy(self.data["beta_raw"])
52
72
  cloud_ind, cloud_values, cloud_limit = _estimate_clouds_from_beta(beta)
53
73
  beta_raw[cloud_ind] = cloud_limit
54
74
  sigma = calc_sigma_units(self.data["time"], self.data["range"])
55
75
  beta_raw_smooth = gaussian_filter(beta_raw, sigma)
56
76
  beta_raw_smooth[cloud_ind] = cloud_values
57
- beta_smooth = noisy_data.screen_data(
58
- beta_raw_smooth, is_smoothed=True, snr_limit=snr_limit
77
+ return noisy_data.screen_data(
78
+ beta_raw_smooth,
79
+ is_smoothed=True,
80
+ snr_limit=snr_limit,
81
+ n_negatives=n_negatives,
59
82
  )
60
- return beta_smooth
61
83
 
62
- def prepare_data(self):
84
+ def prepare_data(self) -> None:
63
85
  """Add common additional data / metadata and convert into CloudnetArrays."""
64
86
  zenith_angle = self.data["zenith_angle"]
65
87
  self.data["height"] = np.array(
66
88
  self.site_meta["altitude"]
67
- + utils.range_to_height(self.data["range"], zenith_angle)
89
+ + utils.range_to_height(self.data["range"], zenith_angle),
68
90
  )
69
91
  for key in ("time", "range"):
70
92
  self.data[key] = np.array(self.data[key])
71
- assert self.instrument is not None
72
- assert self.instrument.wavelength is not None
93
+ if self.instrument is None or self.instrument.wavelength is None:
94
+ msg = "Instrument wavelength not defined"
95
+ raise RuntimeError(msg)
73
96
  self.data["wavelength"] = float(self.instrument.wavelength)
74
- for key in ("latitude", "longitude", "altitude"):
75
- if key in self.site_meta:
76
- self.data[key] = float(self.site_meta[key])
77
97
 
78
- def get_date_and_time(self, epoch: Epoch) -> None:
98
+ def get_date_and_time(self, epoch: datetime.datetime) -> None:
99
+ if "time" not in self.data:
100
+ msg = "Time array missing from data"
101
+ raise ValidTimeStampError(msg)
79
102
  if self.expected_date is not None:
80
103
  self.data = utils.screen_by_time(self.data, epoch, self.expected_date)
81
- self.date = utils.seconds2date(self.data["time"][0], epoch=epoch)[:3]
104
+ self.date = utils.seconds2date(self.data["time"][0], epoch=epoch).date()
82
105
  self.data["time"] = utils.seconds2hours(self.data["time"])
83
106
 
84
- def remove_raw_data(self):
85
- keys = [key for key in self.data.keys() if "raw" in key]
86
- for key in keys:
87
- del self.data[key]
88
- self.data.pop("x_pol", None)
89
- self.data.pop("p_pol", None)
90
-
91
- def data_to_cloudnet_arrays(self):
107
+ def data_to_cloudnet_arrays(self, time_dtype: str = "f4") -> None:
92
108
  for key, array in self.data.items():
93
- self.data[key] = CloudnetArray(array, key)
109
+ if key == "time":
110
+ self.data[key] = CloudnetArray(array, key, data_type=time_dtype)
111
+ else:
112
+ self.data[key] = CloudnetArray(array, key)
94
113
 
95
- def screen_depol(self):
114
+ def add_site_geolocation(self) -> None:
115
+ utils.add_site_geolocation(self.data, gps=False, site_meta=self.site_meta)
116
+
117
+ def screen_depol(self) -> None:
96
118
  key = "depolarisation"
97
119
  if key in self.data:
98
120
  self.data[key][self.data[key] <= 0] = ma.masked
99
121
  self.data[key][self.data[key] > 1] = ma.masked
100
122
 
101
- def add_snr_info(self, key: str, snr_limit: float):
123
+ def screen_invalid_values(self) -> None:
124
+ for key in self.data:
125
+ try:
126
+ if self.data[key][:].ndim == 2:
127
+ self.data[key] = ma.masked_invalid(self.data[key])
128
+ except (IndexError, TypeError):
129
+ continue
130
+
131
+ def add_snr_info(self, key: str, snr_limit: float) -> None:
102
132
  if key in self.data:
103
133
  self.data[key].comment += f" SNR threshold applied: {snr_limit}."
104
134
 
135
+ def check_beta_raw_shape(self) -> None:
136
+ beta_raw = self.data["beta_raw"]
137
+ if beta_raw.ndim != 2 or (beta_raw.shape[0] == 1 or beta_raw.shape[1] == 1):
138
+ msg = f"Invalid beta_raw shape: {beta_raw.shape}"
139
+ raise ValidTimeStampError(msg)
140
+
141
+ def screen_sunbeam(self) -> None:
142
+ high_alt_mask = self.data["range"] > 10000
143
+ if not np.any(high_alt_mask):
144
+ return
145
+
146
+ is_data = ~self.data["beta"][:, high_alt_mask].mask
147
+ n_bins = 20
148
+
149
+ n_profiles, n_heights = is_data.shape
150
+ bin_size = n_heights // n_bins
151
+ reshaped = is_data[:, : bin_size * n_bins].reshape(n_profiles, n_bins, bin_size)
152
+
153
+ valid_profiles = np.any(reshaped, axis=2).sum(axis=1) < 15
154
+
155
+ for key, value in self.data.items():
156
+ if key == "time" or (isinstance(value, np.ndarray) and value.ndim == 2):
157
+ self.data[key] = value[valid_profiles]
158
+
105
159
 
106
160
  class NoisyData:
107
161
  def __init__(
108
- self, data: dict, noise_param: NoiseParam, range_corrected: bool = True
109
- ):
162
+ self,
163
+ data: dict,
164
+ noise_param: NoiseParam,
165
+ *,
166
+ range_corrected: bool = True,
167
+ instrument: Instrument | None = None,
168
+ ) -> None:
110
169
  self.data = data
111
170
  self.noise_param = noise_param
112
171
  self.range_corrected = range_corrected
172
+ self.instrument = instrument
113
173
 
114
174
  def screen_data(
115
175
  self,
116
- data_in: np.ndarray,
176
+ data_in: npt.NDArray,
117
177
  snr_limit: float = 5,
178
+ n_negatives: int = 5,
179
+ *,
118
180
  is_smoothed: bool = False,
119
181
  keep_negative: bool = False,
120
182
  filter_fog: bool = True,
121
183
  filter_negatives: bool = True,
122
184
  filter_snr: bool = True,
123
- ) -> np.ndarray:
124
- data = ma.copy(data_in)
185
+ ) -> npt.NDArray:
186
+ data: npt.NDArray = ma.copy(data_in)
125
187
  self._calc_range_uncorrected(data)
126
188
  noise = _estimate_background_noise(data)
127
- noise = self._adjust_noise(noise, is_smoothed)
189
+ noise = self._adjust_noise(noise, is_smoothed=is_smoothed)
128
190
  if filter_negatives is True:
129
- is_negative = self._mask_low_values_above_consequent_negatives(data)
191
+ is_negative = self._mask_low_values_above_consequent_negatives(
192
+ data,
193
+ n_negatives=n_negatives,
194
+ )
130
195
  noise[is_negative] = 1e-12
131
196
  if filter_fog is True:
132
197
  is_fog = self._find_fog_profiles()
133
198
  self._clean_fog_profiles(data, is_fog)
134
199
  noise[is_fog] = 1e-12
135
200
  if filter_snr is True:
136
- data = self._remove_noise(data, noise, keep_negative, snr_limit)
201
+ data = self._remove_noise(
202
+ data,
203
+ noise,
204
+ keep_negative=keep_negative,
205
+ snr_limit=snr_limit,
206
+ )
137
207
  self._calc_range_corrected(data)
138
208
  return data
139
209
 
140
- def _adjust_noise(self, noise: np.ndarray, is_smoothed: bool) -> np.ndarray:
210
+ def _adjust_noise(self, noise: npt.NDArray, *, is_smoothed: bool) -> npt.NDArray:
141
211
  noise_min = (
142
212
  self.noise_param.noise_smooth_min
143
213
  if is_smoothed is True
144
214
  else self.noise_param.noise_min
145
215
  )
146
216
  noise_below_threshold = noise < noise_min
147
- logging.debug(f"Adjusted noise of {sum(noise_below_threshold)} profiles")
217
+ logging.debug(
218
+ "Adjusted noise of %s profiles",
219
+ sum(np.array(noise_below_threshold)),
220
+ )
148
221
  noise[noise_below_threshold] = noise_min
149
222
  return noise
150
223
 
151
224
  @staticmethod
152
225
  def _mask_low_values_above_consequent_negatives(
153
- data: np.ndarray,
226
+ data: npt.NDArray,
154
227
  n_negatives: int = 5,
155
228
  threshold: float = 8e-6,
156
229
  n_gates: int = 95,
157
230
  n_skip_lowest: int = 5,
158
- ) -> np.ndarray:
231
+ ) -> npt.NDArray:
159
232
  negative_data = data[:, n_skip_lowest : n_gates + n_skip_lowest] < 0
160
233
  n_consequent_negatives = utils.cumsumr(negative_data, axis=1)
161
234
  time_indices, alt_indices = np.where(n_consequent_negatives > n_negatives)
162
235
  alt_indices += n_skip_lowest
163
- for time_ind, alt_ind in zip(time_indices, alt_indices):
236
+ for time_ind, alt_ind in zip(time_indices, alt_indices, strict=True):
164
237
  profile = data[time_ind, alt_ind:]
165
238
  profile[profile < threshold] = ma.masked
166
239
  cleaned_time_indices = np.unique(time_indices)
167
240
  logging.debug(
168
- f"Cleaned {len(cleaned_time_indices)} profiles with negative filter"
241
+ "Cleaned %s profiles with negative filter",
242
+ len(cleaned_time_indices),
169
243
  )
170
244
  return cleaned_time_indices
171
245
 
@@ -174,23 +248,25 @@ class NoisyData:
174
248
  n_gates_for_signal_sum: int = 20,
175
249
  signal_sum_threshold: float = 1e-3,
176
250
  variance_threshold: float = 1e-15,
177
- ) -> np.ndarray:
251
+ ) -> npt.NDArray:
178
252
  """Finds saturated (usually fog) profiles from beta_raw."""
179
253
  signal_sum = ma.sum(
180
- ma.abs(self.data["beta_raw"][:, :n_gates_for_signal_sum]), axis=1
254
+ ma.abs(self.data["beta_raw"][:, :n_gates_for_signal_sum]),
255
+ axis=1,
181
256
  )
182
257
  variance = _calc_var_from_top_gates(self.data["beta_raw"])
183
258
  is_fog = (signal_sum > signal_sum_threshold) | (variance < variance_threshold)
184
- logging.debug(f"Cleaned {sum(is_fog)} profiles with fog filter")
259
+ logging.debug("Cleaned %s profiles with fog filter", sum(is_fog))
185
260
  return is_fog
186
261
 
187
262
  def _remove_noise(
188
263
  self,
189
- array: np.ndarray,
190
- noise: np.ndarray,
264
+ array: npt.NDArray,
265
+ noise: npt.NDArray,
266
+ *,
191
267
  keep_negative: bool,
192
268
  snr_limit: float,
193
- ) -> np.ndarray:
269
+ ) -> npt.NDArray:
194
270
  snr = array / utils.transpose(noise)
195
271
  if self.range_corrected is False:
196
272
  snr_scale_factor = 6
@@ -204,32 +280,38 @@ class NoisyData:
204
280
  array[snr < snr_limit] = ma.masked
205
281
  return array
206
282
 
207
- def _calc_range_uncorrected(self, data: np.ndarray) -> None:
283
+ def _calc_range_uncorrected(self, data: npt.NDArray) -> None:
208
284
  ind = self._get_altitude_ind()
209
285
  data[:, ind] = data[:, ind] / self._get_range_squared()[ind]
210
286
 
211
- def _calc_range_corrected(self, data: np.ndarray) -> None:
287
+ def _calc_range_corrected(self, data: npt.NDArray) -> None:
212
288
  ind = self._get_altitude_ind()
213
289
  data[:, ind] = data[:, ind] * self._get_range_squared()[ind]
214
290
 
215
291
  def _get_altitude_ind(self) -> tuple:
216
- if self.range_corrected is False:
217
- alt_limit = 2400.0
218
- logging.warning(
219
- f"Raw data not range-corrected, correcting below {alt_limit} m"
220
- )
221
- else:
222
- alt_limit = 1e12
292
+ alt_limit = 1e12 # All altitudes
293
+ if (
294
+ self.range_corrected is False
295
+ and self.instrument is not None
296
+ and self.instrument.model is not None
297
+ ):
298
+ model = self.instrument.model.lower()
299
+ if model == "ct25k":
300
+ alt_limit = 0.0
301
+ elif model in ("cl31", "cl51"):
302
+ alt_limit = 2400.0
223
303
  return np.where(self.data["range"] < alt_limit)
224
304
 
225
- def _get_range_squared(self) -> np.ndarray:
305
+ def _get_range_squared(self) -> npt.NDArray:
226
306
  """Returns range (m), squared and converted to km."""
227
307
  m2km = 0.001
228
308
  return (self.data["range"] * m2km) ** 2
229
309
 
230
310
  @staticmethod
231
311
  def _clean_fog_profiles(
232
- data: np.ndarray, is_fog: np.ndarray, threshold: float = 2e-6
312
+ data: npt.NDArray,
313
+ is_fog: npt.NDArray,
314
+ threshold: float = 2e-6,
233
315
  ) -> None:
234
316
  """Removes values in saturated (e.g. fog) profiles above peak."""
235
317
  for time_ind in np.where(is_fog)[0]:
@@ -238,19 +320,22 @@ class NoisyData:
238
320
  profile[peak_ind:][profile[peak_ind:] < threshold] = ma.masked
239
321
 
240
322
 
241
- def _estimate_background_noise(data: np.ndarray) -> np.ndarray:
323
+ def _estimate_background_noise(data: npt.NDArray) -> npt.NDArray:
242
324
  var = _calc_var_from_top_gates(data)
243
325
  return np.sqrt(var)
244
326
 
245
327
 
246
- def _calc_var_from_top_gates(data: np.ndarray) -> np.ndarray:
328
+ def _calc_var_from_top_gates(data: npt.NDArray) -> npt.NDArray:
247
329
  fraction = 0.1
248
330
  n_gates = round(data.shape[1] * fraction)
249
331
  return ma.var(data[:, -n_gates:], axis=1)
250
332
 
251
333
 
252
334
  def calc_sigma_units(
253
- time_vector: np.ndarray, range_los: np.ndarray
335
+ time_vector: npt.NDArray,
336
+ range_los: npt.NDArray,
337
+ sigma_minutes: float = 1,
338
+ sigma_metres: float = 10,
254
339
  ) -> tuple[float, float]:
255
340
  """Calculates Gaussian peak std parameters.
256
341
 
@@ -260,6 +345,8 @@ def calc_sigma_units(
260
345
  Args:
261
346
  time_vector: 1D vector (fraction hour).
262
347
  range_los: 1D vector (m).
348
+ sigma_minutes: Smoothing in minutes.
349
+ sigma_metres: Smoothing in metres.
263
350
 
264
351
  Returns:
265
352
  tuple: Two element tuple containing number of steps in time and height to
@@ -267,10 +354,9 @@ def calc_sigma_units(
267
354
 
268
355
  """
269
356
  if len(time_vector) == 0 or np.max(time_vector) > 24:
270
- raise ValueError("Invalid time vector")
357
+ msg = "Invalid time vector"
358
+ raise ValueError(msg)
271
359
  minutes_in_hour = 60
272
- sigma_minutes = 2
273
- sigma_metres = 5
274
360
  time_step = utils.mdiff(time_vector) * minutes_in_hour
275
361
  alt_step = utils.mdiff(range_los)
276
362
  x_std = sigma_minutes / time_step
@@ -278,9 +364,7 @@ def calc_sigma_units(
278
364
  return x_std, y_std
279
365
 
280
366
 
281
- def _estimate_clouds_from_beta(
282
- beta: np.ndarray,
283
- ) -> tuple[tuple, np.ndarray, float]:
367
+ def _estimate_clouds_from_beta(beta: npt.NDArray) -> tuple[tuple, npt.NDArray, float]:
284
368
  """Naively finds strong clouds from ceilometer backscatter."""
285
369
  cloud_limit = 1e-6
286
370
  cloud_ind = np.where(beta > cloud_limit)
@@ -1,8 +1,11 @@
1
- """Module with a class for Lufft chm15k ceilometer."""
1
+ import datetime
2
2
  import logging
3
+ from os import PathLike
3
4
 
4
5
  import netCDF4
5
6
 
7
+ from cloudnetpy import utils
8
+ from cloudnetpy.exceptions import LidarDataError
6
9
  from cloudnetpy.instruments import instruments
7
10
  from cloudnetpy.instruments.nc_lidar import NcLidar
8
11
 
@@ -11,8 +14,11 @@ class Cl61d(NcLidar):
11
14
  """Class for Vaisala CL61d ceilometer."""
12
15
 
13
16
  def __init__(
14
- self, file_name: str, site_meta: dict, expected_date: str | None = None
15
- ):
17
+ self,
18
+ file_name: str | PathLike,
19
+ site_meta: dict,
20
+ expected_date: datetime.date | None = None,
21
+ ) -> None:
16
22
  super().__init__()
17
23
  self.file_name = file_name
18
24
  self.site_meta = site_meta
@@ -23,15 +29,21 @@ class Cl61d(NcLidar):
23
29
  """Reads data and metadata from concatenated Vaisala CL61d netCDF file."""
24
30
  with netCDF4.Dataset(self.file_name) as dataset:
25
31
  self.dataset = dataset
26
- self._fetch_zenith_angle("zenith", default=3.0)
32
+ self._fetch_attributes()
33
+ self._fetch_zenith_angle("tilt_angle", default=3.0)
27
34
  self._fetch_range(reference="lower")
28
35
  self._fetch_lidar_variables(calibration_factor)
29
36
  self._fetch_time_and_date()
30
37
  self.dataset = None
31
38
 
32
39
  def _fetch_lidar_variables(self, calibration_factor: float | None = None) -> None:
33
- assert self.dataset is not None
40
+ if self.dataset is None:
41
+ msg = "No dataset found"
42
+ raise RuntimeError(msg)
34
43
  beta_raw = self.dataset.variables["beta_att"][:]
44
+ if utils.is_all_masked(beta_raw):
45
+ msg = "All beta_raw values are masked. Check the input file(s)."
46
+ raise LidarDataError(msg)
35
47
  if calibration_factor is None:
36
48
  logging.warning("Using default calibration factor")
37
49
  calibration_factor = 1
@@ -41,3 +53,7 @@ class Cl61d(NcLidar):
41
53
  self.data["depolarisation"] = (
42
54
  self.dataset.variables["x_pol"][:] / self.dataset.variables["p_pol"][:]
43
55
  )
56
+ self.data["depolarisation_raw"] = self.data["depolarisation"].copy()
57
+
58
+ def _fetch_attributes(self) -> None:
59
+ self.serial_number = getattr(self.dataset, "instrument_serial_number", None)
@@ -1,57 +1,44 @@
1
1
  import logging
2
+ from typing import TYPE_CHECKING
2
3
 
3
- import netCDF4
4
4
  import numpy as np
5
+ import numpy.typing as npt
5
6
  from numpy import ma
6
7
 
7
8
  from cloudnetpy import utils
8
9
  from cloudnetpy.cloudnetarray import CloudnetArray
10
+ from cloudnetpy.exceptions import ValidTimeStampError
11
+ from cloudnetpy.instruments.instruments import BASTA, FMCW35, FMCW94, HATPRO, Instrument
12
+
13
+ if TYPE_CHECKING:
14
+ import netCDF4
9
15
 
10
16
 
11
17
  class CloudnetInstrument:
12
- def __init__(self):
13
- self.dataset: netCDF4.Dataset | None = None
14
- self.time: np.ndarray = np.array([])
18
+ def __init__(self) -> None:
19
+ self.dataset: netCDF4.Dataset
20
+ self.time: npt.NDArray = np.array([])
15
21
  self.site_meta: dict = {}
16
22
  self.data: dict = {}
23
+ self.serial_number: str | None = None
24
+ self.instrument: Instrument | None = None
17
25
 
18
26
  def add_site_geolocation(self) -> None:
19
- for key in ("latitude", "longitude", "altitude"):
20
- value = None
21
- # User-supplied:
22
- if key in self.site_meta:
23
- value = self.site_meta[key]
24
- # From source global attributes (MIRA):
25
- elif isinstance(self.dataset, netCDF4.Dataset) and hasattr(
26
- self.dataset, key.capitalize()
27
- ):
28
- value = self.parse_global_attribute_numeral(key.capitalize())
29
- # From source data (BASTA / RPG):
30
- elif (
31
- isinstance(self.dataset, netCDF4.Dataset)
32
- and key in self.dataset.variables
33
- ):
34
- value = self.dataset.variables[key][:]
35
- if value is not None:
36
- value = float(ma.mean(value))
37
- self.data[key] = CloudnetArray(value, key)
38
-
39
- def parse_global_attribute_numeral(self, key: str) -> float:
40
- new_str = ""
41
- for char in getattr(self.dataset, key):
42
- if char.isdigit() or char == ".":
43
- new_str += char
44
- return float(new_str)
27
+ has_gps = self.instrument in (BASTA, FMCW94, FMCW35, HATPRO)
28
+ utils.add_site_geolocation(
29
+ self.data,
30
+ gps=has_gps,
31
+ site_meta=self.site_meta,
32
+ dataset=self.dataset if hasattr(self, "dataset") else None,
33
+ )
45
34
 
46
35
  def add_height(self) -> None:
47
- try:
48
- zenith_angle = ma.median(self.data["zenith_angle"].data)
49
- except RuntimeError:
36
+ zenith_angle = self._get_zenith_angle()
37
+ if zenith_angle is None:
50
38
  logging.warning("Assuming 0 deg zenith_angle")
51
39
  zenith_angle = 0
52
40
  height = utils.range_to_height(self.data["range"].data, zenith_angle)
53
41
  height += self.data["altitude"].data
54
- height = np.array(height)
55
42
  self.data["height"] = CloudnetArray(height, "height")
56
43
 
57
44
  def linear_to_db(self, variables_to_log: tuple) -> None:
@@ -69,9 +56,17 @@ class CloudnetInstrument:
69
56
  ind = time.argsort()
70
57
  self.screen_time_indices(ind)
71
58
 
72
- def screen_time_indices(self, valid_indices: list | np.ndarray) -> None:
59
+ def screen_time_indices(self, valid_indices: list | npt.NDArray) -> None:
73
60
  time = self._get_time()
74
61
  n_time = len(time)
62
+ if len(valid_indices) == 0 or (
63
+ isinstance(valid_indices, np.ndarray)
64
+ and valid_indices.dtype == np.bool_
65
+ and valid_indices.shape == time.shape
66
+ and not np.any(valid_indices)
67
+ ):
68
+ msg = "All timestamps screened"
69
+ raise ValidTimeStampError(msg)
75
70
  for cloudnet_array in self.data.values():
76
71
  array = cloudnet_array.data
77
72
  if not utils.isscalar(array) and array.shape[0] == n_time:
@@ -85,8 +80,51 @@ class CloudnetInstrument:
85
80
  if self.time.size > 0:
86
81
  self.time = self.time[valid_indices]
87
82
 
88
- def _get_time(self) -> np.ndarray:
83
+ def _get_time(self) -> npt.NDArray:
89
84
  try:
90
85
  return self.data["time"].data[:]
91
86
  except KeyError:
92
87
  return self.time
88
+
89
+ def _get_zenith_angle(self) -> float | None:
90
+ if "zenith_angle" not in self.data or self.data["zenith_angle"].data.size == 0:
91
+ return None
92
+ zenith_angle = ma.median(self.data["zenith_angle"].data)
93
+ if np.isnan(zenith_angle) or zenith_angle is ma.masked:
94
+ return None
95
+ return zenith_angle
96
+
97
+
98
+ class CSVFile(CloudnetInstrument):
99
+ def __init__(self, site_meta: dict) -> None:
100
+ super().__init__()
101
+ self.site_meta = site_meta
102
+ self._data: dict = {}
103
+
104
+ def add_date(self) -> None:
105
+ self.date = self._data["time"][0].date()
106
+
107
+ def add_data(self) -> None:
108
+ for key, value in self._data.items():
109
+ parsed = (
110
+ utils.datetime2decimal_hours(value)
111
+ if key == "time"
112
+ else ma.masked_invalid(value)
113
+ )
114
+ self.data[key] = CloudnetArray(parsed, key)
115
+
116
+ def normalize_cumulative_amount(self, key: str) -> None:
117
+ if key not in self.data:
118
+ return
119
+ amount = self.data[key][:]
120
+ offset = 0
121
+ last_valid = 0
122
+ for i in range(1, len(amount)):
123
+ if amount[i] is ma.masked:
124
+ continue
125
+ if amount[i] + offset < amount[last_valid]:
126
+ offset += amount[last_valid]
127
+ amount[i] += offset
128
+ last_valid = i
129
+ amount -= amount[0]
130
+ self.data[key].data = amount