cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. cloudnetpy/categorize/__init__.py +1 -2
  2. cloudnetpy/categorize/atmos_utils.py +297 -67
  3. cloudnetpy/categorize/attenuation.py +31 -0
  4. cloudnetpy/categorize/attenuations/__init__.py +37 -0
  5. cloudnetpy/categorize/attenuations/gas_attenuation.py +30 -0
  6. cloudnetpy/categorize/attenuations/liquid_attenuation.py +84 -0
  7. cloudnetpy/categorize/attenuations/melting_attenuation.py +78 -0
  8. cloudnetpy/categorize/attenuations/rain_attenuation.py +84 -0
  9. cloudnetpy/categorize/categorize.py +332 -156
  10. cloudnetpy/categorize/classify.py +127 -125
  11. cloudnetpy/categorize/containers.py +107 -76
  12. cloudnetpy/categorize/disdrometer.py +40 -0
  13. cloudnetpy/categorize/droplet.py +23 -21
  14. cloudnetpy/categorize/falling.py +53 -24
  15. cloudnetpy/categorize/freezing.py +25 -12
  16. cloudnetpy/categorize/insects.py +35 -23
  17. cloudnetpy/categorize/itu.py +243 -0
  18. cloudnetpy/categorize/lidar.py +36 -41
  19. cloudnetpy/categorize/melting.py +34 -26
  20. cloudnetpy/categorize/model.py +84 -37
  21. cloudnetpy/categorize/mwr.py +18 -14
  22. cloudnetpy/categorize/radar.py +215 -102
  23. cloudnetpy/cli.py +578 -0
  24. cloudnetpy/cloudnetarray.py +43 -89
  25. cloudnetpy/concat_lib.py +218 -78
  26. cloudnetpy/constants.py +28 -10
  27. cloudnetpy/datasource.py +61 -86
  28. cloudnetpy/exceptions.py +49 -20
  29. cloudnetpy/instruments/__init__.py +5 -0
  30. cloudnetpy/instruments/basta.py +29 -12
  31. cloudnetpy/instruments/bowtie.py +135 -0
  32. cloudnetpy/instruments/ceilo.py +138 -115
  33. cloudnetpy/instruments/ceilometer.py +164 -80
  34. cloudnetpy/instruments/cl61d.py +21 -5
  35. cloudnetpy/instruments/cloudnet_instrument.py +74 -36
  36. cloudnetpy/instruments/copernicus.py +108 -30
  37. cloudnetpy/instruments/da10.py +54 -0
  38. cloudnetpy/instruments/disdrometer/common.py +126 -223
  39. cloudnetpy/instruments/disdrometer/parsivel.py +453 -94
  40. cloudnetpy/instruments/disdrometer/thies.py +254 -87
  41. cloudnetpy/instruments/fd12p.py +201 -0
  42. cloudnetpy/instruments/galileo.py +65 -23
  43. cloudnetpy/instruments/hatpro.py +123 -49
  44. cloudnetpy/instruments/instruments.py +113 -1
  45. cloudnetpy/instruments/lufft.py +39 -17
  46. cloudnetpy/instruments/mira.py +268 -61
  47. cloudnetpy/instruments/mrr.py +187 -0
  48. cloudnetpy/instruments/nc_lidar.py +19 -8
  49. cloudnetpy/instruments/nc_radar.py +109 -55
  50. cloudnetpy/instruments/pollyxt.py +135 -51
  51. cloudnetpy/instruments/radiometrics.py +313 -59
  52. cloudnetpy/instruments/rain_e_h3.py +171 -0
  53. cloudnetpy/instruments/rpg.py +321 -189
  54. cloudnetpy/instruments/rpg_reader.py +74 -40
  55. cloudnetpy/instruments/toa5.py +49 -0
  56. cloudnetpy/instruments/vaisala.py +95 -343
  57. cloudnetpy/instruments/weather_station.py +774 -105
  58. cloudnetpy/metadata.py +90 -19
  59. cloudnetpy/model_evaluation/file_handler.py +55 -52
  60. cloudnetpy/model_evaluation/metadata.py +46 -20
  61. cloudnetpy/model_evaluation/model_metadata.py +1 -1
  62. cloudnetpy/model_evaluation/plotting/plot_tools.py +32 -37
  63. cloudnetpy/model_evaluation/plotting/plotting.py +327 -117
  64. cloudnetpy/model_evaluation/products/advance_methods.py +92 -83
  65. cloudnetpy/model_evaluation/products/grid_methods.py +88 -63
  66. cloudnetpy/model_evaluation/products/model_products.py +43 -35
  67. cloudnetpy/model_evaluation/products/observation_products.py +41 -35
  68. cloudnetpy/model_evaluation/products/product_resampling.py +17 -7
  69. cloudnetpy/model_evaluation/products/tools.py +29 -20
  70. cloudnetpy/model_evaluation/statistics/statistical_methods.py +30 -20
  71. cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
  72. cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
  73. cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +15 -14
  74. cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
  75. cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +15 -14
  76. cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
  77. cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +15 -14
  78. cloudnetpy/model_evaluation/tests/unit/conftest.py +42 -41
  79. cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +41 -48
  80. cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +216 -194
  81. cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
  82. cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +37 -38
  83. cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +43 -40
  84. cloudnetpy/model_evaluation/tests/unit/test_plotting.py +30 -36
  85. cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +68 -31
  86. cloudnetpy/model_evaluation/tests/unit/test_tools.py +33 -26
  87. cloudnetpy/model_evaluation/utils.py +2 -1
  88. cloudnetpy/output.py +170 -111
  89. cloudnetpy/plotting/__init__.py +2 -1
  90. cloudnetpy/plotting/plot_meta.py +562 -822
  91. cloudnetpy/plotting/plotting.py +1142 -704
  92. cloudnetpy/products/__init__.py +1 -0
  93. cloudnetpy/products/classification.py +370 -88
  94. cloudnetpy/products/der.py +85 -55
  95. cloudnetpy/products/drizzle.py +77 -34
  96. cloudnetpy/products/drizzle_error.py +15 -11
  97. cloudnetpy/products/drizzle_tools.py +79 -59
  98. cloudnetpy/products/epsilon.py +211 -0
  99. cloudnetpy/products/ier.py +27 -50
  100. cloudnetpy/products/iwc.py +55 -48
  101. cloudnetpy/products/lwc.py +96 -70
  102. cloudnetpy/products/mwr_tools.py +186 -0
  103. cloudnetpy/products/product_tools.py +170 -128
  104. cloudnetpy/utils.py +455 -240
  105. cloudnetpy/version.py +2 -2
  106. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/METADATA +44 -40
  107. cloudnetpy-1.87.3.dist-info/RECORD +127 -0
  108. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/WHEEL +1 -1
  109. cloudnetpy-1.87.3.dist-info/entry_points.txt +2 -0
  110. docs/source/conf.py +2 -2
  111. cloudnetpy/categorize/atmos.py +0 -361
  112. cloudnetpy/products/mwr_multi.py +0 -68
  113. cloudnetpy/products/mwr_single.py +0 -75
  114. cloudnetpy-1.49.9.dist-info/RECORD +0 -112
  115. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info/licenses}/LICENSE +0 -0
  116. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/top_level.txt +0 -0
@@ -1,20 +1,28 @@
1
1
  """Module for creating Cloudnet liquid water content file using scaled-adiabatic
2
2
  method.
3
3
  """
4
+
5
+ from os import PathLike
6
+ from uuid import UUID
7
+
4
8
  import numpy as np
9
+ import numpy.typing as npt
5
10
  from numpy import ma
6
11
 
7
12
  from cloudnetpy import output, utils
8
- from cloudnetpy.categorize import atmos
13
+ from cloudnetpy.categorize import atmos_utils
9
14
  from cloudnetpy.datasource import DataSource
15
+ from cloudnetpy.exceptions import InvalidSourceFileError
10
16
  from cloudnetpy.metadata import MetaData
11
17
  from cloudnetpy.products import product_tools as p_tools
12
18
  from cloudnetpy.products.product_tools import CategorizeBits, get_is_rain
13
19
 
14
20
 
15
21
  def generate_lwc(
16
- categorize_file: str, output_file: str, uuid: str | None = None
17
- ) -> str:
22
+ categorize_file: str | PathLike,
23
+ output_file: str | PathLike,
24
+ uuid: str | UUID | None = None,
25
+ ) -> UUID:
18
26
  """Generates Cloudnet liquid water content product.
19
27
 
20
28
  This function calculates cloud liquid water content using the so-called
@@ -43,6 +51,7 @@ def generate_lwc(
43
51
  Bull. Amer. Meteor. Soc., 88, 883–898, https://doi.org/10.1175/BAMS-88-6-883
44
52
 
45
53
  """
54
+ uuid = utils.get_uuid(uuid)
46
55
  with LwcSource(categorize_file) as lwc_source:
47
56
  lwc = Lwc(lwc_source)
48
57
  clouds = CloudAdjustor(lwc_source, lwc)
@@ -51,7 +60,7 @@ def generate_lwc(
51
60
  date = lwc_source.get_date()
52
61
  attributes = output.add_time_attribute(LWC_ATTRIBUTES, date)
53
62
  output.update_attributes(lwc_source.data, attributes)
54
- uuid = output.save_product_file(
63
+ output.save_product_file(
55
64
  "lwc",
56
65
  lwc_source,
57
66
  output_file,
@@ -61,7 +70,7 @@ def generate_lwc(
61
70
  "lwp_error",
62
71
  ),
63
72
  )
64
- return uuid
73
+ return uuid
65
74
 
66
75
 
67
76
  class LwcSource(DataSource):
@@ -77,32 +86,41 @@ class LwcSource(DataSource):
77
86
  lwp (ndarray): 1D liquid water path.
78
87
  lwp_error (ndarray): 1D error of liquid water path.
79
88
  is_rain (ndarray): 1D array denoting presence of rain.
80
- dheight (float): Median difference in height vector.
89
+ path_lengths (ndarray): 1D array of path lengths.
81
90
  atmosphere (dict): Dictionary containing interpolated fields `temperature`
82
91
  and `pressure`.
83
92
  categorize_bits (CategorizeBits): The :class:`CategorizeBits` instance.
84
93
 
85
94
  """
86
95
 
87
- def __init__(self, categorize_file: str):
96
+ def __init__(self, categorize_file: str | PathLike) -> None:
88
97
  super().__init__(categorize_file)
98
+ if "lwp" not in self.dataset.variables:
99
+ msg = "Liquid water path missing from the categorize file."
100
+ raise InvalidSourceFileError(msg)
89
101
  self.lwp = self.getvar("lwp")
90
102
  self.lwp[self.lwp < 0] = 0
91
103
  self.lwp_error = self.getvar("lwp_error")
92
104
  self.is_rain = get_is_rain(categorize_file)
93
- self.dheight = utils.mdiff(self.getvar("height"))
105
+ self.height_agl: npt.NDArray
106
+ self.path_lengths = utils.path_lengths_from_ground(self.height_agl)
94
107
  self.atmosphere = self._get_atmosphere(categorize_file)
95
108
  self.categorize_bits = CategorizeBits(categorize_file)
96
109
 
97
110
  def append_results(
98
- self, lwc: np.ndarray, status: np.ndarray, error: np.ndarray
111
+ self,
112
+ lwc: npt.NDArray,
113
+ status: npt.NDArray,
114
+ error: npt.NDArray,
99
115
  ) -> None:
100
116
  self.append_data(lwc, "lwc", units="kg m-3")
101
117
  self.append_data(status, "lwc_retrieval_status")
102
118
  self.append_data(error, "lwc_error", units="dB")
103
119
 
104
120
  @staticmethod
105
- def _get_atmosphere(categorize_file: str) -> tuple[np.ndarray, np.ndarray]:
121
+ def _get_atmosphere(
122
+ categorize_file: str | PathLike,
123
+ ) -> tuple[npt.NDArray, npt.NDArray]:
106
124
  fields = ["temperature", "pressure"]
107
125
  atmosphere = p_tools.interpolate_model(categorize_file, fields)
108
126
  return atmosphere["temperature"], atmosphere["pressure"]
@@ -116,41 +134,42 @@ class Lwc:
116
134
 
117
135
  Attributes:
118
136
  lwc_source (LwcSource): The :class:`LwcSource` instance.
119
- dheight (float): Median difference in height vector.
120
137
  is_liquid (ndarray): 2D array denoting liquid.
121
138
  lwc_adiabatic (ndarray): 2D array storing adiabatic lwc.
122
139
  lwc (ndarray): 2D array of liquid water content (scaled with lwp).
123
140
 
124
141
  """
125
142
 
126
- def __init__(self, lwc_source: LwcSource):
143
+ def __init__(self, lwc_source: LwcSource) -> None:
127
144
  self.lwc_source = lwc_source
128
- self.dheight = self.lwc_source.dheight
145
+ self.height_agl = lwc_source.height_agl
129
146
  self.is_liquid = self._get_liquid()
130
147
  self.lwc_adiabatic = self._init_lwc_adiabatic()
131
148
  self.lwc = self._adiabatic_lwc_to_lwc()
132
149
  self._mask_rain()
133
150
 
134
- def _get_liquid(self) -> np.ndarray:
151
+ def _get_liquid(self) -> npt.NDArray:
135
152
  category_bits = self.lwc_source.categorize_bits.category_bits
136
- return category_bits["droplet"]
153
+ return category_bits.droplet
137
154
 
138
- def _init_lwc_adiabatic(self) -> np.ndarray:
155
+ def _init_lwc_adiabatic(self) -> npt.NDArray:
139
156
  """Returns theoretical adiabatic lwc in liquid clouds (kg/m3)."""
140
- lwc_dz = atmos.fill_clouds_with_lwc_dz(
141
- self.lwc_source.atmosphere, self.is_liquid
157
+ lwc_dz = atmos_utils.fill_clouds_with_lwc_dz(
158
+ *self.lwc_source.atmosphere,
159
+ self.is_liquid,
142
160
  )
143
- return atmos.calc_adiabatic_lwc(lwc_dz, self.dheight)
161
+ return atmos_utils.calc_adiabatic_lwc(lwc_dz, self.height_agl)
144
162
 
145
- def _adiabatic_lwc_to_lwc(self) -> np.ndarray:
163
+ def _adiabatic_lwc_to_lwc(self) -> npt.NDArray:
146
164
  """Initialises liquid water content (kg/m3).
147
165
 
148
166
  Calculates LWC for ALL profiles (rain, lwp > theoretical, etc.),
149
167
  """
150
- lwc_scaled = atmos.distribute_lwp_to_liquid_clouds(
151
- self.lwc_adiabatic, self.lwc_source.lwp
168
+ return atmos_utils.normalize_lwc_by_lwp(
169
+ self.lwc_adiabatic,
170
+ self.lwc_source.lwp,
171
+ self.height_agl,
152
172
  )
153
- return lwc_scaled / self.dheight
154
173
 
155
174
  def _mask_rain(self) -> None:
156
175
  is_rain = self.lwc_source.is_rain.astype(bool)
@@ -174,7 +193,7 @@ class CloudAdjustor:
174
193
 
175
194
  """
176
195
 
177
- def __init__(self, lwc_source: LwcSource, lwc: Lwc):
196
+ def __init__(self, lwc_source: LwcSource, lwc: Lwc) -> None:
178
197
  self.lwc_source = lwc_source
179
198
  self.lwc = lwc.lwc
180
199
  self.is_liquid = lwc.is_liquid
@@ -187,22 +206,23 @@ class CloudAdjustor:
187
206
 
188
207
  def _get_echo(self) -> dict:
189
208
  quality_bits = self.lwc_source.categorize_bits.quality_bits
190
- return {key: quality_bits[key] for key in ("radar", "lidar")}
209
+ return {"radar": quality_bits.radar, "lidar": quality_bits.lidar}
191
210
 
192
211
  def _init_status(self) -> ma.MaskedArray:
193
212
  status = ma.zeros(self.is_liquid.shape, dtype=int)
194
213
  status[self.is_liquid] = 1
195
214
  return status
196
215
 
197
- def _adjust_cloud_tops(self, adjustable_clouds: np.ndarray) -> None:
216
+ def _adjust_cloud_tops(self, adjustable_clouds: npt.NDArray) -> None:
198
217
  """Adjusts cloud top index so that measured lwc corresponds to theoretical
199
- value."""
218
+ value.
219
+ """
200
220
  for time_index in np.unique(np.where(adjustable_clouds)[0]):
201
221
  base_index = np.where(adjustable_clouds[time_index, :])[0][0]
202
222
  self._update_status(time_index)
203
223
  self._adjust_lwc(time_index, base_index)
204
224
 
205
- def _update_status(self, time_ind: np.ndarray) -> None:
225
+ def _update_status(self, time_ind: npt.NDArray) -> None:
206
226
  alt_indices = np.where(self.is_liquid[time_ind, :])[0]
207
227
  self.status[time_ind, alt_indices] = 2
208
228
 
@@ -220,24 +240,21 @@ class CloudAdjustor:
220
240
  distance_from_base += 1
221
241
 
222
242
  def _has_converged(self, ind: int) -> bool:
223
- lwc_sum = ma.sum(self.lwc_adiabatic[ind, :])
224
- if lwc_sum * self.lwc_source.dheight > self.lwc_source.lwp[ind]:
225
- return True
226
- return False
243
+ lwc_sum = ma.sum(self.lwc_adiabatic[ind, :] * self.lwc_source.path_lengths)
244
+ return lwc_sum > self.lwc_source.lwp[ind]
227
245
 
228
246
  def _out_of_bound(self, ind: int) -> bool:
229
247
  return ind >= self.lwc.shape[1] - 1
230
248
 
231
- def _find_adjustable_clouds(self) -> np.ndarray:
249
+ def _find_adjustable_clouds(self) -> npt.NDArray:
232
250
  top_clouds = self._find_topmost_clouds()
233
251
  detection_type = self._find_echo_combinations_in_liquid()
234
252
  detection_type[~top_clouds] = 0
235
253
  lidar_only_clouds = self._find_lidar_only_clouds(detection_type)
236
254
  top_clouds[~lidar_only_clouds, :] = 0
237
- top_clouds = self._remove_good_profiles(top_clouds)
238
- return top_clouds
255
+ return self._remove_good_profiles(top_clouds)
239
256
 
240
- def _find_topmost_clouds(self) -> np.ndarray:
257
+ def _find_topmost_clouds(self) -> npt.NDArray:
241
258
  top_clouds = np.copy(self.is_liquid)
242
259
  cloud_edges = top_clouds[:, :-1][:, ::-1] < top_clouds[:, 1:][:, ::-1]
243
260
  topmost_bases = self.is_liquid.shape[1] - 1 - np.argmax(cloud_edges, axis=1)
@@ -245,14 +262,14 @@ class CloudAdjustor:
245
262
  top_clouds[n, :base] = 0
246
263
  return top_clouds
247
264
 
248
- def _find_echo_combinations_in_liquid(self) -> np.ndarray:
265
+ def _find_echo_combinations_in_liquid(self) -> npt.NDArray:
249
266
  """Classifies liquid clouds by detection type: 1=lidar, 2=radar, 3=both."""
250
267
  lidar_detected = (self.is_liquid & self.echo["lidar"]).astype(int)
251
268
  radar_detected = (self.is_liquid & self.echo["radar"]).astype(int) * 2
252
269
  return lidar_detected + radar_detected
253
270
 
254
271
  @staticmethod
255
- def _find_lidar_only_clouds(detection: np.ndarray) -> np.ndarray:
272
+ def _find_lidar_only_clouds(detection: npt.NDArray) -> npt.NDArray:
256
273
  """Finds top clouds that contain only lidar-detected pixels.
257
274
 
258
275
  Args:
@@ -266,20 +283,20 @@ class CloudAdjustor:
266
283
  sum_of_detection_type = ma.sum(detection, axis=1)
267
284
  return sum_of_cloud_pixels / sum_of_detection_type == 1
268
285
 
269
- def _remove_good_profiles(self, top_clouds: np.ndarray) -> np.ndarray:
286
+ def _remove_good_profiles(self, top_clouds: npt.NDArray) -> npt.NDArray:
270
287
  no_rain = ~self.lwc_source.is_rain.astype(bool)
271
288
  lwp_difference = self._find_lwp_difference()
272
289
  dubious_profiles = (lwp_difference < 0) & no_rain
273
290
  top_clouds[~dubious_profiles, :] = 0
274
291
  return top_clouds
275
292
 
276
- def _find_lwp_difference(self) -> np.ndarray:
293
+ def _find_lwp_difference(self) -> npt.NDArray:
277
294
  """Returns difference of theoretical LWP and measured LWP (g/m2).
278
295
 
279
296
  In theory, this difference should be always positive. Negative values
280
297
  indicate missing (or too narrow) liquid clouds.
281
298
  """
282
- lwc_sum = ma.sum(self.lwc_adiabatic, axis=1) * self.lwc_source.dheight
299
+ lwc_sum = ma.sum(self.lwc_adiabatic * self.lwc_source.path_lengths, axis=1)
283
300
  return lwc_sum - self.lwc_source.lwp
284
301
 
285
302
  def _mask_rain(self) -> None:
@@ -305,47 +322,51 @@ class LwcError:
305
322
 
306
323
  """
307
324
 
308
- def __init__(self, lwc_source: LwcSource, lwc: Lwc):
325
+ def __init__(self, lwc_source: LwcSource, lwc: Lwc) -> None:
309
326
  self.lwc = lwc.lwc
310
327
  self.lwc_source = lwc_source
311
328
  self.error = self._calculate_lwc_error()
312
329
  self._mask_rain()
313
330
 
314
- def _calculate_lwc_error(self) -> np.ndarray:
331
+ def _calculate_lwc_error(self) -> npt.NDArray:
315
332
  lwc_relative_error = self._calc_lwc_relative_error()
316
333
  lwp_relative_error = self._calc_lwp_relative_error()
317
334
  combined_error = self._calc_combined_error(
318
- lwc_relative_error, lwp_relative_error
335
+ lwc_relative_error,
336
+ lwp_relative_error,
319
337
  )
320
338
  return self._fill_error_array(combined_error)
321
339
 
322
- def _calc_lwc_relative_error(self) -> np.ndarray:
340
+ def _calc_lwc_relative_error(self) -> npt.NDArray:
323
341
  lwc_gradient = self._calc_lwc_gradient()
324
342
  error = lwc_gradient / self.lwc / 2
325
343
  return self._limit_error(error, 5)
326
344
 
327
- def _calc_lwc_gradient(self) -> np.ndarray:
328
- assert isinstance(self.lwc, ma.MaskedArray)
345
+ def _calc_lwc_gradient(self) -> npt.NDArray:
346
+ if not isinstance(self.lwc, ma.MaskedArray):
347
+ self.lwc = ma.masked_array(self.lwc)
329
348
  gradient_elements = np.gradient(self.lwc.filled(0))
330
349
  return utils.l2norm(*gradient_elements)
331
350
 
332
- def _calc_lwp_relative_error(self) -> np.ndarray:
351
+ def _calc_lwp_relative_error(self) -> npt.NDArray:
333
352
  err = self.lwc_source.lwp_error
334
353
  value = self.lwc_source.lwp
335
354
  error = np.divide(err, value, out=np.zeros_like(err), where=value != 0)
336
355
  return self._limit_error(error, 10)
337
356
 
338
357
  @staticmethod
339
- def _limit_error(error: np.ndarray, max_value: float) -> np.ndarray:
358
+ def _limit_error(error: npt.NDArray, max_value: float) -> npt.NDArray:
340
359
  error[error > max_value] = max_value
341
360
  return error
342
361
 
343
362
  @staticmethod
344
- def _calc_combined_error(error_2d: np.ndarray, error_1d: np.ndarray) -> np.ndarray:
363
+ def _calc_combined_error(
364
+ error_2d: npt.NDArray, error_1d: npt.NDArray
365
+ ) -> npt.NDArray:
345
366
  error_1d_transposed = utils.transpose(error_1d)
346
367
  return utils.l2norm(error_2d, error_1d_transposed)
347
368
 
348
- def _fill_error_array(self, error_in: np.ndarray) -> ma.MaskedArray:
369
+ def _fill_error_array(self, error_in: npt.NDArray) -> ma.MaskedArray:
349
370
  lwc_error = ma.masked_all(self.lwc.shape)
350
371
  ind = ma.where(self.lwc)
351
372
  lwc_error[ind] = error_in[ind]
@@ -406,25 +427,26 @@ COMMENTS = {
406
427
  }
407
428
 
408
429
  DEFINITIONS = {
409
- "lwc_retrieval_status": (
410
- "\n"
411
- "Value 0: No liquid water detected.\n"
412
- "Value 1: Reliable retrieval.\n"
413
- "Value 2: Adiabatic retrieval where cloud top has been adjusted to match\n"
414
- " liquid water path from microwave radiometer because layer is not\n"
415
- " detected by radar."
416
- "Value 3: Adiabatic retrieval: new cloud pixels where cloud top has been\n"
417
- " adjusted to match liquid water path from microwave radiometer\n"
418
- " because layer is not detected by radar.\n"
419
- "Value 4: No retrieval: either no liquid water path is available or liquid\n"
420
- " water path is uncertain.\n"
421
- "Value 5: No retrieval: liquid water layer detected only by the lidar and\n"
422
- " liquid water path is unavailable or uncertain: cloud top may be\n"
423
- " higher than diagnosed cloud top since lidar signal has been\n"
424
- " attenuated."
425
- "Value 6: Rain present: cloud extent is difficult to ascertain and liquid\n"
426
- " water path also uncertain."
427
- )
430
+ "lwc_retrieval_status": utils.status_field_definition(
431
+ {
432
+ 0: """No liquid water detected.""",
433
+ 1: """Reliable retrieval.""",
434
+ 2: """Adiabatic retrieval where cloud top has been adjusted to match
435
+ liquid water path from microwave radiometer because layer is
436
+ not detected by radar.""",
437
+ 3: """Adiabatic retrieval: new cloud pixels where cloud top has been
438
+ adjusted to match liquid water path from microwave radiometer
439
+ because layer is not detected by radar.""",
440
+ 4: """No retrieval: either no liquid water path is available or
441
+ liquid water path is uncertain.""",
442
+ 5: """No retrieval: liquid water layer detected only by the lidar
443
+ and liquid water path is unavailable or uncertain: cloud top
444
+ may be higher than diagnosed cloud top since lidar signal has
445
+ been attenuated.""",
446
+ 6: """Rain present: cloud extent is difficult to ascertain and
447
+ liquid water path also uncertain.""",
448
+ }
449
+ ),
428
450
  }
429
451
 
430
452
 
@@ -433,16 +455,20 @@ LWC_ATTRIBUTES = {
433
455
  long_name="Liquid water content",
434
456
  comment=COMMENTS["lwc"],
435
457
  ancillary_variables="lwc_error",
458
+ standard_name="mass_concentration_of_liquid_water_in_air",
459
+ dimensions=("time", "height"),
436
460
  ),
437
461
  "lwc_error": MetaData(
438
462
  long_name="Random error in liquid water content, one standard deviation",
439
463
  comment=COMMENTS["lwc_error"],
440
464
  units="dB",
465
+ dimensions=("time", "height"),
441
466
  ),
442
467
  "lwc_retrieval_status": MetaData(
443
468
  long_name="Liquid water content retrieval status",
444
469
  comment=COMMENTS["lwc_retrieval_status"],
445
470
  definition=DEFINITIONS["lwc_retrieval_status"],
446
471
  units="1",
472
+ dimensions=("time", "height"),
447
473
  ),
448
474
  }
@@ -0,0 +1,186 @@
1
+ import os
2
+ import tempfile
3
+ from os import PathLike
4
+ from typing import Literal
5
+ from uuid import UUID
6
+
7
+ import netCDF4
8
+ import numpy as np
9
+ import requests
10
+ from mwrpy.level2.lev2_collocated import generate_lev2_lhumpro as gen_lhumpro
11
+ from mwrpy.level2.lev2_collocated import generate_lev2_multi as gen_multi
12
+ from mwrpy.level2.lev2_collocated import generate_lev2_single as gen_single
13
+ from mwrpy.level2.write_lev2_nc import MissingInputData
14
+ from mwrpy.version import __version__ as mwrpy_version
15
+
16
+ from cloudnetpy import output, utils
17
+ from cloudnetpy.exceptions import ValidTimeStampError
18
+
19
+
20
+ def generate_mwr_single(
21
+ mwr_l1c_file: str | PathLike,
22
+ output_file: str | PathLike,
23
+ uuid: str | UUID | None = None,
24
+ lwp_offset: tuple[float | None, float | None] = (None, None),
25
+ ) -> UUID:
26
+ """Generates MWR single-pointing product including liquid water path, integrated
27
+ water vapor, etc. from zenith measurements.
28
+
29
+ Args:
30
+ mwr_l1c_file: The Level 1C MWR file to be processed.
31
+ output_file: The file path where the output file should be saved.
32
+ uuid: The UUID, if any, associated with the output file. Defaults to None.
33
+ lwp_offset: Optional offset to apply to the liquid water path.
34
+
35
+ Returns:
36
+ UUID of generated file.
37
+
38
+ Example:
39
+ >>> generate_mwr_single('input_mwr_l1c_file', 'output_file', 'abcdefg1234567')
40
+ """
41
+ return _generate_product(mwr_l1c_file, output_file, uuid, "single", lwp_offset)
42
+
43
+
44
+ def generate_mwr_lhumpro(
45
+ mwr_l1c_file: str | PathLike,
46
+ output_file: str | PathLike,
47
+ uuid: str | UUID | None = None,
48
+ lwp_offset: tuple[float | None, float | None] = (None, None),
49
+ ) -> UUID:
50
+ """Generates LHUMPRO single-pointing product including liquid water path, integrated
51
+ water vapor, etc. from zenith measurements.
52
+
53
+ Args:
54
+ mwr_l1c_file: The Level 1C MWR file to be processed.
55
+ output_file: The file path where the output file should be saved.
56
+ uuid: The UUID, if any, associated with the output file. Defaults to None.
57
+ lwp_offset: Optional offset to apply to the liquid water path.
58
+
59
+ Returns:
60
+ UUID of generated file.
61
+
62
+ Example:
63
+ >>> generate_mwr_lhumpro('input_mwr_l1c_file', 'output_file', 'abcdefg1234567')
64
+ """
65
+ return _generate_product(mwr_l1c_file, output_file, uuid, "lhumpro", lwp_offset)
66
+
67
+
68
+ def generate_mwr_multi(
69
+ mwr_l1c_file: str | PathLike,
70
+ output_file: str | PathLike,
71
+ uuid: str | UUID | None = None,
72
+ ) -> UUID:
73
+ """Generates MWR multiple-pointing product, including relative humidity profiles,
74
+ etc. from scanning measurements.
75
+
76
+ Args:
77
+ mwr_l1c_file: The input file in MWR L1C format.
78
+ output_file: The location where the output file should be generated.
79
+ uuid: The UUID for the MWR multi product, defaults to None if
80
+ not provided.
81
+
82
+ Returns:
83
+ UUID of generated file.
84
+ """
85
+ return _generate_product(mwr_l1c_file, output_file, uuid, "multi")
86
+
87
+
88
+ def _generate_product(
89
+ mwr_l1c_file: str | PathLike,
90
+ output_file: str | PathLike,
91
+ uuid: str | UUID | None,
92
+ product: Literal["single", "multi", "lhumpro"],
93
+ lwp_offset: tuple[float | None, float | None] = (None, None),
94
+ ) -> UUID:
95
+ uuid = utils.get_uuid(uuid)
96
+ with tempfile.TemporaryDirectory() as temp_dir:
97
+ coeffs = _read_mwrpy_coeffs(mwr_l1c_file, temp_dir)
98
+ try:
99
+ if product == "multi":
100
+ gen_multi(None, mwr_l1c_file, output_file, coeffs)
101
+ elif product == "single":
102
+ gen_single(None, mwr_l1c_file, output_file, lwp_offset, coeffs)
103
+ else:
104
+ gen_lhumpro(None, mwr_l1c_file, output_file, lwp_offset, coeffs)
105
+ product = "single"
106
+ except MissingInputData as err:
107
+ raise ValidTimeStampError from err
108
+ with (
109
+ netCDF4.Dataset(mwr_l1c_file, "r") as nc_input,
110
+ netCDF4.Dataset(output_file, "r+") as nc_output,
111
+ ):
112
+ flag_variable = "lwp" if product == "single" else "temperature"
113
+ flag_name = f"{flag_variable}_quality_flag"
114
+ flags = nc_output.variables[flag_name][:]
115
+ if not np.any(flags == 0):
116
+ msg = f"All {flag_variable} data are flagged."
117
+ raise ValidTimeStampError(msg)
118
+ mwr = Mwr(nc_input, nc_output, uuid)
119
+ return mwr.harmonize(product)
120
+
121
+
122
+ class Mwr:
123
+ def __init__(
124
+ self, nc_l1c: netCDF4.Dataset, nc_l2: netCDF4.Dataset, uuid: UUID
125
+ ) -> None:
126
+ self.nc_l1c = nc_l1c
127
+ self.nc_l2 = nc_l2
128
+ self.uuid = uuid
129
+
130
+ def harmonize(self, product: Literal["multi", "single"]) -> UUID:
131
+ self._truncate_global_attributes()
132
+ self._copy_global_attributes()
133
+ self._fix_variable_attributes()
134
+ self._write_missing_global_attributes(product)
135
+ return self.uuid
136
+
137
+ def _truncate_global_attributes(self) -> None:
138
+ for attr in self.nc_l2.ncattrs():
139
+ delattr(self.nc_l2, attr)
140
+
141
+ def _copy_global_attributes(self) -> None:
142
+ keys = ("year", "month", "day", "location", "source")
143
+ output.copy_global(self.nc_l1c, self.nc_l2, keys)
144
+
145
+ def _fix_variable_attributes(self) -> None:
146
+ output.replace_attribute_with_standard_value(
147
+ self.nc_l2,
148
+ (
149
+ "lwp",
150
+ "iwv",
151
+ "temperature",
152
+ "azimuth_angle",
153
+ "latitude",
154
+ "longitude",
155
+ "altitude",
156
+ ),
157
+ ("units", "long_name", "standard_name"),
158
+ )
159
+
160
+ def _write_missing_global_attributes(
161
+ self, product: Literal["multi", "single"]
162
+ ) -> None:
163
+ output.add_standard_global_attributes(self.nc_l2, self.uuid)
164
+ product_type = "multiple-pointing" if product == "multi" else "single-pointing"
165
+ self.nc_l2.title = f"MWR {product_type} from {self.nc_l1c.location}"
166
+ self.nc_l2.cloudnet_file_type = f"mwr-{product}"
167
+ output.fix_time_attributes(self.nc_l2)
168
+ self.nc_l2.history = (
169
+ f"{utils.get_time()} - MWR {product_type} file created \n"
170
+ f"{self.nc_l1c.history}"
171
+ )
172
+ self.nc_l2.source_file_uuids = self.nc_l1c.file_uuid
173
+ self.nc_l2.mwrpy_version = mwrpy_version
174
+ self.nc_l2.instrument_pid = self.nc_l1c.instrument_pid
175
+
176
+
177
+ def _read_mwrpy_coeffs(mwr_l1c_file: str | PathLike, folder: str) -> list[str]:
178
+ with netCDF4.Dataset(mwr_l1c_file) as nc:
179
+ links = nc.mwrpy_coefficients.split(", ")
180
+ coeffs = []
181
+ for link in links:
182
+ full_path = os.path.join(folder, link.split("/")[-1])
183
+ with open(full_path, "wb") as f:
184
+ f.write(requests.get(link, timeout=10).content)
185
+ coeffs.append(full_path)
186
+ return coeffs