cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudnetpy/categorize/__init__.py +1 -2
- cloudnetpy/categorize/atmos_utils.py +297 -67
- cloudnetpy/categorize/attenuation.py +31 -0
- cloudnetpy/categorize/attenuations/__init__.py +37 -0
- cloudnetpy/categorize/attenuations/gas_attenuation.py +30 -0
- cloudnetpy/categorize/attenuations/liquid_attenuation.py +84 -0
- cloudnetpy/categorize/attenuations/melting_attenuation.py +78 -0
- cloudnetpy/categorize/attenuations/rain_attenuation.py +84 -0
- cloudnetpy/categorize/categorize.py +332 -156
- cloudnetpy/categorize/classify.py +127 -125
- cloudnetpy/categorize/containers.py +107 -76
- cloudnetpy/categorize/disdrometer.py +40 -0
- cloudnetpy/categorize/droplet.py +23 -21
- cloudnetpy/categorize/falling.py +53 -24
- cloudnetpy/categorize/freezing.py +25 -12
- cloudnetpy/categorize/insects.py +35 -23
- cloudnetpy/categorize/itu.py +243 -0
- cloudnetpy/categorize/lidar.py +36 -41
- cloudnetpy/categorize/melting.py +34 -26
- cloudnetpy/categorize/model.py +84 -37
- cloudnetpy/categorize/mwr.py +18 -14
- cloudnetpy/categorize/radar.py +215 -102
- cloudnetpy/cli.py +578 -0
- cloudnetpy/cloudnetarray.py +43 -89
- cloudnetpy/concat_lib.py +218 -78
- cloudnetpy/constants.py +28 -10
- cloudnetpy/datasource.py +61 -86
- cloudnetpy/exceptions.py +49 -20
- cloudnetpy/instruments/__init__.py +5 -0
- cloudnetpy/instruments/basta.py +29 -12
- cloudnetpy/instruments/bowtie.py +135 -0
- cloudnetpy/instruments/ceilo.py +138 -115
- cloudnetpy/instruments/ceilometer.py +164 -80
- cloudnetpy/instruments/cl61d.py +21 -5
- cloudnetpy/instruments/cloudnet_instrument.py +74 -36
- cloudnetpy/instruments/copernicus.py +108 -30
- cloudnetpy/instruments/da10.py +54 -0
- cloudnetpy/instruments/disdrometer/common.py +126 -223
- cloudnetpy/instruments/disdrometer/parsivel.py +453 -94
- cloudnetpy/instruments/disdrometer/thies.py +254 -87
- cloudnetpy/instruments/fd12p.py +201 -0
- cloudnetpy/instruments/galileo.py +65 -23
- cloudnetpy/instruments/hatpro.py +123 -49
- cloudnetpy/instruments/instruments.py +113 -1
- cloudnetpy/instruments/lufft.py +39 -17
- cloudnetpy/instruments/mira.py +268 -61
- cloudnetpy/instruments/mrr.py +187 -0
- cloudnetpy/instruments/nc_lidar.py +19 -8
- cloudnetpy/instruments/nc_radar.py +109 -55
- cloudnetpy/instruments/pollyxt.py +135 -51
- cloudnetpy/instruments/radiometrics.py +313 -59
- cloudnetpy/instruments/rain_e_h3.py +171 -0
- cloudnetpy/instruments/rpg.py +321 -189
- cloudnetpy/instruments/rpg_reader.py +74 -40
- cloudnetpy/instruments/toa5.py +49 -0
- cloudnetpy/instruments/vaisala.py +95 -343
- cloudnetpy/instruments/weather_station.py +774 -105
- cloudnetpy/metadata.py +90 -19
- cloudnetpy/model_evaluation/file_handler.py +55 -52
- cloudnetpy/model_evaluation/metadata.py +46 -20
- cloudnetpy/model_evaluation/model_metadata.py +1 -1
- cloudnetpy/model_evaluation/plotting/plot_tools.py +32 -37
- cloudnetpy/model_evaluation/plotting/plotting.py +327 -117
- cloudnetpy/model_evaluation/products/advance_methods.py +92 -83
- cloudnetpy/model_evaluation/products/grid_methods.py +88 -63
- cloudnetpy/model_evaluation/products/model_products.py +43 -35
- cloudnetpy/model_evaluation/products/observation_products.py +41 -35
- cloudnetpy/model_evaluation/products/product_resampling.py +17 -7
- cloudnetpy/model_evaluation/products/tools.py +29 -20
- cloudnetpy/model_evaluation/statistics/statistical_methods.py +30 -20
- cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
- cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/unit/conftest.py +42 -41
- cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +41 -48
- cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +216 -194
- cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
- cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +37 -38
- cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +43 -40
- cloudnetpy/model_evaluation/tests/unit/test_plotting.py +30 -36
- cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +68 -31
- cloudnetpy/model_evaluation/tests/unit/test_tools.py +33 -26
- cloudnetpy/model_evaluation/utils.py +2 -1
- cloudnetpy/output.py +170 -111
- cloudnetpy/plotting/__init__.py +2 -1
- cloudnetpy/plotting/plot_meta.py +562 -822
- cloudnetpy/plotting/plotting.py +1142 -704
- cloudnetpy/products/__init__.py +1 -0
- cloudnetpy/products/classification.py +370 -88
- cloudnetpy/products/der.py +85 -55
- cloudnetpy/products/drizzle.py +77 -34
- cloudnetpy/products/drizzle_error.py +15 -11
- cloudnetpy/products/drizzle_tools.py +79 -59
- cloudnetpy/products/epsilon.py +211 -0
- cloudnetpy/products/ier.py +27 -50
- cloudnetpy/products/iwc.py +55 -48
- cloudnetpy/products/lwc.py +96 -70
- cloudnetpy/products/mwr_tools.py +186 -0
- cloudnetpy/products/product_tools.py +170 -128
- cloudnetpy/utils.py +455 -240
- cloudnetpy/version.py +2 -2
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/METADATA +44 -40
- cloudnetpy-1.87.3.dist-info/RECORD +127 -0
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/WHEEL +1 -1
- cloudnetpy-1.87.3.dist-info/entry_points.txt +2 -0
- docs/source/conf.py +2 -2
- cloudnetpy/categorize/atmos.py +0 -361
- cloudnetpy/products/mwr_multi.py +0 -68
- cloudnetpy/products/mwr_single.py +0 -75
- cloudnetpy-1.49.9.dist-info/RECORD +0 -112
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info/licenses}/LICENSE +0 -0
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/top_level.txt +0 -0
cloudnetpy/products/lwc.py
CHANGED
|
@@ -1,20 +1,28 @@
|
|
|
1
1
|
"""Module for creating Cloudnet liquid water content file using scaled-adiabatic
|
|
2
2
|
method.
|
|
3
3
|
"""
|
|
4
|
+
|
|
5
|
+
from os import PathLike
|
|
6
|
+
from uuid import UUID
|
|
7
|
+
|
|
4
8
|
import numpy as np
|
|
9
|
+
import numpy.typing as npt
|
|
5
10
|
from numpy import ma
|
|
6
11
|
|
|
7
12
|
from cloudnetpy import output, utils
|
|
8
|
-
from cloudnetpy.categorize import
|
|
13
|
+
from cloudnetpy.categorize import atmos_utils
|
|
9
14
|
from cloudnetpy.datasource import DataSource
|
|
15
|
+
from cloudnetpy.exceptions import InvalidSourceFileError
|
|
10
16
|
from cloudnetpy.metadata import MetaData
|
|
11
17
|
from cloudnetpy.products import product_tools as p_tools
|
|
12
18
|
from cloudnetpy.products.product_tools import CategorizeBits, get_is_rain
|
|
13
19
|
|
|
14
20
|
|
|
15
21
|
def generate_lwc(
|
|
16
|
-
categorize_file: str
|
|
17
|
-
|
|
22
|
+
categorize_file: str | PathLike,
|
|
23
|
+
output_file: str | PathLike,
|
|
24
|
+
uuid: str | UUID | None = None,
|
|
25
|
+
) -> UUID:
|
|
18
26
|
"""Generates Cloudnet liquid water content product.
|
|
19
27
|
|
|
20
28
|
This function calculates cloud liquid water content using the so-called
|
|
@@ -43,6 +51,7 @@ def generate_lwc(
|
|
|
43
51
|
Bull. Amer. Meteor. Soc., 88, 883–898, https://doi.org/10.1175/BAMS-88-6-883
|
|
44
52
|
|
|
45
53
|
"""
|
|
54
|
+
uuid = utils.get_uuid(uuid)
|
|
46
55
|
with LwcSource(categorize_file) as lwc_source:
|
|
47
56
|
lwc = Lwc(lwc_source)
|
|
48
57
|
clouds = CloudAdjustor(lwc_source, lwc)
|
|
@@ -51,7 +60,7 @@ def generate_lwc(
|
|
|
51
60
|
date = lwc_source.get_date()
|
|
52
61
|
attributes = output.add_time_attribute(LWC_ATTRIBUTES, date)
|
|
53
62
|
output.update_attributes(lwc_source.data, attributes)
|
|
54
|
-
|
|
63
|
+
output.save_product_file(
|
|
55
64
|
"lwc",
|
|
56
65
|
lwc_source,
|
|
57
66
|
output_file,
|
|
@@ -61,7 +70,7 @@ def generate_lwc(
|
|
|
61
70
|
"lwp_error",
|
|
62
71
|
),
|
|
63
72
|
)
|
|
64
|
-
|
|
73
|
+
return uuid
|
|
65
74
|
|
|
66
75
|
|
|
67
76
|
class LwcSource(DataSource):
|
|
@@ -77,32 +86,41 @@ class LwcSource(DataSource):
|
|
|
77
86
|
lwp (ndarray): 1D liquid water path.
|
|
78
87
|
lwp_error (ndarray): 1D error of liquid water path.
|
|
79
88
|
is_rain (ndarray): 1D array denoting presence of rain.
|
|
80
|
-
|
|
89
|
+
path_lengths (ndarray): 1D array of path lengths.
|
|
81
90
|
atmosphere (dict): Dictionary containing interpolated fields `temperature`
|
|
82
91
|
and `pressure`.
|
|
83
92
|
categorize_bits (CategorizeBits): The :class:`CategorizeBits` instance.
|
|
84
93
|
|
|
85
94
|
"""
|
|
86
95
|
|
|
87
|
-
def __init__(self, categorize_file: str):
|
|
96
|
+
def __init__(self, categorize_file: str | PathLike) -> None:
|
|
88
97
|
super().__init__(categorize_file)
|
|
98
|
+
if "lwp" not in self.dataset.variables:
|
|
99
|
+
msg = "Liquid water path missing from the categorize file."
|
|
100
|
+
raise InvalidSourceFileError(msg)
|
|
89
101
|
self.lwp = self.getvar("lwp")
|
|
90
102
|
self.lwp[self.lwp < 0] = 0
|
|
91
103
|
self.lwp_error = self.getvar("lwp_error")
|
|
92
104
|
self.is_rain = get_is_rain(categorize_file)
|
|
93
|
-
self.
|
|
105
|
+
self.height_agl: npt.NDArray
|
|
106
|
+
self.path_lengths = utils.path_lengths_from_ground(self.height_agl)
|
|
94
107
|
self.atmosphere = self._get_atmosphere(categorize_file)
|
|
95
108
|
self.categorize_bits = CategorizeBits(categorize_file)
|
|
96
109
|
|
|
97
110
|
def append_results(
|
|
98
|
-
self,
|
|
111
|
+
self,
|
|
112
|
+
lwc: npt.NDArray,
|
|
113
|
+
status: npt.NDArray,
|
|
114
|
+
error: npt.NDArray,
|
|
99
115
|
) -> None:
|
|
100
116
|
self.append_data(lwc, "lwc", units="kg m-3")
|
|
101
117
|
self.append_data(status, "lwc_retrieval_status")
|
|
102
118
|
self.append_data(error, "lwc_error", units="dB")
|
|
103
119
|
|
|
104
120
|
@staticmethod
|
|
105
|
-
def _get_atmosphere(
|
|
121
|
+
def _get_atmosphere(
|
|
122
|
+
categorize_file: str | PathLike,
|
|
123
|
+
) -> tuple[npt.NDArray, npt.NDArray]:
|
|
106
124
|
fields = ["temperature", "pressure"]
|
|
107
125
|
atmosphere = p_tools.interpolate_model(categorize_file, fields)
|
|
108
126
|
return atmosphere["temperature"], atmosphere["pressure"]
|
|
@@ -116,41 +134,42 @@ class Lwc:
|
|
|
116
134
|
|
|
117
135
|
Attributes:
|
|
118
136
|
lwc_source (LwcSource): The :class:`LwcSource` instance.
|
|
119
|
-
dheight (float): Median difference in height vector.
|
|
120
137
|
is_liquid (ndarray): 2D array denoting liquid.
|
|
121
138
|
lwc_adiabatic (ndarray): 2D array storing adiabatic lwc.
|
|
122
139
|
lwc (ndarray): 2D array of liquid water content (scaled with lwp).
|
|
123
140
|
|
|
124
141
|
"""
|
|
125
142
|
|
|
126
|
-
def __init__(self, lwc_source: LwcSource):
|
|
143
|
+
def __init__(self, lwc_source: LwcSource) -> None:
|
|
127
144
|
self.lwc_source = lwc_source
|
|
128
|
-
self.
|
|
145
|
+
self.height_agl = lwc_source.height_agl
|
|
129
146
|
self.is_liquid = self._get_liquid()
|
|
130
147
|
self.lwc_adiabatic = self._init_lwc_adiabatic()
|
|
131
148
|
self.lwc = self._adiabatic_lwc_to_lwc()
|
|
132
149
|
self._mask_rain()
|
|
133
150
|
|
|
134
|
-
def _get_liquid(self) ->
|
|
151
|
+
def _get_liquid(self) -> npt.NDArray:
|
|
135
152
|
category_bits = self.lwc_source.categorize_bits.category_bits
|
|
136
|
-
return category_bits
|
|
153
|
+
return category_bits.droplet
|
|
137
154
|
|
|
138
|
-
def _init_lwc_adiabatic(self) ->
|
|
155
|
+
def _init_lwc_adiabatic(self) -> npt.NDArray:
|
|
139
156
|
"""Returns theoretical adiabatic lwc in liquid clouds (kg/m3)."""
|
|
140
|
-
lwc_dz =
|
|
141
|
-
self.lwc_source.atmosphere,
|
|
157
|
+
lwc_dz = atmos_utils.fill_clouds_with_lwc_dz(
|
|
158
|
+
*self.lwc_source.atmosphere,
|
|
159
|
+
self.is_liquid,
|
|
142
160
|
)
|
|
143
|
-
return
|
|
161
|
+
return atmos_utils.calc_adiabatic_lwc(lwc_dz, self.height_agl)
|
|
144
162
|
|
|
145
|
-
def _adiabatic_lwc_to_lwc(self) ->
|
|
163
|
+
def _adiabatic_lwc_to_lwc(self) -> npt.NDArray:
|
|
146
164
|
"""Initialises liquid water content (kg/m3).
|
|
147
165
|
|
|
148
166
|
Calculates LWC for ALL profiles (rain, lwp > theoretical, etc.),
|
|
149
167
|
"""
|
|
150
|
-
|
|
151
|
-
self.lwc_adiabatic,
|
|
168
|
+
return atmos_utils.normalize_lwc_by_lwp(
|
|
169
|
+
self.lwc_adiabatic,
|
|
170
|
+
self.lwc_source.lwp,
|
|
171
|
+
self.height_agl,
|
|
152
172
|
)
|
|
153
|
-
return lwc_scaled / self.dheight
|
|
154
173
|
|
|
155
174
|
def _mask_rain(self) -> None:
|
|
156
175
|
is_rain = self.lwc_source.is_rain.astype(bool)
|
|
@@ -174,7 +193,7 @@ class CloudAdjustor:
|
|
|
174
193
|
|
|
175
194
|
"""
|
|
176
195
|
|
|
177
|
-
def __init__(self, lwc_source: LwcSource, lwc: Lwc):
|
|
196
|
+
def __init__(self, lwc_source: LwcSource, lwc: Lwc) -> None:
|
|
178
197
|
self.lwc_source = lwc_source
|
|
179
198
|
self.lwc = lwc.lwc
|
|
180
199
|
self.is_liquid = lwc.is_liquid
|
|
@@ -187,22 +206,23 @@ class CloudAdjustor:
|
|
|
187
206
|
|
|
188
207
|
def _get_echo(self) -> dict:
|
|
189
208
|
quality_bits = self.lwc_source.categorize_bits.quality_bits
|
|
190
|
-
return {
|
|
209
|
+
return {"radar": quality_bits.radar, "lidar": quality_bits.lidar}
|
|
191
210
|
|
|
192
211
|
def _init_status(self) -> ma.MaskedArray:
|
|
193
212
|
status = ma.zeros(self.is_liquid.shape, dtype=int)
|
|
194
213
|
status[self.is_liquid] = 1
|
|
195
214
|
return status
|
|
196
215
|
|
|
197
|
-
def _adjust_cloud_tops(self, adjustable_clouds:
|
|
216
|
+
def _adjust_cloud_tops(self, adjustable_clouds: npt.NDArray) -> None:
|
|
198
217
|
"""Adjusts cloud top index so that measured lwc corresponds to theoretical
|
|
199
|
-
value.
|
|
218
|
+
value.
|
|
219
|
+
"""
|
|
200
220
|
for time_index in np.unique(np.where(adjustable_clouds)[0]):
|
|
201
221
|
base_index = np.where(adjustable_clouds[time_index, :])[0][0]
|
|
202
222
|
self._update_status(time_index)
|
|
203
223
|
self._adjust_lwc(time_index, base_index)
|
|
204
224
|
|
|
205
|
-
def _update_status(self, time_ind:
|
|
225
|
+
def _update_status(self, time_ind: npt.NDArray) -> None:
|
|
206
226
|
alt_indices = np.where(self.is_liquid[time_ind, :])[0]
|
|
207
227
|
self.status[time_ind, alt_indices] = 2
|
|
208
228
|
|
|
@@ -220,24 +240,21 @@ class CloudAdjustor:
|
|
|
220
240
|
distance_from_base += 1
|
|
221
241
|
|
|
222
242
|
def _has_converged(self, ind: int) -> bool:
|
|
223
|
-
lwc_sum = ma.sum(self.lwc_adiabatic[ind, :])
|
|
224
|
-
|
|
225
|
-
return True
|
|
226
|
-
return False
|
|
243
|
+
lwc_sum = ma.sum(self.lwc_adiabatic[ind, :] * self.lwc_source.path_lengths)
|
|
244
|
+
return lwc_sum > self.lwc_source.lwp[ind]
|
|
227
245
|
|
|
228
246
|
def _out_of_bound(self, ind: int) -> bool:
|
|
229
247
|
return ind >= self.lwc.shape[1] - 1
|
|
230
248
|
|
|
231
|
-
def _find_adjustable_clouds(self) ->
|
|
249
|
+
def _find_adjustable_clouds(self) -> npt.NDArray:
|
|
232
250
|
top_clouds = self._find_topmost_clouds()
|
|
233
251
|
detection_type = self._find_echo_combinations_in_liquid()
|
|
234
252
|
detection_type[~top_clouds] = 0
|
|
235
253
|
lidar_only_clouds = self._find_lidar_only_clouds(detection_type)
|
|
236
254
|
top_clouds[~lidar_only_clouds, :] = 0
|
|
237
|
-
|
|
238
|
-
return top_clouds
|
|
255
|
+
return self._remove_good_profiles(top_clouds)
|
|
239
256
|
|
|
240
|
-
def _find_topmost_clouds(self) ->
|
|
257
|
+
def _find_topmost_clouds(self) -> npt.NDArray:
|
|
241
258
|
top_clouds = np.copy(self.is_liquid)
|
|
242
259
|
cloud_edges = top_clouds[:, :-1][:, ::-1] < top_clouds[:, 1:][:, ::-1]
|
|
243
260
|
topmost_bases = self.is_liquid.shape[1] - 1 - np.argmax(cloud_edges, axis=1)
|
|
@@ -245,14 +262,14 @@ class CloudAdjustor:
|
|
|
245
262
|
top_clouds[n, :base] = 0
|
|
246
263
|
return top_clouds
|
|
247
264
|
|
|
248
|
-
def _find_echo_combinations_in_liquid(self) ->
|
|
265
|
+
def _find_echo_combinations_in_liquid(self) -> npt.NDArray:
|
|
249
266
|
"""Classifies liquid clouds by detection type: 1=lidar, 2=radar, 3=both."""
|
|
250
267
|
lidar_detected = (self.is_liquid & self.echo["lidar"]).astype(int)
|
|
251
268
|
radar_detected = (self.is_liquid & self.echo["radar"]).astype(int) * 2
|
|
252
269
|
return lidar_detected + radar_detected
|
|
253
270
|
|
|
254
271
|
@staticmethod
|
|
255
|
-
def _find_lidar_only_clouds(detection:
|
|
272
|
+
def _find_lidar_only_clouds(detection: npt.NDArray) -> npt.NDArray:
|
|
256
273
|
"""Finds top clouds that contain only lidar-detected pixels.
|
|
257
274
|
|
|
258
275
|
Args:
|
|
@@ -266,20 +283,20 @@ class CloudAdjustor:
|
|
|
266
283
|
sum_of_detection_type = ma.sum(detection, axis=1)
|
|
267
284
|
return sum_of_cloud_pixels / sum_of_detection_type == 1
|
|
268
285
|
|
|
269
|
-
def _remove_good_profiles(self, top_clouds:
|
|
286
|
+
def _remove_good_profiles(self, top_clouds: npt.NDArray) -> npt.NDArray:
|
|
270
287
|
no_rain = ~self.lwc_source.is_rain.astype(bool)
|
|
271
288
|
lwp_difference = self._find_lwp_difference()
|
|
272
289
|
dubious_profiles = (lwp_difference < 0) & no_rain
|
|
273
290
|
top_clouds[~dubious_profiles, :] = 0
|
|
274
291
|
return top_clouds
|
|
275
292
|
|
|
276
|
-
def _find_lwp_difference(self) ->
|
|
293
|
+
def _find_lwp_difference(self) -> npt.NDArray:
|
|
277
294
|
"""Returns difference of theoretical LWP and measured LWP (g/m2).
|
|
278
295
|
|
|
279
296
|
In theory, this difference should be always positive. Negative values
|
|
280
297
|
indicate missing (or too narrow) liquid clouds.
|
|
281
298
|
"""
|
|
282
|
-
lwc_sum = ma.sum(self.lwc_adiabatic
|
|
299
|
+
lwc_sum = ma.sum(self.lwc_adiabatic * self.lwc_source.path_lengths, axis=1)
|
|
283
300
|
return lwc_sum - self.lwc_source.lwp
|
|
284
301
|
|
|
285
302
|
def _mask_rain(self) -> None:
|
|
@@ -305,47 +322,51 @@ class LwcError:
|
|
|
305
322
|
|
|
306
323
|
"""
|
|
307
324
|
|
|
308
|
-
def __init__(self, lwc_source: LwcSource, lwc: Lwc):
|
|
325
|
+
def __init__(self, lwc_source: LwcSource, lwc: Lwc) -> None:
|
|
309
326
|
self.lwc = lwc.lwc
|
|
310
327
|
self.lwc_source = lwc_source
|
|
311
328
|
self.error = self._calculate_lwc_error()
|
|
312
329
|
self._mask_rain()
|
|
313
330
|
|
|
314
|
-
def _calculate_lwc_error(self) ->
|
|
331
|
+
def _calculate_lwc_error(self) -> npt.NDArray:
|
|
315
332
|
lwc_relative_error = self._calc_lwc_relative_error()
|
|
316
333
|
lwp_relative_error = self._calc_lwp_relative_error()
|
|
317
334
|
combined_error = self._calc_combined_error(
|
|
318
|
-
lwc_relative_error,
|
|
335
|
+
lwc_relative_error,
|
|
336
|
+
lwp_relative_error,
|
|
319
337
|
)
|
|
320
338
|
return self._fill_error_array(combined_error)
|
|
321
339
|
|
|
322
|
-
def _calc_lwc_relative_error(self) ->
|
|
340
|
+
def _calc_lwc_relative_error(self) -> npt.NDArray:
|
|
323
341
|
lwc_gradient = self._calc_lwc_gradient()
|
|
324
342
|
error = lwc_gradient / self.lwc / 2
|
|
325
343
|
return self._limit_error(error, 5)
|
|
326
344
|
|
|
327
|
-
def _calc_lwc_gradient(self) ->
|
|
328
|
-
|
|
345
|
+
def _calc_lwc_gradient(self) -> npt.NDArray:
|
|
346
|
+
if not isinstance(self.lwc, ma.MaskedArray):
|
|
347
|
+
self.lwc = ma.masked_array(self.lwc)
|
|
329
348
|
gradient_elements = np.gradient(self.lwc.filled(0))
|
|
330
349
|
return utils.l2norm(*gradient_elements)
|
|
331
350
|
|
|
332
|
-
def _calc_lwp_relative_error(self) ->
|
|
351
|
+
def _calc_lwp_relative_error(self) -> npt.NDArray:
|
|
333
352
|
err = self.lwc_source.lwp_error
|
|
334
353
|
value = self.lwc_source.lwp
|
|
335
354
|
error = np.divide(err, value, out=np.zeros_like(err), where=value != 0)
|
|
336
355
|
return self._limit_error(error, 10)
|
|
337
356
|
|
|
338
357
|
@staticmethod
|
|
339
|
-
def _limit_error(error:
|
|
358
|
+
def _limit_error(error: npt.NDArray, max_value: float) -> npt.NDArray:
|
|
340
359
|
error[error > max_value] = max_value
|
|
341
360
|
return error
|
|
342
361
|
|
|
343
362
|
@staticmethod
|
|
344
|
-
def _calc_combined_error(
|
|
363
|
+
def _calc_combined_error(
|
|
364
|
+
error_2d: npt.NDArray, error_1d: npt.NDArray
|
|
365
|
+
) -> npt.NDArray:
|
|
345
366
|
error_1d_transposed = utils.transpose(error_1d)
|
|
346
367
|
return utils.l2norm(error_2d, error_1d_transposed)
|
|
347
368
|
|
|
348
|
-
def _fill_error_array(self, error_in:
|
|
369
|
+
def _fill_error_array(self, error_in: npt.NDArray) -> ma.MaskedArray:
|
|
349
370
|
lwc_error = ma.masked_all(self.lwc.shape)
|
|
350
371
|
ind = ma.where(self.lwc)
|
|
351
372
|
lwc_error[ind] = error_in[ind]
|
|
@@ -406,25 +427,26 @@ COMMENTS = {
|
|
|
406
427
|
}
|
|
407
428
|
|
|
408
429
|
DEFINITIONS = {
|
|
409
|
-
"lwc_retrieval_status": (
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
430
|
+
"lwc_retrieval_status": utils.status_field_definition(
|
|
431
|
+
{
|
|
432
|
+
0: """No liquid water detected.""",
|
|
433
|
+
1: """Reliable retrieval.""",
|
|
434
|
+
2: """Adiabatic retrieval where cloud top has been adjusted to match
|
|
435
|
+
liquid water path from microwave radiometer because layer is
|
|
436
|
+
not detected by radar.""",
|
|
437
|
+
3: """Adiabatic retrieval: new cloud pixels where cloud top has been
|
|
438
|
+
adjusted to match liquid water path from microwave radiometer
|
|
439
|
+
because layer is not detected by radar.""",
|
|
440
|
+
4: """No retrieval: either no liquid water path is available or
|
|
441
|
+
liquid water path is uncertain.""",
|
|
442
|
+
5: """No retrieval: liquid water layer detected only by the lidar
|
|
443
|
+
and liquid water path is unavailable or uncertain: cloud top
|
|
444
|
+
may be higher than diagnosed cloud top since lidar signal has
|
|
445
|
+
been attenuated.""",
|
|
446
|
+
6: """Rain present: cloud extent is difficult to ascertain and
|
|
447
|
+
liquid water path also uncertain.""",
|
|
448
|
+
}
|
|
449
|
+
),
|
|
428
450
|
}
|
|
429
451
|
|
|
430
452
|
|
|
@@ -433,16 +455,20 @@ LWC_ATTRIBUTES = {
|
|
|
433
455
|
long_name="Liquid water content",
|
|
434
456
|
comment=COMMENTS["lwc"],
|
|
435
457
|
ancillary_variables="lwc_error",
|
|
458
|
+
standard_name="mass_concentration_of_liquid_water_in_air",
|
|
459
|
+
dimensions=("time", "height"),
|
|
436
460
|
),
|
|
437
461
|
"lwc_error": MetaData(
|
|
438
462
|
long_name="Random error in liquid water content, one standard deviation",
|
|
439
463
|
comment=COMMENTS["lwc_error"],
|
|
440
464
|
units="dB",
|
|
465
|
+
dimensions=("time", "height"),
|
|
441
466
|
),
|
|
442
467
|
"lwc_retrieval_status": MetaData(
|
|
443
468
|
long_name="Liquid water content retrieval status",
|
|
444
469
|
comment=COMMENTS["lwc_retrieval_status"],
|
|
445
470
|
definition=DEFINITIONS["lwc_retrieval_status"],
|
|
446
471
|
units="1",
|
|
472
|
+
dimensions=("time", "height"),
|
|
447
473
|
),
|
|
448
474
|
}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import tempfile
|
|
3
|
+
from os import PathLike
|
|
4
|
+
from typing import Literal
|
|
5
|
+
from uuid import UUID
|
|
6
|
+
|
|
7
|
+
import netCDF4
|
|
8
|
+
import numpy as np
|
|
9
|
+
import requests
|
|
10
|
+
from mwrpy.level2.lev2_collocated import generate_lev2_lhumpro as gen_lhumpro
|
|
11
|
+
from mwrpy.level2.lev2_collocated import generate_lev2_multi as gen_multi
|
|
12
|
+
from mwrpy.level2.lev2_collocated import generate_lev2_single as gen_single
|
|
13
|
+
from mwrpy.level2.write_lev2_nc import MissingInputData
|
|
14
|
+
from mwrpy.version import __version__ as mwrpy_version
|
|
15
|
+
|
|
16
|
+
from cloudnetpy import output, utils
|
|
17
|
+
from cloudnetpy.exceptions import ValidTimeStampError
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def generate_mwr_single(
|
|
21
|
+
mwr_l1c_file: str | PathLike,
|
|
22
|
+
output_file: str | PathLike,
|
|
23
|
+
uuid: str | UUID | None = None,
|
|
24
|
+
lwp_offset: tuple[float | None, float | None] = (None, None),
|
|
25
|
+
) -> UUID:
|
|
26
|
+
"""Generates MWR single-pointing product including liquid water path, integrated
|
|
27
|
+
water vapor, etc. from zenith measurements.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
mwr_l1c_file: The Level 1C MWR file to be processed.
|
|
31
|
+
output_file: The file path where the output file should be saved.
|
|
32
|
+
uuid: The UUID, if any, associated with the output file. Defaults to None.
|
|
33
|
+
lwp_offset: Optional offset to apply to the liquid water path.
|
|
34
|
+
|
|
35
|
+
Returns:
|
|
36
|
+
UUID of generated file.
|
|
37
|
+
|
|
38
|
+
Example:
|
|
39
|
+
>>> generate_mwr_single('input_mwr_l1c_file', 'output_file', 'abcdefg1234567')
|
|
40
|
+
"""
|
|
41
|
+
return _generate_product(mwr_l1c_file, output_file, uuid, "single", lwp_offset)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def generate_mwr_lhumpro(
|
|
45
|
+
mwr_l1c_file: str | PathLike,
|
|
46
|
+
output_file: str | PathLike,
|
|
47
|
+
uuid: str | UUID | None = None,
|
|
48
|
+
lwp_offset: tuple[float | None, float | None] = (None, None),
|
|
49
|
+
) -> UUID:
|
|
50
|
+
"""Generates LHUMPRO single-pointing product including liquid water path, integrated
|
|
51
|
+
water vapor, etc. from zenith measurements.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
mwr_l1c_file: The Level 1C MWR file to be processed.
|
|
55
|
+
output_file: The file path where the output file should be saved.
|
|
56
|
+
uuid: The UUID, if any, associated with the output file. Defaults to None.
|
|
57
|
+
lwp_offset: Optional offset to apply to the liquid water path.
|
|
58
|
+
|
|
59
|
+
Returns:
|
|
60
|
+
UUID of generated file.
|
|
61
|
+
|
|
62
|
+
Example:
|
|
63
|
+
>>> generate_mwr_lhumpro('input_mwr_l1c_file', 'output_file', 'abcdefg1234567')
|
|
64
|
+
"""
|
|
65
|
+
return _generate_product(mwr_l1c_file, output_file, uuid, "lhumpro", lwp_offset)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def generate_mwr_multi(
|
|
69
|
+
mwr_l1c_file: str | PathLike,
|
|
70
|
+
output_file: str | PathLike,
|
|
71
|
+
uuid: str | UUID | None = None,
|
|
72
|
+
) -> UUID:
|
|
73
|
+
"""Generates MWR multiple-pointing product, including relative humidity profiles,
|
|
74
|
+
etc. from scanning measurements.
|
|
75
|
+
|
|
76
|
+
Args:
|
|
77
|
+
mwr_l1c_file: The input file in MWR L1C format.
|
|
78
|
+
output_file: The location where the output file should be generated.
|
|
79
|
+
uuid: The UUID for the MWR multi product, defaults to None if
|
|
80
|
+
not provided.
|
|
81
|
+
|
|
82
|
+
Returns:
|
|
83
|
+
UUID of generated file.
|
|
84
|
+
"""
|
|
85
|
+
return _generate_product(mwr_l1c_file, output_file, uuid, "multi")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _generate_product(
|
|
89
|
+
mwr_l1c_file: str | PathLike,
|
|
90
|
+
output_file: str | PathLike,
|
|
91
|
+
uuid: str | UUID | None,
|
|
92
|
+
product: Literal["single", "multi", "lhumpro"],
|
|
93
|
+
lwp_offset: tuple[float | None, float | None] = (None, None),
|
|
94
|
+
) -> UUID:
|
|
95
|
+
uuid = utils.get_uuid(uuid)
|
|
96
|
+
with tempfile.TemporaryDirectory() as temp_dir:
|
|
97
|
+
coeffs = _read_mwrpy_coeffs(mwr_l1c_file, temp_dir)
|
|
98
|
+
try:
|
|
99
|
+
if product == "multi":
|
|
100
|
+
gen_multi(None, mwr_l1c_file, output_file, coeffs)
|
|
101
|
+
elif product == "single":
|
|
102
|
+
gen_single(None, mwr_l1c_file, output_file, lwp_offset, coeffs)
|
|
103
|
+
else:
|
|
104
|
+
gen_lhumpro(None, mwr_l1c_file, output_file, lwp_offset, coeffs)
|
|
105
|
+
product = "single"
|
|
106
|
+
except MissingInputData as err:
|
|
107
|
+
raise ValidTimeStampError from err
|
|
108
|
+
with (
|
|
109
|
+
netCDF4.Dataset(mwr_l1c_file, "r") as nc_input,
|
|
110
|
+
netCDF4.Dataset(output_file, "r+") as nc_output,
|
|
111
|
+
):
|
|
112
|
+
flag_variable = "lwp" if product == "single" else "temperature"
|
|
113
|
+
flag_name = f"{flag_variable}_quality_flag"
|
|
114
|
+
flags = nc_output.variables[flag_name][:]
|
|
115
|
+
if not np.any(flags == 0):
|
|
116
|
+
msg = f"All {flag_variable} data are flagged."
|
|
117
|
+
raise ValidTimeStampError(msg)
|
|
118
|
+
mwr = Mwr(nc_input, nc_output, uuid)
|
|
119
|
+
return mwr.harmonize(product)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class Mwr:
|
|
123
|
+
def __init__(
|
|
124
|
+
self, nc_l1c: netCDF4.Dataset, nc_l2: netCDF4.Dataset, uuid: UUID
|
|
125
|
+
) -> None:
|
|
126
|
+
self.nc_l1c = nc_l1c
|
|
127
|
+
self.nc_l2 = nc_l2
|
|
128
|
+
self.uuid = uuid
|
|
129
|
+
|
|
130
|
+
def harmonize(self, product: Literal["multi", "single"]) -> UUID:
|
|
131
|
+
self._truncate_global_attributes()
|
|
132
|
+
self._copy_global_attributes()
|
|
133
|
+
self._fix_variable_attributes()
|
|
134
|
+
self._write_missing_global_attributes(product)
|
|
135
|
+
return self.uuid
|
|
136
|
+
|
|
137
|
+
def _truncate_global_attributes(self) -> None:
|
|
138
|
+
for attr in self.nc_l2.ncattrs():
|
|
139
|
+
delattr(self.nc_l2, attr)
|
|
140
|
+
|
|
141
|
+
def _copy_global_attributes(self) -> None:
|
|
142
|
+
keys = ("year", "month", "day", "location", "source")
|
|
143
|
+
output.copy_global(self.nc_l1c, self.nc_l2, keys)
|
|
144
|
+
|
|
145
|
+
def _fix_variable_attributes(self) -> None:
|
|
146
|
+
output.replace_attribute_with_standard_value(
|
|
147
|
+
self.nc_l2,
|
|
148
|
+
(
|
|
149
|
+
"lwp",
|
|
150
|
+
"iwv",
|
|
151
|
+
"temperature",
|
|
152
|
+
"azimuth_angle",
|
|
153
|
+
"latitude",
|
|
154
|
+
"longitude",
|
|
155
|
+
"altitude",
|
|
156
|
+
),
|
|
157
|
+
("units", "long_name", "standard_name"),
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
def _write_missing_global_attributes(
|
|
161
|
+
self, product: Literal["multi", "single"]
|
|
162
|
+
) -> None:
|
|
163
|
+
output.add_standard_global_attributes(self.nc_l2, self.uuid)
|
|
164
|
+
product_type = "multiple-pointing" if product == "multi" else "single-pointing"
|
|
165
|
+
self.nc_l2.title = f"MWR {product_type} from {self.nc_l1c.location}"
|
|
166
|
+
self.nc_l2.cloudnet_file_type = f"mwr-{product}"
|
|
167
|
+
output.fix_time_attributes(self.nc_l2)
|
|
168
|
+
self.nc_l2.history = (
|
|
169
|
+
f"{utils.get_time()} - MWR {product_type} file created \n"
|
|
170
|
+
f"{self.nc_l1c.history}"
|
|
171
|
+
)
|
|
172
|
+
self.nc_l2.source_file_uuids = self.nc_l1c.file_uuid
|
|
173
|
+
self.nc_l2.mwrpy_version = mwrpy_version
|
|
174
|
+
self.nc_l2.instrument_pid = self.nc_l1c.instrument_pid
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _read_mwrpy_coeffs(mwr_l1c_file: str | PathLike, folder: str) -> list[str]:
|
|
178
|
+
with netCDF4.Dataset(mwr_l1c_file) as nc:
|
|
179
|
+
links = nc.mwrpy_coefficients.split(", ")
|
|
180
|
+
coeffs = []
|
|
181
|
+
for link in links:
|
|
182
|
+
full_path = os.path.join(folder, link.split("/")[-1])
|
|
183
|
+
with open(full_path, "wb") as f:
|
|
184
|
+
f.write(requests.get(link, timeout=10).content)
|
|
185
|
+
coeffs.append(full_path)
|
|
186
|
+
return coeffs
|