cloudnetpy 1.55.20__py3-none-any.whl → 1.55.22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudnetpy/categorize/atmos.py +46 -14
- cloudnetpy/categorize/atmos_utils.py +11 -1
- cloudnetpy/categorize/categorize.py +38 -21
- cloudnetpy/categorize/classify.py +31 -9
- cloudnetpy/categorize/containers.py +19 -7
- cloudnetpy/categorize/droplet.py +24 -8
- cloudnetpy/categorize/falling.py +17 -7
- cloudnetpy/categorize/freezing.py +19 -5
- cloudnetpy/categorize/insects.py +27 -14
- cloudnetpy/categorize/lidar.py +38 -36
- cloudnetpy/categorize/melting.py +19 -9
- cloudnetpy/categorize/model.py +28 -9
- cloudnetpy/categorize/mwr.py +4 -2
- cloudnetpy/categorize/radar.py +58 -22
- cloudnetpy/cloudnetarray.py +15 -6
- cloudnetpy/concat_lib.py +39 -16
- cloudnetpy/constants.py +7 -0
- cloudnetpy/datasource.py +39 -19
- cloudnetpy/instruments/basta.py +6 -2
- cloudnetpy/instruments/campbell_scientific.py +33 -16
- cloudnetpy/instruments/ceilo.py +30 -13
- cloudnetpy/instruments/ceilometer.py +76 -37
- cloudnetpy/instruments/cl61d.py +8 -3
- cloudnetpy/instruments/cloudnet_instrument.py +2 -1
- cloudnetpy/instruments/copernicus.py +27 -14
- cloudnetpy/instruments/disdrometer/common.py +51 -32
- cloudnetpy/instruments/disdrometer/parsivel.py +79 -48
- cloudnetpy/instruments/disdrometer/thies.py +10 -6
- cloudnetpy/instruments/galileo.py +23 -12
- cloudnetpy/instruments/hatpro.py +27 -11
- cloudnetpy/instruments/instruments.py +4 -1
- cloudnetpy/instruments/lufft.py +20 -11
- cloudnetpy/instruments/mira.py +60 -49
- cloudnetpy/instruments/mrr.py +31 -20
- cloudnetpy/instruments/nc_lidar.py +15 -6
- cloudnetpy/instruments/nc_radar.py +31 -22
- cloudnetpy/instruments/pollyxt.py +36 -21
- cloudnetpy/instruments/radiometrics.py +32 -18
- cloudnetpy/instruments/rpg.py +48 -22
- cloudnetpy/instruments/rpg_reader.py +39 -30
- cloudnetpy/instruments/vaisala.py +39 -27
- cloudnetpy/instruments/weather_station.py +15 -11
- cloudnetpy/metadata.py +3 -1
- cloudnetpy/model_evaluation/file_handler.py +31 -21
- cloudnetpy/model_evaluation/metadata.py +3 -1
- cloudnetpy/model_evaluation/model_metadata.py +1 -1
- cloudnetpy/model_evaluation/plotting/plot_tools.py +20 -15
- cloudnetpy/model_evaluation/plotting/plotting.py +114 -64
- cloudnetpy/model_evaluation/products/advance_methods.py +48 -28
- cloudnetpy/model_evaluation/products/grid_methods.py +44 -19
- cloudnetpy/model_evaluation/products/model_products.py +22 -18
- cloudnetpy/model_evaluation/products/observation_products.py +15 -9
- cloudnetpy/model_evaluation/products/product_resampling.py +14 -4
- cloudnetpy/model_evaluation/products/tools.py +16 -7
- cloudnetpy/model_evaluation/statistics/statistical_methods.py +28 -15
- cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
- cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +14 -13
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +14 -13
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +14 -13
- cloudnetpy/model_evaluation/tests/unit/conftest.py +11 -11
- cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +33 -27
- cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +83 -83
- cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
- cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +24 -25
- cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +40 -39
- cloudnetpy/model_evaluation/tests/unit/test_plotting.py +12 -11
- cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +30 -30
- cloudnetpy/model_evaluation/tests/unit/test_tools.py +18 -17
- cloudnetpy/model_evaluation/utils.py +3 -2
- cloudnetpy/output.py +45 -19
- cloudnetpy/plotting/plot_meta.py +35 -11
- cloudnetpy/plotting/plotting.py +172 -104
- cloudnetpy/products/classification.py +20 -8
- cloudnetpy/products/der.py +25 -10
- cloudnetpy/products/drizzle.py +41 -26
- cloudnetpy/products/drizzle_error.py +10 -5
- cloudnetpy/products/drizzle_tools.py +43 -24
- cloudnetpy/products/ier.py +10 -5
- cloudnetpy/products/iwc.py +16 -9
- cloudnetpy/products/lwc.py +34 -12
- cloudnetpy/products/mwr_multi.py +4 -1
- cloudnetpy/products/mwr_single.py +4 -1
- cloudnetpy/products/product_tools.py +33 -10
- cloudnetpy/utils.py +175 -74
- cloudnetpy/version.py +1 -1
- {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/METADATA +11 -10
- cloudnetpy-1.55.22.dist-info/RECORD +114 -0
- docs/source/conf.py +2 -2
- cloudnetpy-1.55.20.dist-info/RECORD +0 -114
- {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/LICENSE +0 -0
- {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/WHEEL +0 -0
- {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/top_level.txt +0 -0
@@ -17,6 +17,7 @@ class AdvanceProductMethods(DataSource):
|
|
17
17
|
assumptions of model or observation data.
|
18
18
|
|
19
19
|
Args:
|
20
|
+
----
|
20
21
|
model_obj (object): The :class:'ModelManager' object.
|
21
22
|
obs_obj (object): The :class:'ObservationManager' object.
|
22
23
|
"""
|
@@ -38,30 +39,30 @@ class AdvanceProductMethods(DataSource):
|
|
38
39
|
self._model_height = model_obj.data[model_obj.keys["height"]][:]
|
39
40
|
self.generate_products()
|
40
41
|
|
41
|
-
def generate_products(self):
|
42
|
-
cls =
|
42
|
+
def generate_products(self) -> None:
|
43
|
+
cls = importlib.import_module(__name__).AdvanceProductMethods
|
43
44
|
try:
|
44
45
|
name = f"get_advance_{self.product}"
|
45
46
|
getattr(cls, name)(self)
|
46
47
|
except AttributeError as error:
|
47
|
-
logging.warning(
|
48
|
+
logging.warning("No advance method for %s: %s", self.product, error)
|
48
49
|
|
49
|
-
def get_advance_cf(self):
|
50
|
+
def get_advance_cf(self) -> None:
|
50
51
|
self.cf_cirrus_filter()
|
51
52
|
|
52
|
-
def cf_cirrus_filter(self):
|
53
|
+
def cf_cirrus_filter(self) -> None:
|
53
54
|
cf = self.getvar_from_object("cf")
|
54
55
|
h = self.getvar_from_object("h")
|
55
56
|
temperature = self.getvar("temperature")
|
56
57
|
t_screened = self.remove_extra_levels(temperature - 273.15)
|
57
|
-
iwc, lwc =
|
58
|
+
iwc, lwc = (self._model_obj.get_water_content(var) for var in ["iwc", "lwc"])
|
58
59
|
tZT, tT, tZ, t = self.set_frequency_parameters()
|
59
60
|
z_sen = self.fit_z_sensitivity(h)
|
60
61
|
cf_filtered = self.filter_high_iwc_low_cf(cf, iwc, lwc)
|
61
62
|
cloud_iwc, ice_ind = self.find_ice_in_clouds(cf_filtered, iwc, lwc)
|
62
63
|
variance_iwc = self.iwc_variance(h, ice_ind)
|
63
64
|
# Looks suspicious, check me:
|
64
|
-
for i, ind in enumerate(zip(ice_ind[0], ice_ind[-1])):
|
65
|
+
for i, ind in enumerate(zip(ice_ind[0], ice_ind[-1], strict=True)):
|
65
66
|
iwc_dist = self.calculate_iwc_distribution(cloud_iwc[i], variance_iwc[i])
|
66
67
|
p_iwc = self.gamma_distribution(iwc_dist, variance_iwc[i], cloud_iwc[i])
|
67
68
|
if np.sum(p_iwc) == 0 or p_iwc[-1] > 0.01 * np.sum(p_iwc):
|
@@ -92,7 +93,9 @@ class AdvanceProductMethods(DataSource):
|
|
92
93
|
return self._model_obj.cut_off_extra_levels(arg)
|
93
94
|
|
94
95
|
def set_frequency_parameters(self) -> tuple:
|
95
|
-
|
96
|
+
if self._obs_obj.radar_freq is None:
|
97
|
+
msg = "No radar frequency in observation file"
|
98
|
+
raise ValueError(msg)
|
96
99
|
if 30 <= self._obs_obj.radar_freq <= 40:
|
97
100
|
return 0.000242, -0.0186, 0.0699, -1.63
|
98
101
|
if 90 <= float(self._obs_obj.radar_freq) <= 100:
|
@@ -100,8 +103,12 @@ class AdvanceProductMethods(DataSource):
|
|
100
103
|
raise ValueError
|
101
104
|
|
102
105
|
def fit_z_sensitivity(self, h: np.ndarray) -> np.ndarray:
|
103
|
-
|
104
|
-
|
106
|
+
if self._obs_obj.z_sensitivity is None:
|
107
|
+
msg = "No z_sensitivity in observation file"
|
108
|
+
raise ValueError(msg)
|
109
|
+
if self._obs_obj.height is None:
|
110
|
+
msg = "No height in observation file"
|
111
|
+
raise ValueError(msg)
|
105
112
|
z_sen = [
|
106
113
|
cl_tools.rebin_1d(self._obs_obj.height, self._obs_obj.z_sensitivity, h[i])
|
107
114
|
for i in range(len(h))
|
@@ -109,16 +116,22 @@ class AdvanceProductMethods(DataSource):
|
|
109
116
|
return np.asarray(z_sen)
|
110
117
|
|
111
118
|
def filter_high_iwc_low_cf(
|
112
|
-
self,
|
119
|
+
self,
|
120
|
+
cf: np.ndarray,
|
121
|
+
iwc: np.ndarray,
|
122
|
+
lwc: np.ndarray,
|
113
123
|
) -> np.ndarray:
|
114
124
|
cf_filtered = self.mask_weird_indices(cf, iwc, lwc)
|
115
125
|
if np.sum((iwc > 0) & (lwc < iwc / 10) & (cf_filtered > 0)) == 0:
|
116
|
-
|
126
|
+
msg = "No ice clouds in a input data"
|
127
|
+
raise ValueError(msg)
|
117
128
|
return cf_filtered
|
118
129
|
|
119
130
|
@staticmethod
|
120
131
|
def mask_weird_indices(
|
121
|
-
cf: np.ndarray,
|
132
|
+
cf: np.ndarray,
|
133
|
+
iwc: np.ndarray,
|
134
|
+
lwc: np.ndarray,
|
122
135
|
) -> np.ndarray:
|
123
136
|
cf_filtered = np.copy(cf)
|
124
137
|
weird_ind = (iwc / cf > 0.5e-3) & (cf < 0.001)
|
@@ -127,7 +140,10 @@ class AdvanceProductMethods(DataSource):
|
|
127
140
|
return cf_filtered
|
128
141
|
|
129
142
|
def find_ice_in_clouds(
|
130
|
-
self,
|
143
|
+
self,
|
144
|
+
cf_filtered: np.ndarray,
|
145
|
+
iwc: np.ndarray,
|
146
|
+
lwc: np.ndarray,
|
131
147
|
) -> tuple[np.ndarray, tuple]:
|
132
148
|
ice_ind = self.get_ice_indices(cf_filtered, iwc, lwc)
|
133
149
|
cloud_iwc = iwc[ice_ind] / cf_filtered[ice_ind] * 1e3
|
@@ -135,7 +151,9 @@ class AdvanceProductMethods(DataSource):
|
|
135
151
|
|
136
152
|
@staticmethod
|
137
153
|
def get_ice_indices(
|
138
|
-
cf_filtered: np.ndarray,
|
154
|
+
cf_filtered: np.ndarray,
|
155
|
+
iwc: np.ndarray,
|
156
|
+
lwc: np.ndarray,
|
139
157
|
) -> tuple:
|
140
158
|
return tuple(np.where((cf_filtered > 0) & (iwc > 0) & (lwc < iwc / 10)))
|
141
159
|
|
@@ -145,8 +163,7 @@ class AdvanceProductMethods(DataSource):
|
|
145
163
|
u = self.remove_extra_levels(u)
|
146
164
|
v = self.remove_extra_levels(v)
|
147
165
|
w_shear = self.calculate_wind_shear(self._model_obj.wind, u, v, height)
|
148
|
-
|
149
|
-
return variance_iwc
|
166
|
+
return self.calculate_variance_iwc(w_shear, ice_ind)
|
150
167
|
|
151
168
|
def calculate_variance_iwc(self, w_shear: np.ndarray, ice_ind: tuple) -> np.ndarray:
|
152
169
|
return 10 ** (
|
@@ -157,7 +174,10 @@ class AdvanceProductMethods(DataSource):
|
|
157
174
|
|
158
175
|
@staticmethod
|
159
176
|
def calculate_wind_shear(
|
160
|
-
wind,
|
177
|
+
wind,
|
178
|
+
u: np.ndarray,
|
179
|
+
v: np.ndarray,
|
180
|
+
height: np.ndarray,
|
161
181
|
) -> np.ndarray:
|
162
182
|
grand_winds = []
|
163
183
|
for w in (wind, u, v):
|
@@ -189,9 +209,11 @@ class AdvanceProductMethods(DataSource):
|
|
189
209
|
|
190
210
|
@staticmethod
|
191
211
|
def gamma_distribution(
|
192
|
-
iwc_dist: np.ndarray,
|
212
|
+
iwc_dist: np.ndarray,
|
213
|
+
f_variance_iwc: float,
|
214
|
+
cloud_iwc: float,
|
193
215
|
) -> np.ndarray:
|
194
|
-
def calculate_gamma_dist():
|
216
|
+
def calculate_gamma_dist() -> float:
|
195
217
|
alpha = 1 / f_variance_iwc
|
196
218
|
return (
|
197
219
|
1
|
@@ -216,18 +238,16 @@ class AdvanceProductMethods(DataSource):
|
|
216
238
|
temperature: float,
|
217
239
|
z_sen: float,
|
218
240
|
) -> np.ndarray:
|
219
|
-
def calculate_min_iwc():
|
220
|
-
|
221
|
-
tZT * z_sen * temperature + tT * temperature + tZ * z_sen + t
|
222
|
-
)
|
223
|
-
return min_iwc
|
241
|
+
def calculate_min_iwc() -> np.ndarray:
|
242
|
+
return 10 ** (tZT * z_sen * temperature + tT * temperature + tZ * z_sen + t)
|
224
243
|
|
225
244
|
iwc_min = calculate_min_iwc()
|
226
|
-
|
227
|
-
return obs_index
|
245
|
+
return iwc_dist > iwc_min
|
228
246
|
|
229
247
|
@staticmethod
|
230
248
|
def filter_cirrus(
|
231
|
-
p_iwc: np.ndarray,
|
249
|
+
p_iwc: np.ndarray,
|
250
|
+
obs_index: np.ndarray,
|
251
|
+
cf_filtered: np.ndarray,
|
232
252
|
) -> np.ndarray:
|
233
253
|
return (np.sum(p_iwc * obs_index) / np.sum(p_iwc)) * cf_filtered
|
@@ -11,9 +11,12 @@ class ProductGrid:
|
|
11
11
|
"""Class to generate downsampling of observation product to model grid.
|
12
12
|
|
13
13
|
Args:
|
14
|
+
----
|
14
15
|
model_obj (object): The :class:'ModelManager' object.
|
15
16
|
obs_obj (object): The :class:'ObservationManager' object.
|
17
|
+
|
16
18
|
Notes:
|
19
|
+
-----
|
17
20
|
Downsampled observation products data is added to a ModelManager
|
18
21
|
object which is used for nc-file creation and writing
|
19
22
|
"""
|
@@ -28,13 +31,15 @@ class ProductGrid:
|
|
28
31
|
self._model_time = model_obj.time
|
29
32
|
self._model_height = model_obj.data[model_obj.keys["height"]][:]
|
30
33
|
self._time_adv = tl.calculate_advection_time(
|
31
|
-
int(model_obj.resolution_h),
|
34
|
+
int(model_obj.resolution_h),
|
35
|
+
ma.array(model_obj.wind),
|
36
|
+
1,
|
32
37
|
)
|
33
38
|
time_steps = utils.binvec(self._model_time)
|
34
39
|
self._time_steps = tl.time2datetime(time_steps, self._date)
|
35
40
|
self._generate_downsample_product()
|
36
41
|
|
37
|
-
def _generate_downsample_product(self):
|
42
|
+
def _generate_downsample_product(self) -> None:
|
38
43
|
"""Downsampling products are generated with different averaging methods
|
39
44
|
for a selected size of model time-height window.
|
40
45
|
"""
|
@@ -42,7 +47,8 @@ class ProductGrid:
|
|
42
47
|
model_t = tl.time2datetime(self._model_time, self._date)
|
43
48
|
for i in range(len(self._time_steps) - 1):
|
44
49
|
x_ind = tl.get_1d_indices(
|
45
|
-
(self._time_steps[i], self._time_steps[i + 1]),
|
50
|
+
(self._time_steps[i], self._time_steps[i + 1]),
|
51
|
+
self._obs_time,
|
46
52
|
)
|
47
53
|
if self._obs_obj.obs == "iwc":
|
48
54
|
x_ind_no_rain = tl.get_1d_indices(
|
@@ -53,10 +59,13 @@ class ProductGrid:
|
|
53
59
|
y_steps = tl.rebin_edges(self._model_height[i])
|
54
60
|
for j in range(len(y_steps) - 1):
|
55
61
|
x_ind_adv = tl.get_adv_indices(
|
56
|
-
model_t[i],
|
62
|
+
model_t[i],
|
63
|
+
self._time_adv[i, j],
|
64
|
+
self._obs_time,
|
57
65
|
)
|
58
66
|
y_ind = tl.get_1d_indices(
|
59
|
-
(y_steps[j], y_steps[j + 1]),
|
67
|
+
(y_steps[j], y_steps[j + 1]),
|
68
|
+
self._obs_height,
|
60
69
|
)
|
61
70
|
ind = np.outer(x_ind, y_ind)
|
62
71
|
ind_avd = np.outer(x_ind_adv, y_ind)
|
@@ -66,7 +75,9 @@ class ProductGrid:
|
|
66
75
|
continue
|
67
76
|
product_dict = self._regrid_cf(product_dict, i, j, data)
|
68
77
|
data_adv = self._reshape_data_to_window(ind_avd, x_ind_adv, y_ind)
|
69
|
-
|
78
|
+
if data_adv is None:
|
79
|
+
msg = "No data for advection"
|
80
|
+
raise RuntimeError(msg)
|
70
81
|
product_adv_dict = self._regrid_cf(product_adv_dict, i, j, data_adv)
|
71
82
|
elif self._obs_obj.obs == "iwc":
|
72
83
|
x_ind_no_rain_adv = tl.get_adv_indices(
|
@@ -78,19 +89,30 @@ class ProductGrid:
|
|
78
89
|
ind_no_rain = np.outer(x_ind_no_rain, y_ind)
|
79
90
|
ind_no_rain_adv = np.outer(x_ind_no_rain_adv, y_ind)
|
80
91
|
product_dict = self._regrid_iwc(
|
81
|
-
product_dict,
|
92
|
+
product_dict,
|
93
|
+
i,
|
94
|
+
j,
|
95
|
+
ind,
|
96
|
+
ind_no_rain,
|
82
97
|
)
|
83
98
|
product_adv_dict = self._regrid_iwc(
|
84
|
-
product_adv_dict,
|
99
|
+
product_adv_dict,
|
100
|
+
i,
|
101
|
+
j,
|
102
|
+
ind_avd,
|
103
|
+
ind_no_rain_adv,
|
85
104
|
)
|
86
105
|
else:
|
87
106
|
product_dict = self._regrid_product(product_dict, i, j, ind)
|
88
107
|
product_adv_dict = self._regrid_product(
|
89
|
-
product_adv_dict,
|
108
|
+
product_adv_dict,
|
109
|
+
i,
|
110
|
+
j,
|
111
|
+
ind_avd,
|
90
112
|
)
|
91
113
|
self._append_data2object([product_dict, product_adv_dict])
|
92
114
|
|
93
|
-
def _get_method_storage(self):
|
115
|
+
def _get_method_storage(self) -> tuple[dict, dict]:
|
94
116
|
if self._obs_obj.obs == "cf":
|
95
117
|
return self._cf_method_storage()
|
96
118
|
if self._obs_obj.obs == "iwc":
|
@@ -124,7 +146,7 @@ class ProductGrid:
|
|
124
146
|
def _product_method_storage(self) -> tuple[dict, dict]:
|
125
147
|
product_dict = {f"{self._obs_obj.obs}": np.zeros(self._model_height.shape)}
|
126
148
|
product_adv_dict = {
|
127
|
-
f"{self._obs_obj.obs}_adv": np.zeros(self._model_height.shape)
|
149
|
+
f"{self._obs_obj.obs}_adv": np.zeros(self._model_height.shape),
|
128
150
|
}
|
129
151
|
return product_dict, product_adv_dict
|
130
152
|
|
@@ -139,18 +161,21 @@ class ProductGrid:
|
|
139
161
|
downsample[i, j] = np.nanmean(data)
|
140
162
|
else:
|
141
163
|
downsample[i, j] = np.nan
|
142
|
-
if "_A" in key
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
164
|
+
if "_A" in key and (
|
165
|
+
not np.isnan(data).all()
|
166
|
+
and not (isinstance(data, ma.MaskedArray) and data.mask.all())
|
167
|
+
):
|
168
|
+
downsample[i, j] = tl.average_column_sum(data)
|
147
169
|
else:
|
148
170
|
downsample[i, j] = np.nan
|
149
171
|
storage[key] = downsample
|
150
172
|
return storage
|
151
173
|
|
152
174
|
def _reshape_data_to_window(
|
153
|
-
self,
|
175
|
+
self,
|
176
|
+
ind: np.ndarray,
|
177
|
+
x_ind: np.ndarray,
|
178
|
+
y_ind: np.ndarray,
|
154
179
|
) -> np.ndarray | None:
|
155
180
|
"""Reshapes True observation values to windows shape"""
|
156
181
|
window_size = tl.get_obs_window_size(x_ind, y_ind)
|
@@ -203,9 +228,9 @@ class ProductGrid:
|
|
203
228
|
storage[key] = down_sample
|
204
229
|
return storage
|
205
230
|
|
206
|
-
def _append_data2object(self, data_storage: list):
|
231
|
+
def _append_data2object(self, data_storage: list) -> None:
|
207
232
|
for storage in data_storage:
|
208
|
-
for key in storage
|
233
|
+
for key in storage:
|
209
234
|
down_sample = storage[key]
|
210
235
|
self.model_obj.append_data(
|
211
236
|
down_sample,
|
@@ -14,12 +14,14 @@ class ModelManager(DataSource):
|
|
14
14
|
"""Class to collect and manage model data.
|
15
15
|
|
16
16
|
Args:
|
17
|
+
----
|
17
18
|
model_file (str): Path to source model file.
|
18
19
|
model (str): Name of model
|
19
20
|
output_file (str): name of output file name and path to save data
|
20
21
|
product (str): name of product to generate
|
21
22
|
|
22
23
|
Notes:
|
24
|
+
-----
|
23
25
|
For this class to work, needed information of model in use should be found in
|
24
26
|
model_metadata.py
|
25
27
|
|
@@ -36,6 +38,7 @@ class ModelManager(DataSource):
|
|
36
38
|
model: str,
|
37
39
|
output_file: str,
|
38
40
|
product: str,
|
41
|
+
*,
|
39
42
|
check_file: bool = True,
|
40
43
|
):
|
41
44
|
super().__init__(model_file)
|
@@ -52,7 +55,7 @@ class ModelManager(DataSource):
|
|
52
55
|
self.wind = self._calculate_wind_speed()
|
53
56
|
self.resolution_h = self._get_horizontal_resolution()
|
54
57
|
|
55
|
-
def _read_cycle_name(self, model_file: str):
|
58
|
+
def _read_cycle_name(self, model_file: str) -> str:
|
56
59
|
"""Get cycle name from model_metadata.py for saving variable name(s)"""
|
57
60
|
try:
|
58
61
|
cycles = self.model_info.cycle
|
@@ -66,17 +69,18 @@ class ModelManager(DataSource):
|
|
66
69
|
return ""
|
67
70
|
return ""
|
68
71
|
|
69
|
-
def _generate_products(self):
|
72
|
+
def _generate_products(self) -> None:
|
70
73
|
"""Process needed data of model to a ModelManager object"""
|
71
|
-
cls =
|
74
|
+
cls = importlib.import_module(__name__).ModelManager
|
72
75
|
try:
|
73
76
|
name = f"_get_{self._product}"
|
74
77
|
getattr(cls, name)(self)
|
75
|
-
except AttributeError
|
76
|
-
|
78
|
+
except AttributeError:
|
79
|
+
msg = f"Invalid product name: {self._product}"
|
80
|
+
logging.exception(msg)
|
77
81
|
raise
|
78
82
|
|
79
|
-
def _get_cf(self):
|
83
|
+
def _get_cf(self) -> None:
|
80
84
|
"""Collect cloud fraction straight from model file."""
|
81
85
|
cf_name = self.get_model_var_names(("cf",))[0]
|
82
86
|
cf = self.getvar(cf_name)
|
@@ -85,13 +89,13 @@ class ModelManager(DataSource):
|
|
85
89
|
self.append_data(cf, f"{self.model}{self.cycle}_cf")
|
86
90
|
self.keys[self._product] = f"{self.model}{self.cycle}_cf"
|
87
91
|
|
88
|
-
def _get_iwc(self):
|
92
|
+
def _get_iwc(self) -> None:
|
89
93
|
iwc = self.get_water_content("iwc")
|
90
94
|
iwc[iwc < 1e-7] = ma.masked
|
91
95
|
self.append_data(iwc, f"{self.model}{self.cycle}_iwc")
|
92
96
|
self.keys[self._product] = f"{self.model}{self.cycle}_iwc"
|
93
97
|
|
94
|
-
def _get_lwc(self):
|
98
|
+
def _get_lwc(self) -> None:
|
95
99
|
lwc = self.get_water_content("lwc")
|
96
100
|
lwc[lwc < 1e-5] = ma.masked
|
97
101
|
self.append_data(lwc, f"{self.model}{self.cycle}_lwc")
|
@@ -120,13 +124,15 @@ class ModelManager(DataSource):
|
|
120
124
|
def _calc_water_content(q: np.ndarray, p: np.ndarray, t: np.ndarray) -> np.ndarray:
|
121
125
|
return q * p / (287 * t)
|
122
126
|
|
123
|
-
def _add_variables(self):
|
127
|
+
def _add_variables(self) -> None:
|
124
128
|
"""Add basic variables off model and cycle"""
|
125
129
|
|
126
|
-
def _add_common_variables():
|
130
|
+
def _add_common_variables() -> None:
|
127
131
|
"""Model variables that are always the same within cycles"""
|
128
132
|
wanted_vars = self.model_vars.common_var
|
129
|
-
|
133
|
+
if wanted_vars is None:
|
134
|
+
msg = f"Model {self.model} has no common variables"
|
135
|
+
raise ValueError(msg)
|
130
136
|
wanted_vars_split = [x.strip() for x in wanted_vars.split(",")]
|
131
137
|
for var in wanted_vars_split:
|
132
138
|
if var in self.dataset.variables:
|
@@ -135,10 +141,12 @@ class ModelManager(DataSource):
|
|
135
141
|
data = self.cut_off_extra_levels(self.dataset.variables[var][:])
|
136
142
|
self.append_data(data, f"{var}")
|
137
143
|
|
138
|
-
def _add_cycle_variables():
|
144
|
+
def _add_cycle_variables() -> None:
|
139
145
|
"""Add cycle depending variables"""
|
140
146
|
wanted_vars = self.model_vars.cycle_var
|
141
|
-
|
147
|
+
if wanted_vars is None:
|
148
|
+
msg = f"Model {self.model} has no cycle variables"
|
149
|
+
raise ValueError(msg)
|
142
150
|
wanted_vars_split = [x.strip() for x in wanted_vars.split(",")]
|
143
151
|
for var in wanted_vars_split:
|
144
152
|
if var in self.dataset.variables:
|
@@ -160,11 +168,7 @@ class ModelManager(DataSource):
|
|
160
168
|
except KeyError:
|
161
169
|
return data
|
162
170
|
|
163
|
-
if data.ndim > 1:
|
164
|
-
data = data[:, :level]
|
165
|
-
else:
|
166
|
-
data = data[:level]
|
167
|
-
return data
|
171
|
+
return data[:, :level] if data.ndim > 1 else data[:level]
|
168
172
|
|
169
173
|
def _calculate_wind_speed(self) -> np.ndarray:
|
170
174
|
"""Real wind from x- and y-components"""
|
@@ -1,5 +1,5 @@
|
|
1
1
|
import logging
|
2
|
-
from datetime import datetime
|
2
|
+
from datetime import datetime, timezone
|
3
3
|
|
4
4
|
import numpy as np
|
5
5
|
from numpy import ma
|
@@ -13,10 +13,12 @@ class ObservationManager(DataSource):
|
|
13
13
|
"""Class to collect and manage observations for downsampling.
|
14
14
|
|
15
15
|
Args:
|
16
|
+
----
|
16
17
|
obs (str): Name of observation product
|
17
18
|
obs_file (str): Path to source observation file
|
18
19
|
|
19
20
|
Notes:
|
21
|
+
-----
|
20
22
|
Output is ObservationManager object where all product data and
|
21
23
|
information is included.
|
22
24
|
|
@@ -42,6 +44,7 @@ class ObservationManager(DataSource):
|
|
42
44
|
0,
|
43
45
|
0,
|
44
46
|
0,
|
47
|
+
tzinfo=timezone.utc,
|
45
48
|
)
|
46
49
|
|
47
50
|
def _get_radar_frequency(self) -> np.ndarray | None:
|
@@ -56,7 +59,7 @@ class ObservationManager(DataSource):
|
|
56
59
|
except (KeyError, RuntimeError):
|
57
60
|
return None
|
58
61
|
|
59
|
-
def _generate_product(self):
|
62
|
+
def _generate_product(self) -> None:
|
60
63
|
"""Process needed data of observation to a ObservationManager object"""
|
61
64
|
try:
|
62
65
|
if self.obs == "cf":
|
@@ -66,16 +69,16 @@ class ObservationManager(DataSource):
|
|
66
69
|
if self.obs == "iwc":
|
67
70
|
self._generate_iwc_masks()
|
68
71
|
self.append_data(self.getvar("height"), "height")
|
69
|
-
except (KeyError, RuntimeError)
|
70
|
-
|
72
|
+
except (KeyError, RuntimeError):
|
73
|
+
msg = f"Failed to read {self.obs} from {self._file}"
|
74
|
+
logging.exception(msg)
|
71
75
|
raise
|
72
76
|
|
73
77
|
def _generate_cf(self) -> np.ndarray:
|
74
78
|
"""Generates cloud fractions using categorize bits and masking conditions"""
|
75
79
|
categorize_bits = CategorizeBits(self._file)
|
76
80
|
cloud_mask = self._classify_basic_mask(categorize_bits.category_bits)
|
77
|
-
|
78
|
-
return cloud_mask
|
81
|
+
return self._mask_cloud_bits(cloud_mask)
|
79
82
|
|
80
83
|
@staticmethod
|
81
84
|
def _classify_basic_mask(bits: dict) -> np.ndarray:
|
@@ -101,12 +104,14 @@ class ObservationManager(DataSource):
|
|
101
104
|
"""Check if rainrate in file"""
|
102
105
|
try:
|
103
106
|
self.getvar("rainrate")
|
104
|
-
return True
|
105
107
|
except RuntimeError:
|
106
108
|
return False
|
109
|
+
return True
|
107
110
|
|
108
111
|
def _get_rainrate_threshold(self) -> int:
|
109
|
-
|
112
|
+
if self.radar_freq is None:
|
113
|
+
msg = "Radar frequency not found from file"
|
114
|
+
raise RuntimeError(msg)
|
110
115
|
wband = utils.get_wl_band(float(self.radar_freq))
|
111
116
|
rainrate_threshold = 8
|
112
117
|
if 90 < wband < 100:
|
@@ -135,7 +140,8 @@ class ObservationManager(DataSource):
|
|
135
140
|
|
136
141
|
def _mask_iwc_att(self, iwc: np.ndarray, iwc_status: np.ndarray) -> None:
|
137
142
|
"""Leaves only where reliable data, corrected liquid attenuation
|
138
|
-
and uncorrected liquid attenuation
|
143
|
+
and uncorrected liquid attenuation
|
144
|
+
"""
|
139
145
|
iwc_att = ma.copy(iwc)
|
140
146
|
iwc_att[iwc_status > 3] = ma.masked
|
141
147
|
self.append_data(iwc_att, "iwc_att")
|
@@ -21,14 +21,17 @@ def process_L3_day_product(
|
|
21
21
|
product_file: str,
|
22
22
|
output_file: str,
|
23
23
|
uuid: str | None = None,
|
24
|
+
*,
|
24
25
|
overwrite: bool = False,
|
25
|
-
):
|
26
|
+
) -> str:
|
26
27
|
"""Main function to generate downsample of observations to match model grid.
|
27
28
|
|
28
29
|
This function will generate a L3 product nc-file. It includes the information of
|
29
30
|
downsampled observation products for each model cycles and model products
|
30
31
|
and other variables of each cycles.
|
32
|
+
|
31
33
|
Args:
|
34
|
+
----
|
32
35
|
model (str): Name of model
|
33
36
|
obs (str): Name of product to generate
|
34
37
|
model_files (list): List of model + cycles file path(s) to be generated
|
@@ -41,16 +44,19 @@ def process_L3_day_product(
|
|
41
44
|
default False
|
42
45
|
|
43
46
|
Raises:
|
47
|
+
------
|
44
48
|
RuntimeError: Failed to create the L3 product file.
|
45
49
|
ValueError (Warning): No ice clouds in model data
|
46
50
|
|
47
51
|
Notes:
|
52
|
+
-----
|
48
53
|
Model file(s) are given as a list to make all different cycles to be at same
|
49
54
|
nc-file. If list includes more than one model file, nc-file is created within
|
50
55
|
the first round. With rest of rounds, downsample observation and model data
|
51
56
|
is added to same L3 day nc-file.
|
52
57
|
|
53
58
|
Examples:
|
59
|
+
--------
|
54
60
|
>>> from cloudnetpy.model_evaluation.products.product_resampling import \
|
55
61
|
process_L3_day_product
|
56
62
|
>>> product = 'cf'
|
@@ -65,7 +71,11 @@ def process_L3_day_product(
|
|
65
71
|
tl.check_model_file_list(model, model_files)
|
66
72
|
for m_file in model_files:
|
67
73
|
model_obj = ModelManager(
|
68
|
-
m_file,
|
74
|
+
m_file,
|
75
|
+
model,
|
76
|
+
output_file,
|
77
|
+
obs,
|
78
|
+
check_file=not overwrite,
|
69
79
|
)
|
70
80
|
try:
|
71
81
|
AdvanceProductMethods(model_obj, m_file, product_obj)
|
@@ -76,7 +86,7 @@ def process_L3_day_product(
|
|
76
86
|
update_attributes(model_obj.data, attributes)
|
77
87
|
if not file_exists(output_file) or overwrite:
|
78
88
|
tl.add_date(model_obj, product_obj)
|
79
|
-
|
89
|
+
uuid_out = save_downsampled_file(
|
80
90
|
f"{obs}_{model}",
|
81
91
|
output_file,
|
82
92
|
(model_obj, product_obj),
|
@@ -85,4 +95,4 @@ def process_L3_day_product(
|
|
85
95
|
)
|
86
96
|
else:
|
87
97
|
add_var2ncfile(model_obj, output_file)
|
88
|
-
return
|
98
|
+
return uuid_out
|
@@ -14,7 +14,8 @@ def check_model_file_list(name: str, models: list) -> None:
|
|
14
14
|
for m in models:
|
15
15
|
if name not in m:
|
16
16
|
logging.error("Invalid model file set")
|
17
|
-
|
17
|
+
msg = f"{m} not from {name}"
|
18
|
+
raise AttributeError(msg)
|
18
19
|
|
19
20
|
|
20
21
|
def time2datetime(time: np.ndarray, date: datetime.datetime) -> np.ndarray:
|
@@ -30,18 +31,22 @@ def rebin_edges(arr: np.ndarray) -> np.ndarray:
|
|
30
31
|
|
31
32
|
|
32
33
|
def calculate_advection_time(
|
33
|
-
resolution: int,
|
34
|
+
resolution: int,
|
35
|
+
wind: ma.MaskedArray,
|
36
|
+
sampling: int,
|
34
37
|
) -> np.ndarray:
|
35
38
|
"""Calculates time which variable takes to go through the time window
|
36
39
|
|
37
|
-
Notes
|
40
|
+
Notes
|
41
|
+
-----
|
38
42
|
Wind speed is stronger in upper levels, so advection time is more
|
39
43
|
there then lower levels. Effect is small in a mid-latitudes,
|
40
44
|
but visible in a tropics.
|
41
45
|
|
42
46
|
sampling = 1 -> hour, sampling 1/6 -> 10min
|
43
47
|
|
44
|
-
References
|
48
|
+
References
|
49
|
+
----------
|
45
50
|
"""
|
46
51
|
t_adv = resolution * 1000 / wind / 60**2
|
47
52
|
t_adv[t_adv.mask] = 0
|
@@ -49,8 +54,12 @@ def calculate_advection_time(
|
|
49
54
|
return np.asarray([[timedelta(hours=float(t)) for t in time] for time in t_adv])
|
50
55
|
|
51
56
|
|
52
|
-
def get_1d_indices(
|
53
|
-
|
57
|
+
def get_1d_indices(
|
58
|
+
window: tuple,
|
59
|
+
data: np.ndarray,
|
60
|
+
mask: np.ndarray | None = None,
|
61
|
+
) -> np.ndarray:
|
62
|
+
indices: np.ndarray = np.array((window[0] <= data) & (data < window[-1]))
|
54
63
|
if mask is not None:
|
55
64
|
indices[mask] = ma.masked
|
56
65
|
return indices
|
@@ -61,7 +70,7 @@ def get_adv_indices(
|
|
61
70
|
adv_t: float,
|
62
71
|
data: np.ndarray,
|
63
72
|
mask: np.ndarray | None = None,
|
64
|
-
):
|
73
|
+
) -> np.ndarray:
|
65
74
|
adv_indices = ((model_t - adv_t / 2) <= data) & (data < (model_t + adv_t / 2))
|
66
75
|
if mask is not None:
|
67
76
|
adv_indices[mask] = ma.masked
|