cloudnetpy 1.55.20__py3-none-any.whl → 1.55.22__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. cloudnetpy/categorize/atmos.py +46 -14
  2. cloudnetpy/categorize/atmos_utils.py +11 -1
  3. cloudnetpy/categorize/categorize.py +38 -21
  4. cloudnetpy/categorize/classify.py +31 -9
  5. cloudnetpy/categorize/containers.py +19 -7
  6. cloudnetpy/categorize/droplet.py +24 -8
  7. cloudnetpy/categorize/falling.py +17 -7
  8. cloudnetpy/categorize/freezing.py +19 -5
  9. cloudnetpy/categorize/insects.py +27 -14
  10. cloudnetpy/categorize/lidar.py +38 -36
  11. cloudnetpy/categorize/melting.py +19 -9
  12. cloudnetpy/categorize/model.py +28 -9
  13. cloudnetpy/categorize/mwr.py +4 -2
  14. cloudnetpy/categorize/radar.py +58 -22
  15. cloudnetpy/cloudnetarray.py +15 -6
  16. cloudnetpy/concat_lib.py +39 -16
  17. cloudnetpy/constants.py +7 -0
  18. cloudnetpy/datasource.py +39 -19
  19. cloudnetpy/instruments/basta.py +6 -2
  20. cloudnetpy/instruments/campbell_scientific.py +33 -16
  21. cloudnetpy/instruments/ceilo.py +30 -13
  22. cloudnetpy/instruments/ceilometer.py +76 -37
  23. cloudnetpy/instruments/cl61d.py +8 -3
  24. cloudnetpy/instruments/cloudnet_instrument.py +2 -1
  25. cloudnetpy/instruments/copernicus.py +27 -14
  26. cloudnetpy/instruments/disdrometer/common.py +51 -32
  27. cloudnetpy/instruments/disdrometer/parsivel.py +79 -48
  28. cloudnetpy/instruments/disdrometer/thies.py +10 -6
  29. cloudnetpy/instruments/galileo.py +23 -12
  30. cloudnetpy/instruments/hatpro.py +27 -11
  31. cloudnetpy/instruments/instruments.py +4 -1
  32. cloudnetpy/instruments/lufft.py +20 -11
  33. cloudnetpy/instruments/mira.py +60 -49
  34. cloudnetpy/instruments/mrr.py +31 -20
  35. cloudnetpy/instruments/nc_lidar.py +15 -6
  36. cloudnetpy/instruments/nc_radar.py +31 -22
  37. cloudnetpy/instruments/pollyxt.py +36 -21
  38. cloudnetpy/instruments/radiometrics.py +32 -18
  39. cloudnetpy/instruments/rpg.py +48 -22
  40. cloudnetpy/instruments/rpg_reader.py +39 -30
  41. cloudnetpy/instruments/vaisala.py +39 -27
  42. cloudnetpy/instruments/weather_station.py +15 -11
  43. cloudnetpy/metadata.py +3 -1
  44. cloudnetpy/model_evaluation/file_handler.py +31 -21
  45. cloudnetpy/model_evaluation/metadata.py +3 -1
  46. cloudnetpy/model_evaluation/model_metadata.py +1 -1
  47. cloudnetpy/model_evaluation/plotting/plot_tools.py +20 -15
  48. cloudnetpy/model_evaluation/plotting/plotting.py +114 -64
  49. cloudnetpy/model_evaluation/products/advance_methods.py +48 -28
  50. cloudnetpy/model_evaluation/products/grid_methods.py +44 -19
  51. cloudnetpy/model_evaluation/products/model_products.py +22 -18
  52. cloudnetpy/model_evaluation/products/observation_products.py +15 -9
  53. cloudnetpy/model_evaluation/products/product_resampling.py +14 -4
  54. cloudnetpy/model_evaluation/products/tools.py +16 -7
  55. cloudnetpy/model_evaluation/statistics/statistical_methods.py +28 -15
  56. cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
  57. cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
  58. cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +14 -13
  59. cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
  60. cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +14 -13
  61. cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
  62. cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +14 -13
  63. cloudnetpy/model_evaluation/tests/unit/conftest.py +11 -11
  64. cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +33 -27
  65. cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +83 -83
  66. cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
  67. cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +24 -25
  68. cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +40 -39
  69. cloudnetpy/model_evaluation/tests/unit/test_plotting.py +12 -11
  70. cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +30 -30
  71. cloudnetpy/model_evaluation/tests/unit/test_tools.py +18 -17
  72. cloudnetpy/model_evaluation/utils.py +3 -2
  73. cloudnetpy/output.py +45 -19
  74. cloudnetpy/plotting/plot_meta.py +35 -11
  75. cloudnetpy/plotting/plotting.py +172 -104
  76. cloudnetpy/products/classification.py +20 -8
  77. cloudnetpy/products/der.py +25 -10
  78. cloudnetpy/products/drizzle.py +41 -26
  79. cloudnetpy/products/drizzle_error.py +10 -5
  80. cloudnetpy/products/drizzle_tools.py +43 -24
  81. cloudnetpy/products/ier.py +10 -5
  82. cloudnetpy/products/iwc.py +16 -9
  83. cloudnetpy/products/lwc.py +34 -12
  84. cloudnetpy/products/mwr_multi.py +4 -1
  85. cloudnetpy/products/mwr_single.py +4 -1
  86. cloudnetpy/products/product_tools.py +33 -10
  87. cloudnetpy/utils.py +175 -74
  88. cloudnetpy/version.py +1 -1
  89. {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/METADATA +11 -10
  90. cloudnetpy-1.55.22.dist-info/RECORD +114 -0
  91. docs/source/conf.py +2 -2
  92. cloudnetpy-1.55.20.dist-info/RECORD +0 -114
  93. {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/LICENSE +0 -0
  94. {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/WHEEL +0 -0
  95. {cloudnetpy-1.55.20.dist-info → cloudnetpy-1.55.22.dist-info}/top_level.txt +0 -0
@@ -17,6 +17,7 @@ class AdvanceProductMethods(DataSource):
17
17
  assumptions of model or observation data.
18
18
 
19
19
  Args:
20
+ ----
20
21
  model_obj (object): The :class:'ModelManager' object.
21
22
  obs_obj (object): The :class:'ObservationManager' object.
22
23
  """
@@ -38,30 +39,30 @@ class AdvanceProductMethods(DataSource):
38
39
  self._model_height = model_obj.data[model_obj.keys["height"]][:]
39
40
  self.generate_products()
40
41
 
41
- def generate_products(self):
42
- cls = getattr(importlib.import_module(__name__), "AdvanceProductMethods")
42
+ def generate_products(self) -> None:
43
+ cls = importlib.import_module(__name__).AdvanceProductMethods
43
44
  try:
44
45
  name = f"get_advance_{self.product}"
45
46
  getattr(cls, name)(self)
46
47
  except AttributeError as error:
47
- logging.warning(f"No advance method for {self.product}: {error}")
48
+ logging.warning("No advance method for %s: %s", self.product, error)
48
49
 
49
- def get_advance_cf(self):
50
+ def get_advance_cf(self) -> None:
50
51
  self.cf_cirrus_filter()
51
52
 
52
- def cf_cirrus_filter(self):
53
+ def cf_cirrus_filter(self) -> None:
53
54
  cf = self.getvar_from_object("cf")
54
55
  h = self.getvar_from_object("h")
55
56
  temperature = self.getvar("temperature")
56
57
  t_screened = self.remove_extra_levels(temperature - 273.15)
57
- iwc, lwc = [self._model_obj.get_water_content(var) for var in ["iwc", "lwc"]]
58
+ iwc, lwc = (self._model_obj.get_water_content(var) for var in ["iwc", "lwc"])
58
59
  tZT, tT, tZ, t = self.set_frequency_parameters()
59
60
  z_sen = self.fit_z_sensitivity(h)
60
61
  cf_filtered = self.filter_high_iwc_low_cf(cf, iwc, lwc)
61
62
  cloud_iwc, ice_ind = self.find_ice_in_clouds(cf_filtered, iwc, lwc)
62
63
  variance_iwc = self.iwc_variance(h, ice_ind)
63
64
  # Looks suspicious, check me:
64
- for i, ind in enumerate(zip(ice_ind[0], ice_ind[-1])):
65
+ for i, ind in enumerate(zip(ice_ind[0], ice_ind[-1], strict=True)):
65
66
  iwc_dist = self.calculate_iwc_distribution(cloud_iwc[i], variance_iwc[i])
66
67
  p_iwc = self.gamma_distribution(iwc_dist, variance_iwc[i], cloud_iwc[i])
67
68
  if np.sum(p_iwc) == 0 or p_iwc[-1] > 0.01 * np.sum(p_iwc):
@@ -92,7 +93,9 @@ class AdvanceProductMethods(DataSource):
92
93
  return self._model_obj.cut_off_extra_levels(arg)
93
94
 
94
95
  def set_frequency_parameters(self) -> tuple:
95
- assert self._obs_obj.radar_freq is not None
96
+ if self._obs_obj.radar_freq is None:
97
+ msg = "No radar frequency in observation file"
98
+ raise ValueError(msg)
96
99
  if 30 <= self._obs_obj.radar_freq <= 40:
97
100
  return 0.000242, -0.0186, 0.0699, -1.63
98
101
  if 90 <= float(self._obs_obj.radar_freq) <= 100:
@@ -100,8 +103,12 @@ class AdvanceProductMethods(DataSource):
100
103
  raise ValueError
101
104
 
102
105
  def fit_z_sensitivity(self, h: np.ndarray) -> np.ndarray:
103
- assert self._obs_obj.z_sensitivity is not None
104
- assert self._obs_obj.height is not None
106
+ if self._obs_obj.z_sensitivity is None:
107
+ msg = "No z_sensitivity in observation file"
108
+ raise ValueError(msg)
109
+ if self._obs_obj.height is None:
110
+ msg = "No height in observation file"
111
+ raise ValueError(msg)
105
112
  z_sen = [
106
113
  cl_tools.rebin_1d(self._obs_obj.height, self._obs_obj.z_sensitivity, h[i])
107
114
  for i in range(len(h))
@@ -109,16 +116,22 @@ class AdvanceProductMethods(DataSource):
109
116
  return np.asarray(z_sen)
110
117
 
111
118
  def filter_high_iwc_low_cf(
112
- self, cf: np.ndarray, iwc: np.ndarray, lwc: np.ndarray
119
+ self,
120
+ cf: np.ndarray,
121
+ iwc: np.ndarray,
122
+ lwc: np.ndarray,
113
123
  ) -> np.ndarray:
114
124
  cf_filtered = self.mask_weird_indices(cf, iwc, lwc)
115
125
  if np.sum((iwc > 0) & (lwc < iwc / 10) & (cf_filtered > 0)) == 0:
116
- raise ValueError("No ice clouds in a input data")
126
+ msg = "No ice clouds in a input data"
127
+ raise ValueError(msg)
117
128
  return cf_filtered
118
129
 
119
130
  @staticmethod
120
131
  def mask_weird_indices(
121
- cf: np.ndarray, iwc: np.ndarray, lwc: np.ndarray
132
+ cf: np.ndarray,
133
+ iwc: np.ndarray,
134
+ lwc: np.ndarray,
122
135
  ) -> np.ndarray:
123
136
  cf_filtered = np.copy(cf)
124
137
  weird_ind = (iwc / cf > 0.5e-3) & (cf < 0.001)
@@ -127,7 +140,10 @@ class AdvanceProductMethods(DataSource):
127
140
  return cf_filtered
128
141
 
129
142
  def find_ice_in_clouds(
130
- self, cf_filtered: np.ndarray, iwc: np.ndarray, lwc: np.ndarray
143
+ self,
144
+ cf_filtered: np.ndarray,
145
+ iwc: np.ndarray,
146
+ lwc: np.ndarray,
131
147
  ) -> tuple[np.ndarray, tuple]:
132
148
  ice_ind = self.get_ice_indices(cf_filtered, iwc, lwc)
133
149
  cloud_iwc = iwc[ice_ind] / cf_filtered[ice_ind] * 1e3
@@ -135,7 +151,9 @@ class AdvanceProductMethods(DataSource):
135
151
 
136
152
  @staticmethod
137
153
  def get_ice_indices(
138
- cf_filtered: np.ndarray, iwc: np.ndarray, lwc: np.ndarray
154
+ cf_filtered: np.ndarray,
155
+ iwc: np.ndarray,
156
+ lwc: np.ndarray,
139
157
  ) -> tuple:
140
158
  return tuple(np.where((cf_filtered > 0) & (iwc > 0) & (lwc < iwc / 10)))
141
159
 
@@ -145,8 +163,7 @@ class AdvanceProductMethods(DataSource):
145
163
  u = self.remove_extra_levels(u)
146
164
  v = self.remove_extra_levels(v)
147
165
  w_shear = self.calculate_wind_shear(self._model_obj.wind, u, v, height)
148
- variance_iwc = self.calculate_variance_iwc(w_shear, ice_ind)
149
- return variance_iwc
166
+ return self.calculate_variance_iwc(w_shear, ice_ind)
150
167
 
151
168
  def calculate_variance_iwc(self, w_shear: np.ndarray, ice_ind: tuple) -> np.ndarray:
152
169
  return 10 ** (
@@ -157,7 +174,10 @@ class AdvanceProductMethods(DataSource):
157
174
 
158
175
  @staticmethod
159
176
  def calculate_wind_shear(
160
- wind, u: np.ndarray, v: np.ndarray, height: np.ndarray
177
+ wind,
178
+ u: np.ndarray,
179
+ v: np.ndarray,
180
+ height: np.ndarray,
161
181
  ) -> np.ndarray:
162
182
  grand_winds = []
163
183
  for w in (wind, u, v):
@@ -189,9 +209,11 @@ class AdvanceProductMethods(DataSource):
189
209
 
190
210
  @staticmethod
191
211
  def gamma_distribution(
192
- iwc_dist: np.ndarray, f_variance_iwc: float, cloud_iwc: float
212
+ iwc_dist: np.ndarray,
213
+ f_variance_iwc: float,
214
+ cloud_iwc: float,
193
215
  ) -> np.ndarray:
194
- def calculate_gamma_dist():
216
+ def calculate_gamma_dist() -> float:
195
217
  alpha = 1 / f_variance_iwc
196
218
  return (
197
219
  1
@@ -216,18 +238,16 @@ class AdvanceProductMethods(DataSource):
216
238
  temperature: float,
217
239
  z_sen: float,
218
240
  ) -> np.ndarray:
219
- def calculate_min_iwc():
220
- min_iwc = 10 ** (
221
- tZT * z_sen * temperature + tT * temperature + tZ * z_sen + t
222
- )
223
- return min_iwc
241
+ def calculate_min_iwc() -> np.ndarray:
242
+ return 10 ** (tZT * z_sen * temperature + tT * temperature + tZ * z_sen + t)
224
243
 
225
244
  iwc_min = calculate_min_iwc()
226
- obs_index = iwc_dist > iwc_min
227
- return obs_index
245
+ return iwc_dist > iwc_min
228
246
 
229
247
  @staticmethod
230
248
  def filter_cirrus(
231
- p_iwc: np.ndarray, obs_index: np.ndarray, cf_filtered: np.ndarray
249
+ p_iwc: np.ndarray,
250
+ obs_index: np.ndarray,
251
+ cf_filtered: np.ndarray,
232
252
  ) -> np.ndarray:
233
253
  return (np.sum(p_iwc * obs_index) / np.sum(p_iwc)) * cf_filtered
@@ -11,9 +11,12 @@ class ProductGrid:
11
11
  """Class to generate downsampling of observation product to model grid.
12
12
 
13
13
  Args:
14
+ ----
14
15
  model_obj (object): The :class:'ModelManager' object.
15
16
  obs_obj (object): The :class:'ObservationManager' object.
17
+
16
18
  Notes:
19
+ -----
17
20
  Downsampled observation products data is added to a ModelManager
18
21
  object which is used for nc-file creation and writing
19
22
  """
@@ -28,13 +31,15 @@ class ProductGrid:
28
31
  self._model_time = model_obj.time
29
32
  self._model_height = model_obj.data[model_obj.keys["height"]][:]
30
33
  self._time_adv = tl.calculate_advection_time(
31
- int(model_obj.resolution_h), ma.array(model_obj.wind), 1
34
+ int(model_obj.resolution_h),
35
+ ma.array(model_obj.wind),
36
+ 1,
32
37
  )
33
38
  time_steps = utils.binvec(self._model_time)
34
39
  self._time_steps = tl.time2datetime(time_steps, self._date)
35
40
  self._generate_downsample_product()
36
41
 
37
- def _generate_downsample_product(self):
42
+ def _generate_downsample_product(self) -> None:
38
43
  """Downsampling products are generated with different averaging methods
39
44
  for a selected size of model time-height window.
40
45
  """
@@ -42,7 +47,8 @@ class ProductGrid:
42
47
  model_t = tl.time2datetime(self._model_time, self._date)
43
48
  for i in range(len(self._time_steps) - 1):
44
49
  x_ind = tl.get_1d_indices(
45
- (self._time_steps[i], self._time_steps[i + 1]), self._obs_time
50
+ (self._time_steps[i], self._time_steps[i + 1]),
51
+ self._obs_time,
46
52
  )
47
53
  if self._obs_obj.obs == "iwc":
48
54
  x_ind_no_rain = tl.get_1d_indices(
@@ -53,10 +59,13 @@ class ProductGrid:
53
59
  y_steps = tl.rebin_edges(self._model_height[i])
54
60
  for j in range(len(y_steps) - 1):
55
61
  x_ind_adv = tl.get_adv_indices(
56
- model_t[i], self._time_adv[i, j], self._obs_time
62
+ model_t[i],
63
+ self._time_adv[i, j],
64
+ self._obs_time,
57
65
  )
58
66
  y_ind = tl.get_1d_indices(
59
- (y_steps[j], y_steps[j + 1]), self._obs_height
67
+ (y_steps[j], y_steps[j + 1]),
68
+ self._obs_height,
60
69
  )
61
70
  ind = np.outer(x_ind, y_ind)
62
71
  ind_avd = np.outer(x_ind_adv, y_ind)
@@ -66,7 +75,9 @@ class ProductGrid:
66
75
  continue
67
76
  product_dict = self._regrid_cf(product_dict, i, j, data)
68
77
  data_adv = self._reshape_data_to_window(ind_avd, x_ind_adv, y_ind)
69
- assert data_adv is not None
78
+ if data_adv is None:
79
+ msg = "No data for advection"
80
+ raise RuntimeError(msg)
70
81
  product_adv_dict = self._regrid_cf(product_adv_dict, i, j, data_adv)
71
82
  elif self._obs_obj.obs == "iwc":
72
83
  x_ind_no_rain_adv = tl.get_adv_indices(
@@ -78,19 +89,30 @@ class ProductGrid:
78
89
  ind_no_rain = np.outer(x_ind_no_rain, y_ind)
79
90
  ind_no_rain_adv = np.outer(x_ind_no_rain_adv, y_ind)
80
91
  product_dict = self._regrid_iwc(
81
- product_dict, i, j, ind, ind_no_rain
92
+ product_dict,
93
+ i,
94
+ j,
95
+ ind,
96
+ ind_no_rain,
82
97
  )
83
98
  product_adv_dict = self._regrid_iwc(
84
- product_adv_dict, i, j, ind_avd, ind_no_rain_adv
99
+ product_adv_dict,
100
+ i,
101
+ j,
102
+ ind_avd,
103
+ ind_no_rain_adv,
85
104
  )
86
105
  else:
87
106
  product_dict = self._regrid_product(product_dict, i, j, ind)
88
107
  product_adv_dict = self._regrid_product(
89
- product_adv_dict, i, j, ind_avd
108
+ product_adv_dict,
109
+ i,
110
+ j,
111
+ ind_avd,
90
112
  )
91
113
  self._append_data2object([product_dict, product_adv_dict])
92
114
 
93
- def _get_method_storage(self):
115
+ def _get_method_storage(self) -> tuple[dict, dict]:
94
116
  if self._obs_obj.obs == "cf":
95
117
  return self._cf_method_storage()
96
118
  if self._obs_obj.obs == "iwc":
@@ -124,7 +146,7 @@ class ProductGrid:
124
146
  def _product_method_storage(self) -> tuple[dict, dict]:
125
147
  product_dict = {f"{self._obs_obj.obs}": np.zeros(self._model_height.shape)}
126
148
  product_adv_dict = {
127
- f"{self._obs_obj.obs}_adv": np.zeros(self._model_height.shape)
149
+ f"{self._obs_obj.obs}_adv": np.zeros(self._model_height.shape),
128
150
  }
129
151
  return product_dict, product_adv_dict
130
152
 
@@ -139,18 +161,21 @@ class ProductGrid:
139
161
  downsample[i, j] = np.nanmean(data)
140
162
  else:
141
163
  downsample[i, j] = np.nan
142
- if "_A" in key:
143
- if not np.isnan(data).all() and not (
144
- isinstance(data, ma.MaskedArray) and data.mask.all()
145
- ):
146
- downsample[i, j] = tl.average_column_sum(data)
164
+ if "_A" in key and (
165
+ not np.isnan(data).all()
166
+ and not (isinstance(data, ma.MaskedArray) and data.mask.all())
167
+ ):
168
+ downsample[i, j] = tl.average_column_sum(data)
147
169
  else:
148
170
  downsample[i, j] = np.nan
149
171
  storage[key] = downsample
150
172
  return storage
151
173
 
152
174
  def _reshape_data_to_window(
153
- self, ind: np.ndarray, x_ind: np.ndarray, y_ind: np.ndarray
175
+ self,
176
+ ind: np.ndarray,
177
+ x_ind: np.ndarray,
178
+ y_ind: np.ndarray,
154
179
  ) -> np.ndarray | None:
155
180
  """Reshapes True observation values to windows shape"""
156
181
  window_size = tl.get_obs_window_size(x_ind, y_ind)
@@ -203,9 +228,9 @@ class ProductGrid:
203
228
  storage[key] = down_sample
204
229
  return storage
205
230
 
206
- def _append_data2object(self, data_storage: list):
231
+ def _append_data2object(self, data_storage: list) -> None:
207
232
  for storage in data_storage:
208
- for key in storage.keys():
233
+ for key in storage:
209
234
  down_sample = storage[key]
210
235
  self.model_obj.append_data(
211
236
  down_sample,
@@ -14,12 +14,14 @@ class ModelManager(DataSource):
14
14
  """Class to collect and manage model data.
15
15
 
16
16
  Args:
17
+ ----
17
18
  model_file (str): Path to source model file.
18
19
  model (str): Name of model
19
20
  output_file (str): name of output file name and path to save data
20
21
  product (str): name of product to generate
21
22
 
22
23
  Notes:
24
+ -----
23
25
  For this class to work, needed information of model in use should be found in
24
26
  model_metadata.py
25
27
 
@@ -36,6 +38,7 @@ class ModelManager(DataSource):
36
38
  model: str,
37
39
  output_file: str,
38
40
  product: str,
41
+ *,
39
42
  check_file: bool = True,
40
43
  ):
41
44
  super().__init__(model_file)
@@ -52,7 +55,7 @@ class ModelManager(DataSource):
52
55
  self.wind = self._calculate_wind_speed()
53
56
  self.resolution_h = self._get_horizontal_resolution()
54
57
 
55
- def _read_cycle_name(self, model_file: str):
58
+ def _read_cycle_name(self, model_file: str) -> str:
56
59
  """Get cycle name from model_metadata.py for saving variable name(s)"""
57
60
  try:
58
61
  cycles = self.model_info.cycle
@@ -66,17 +69,18 @@ class ModelManager(DataSource):
66
69
  return ""
67
70
  return ""
68
71
 
69
- def _generate_products(self):
72
+ def _generate_products(self) -> None:
70
73
  """Process needed data of model to a ModelManager object"""
71
- cls = getattr(importlib.import_module(__name__), "ModelManager")
74
+ cls = importlib.import_module(__name__).ModelManager
72
75
  try:
73
76
  name = f"_get_{self._product}"
74
77
  getattr(cls, name)(self)
75
- except AttributeError as e:
76
- logging.error(f"Invalid product name: {e}")
78
+ except AttributeError:
79
+ msg = f"Invalid product name: {self._product}"
80
+ logging.exception(msg)
77
81
  raise
78
82
 
79
- def _get_cf(self):
83
+ def _get_cf(self) -> None:
80
84
  """Collect cloud fraction straight from model file."""
81
85
  cf_name = self.get_model_var_names(("cf",))[0]
82
86
  cf = self.getvar(cf_name)
@@ -85,13 +89,13 @@ class ModelManager(DataSource):
85
89
  self.append_data(cf, f"{self.model}{self.cycle}_cf")
86
90
  self.keys[self._product] = f"{self.model}{self.cycle}_cf"
87
91
 
88
- def _get_iwc(self):
92
+ def _get_iwc(self) -> None:
89
93
  iwc = self.get_water_content("iwc")
90
94
  iwc[iwc < 1e-7] = ma.masked
91
95
  self.append_data(iwc, f"{self.model}{self.cycle}_iwc")
92
96
  self.keys[self._product] = f"{self.model}{self.cycle}_iwc"
93
97
 
94
- def _get_lwc(self):
98
+ def _get_lwc(self) -> None:
95
99
  lwc = self.get_water_content("lwc")
96
100
  lwc[lwc < 1e-5] = ma.masked
97
101
  self.append_data(lwc, f"{self.model}{self.cycle}_lwc")
@@ -120,13 +124,15 @@ class ModelManager(DataSource):
120
124
  def _calc_water_content(q: np.ndarray, p: np.ndarray, t: np.ndarray) -> np.ndarray:
121
125
  return q * p / (287 * t)
122
126
 
123
- def _add_variables(self):
127
+ def _add_variables(self) -> None:
124
128
  """Add basic variables off model and cycle"""
125
129
 
126
- def _add_common_variables():
130
+ def _add_common_variables() -> None:
127
131
  """Model variables that are always the same within cycles"""
128
132
  wanted_vars = self.model_vars.common_var
129
- assert wanted_vars is not None
133
+ if wanted_vars is None:
134
+ msg = f"Model {self.model} has no common variables"
135
+ raise ValueError(msg)
130
136
  wanted_vars_split = [x.strip() for x in wanted_vars.split(",")]
131
137
  for var in wanted_vars_split:
132
138
  if var in self.dataset.variables:
@@ -135,10 +141,12 @@ class ModelManager(DataSource):
135
141
  data = self.cut_off_extra_levels(self.dataset.variables[var][:])
136
142
  self.append_data(data, f"{var}")
137
143
 
138
- def _add_cycle_variables():
144
+ def _add_cycle_variables() -> None:
139
145
  """Add cycle depending variables"""
140
146
  wanted_vars = self.model_vars.cycle_var
141
- assert wanted_vars is not None
147
+ if wanted_vars is None:
148
+ msg = f"Model {self.model} has no cycle variables"
149
+ raise ValueError(msg)
142
150
  wanted_vars_split = [x.strip() for x in wanted_vars.split(",")]
143
151
  for var in wanted_vars_split:
144
152
  if var in self.dataset.variables:
@@ -160,11 +168,7 @@ class ModelManager(DataSource):
160
168
  except KeyError:
161
169
  return data
162
170
 
163
- if data.ndim > 1:
164
- data = data[:, :level]
165
- else:
166
- data = data[:level]
167
- return data
171
+ return data[:, :level] if data.ndim > 1 else data[:level]
168
172
 
169
173
  def _calculate_wind_speed(self) -> np.ndarray:
170
174
  """Real wind from x- and y-components"""
@@ -1,5 +1,5 @@
1
1
  import logging
2
- from datetime import datetime
2
+ from datetime import datetime, timezone
3
3
 
4
4
  import numpy as np
5
5
  from numpy import ma
@@ -13,10 +13,12 @@ class ObservationManager(DataSource):
13
13
  """Class to collect and manage observations for downsampling.
14
14
 
15
15
  Args:
16
+ ----
16
17
  obs (str): Name of observation product
17
18
  obs_file (str): Path to source observation file
18
19
 
19
20
  Notes:
21
+ -----
20
22
  Output is ObservationManager object where all product data and
21
23
  information is included.
22
24
 
@@ -42,6 +44,7 @@ class ObservationManager(DataSource):
42
44
  0,
43
45
  0,
44
46
  0,
47
+ tzinfo=timezone.utc,
45
48
  )
46
49
 
47
50
  def _get_radar_frequency(self) -> np.ndarray | None:
@@ -56,7 +59,7 @@ class ObservationManager(DataSource):
56
59
  except (KeyError, RuntimeError):
57
60
  return None
58
61
 
59
- def _generate_product(self):
62
+ def _generate_product(self) -> None:
60
63
  """Process needed data of observation to a ObservationManager object"""
61
64
  try:
62
65
  if self.obs == "cf":
@@ -66,16 +69,16 @@ class ObservationManager(DataSource):
66
69
  if self.obs == "iwc":
67
70
  self._generate_iwc_masks()
68
71
  self.append_data(self.getvar("height"), "height")
69
- except (KeyError, RuntimeError) as e:
70
- logging.error(f"Invalid product name: {e}")
72
+ except (KeyError, RuntimeError):
73
+ msg = f"Failed to read {self.obs} from {self._file}"
74
+ logging.exception(msg)
71
75
  raise
72
76
 
73
77
  def _generate_cf(self) -> np.ndarray:
74
78
  """Generates cloud fractions using categorize bits and masking conditions"""
75
79
  categorize_bits = CategorizeBits(self._file)
76
80
  cloud_mask = self._classify_basic_mask(categorize_bits.category_bits)
77
- cloud_mask = self._mask_cloud_bits(cloud_mask)
78
- return cloud_mask
81
+ return self._mask_cloud_bits(cloud_mask)
79
82
 
80
83
  @staticmethod
81
84
  def _classify_basic_mask(bits: dict) -> np.ndarray:
@@ -101,12 +104,14 @@ class ObservationManager(DataSource):
101
104
  """Check if rainrate in file"""
102
105
  try:
103
106
  self.getvar("rainrate")
104
- return True
105
107
  except RuntimeError:
106
108
  return False
109
+ return True
107
110
 
108
111
  def _get_rainrate_threshold(self) -> int:
109
- assert self.radar_freq is not None
112
+ if self.radar_freq is None:
113
+ msg = "Radar frequency not found from file"
114
+ raise RuntimeError(msg)
110
115
  wband = utils.get_wl_band(float(self.radar_freq))
111
116
  rainrate_threshold = 8
112
117
  if 90 < wband < 100:
@@ -135,7 +140,8 @@ class ObservationManager(DataSource):
135
140
 
136
141
  def _mask_iwc_att(self, iwc: np.ndarray, iwc_status: np.ndarray) -> None:
137
142
  """Leaves only where reliable data, corrected liquid attenuation
138
- and uncorrected liquid attenuation"""
143
+ and uncorrected liquid attenuation
144
+ """
139
145
  iwc_att = ma.copy(iwc)
140
146
  iwc_att[iwc_status > 3] = ma.masked
141
147
  self.append_data(iwc_att, "iwc_att")
@@ -21,14 +21,17 @@ def process_L3_day_product(
21
21
  product_file: str,
22
22
  output_file: str,
23
23
  uuid: str | None = None,
24
+ *,
24
25
  overwrite: bool = False,
25
- ):
26
+ ) -> str:
26
27
  """Main function to generate downsample of observations to match model grid.
27
28
 
28
29
  This function will generate a L3 product nc-file. It includes the information of
29
30
  downsampled observation products for each model cycles and model products
30
31
  and other variables of each cycles.
32
+
31
33
  Args:
34
+ ----
32
35
  model (str): Name of model
33
36
  obs (str): Name of product to generate
34
37
  model_files (list): List of model + cycles file path(s) to be generated
@@ -41,16 +44,19 @@ def process_L3_day_product(
41
44
  default False
42
45
 
43
46
  Raises:
47
+ ------
44
48
  RuntimeError: Failed to create the L3 product file.
45
49
  ValueError (Warning): No ice clouds in model data
46
50
 
47
51
  Notes:
52
+ -----
48
53
  Model file(s) are given as a list to make all different cycles to be at same
49
54
  nc-file. If list includes more than one model file, nc-file is created within
50
55
  the first round. With rest of rounds, downsample observation and model data
51
56
  is added to same L3 day nc-file.
52
57
 
53
58
  Examples:
59
+ --------
54
60
  >>> from cloudnetpy.model_evaluation.products.product_resampling import \
55
61
  process_L3_day_product
56
62
  >>> product = 'cf'
@@ -65,7 +71,11 @@ def process_L3_day_product(
65
71
  tl.check_model_file_list(model, model_files)
66
72
  for m_file in model_files:
67
73
  model_obj = ModelManager(
68
- m_file, model, output_file, obs, check_file=not overwrite
74
+ m_file,
75
+ model,
76
+ output_file,
77
+ obs,
78
+ check_file=not overwrite,
69
79
  )
70
80
  try:
71
81
  AdvanceProductMethods(model_obj, m_file, product_obj)
@@ -76,7 +86,7 @@ def process_L3_day_product(
76
86
  update_attributes(model_obj.data, attributes)
77
87
  if not file_exists(output_file) or overwrite:
78
88
  tl.add_date(model_obj, product_obj)
79
- uuid = save_downsampled_file(
89
+ uuid_out = save_downsampled_file(
80
90
  f"{obs}_{model}",
81
91
  output_file,
82
92
  (model_obj, product_obj),
@@ -85,4 +95,4 @@ def process_L3_day_product(
85
95
  )
86
96
  else:
87
97
  add_var2ncfile(model_obj, output_file)
88
- return uuid
98
+ return uuid_out
@@ -14,7 +14,8 @@ def check_model_file_list(name: str, models: list) -> None:
14
14
  for m in models:
15
15
  if name not in m:
16
16
  logging.error("Invalid model file set")
17
- raise AttributeError(f"{m} not from {name}")
17
+ msg = f"{m} not from {name}"
18
+ raise AttributeError(msg)
18
19
 
19
20
 
20
21
  def time2datetime(time: np.ndarray, date: datetime.datetime) -> np.ndarray:
@@ -30,18 +31,22 @@ def rebin_edges(arr: np.ndarray) -> np.ndarray:
30
31
 
31
32
 
32
33
  def calculate_advection_time(
33
- resolution: int, wind: ma.MaskedArray, sampling: int
34
+ resolution: int,
35
+ wind: ma.MaskedArray,
36
+ sampling: int,
34
37
  ) -> np.ndarray:
35
38
  """Calculates time which variable takes to go through the time window
36
39
 
37
- Notes:
40
+ Notes
41
+ -----
38
42
  Wind speed is stronger in upper levels, so advection time is more
39
43
  there then lower levels. Effect is small in a mid-latitudes,
40
44
  but visible in a tropics.
41
45
 
42
46
  sampling = 1 -> hour, sampling 1/6 -> 10min
43
47
 
44
- References:
48
+ References
49
+ ----------
45
50
  """
46
51
  t_adv = resolution * 1000 / wind / 60**2
47
52
  t_adv[t_adv.mask] = 0
@@ -49,8 +54,12 @@ def calculate_advection_time(
49
54
  return np.asarray([[timedelta(hours=float(t)) for t in time] for time in t_adv])
50
55
 
51
56
 
52
- def get_1d_indices(window: tuple, data: np.ndarray, mask: np.ndarray | None = None):
53
- indices = (window[0] <= data) & (data < window[-1])
57
+ def get_1d_indices(
58
+ window: tuple,
59
+ data: np.ndarray,
60
+ mask: np.ndarray | None = None,
61
+ ) -> np.ndarray:
62
+ indices: np.ndarray = np.array((window[0] <= data) & (data < window[-1]))
54
63
  if mask is not None:
55
64
  indices[mask] = ma.masked
56
65
  return indices
@@ -61,7 +70,7 @@ def get_adv_indices(
61
70
  adv_t: float,
62
71
  data: np.ndarray,
63
72
  mask: np.ndarray | None = None,
64
- ):
73
+ ) -> np.ndarray:
65
74
  adv_indices = ((model_t - adv_t / 2) <= data) & (data < (model_t + adv_t / 2))
66
75
  if mask is not None:
67
76
  adv_indices[mask] = ma.masked