cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/model_evaluation/products/advance_methods.py

@@ -1,9 +1,12 @@
 import importlib
 import logging
+from os import PathLike
 
 import numpy as np
+import numpy.typing as npt
+import scipy.special
+import scipy.stats
 from numpy import ma
-from scipy.special import gamma
 
 import cloudnetpy.utils as cl_tools
 from cloudnetpy.datasource import DataSource
@@ -24,9 +27,9 @@ class AdvanceProductMethods(DataSource):
     def __init__(
         self,
         model_obj: ModelManager,
-        model_file: str,
+        model_file: str | PathLike,
         obs_obj: ObservationManager,
-    ):
+    ) -> None:
         super().__init__(model_file)
         self._obs_obj = obs_obj
         self.product = obs_obj.obs
@@ -38,70 +41,88 @@ class AdvanceProductMethods(DataSource):
         self._model_height = model_obj.data[model_obj.keys["height"]][:]
         self.generate_products()
 
-    def generate_products(self):
-        cls = getattr(importlib.import_module(__name__), "AdvanceProductMethods")
+    def generate_products(self) -> None:
+        cls = importlib.import_module(__name__).AdvanceProductMethods
         try:
             name = f"get_advance_{self.product}"
             getattr(cls, name)(self)
         except AttributeError as error:
-            logging.warning(f"No advance method for {self.product}: {error}")
+            logging.debug("No advance method for %s: %s", self.product, error)
 
-    def get_advance_cf(self):
+    def get_advance_cf(self) -> None:
         self.cf_cirrus_filter()
 
-    def cf_cirrus_filter(self):
+    def cf_cirrus_filter(self) -> None:
         cf = self.getvar_from_object("cf")
         h = self.getvar_from_object("h")
         temperature = self.getvar("temperature")
         t_screened = self.remove_extra_levels(temperature - 273.15)
-        iwc, lwc = [self._model_obj.get_water_continent(var) for var in ["iwc", "lwc"]]
+        iwc, lwc = (self._model_obj.get_water_content(var) for var in ["iwc", "lwc"])
         tZT, tT, tZ, t = self.set_frequency_parameters()
         z_sen = self.fit_z_sensitivity(h)
         cf_filtered = self.filter_high_iwc_low_cf(cf, iwc, lwc)
         cloud_iwc, ice_ind = self.find_ice_in_clouds(cf_filtered, iwc, lwc)
         variance_iwc = self.iwc_variance(h, ice_ind)
-        # Looks suspicious, check me:
-        for i, ind in enumerate(zip(ice_ind[0], ice_ind[-1])):
-            iwc_dist = self.calculate_iwc_distribution(cloud_iwc[i], variance_iwc[i])
+
+        for i, ind in enumerate(zip(ice_ind[0], ice_ind[-1], strict=True)):
+            try:
+                iwc_dist = self.calculate_iwc_distribution(
+                    cloud_iwc[i], variance_iwc[i]
+                )
+            except ValueError:
+                continue
+
             p_iwc = self.gamma_distribution(iwc_dist, variance_iwc[i], cloud_iwc[i])
+
+            if np.isinf(p_iwc).any():
+                cf_filtered[ind] = ma.masked
+                continue
+
             if np.sum(p_iwc) == 0 or p_iwc[-1] > 0.01 * np.sum(p_iwc):
-                cf_filtered[ind] = np.nan
+                cf_filtered[ind] = ma.masked
                 continue
-            obs_index = self.get_observation_index(
-                iwc_dist,
-                tZT,
-                tT,
-                tZ,
-                t,
-                float(t_screened[ind]),
-                float(z_sen[ind]),
+
+            min_iwc = 10 ** (
+                tZT * z_sen[ind] * t_screened[ind]
+                + tT * t_screened[ind]
+                + tZ * z_sen[ind]
+                + t
             )
+            obs_index = iwc_dist > min_iwc
             cf_filtered[ind] = self.filter_cirrus(p_iwc, obs_index, cf_filtered[ind])
+
         cf_filtered[cf_filtered < 0.05] = ma.masked
+
         self._model_obj.append_data(
             cf_filtered,
             f"{self._model_obj.model}{self._model_obj.cycle}_cf_cirrus",
         )
 
-    def getvar_from_object(self, arg: str) -> np.ndarray:
+    def getvar_from_object(self, arg: str) -> npt.NDArray:
         v_name = arg if arg == "cf" else self._model_obj.get_model_var_names((arg,))[0]
         key = f"{self._model_obj.model}{self._model_obj.cycle}_{v_name}"
         return self._model_obj.data[key][:]
 
-    def remove_extra_levels(self, arg: np.ndarray) -> np.ndarray:
+    def remove_extra_levels(self, arg: npt.NDArray) -> npt.NDArray:
         return self._model_obj.cut_off_extra_levels(arg)
 
     def set_frequency_parameters(self) -> tuple:
-        assert self._obs_obj.radar_freq is not None
+        if self._obs_obj.radar_freq is None:
+            msg = "No radar frequency in observation file"
+            raise ValueError(msg)
         if 30 <= self._obs_obj.radar_freq <= 40:
             return 0.000242, -0.0186, 0.0699, -1.63
         if 90 <= float(self._obs_obj.radar_freq) <= 100:
             return 0.00058, -0.00706, 0.0923, -0.992
         raise ValueError
 
-    def fit_z_sensitivity(self, h: np.ndarray) -> np.ndarray:
-        assert self._obs_obj.z_sensitivity is not None
-        assert self._obs_obj.height is not None
+    def fit_z_sensitivity(self, h: npt.NDArray) -> npt.NDArray:
+        if self._obs_obj.z_sensitivity is None:
+            msg = "No z_sensitivity in observation file"
+            raise ValueError(msg)
+        if self._obs_obj.height is None:
+            msg = "No height in observation file"
+            raise ValueError(msg)
         z_sen = [
             cl_tools.rebin_1d(self._obs_obj.height, self._obs_obj.z_sensitivity, h[i])
             for i in range(len(h))
@@ -109,46 +130,58 @@ class AdvanceProductMethods(DataSource):
         return np.asarray(z_sen)
 
     def filter_high_iwc_low_cf(
-        self, cf: np.ndarray, iwc: np.ndarray, lwc: np.ndarray
-    ) -> np.ndarray:
+        self,
+        cf: npt.NDArray,
+        iwc: npt.NDArray,
+        lwc: npt.NDArray,
+    ) -> npt.NDArray:
         cf_filtered = self.mask_weird_indices(cf, iwc, lwc)
         if np.sum((iwc > 0) & (lwc < iwc / 10) & (cf_filtered > 0)) == 0:
-            raise ValueError("No ice clouds in a input data")
+            msg = "No ice clouds in a input data"
+            raise ValueError(msg)
         return cf_filtered
 
     @staticmethod
     def mask_weird_indices(
-        cf: np.ndarray, iwc: np.ndarray, lwc: np.ndarray
-    ) -> np.ndarray:
-        cf_filtered = np.copy(cf)
+        cf: npt.NDArray,
+        iwc: npt.NDArray,
+        lwc: npt.NDArray,
+    ) -> npt.NDArray:
+        cf_filtered = ma.copy(cf)
         weird_ind = (iwc / cf > 0.5e-3) & (cf < 0.001)
         weird_ind = weird_ind | (iwc == 0) & (lwc == 0) & (cf == 0)
         cf_filtered[weird_ind] = ma.masked
         return cf_filtered
 
     def find_ice_in_clouds(
-        self, cf_filtered: np.ndarray, iwc: np.ndarray, lwc: np.ndarray
-    ) -> tuple[np.ndarray, tuple]:
+        self,
+        cf_filtered: npt.NDArray,
+        iwc: npt.NDArray,
+        lwc: npt.NDArray,
+    ) -> tuple[npt.NDArray, tuple]:
         ice_ind = self.get_ice_indices(cf_filtered, iwc, lwc)
         cloud_iwc = iwc[ice_ind] / cf_filtered[ice_ind] * 1e3
         return cloud_iwc, ice_ind
 
     @staticmethod
     def get_ice_indices(
-        cf_filtered: np.ndarray, iwc: np.ndarray, lwc: np.ndarray
+        cf_filtered: npt.NDArray,
+        iwc: npt.NDArray,
+        lwc: npt.NDArray,
     ) -> tuple:
         return tuple(np.where((cf_filtered > 0) & (iwc > 0) & (lwc < iwc / 10)))
 
-    def iwc_variance(self, height: np.ndarray, ice_ind: tuple) -> np.ndarray:
+    def iwc_variance(self, height: npt.NDArray, ice_ind: tuple) -> npt.NDArray:
         u = self.getvar("uwind")
         v = self.getvar("vwind")
         u = self.remove_extra_levels(u)
         v = self.remove_extra_levels(v)
         w_shear = self.calculate_wind_shear(self._model_obj.wind, u, v, height)
-        variance_iwc = self.calculate_variance_iwc(w_shear, ice_ind)
-        return variance_iwc
+        return self.calculate_variance_iwc(w_shear, ice_ind)
 
-    def calculate_variance_iwc(self, w_shear: np.ndarray, ice_ind: tuple) -> np.ndarray:
+    def calculate_variance_iwc(
+        self, w_shear: npt.NDArray, ice_ind: tuple
+    ) -> npt.NDArray:
         return 10 ** (
             0.3 * np.log10(self._model_obj.resolution_h)
             - 0.04 * w_shear[ice_ind]
@@ -157,8 +190,11 @@ class AdvanceProductMethods(DataSource):
 
     @staticmethod
     def calculate_wind_shear(
-        wind, u: np.ndarray, v: np.ndarray, height: np.ndarray
-    ) -> np.ndarray:
+        wind: npt.NDArray,
+        u: npt.NDArray,
+        v: npt.NDArray,
+        height: npt.NDArray,
+    ) -> npt.NDArray:
         grand_winds = []
         for w in (wind, u, v):
             grad_w = np.zeros(w.shape)
@@ -179,8 +215,10 @@ class AdvanceProductMethods(DataSource):
         f_variance_iwc: float,
         n_std: int = 5,
         n_dist: int = 250,
-    ) -> np.ndarray:
-        finish = cloud_iwc + n_std * (np.sqrt(f_variance_iwc) * cloud_iwc)
+    ) -> npt.NDArray:
+        finish = cloud_iwc + n_std * (ma.sqrt(f_variance_iwc) * cloud_iwc)
+        if isinstance(finish, ma.MaskedArray) and finish.mask.all():
+            raise ValueError
         iwc_dist = np.arange(0, finish, finish / (n_dist - 1))
         if cloud_iwc < iwc_dist[2]:
             finish = cloud_iwc * 10
@@ -189,45 +227,16 @@ class AdvanceProductMethods(DataSource):
 
     @staticmethod
     def gamma_distribution(
-        iwc_dist: np.ndarray, f_variance_iwc: float, cloud_iwc: float
-    ) -> np.ndarray:
-        def calculate_gamma_dist():
-            alpha = 1 / f_variance_iwc
-            return (
-                1
-                / gamma(alpha)
-                * (alpha / cloud_iwc) ** alpha
-                * iwc_dist[i] ** (alpha - 1)
-                * ma.exp(-(alpha * iwc_dist[i] / cloud_iwc))
-            )
-
-        p_iwc = np.zeros(iwc_dist.shape)
-        for i in range(len(iwc_dist)):
-            p_iwc[i] = calculate_gamma_dist()
-        return p_iwc
-
-    @staticmethod
-    def get_observation_index(
-        iwc_dist: np.ndarray,
-        tZT: float,
-        tT: float,
-        tZ: float,
-        t: np.ndarray,
-        temperature: float,
-        z_sen: float,
-    ) -> np.ndarray:
-        def calculate_min_iwc():
-            min_iwc = 10 ** (
-                tZT * z_sen * temperature + tT * temperature + tZ * z_sen + t
-            )
-            return min_iwc
-
-        iwc_min = calculate_min_iwc()
-        obs_index = iwc_dist > iwc_min
-        return obs_index
+        iwc_dist: npt.NDArray, f_variance_iwc: float, cloud_iwc: float
+    ) -> npt.NDArray:
+        alpha = 1 / f_variance_iwc
+        theta = cloud_iwc / alpha
+        return scipy.stats.gamma.pdf(iwc_dist, a=alpha, scale=theta)
 
     @staticmethod
     def filter_cirrus(
-        p_iwc: np.ndarray, obs_index: np.ndarray, cf_filtered: np.ndarray
-    ) -> np.ndarray:
-        return (np.sum(p_iwc * obs_index) / np.sum(p_iwc)) * cf_filtered
+        p_iwc: npt.NDArray,
+        obs_index: npt.NDArray,
+        cf_filtered: npt.NDArray,
+    ) -> npt.NDArray:
+        return (ma.sum(p_iwc * obs_index) / ma.sum(p_iwc)) * cf_filtered

cloudnetpy/model_evaluation/products/grid_methods.py

@@ -1,4 +1,5 @@
 import numpy as np
+import numpy.typing as npt
 from numpy import ma
 
 from cloudnetpy import utils
@@ -13,12 +14,13 @@ class ProductGrid:
     Args:
         model_obj (object): The :class:'ModelManager' object.
         obs_obj (object): The :class:'ObservationManager' object.
+
     Notes:
         Downsampled observation products data is added to a ModelManager
         object which is used for nc-file creation and writing
     """
 
-    def __init__(self, model_obj: ModelManager, obs_obj: ObservationManager):
+    def __init__(self, model_obj: ModelManager, obs_obj: ObservationManager) -> None:
         self._obs_obj = obs_obj
         self._date = obs_obj.date
         self._obs_time = tl.time2datetime(obs_obj.time, self._date)
@@ -28,13 +30,15 @@ class ProductGrid:
         self._model_time = model_obj.time
         self._model_height = model_obj.data[model_obj.keys["height"]][:]
         self._time_adv = tl.calculate_advection_time(
-            int(model_obj.resolution_h), ma.array(model_obj.wind), 1
+            int(model_obj.resolution_h),
+            ma.array(model_obj.wind),
+            1,
         )
         time_steps = utils.binvec(self._model_time)
         self._time_steps = tl.time2datetime(time_steps, self._date)
         self._generate_downsample_product()
 
-    def _generate_downsample_product(self):
+    def _generate_downsample_product(self) -> None:
         """Downsampling products are generated with different averaging methods
         for a selected size of model time-height window.
         """
@@ -42,7 +46,8 @@ class ProductGrid:
         model_t = tl.time2datetime(self._model_time, self._date)
         for i in range(len(self._time_steps) - 1):
             x_ind = tl.get_1d_indices(
-                (self._time_steps[i], self._time_steps[i + 1]), self._obs_time
+                (self._time_steps[i], self._time_steps[i + 1]),
+                self._obs_time,
             )
             if self._obs_obj.obs == "iwc":
                 x_ind_no_rain = tl.get_1d_indices(
@@ -53,10 +58,13 @@ class ProductGrid:
             y_steps = tl.rebin_edges(self._model_height[i])
             for j in range(len(y_steps) - 1):
                 x_ind_adv = tl.get_adv_indices(
-                    model_t[i], self._time_adv[i, j], self._obs_time
+                    model_t[i],
+                    self._time_adv[i, j],
+                    self._obs_time,
                 )
                 y_ind = tl.get_1d_indices(
-                    (y_steps[j], y_steps[j + 1]), self._obs_height
+                    (y_steps[j], y_steps[j + 1]),
+                    self._obs_height,
                 )
                 ind = np.outer(x_ind, y_ind)
                 ind_avd = np.outer(x_ind_adv, y_ind)
@@ -66,7 +74,9 @@ class ProductGrid:
                         continue
                     product_dict = self._regrid_cf(product_dict, i, j, data)
                     data_adv = self._reshape_data_to_window(ind_avd, x_ind_adv, y_ind)
-                    assert data_adv is not None
+                    if data_adv is None:
+                        msg = "No data for advection"
+                        raise RuntimeError(msg)
                     product_adv_dict = self._regrid_cf(product_adv_dict, i, j, data_adv)
                 elif self._obs_obj.obs == "iwc":
                     x_ind_no_rain_adv = tl.get_adv_indices(
@@ -78,19 +88,30 @@ class ProductGrid:
                     ind_no_rain = np.outer(x_ind_no_rain, y_ind)
                     ind_no_rain_adv = np.outer(x_ind_no_rain_adv, y_ind)
                     product_dict = self._regrid_iwc(
-                        product_dict, i, j, ind, ind_no_rain
+                        product_dict,
+                        i,
+                        j,
+                        ind,
+                        ind_no_rain,
                     )
                     product_adv_dict = self._regrid_iwc(
-                        product_adv_dict, i, j, ind_avd, ind_no_rain_adv
+                        product_adv_dict,
+                        i,
+                        j,
+                        ind_avd,
+                        ind_no_rain_adv,
                     )
                 else:
                     product_dict = self._regrid_product(product_dict, i, j, ind)
                     product_adv_dict = self._regrid_product(
-                        product_adv_dict, i, j, ind_avd
+                        product_adv_dict,
+                        i,
+                        j,
+                        ind_avd,
                     )
         self._append_data2object([product_dict, product_adv_dict])
 
-    def _get_method_storage(self):
+    def _get_method_storage(self) -> tuple[dict, dict]:
         if self._obs_obj.obs == "cf":
             return self._cf_method_storage()
         if self._obs_obj.obs == "iwc":
@@ -99,95 +120,99 @@ class ProductGrid:
 
     def _cf_method_storage(self) -> tuple[dict, dict]:
         cf_dict = {
-            "cf_V": np.zeros(self._model_height.shape),
-            "cf_A": np.zeros(self._model_height.shape),
+            "cf_V": ma.zeros(self._model_height.shape),
+            "cf_A": ma.zeros(self._model_height.shape),
         }
         cf_adv_dict = {
-            "cf_V_adv": np.zeros(self._model_height.shape),
-            "cf_A_adv": np.zeros(self._model_height.shape),
+            "cf_V_adv": ma.zeros(self._model_height.shape),
+            "cf_A_adv": ma.zeros(self._model_height.shape),
         }
         return cf_dict, cf_adv_dict
 
     def _iwc_method_storage(self) -> tuple[dict, dict]:
         iwc_dict = {
-            "iwc": np.zeros(self._model_height.shape),
-            "iwc_att": np.zeros(self._model_height.shape),
-            "iwc_rain": np.zeros(self._model_height.shape),
+            "iwc": ma.zeros(self._model_height.shape),
+            "iwc_att": ma.zeros(self._model_height.shape),
+            "iwc_rain": ma.zeros(self._model_height.shape),
         }
         iwc_adv_dict = {
-            "iwc_adv": np.zeros(self._model_height.shape),
-            "iwc_att_adv": np.zeros(self._model_height.shape),
-            "iwc_rain_adv": np.zeros(self._model_height.shape),
+            "iwc_adv": ma.zeros(self._model_height.shape),
+            "iwc_att_adv": ma.zeros(self._model_height.shape),
+            "iwc_rain_adv": ma.zeros(self._model_height.shape),
         }
         return iwc_dict, iwc_adv_dict
 
     def _product_method_storage(self) -> tuple[dict, dict]:
-        product_dict = {f"{self._obs_obj.obs}": np.zeros(self._model_height.shape)}
+        product_dict = {f"{self._obs_obj.obs}": ma.zeros(self._model_height.shape)}
         product_adv_dict = {
-            f"{self._obs_obj.obs}_adv": np.zeros(self._model_height.shape)
+            f"{self._obs_obj.obs}_adv": ma.zeros(self._model_height.shape),
         }
         return product_dict, product_adv_dict
 
-    @staticmethod
-    def _regrid_cf(storage: dict, i: int, j: int, data: np.ndarray | None) -> dict:
-        """Calculates average cloud fraction value to grid point"""
-        for key, downsample in storage.items():
-            if data is not None:
-                downsample[i, j] = np.nanmean(data)
-                if "_A" in key:
-                    downsample[i, j] = tl.average_column_sum(data)
-            else:
-                downsample[i, j] = np.nan
-            storage[key] = downsample
-        return storage
-
     def _reshape_data_to_window(
-        self, ind: np.ndarray, x_ind: np.ndarray, y_ind: np.ndarray
-    ) -> np.ndarray | None:
-        """Reshapes True observation values to windows shape"""
+        self,
+        ind: npt.NDArray,
+        x_ind: npt.NDArray,
+        y_ind: npt.NDArray,
+    ) -> npt.NDArray | None:
+        """Reshapes True observation values to windows shape."""
         window_size = tl.get_obs_window_size(x_ind, y_ind)
         if window_size is not None:
             return self._obs_data[ind].reshape(window_size)
         return None
 
+    @staticmethod
+    def _regrid_cf(storage: dict, i: int, j: int, data: npt.NDArray) -> dict:
+        """Calculates average cloud fraction value to grid point."""
+        data_ma = ma.array(data) if not isinstance(data, ma.MaskedArray) else data
+        for key, downsample in storage.items():
+            downsample[i, j] = ma.mean(data_ma)
+            if "_A" in key and not data_ma.mask.all():
+                downsample[i, j] = tl.average_column_sum(data_ma)
+            storage[key] = downsample
+        return storage
+
     def _regrid_iwc(
         self,
         storage: dict,
         i: int,
         j: int,
-        ind_rain: np.ndarray,
-        ind_no_rain: np.ndarray,
+        ind_rain: npt.NDArray,
+        ind_no_rain: npt.NDArray,
     ) -> dict:
-        """Calculates average iwc value for grid point"""
-        for key, downsample in storage.items():
-            if not self._obs_data[ind_no_rain].mask.all():
-                downsample[i, j] = np.nanmean(self._obs_data[ind_no_rain])
-            elif "rain" in key and not self._obs_data[ind_rain].mask.all():
-                downsample[i, j] = np.nanmean(self._obs_data[ind_rain])
-            else:
-                downsample[i, j] = np.nan
+        """Calculates average iwc value for each grid point."""
+        for key, down_sample in storage.items():
+            down_sample[i, j] = ma.masked
+            if "rain" not in key:
+                no_rain_data = self._obs_data[ind_no_rain]
+                if ind_no_rain.any() and not no_rain_data.mask.all():
+                    down_sample[i, j] = ma.mean(no_rain_data)
+            if "rain" in key:
+                rain_data = self._obs_data[ind_rain]
+                if ind_rain.any() and not rain_data.mask.all():
+                    down_sample[i, j] = ma.mean(rain_data)
             if "att" in key:
-                iwc_att = self._obs_obj.data["iwc_att"][:]
-                if iwc_att[ind_no_rain].mask.all():
-                    downsample[i, j] = np.nan
-                else:
-                    downsample[i, j] = np.nanmean(iwc_att[ind_no_rain])
-            storage[key] = downsample
+                no_rain_att_data = self._obs_obj.data["iwc_att"][ind_no_rain]
+                if ind_no_rain.any() and not no_rain_att_data.mask.all():
+                    down_sample[i, j] = ma.mean(no_rain_att_data)
+            storage[key] = down_sample
         return storage
 
-    def _regrid_product(self, storage: dict, i: int, j: int, ind: np.ndarray) -> dict:
-        """Calculates average of standard product value to grid point"""
+    def _regrid_product(self, storage: dict, i: int, j: int, ind: npt.NDArray) -> dict:
+        """Calculates average of standard product value for each grid point."""
         for key, down_sample in storage.items():
-            if not self._obs_data[ind].mask.all() and ind.any():
-                down_sample[i, j] = np.nanmean(self._obs_data[ind])
-            else:
-                down_sample[i, j] = np.nan
+            obs_data_selected = ma.masked_invalid(self._obs_data[ind])
+            down_sample[i, j] = (
+                ma.mean(obs_data_selected)
+                if not obs_data_selected.mask.all()
+                else ma.masked
+            )
             storage[key] = down_sample
         return storage
 
-    def _append_data2object(self, data_storage: list):
+    def _append_data2object(self, data_storage: list) -> None:
         for storage in data_storage:
-            for key in storage.keys():
+            for key in storage:
                 down_sample = storage[key]
                 self.model_obj.append_data(
                     down_sample,