cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/products/der.py

@@ -1,14 +1,19 @@
 """Module for creating Cloudnet droplet effective radius
 using the Frisch et al. 2002 method.
 """
-from collections import namedtuple
+
+from os import PathLike
+from typing import NamedTuple
+from uuid import UUID
 
 import numpy as np
+import numpy.typing as npt
 from numpy import ma
 
 from cloudnetpy import output, utils
-from cloudnetpy.categorize import atmos
+from cloudnetpy.categorize import atmos_utils
 from cloudnetpy.datasource import DataSource
+from cloudnetpy.exceptions import InvalidSourceFileError
 from cloudnetpy.metadata import MetaData
 from cloudnetpy.products.product_tools import (
     CategorizeBits,
@@ -16,17 +21,24 @@ from cloudnetpy.products.product_tools import (
     get_is_rain,
 )
 
-Parameters = namedtuple("Parameters", "ddBZ N dN sigma_x dsigma_x dQ")
+
+class Parameters(NamedTuple):
+    ddBZ: float
+    N: float
+    dN: float
+    sigma_x: float
+    dsigma_x: float
+    dQ: float
 
 
 def generate_der(
-    categorize_file: str,
-    output_file: str,
-    uuid: str | None = None,
+    categorize_file: str | PathLike,
+    output_file: str | PathLike,
+    uuid: str | UUID | None = None,
     parameters: Parameters | None = None,
-) -> str:
+) -> UUID:
     """Generates Cloudnet effective radius of liquid water droplets
-        product acording to Frisch et al. 2002.
+        product according to Frisch et al. 2002.
 
     This function calculates liquid droplet effective radius def
     using the Frisch method. In this method, def is calculated
@@ -37,7 +49,7 @@ def generate_der(
         categorize_file: Categorize file name.
         output_file: Output file name.
         uuid: Set specific UUID for the file.
-        parameters: Tuple of specific fixed paramaters
+        parameters: Tuple of specific fixed parameters
             (ddBZ, N, dN, sigma_x, dsigma_x, dQ)
         used in Frisch approach.
 
@@ -60,16 +72,16 @@ def generate_der(
         https://doi.org/10.1175/1520-0426(2002)019%3C0835:TROSCD%3E2.0.CO;2
 
     """
-    der_source = DerSource(categorize_file, parameters)
-    droplet_classification = DropletClassification(categorize_file)
-    der_source.append_der()
-    der_source.append_retrieval_status(droplet_classification)
-    date = der_source.get_date()
-    attributes = output.add_time_attribute(REFF_ATTRIBUTES, date)
-    attributes = _add_der_error_comment(attributes, der_source)
-    output.update_attributes(der_source.data, attributes)
-    uuid = output.save_product_file("der", der_source, output_file, uuid)
-    der_source.close()
+    uuid = utils.get_uuid(uuid)
+    with DerSource(categorize_file, parameters) as der_source:
+        droplet_classification = DropletClassification(categorize_file)
+        der_source.append_der()
+        der_source.append_retrieval_status(droplet_classification)
+        date = der_source.get_date()
+        attributes = output.add_time_attribute(REFF_ATTRIBUTES, date)
+        attributes = _add_der_error_comment(attributes, der_source)
+        output.update_attributes(der_source.data, attributes)
+        output.save_product_file("der", der_source, output_file, uuid)
     return uuid
 
 
@@ -78,49 +90,53 @@ class DropletClassification(ProductClassification):
     Child of ProductClassification().
     """
 
-    def __init__(self, categorize_file: str):
+    def __init__(self, categorize_file: str | PathLike) -> None:
         super().__init__(categorize_file)
         self.is_mixed = self._find_mixed()
         self.is_droplet = self._find_droplet()
         self.is_ice = self._find_ice()
 
-    def _find_droplet(self) -> np.ndarray:
-        return self.category_bits["droplet"]
+    def _find_droplet(self) -> npt.NDArray:
+        return self.category_bits.droplet
 
-    def _find_mixed(self) -> np.ndarray:
-        return self.category_bits["falling"] & self.category_bits["droplet"]
+    def _find_mixed(self) -> npt.NDArray:
+        return self.category_bits.falling & self.category_bits.droplet
 
-    def _find_ice(self) -> np.ndarray:
+    def _find_ice(self) -> npt.NDArray:
         return (
-            self.category_bits["falling"]
-            & self.category_bits["cold"]
-            & ~self.category_bits["melting"]
-            & ~self.category_bits["droplet"]
-            & ~self.category_bits["insect"]
+            self.category_bits.falling
+            & self.category_bits.freezing
+            & ~self.category_bits.melting
+            & ~self.category_bits.droplet
+            & ~self.category_bits.insect
         )
 
 
 class DerSource(DataSource):
     """Data container for effective radius calculations."""
 
-    def __init__(self, categorize_file: str, parameters: Parameters | None = None):
+    def __init__(
+        self, categorize_file: str | PathLike, parameters: Parameters | None = None
+    ) -> None:
         super().__init__(categorize_file)
+        if "lwp" not in self.dataset.variables:
+            msg = "Liquid water path missing from the categorize file."
+            raise InvalidSourceFileError(msg)
         self.is_rain = get_is_rain(categorize_file)
         self.categorize_bits = CategorizeBits(categorize_file)
+        self.height_agl: npt.NDArray
         if parameters is None:
             # Default parameters from Frisch et al. 2002
             self.parameters = Parameters(2.0, 200.0e6, 200.0e6, 0.35, 0.1, 5.0e-3)
         else:
             self.parameters = parameters
 
-    def append_der(self):
+    def append_der(self) -> None:
         """Estimate liquid droplet effective radius using Frisch et al. 2002."""
-
         params = self.parameters
         rho_l = 1000  # density of liquid water(kg m-3)
 
         var_x = params.sigma_x * params.sigma_x
-        dheight = utils.mdiff(self.getvar("height"))
 
         Z = self.getvar("Z")
         Z = utils.db2lin(Z)
@@ -135,11 +151,17 @@ class DerSource(DataSource):
         der_scaled_error = np.zeros(Z.shape)
         N_scaled = np.zeros(Z.shape)
 
-        is_droplet = self.categorize_bits.category_bits["droplet"]
-        liquid_bases = atmos.find_cloud_bases(is_droplet)
-        liquid_tops = atmos.find_cloud_tops(is_droplet)
+        is_droplet = self.categorize_bits.category_bits.droplet
+        liquid_bases = atmos_utils.find_cloud_bases(is_droplet)
+        liquid_tops = atmos_utils.find_cloud_tops(is_droplet)
 
-        for base, top in zip(zip(*np.where(liquid_bases)), zip(*np.where(liquid_tops))):
+        path_lengths = utils.path_lengths_from_ground(self.height_agl)
+
+        for base, top in zip(
+            zip(*np.where(liquid_bases), strict=True),
+            zip(*np.where(liquid_tops), strict=True),
+            strict=True,
+        ):
             ind_t = base[0]
             idx_layer = np.arange(base[1], top[1] + 1)
             idx_layer = idx_layer[~Z[ind_t, idx_layer].mask]
@@ -147,7 +169,7 @@ class DerSource(DataSource):
             if Z[ind_t, idx_layer].mask.all():
                 continue
 
-            integral = ma.sum(ma.sqrt(Z[ind_t, idx_layer])) * dheight
+            integral = ma.sum(ma.sqrt(Z[ind_t, idx_layer])) * path_lengths[idx_layer]
 
             # der formula (5)
             A = (Z[ind_t, idx_layer] / params.N) ** (1 / 6)
@@ -159,7 +181,7 @@ class DerSource(DataSource):
             B = params.sigma_x * params.dsigma_x
             C = dZ[ind_t, idx_layer] / (6 * Z[ind_t, idx_layer])
             der_error[ind_t, idx_layer] = der[ind_t, idx_layer] * ma.sqrt(
-                A * A + B * B + C * C
+                A * A + B * B + C * C,
             )
 
             # der scaled formula (6)
@@ -176,7 +198,7 @@ class DerSource(DataSource):
             B = 4 * params.sigma_x * params.dsigma_x
             C = params.dQ / (3 * lwp[ind_t])
             der_scaled_error[ind_t, idx_layer] = der_scaled[ind_t, idx_layer] * ma.sqrt(
-                A * A + B * B + C * C
+                A * A + B * B + C * C,
             )
 
         N_scaled = ma.masked_less_equal(ma.masked_invalid(N_scaled), 0.0) * 1.0e-6
@@ -197,13 +219,14 @@ class DerSource(DataSource):
         self.append_data(der_scaled_error, "der_scaled_error")
 
     def append_retrieval_status(
-        self, droplet_classification: DropletClassification
+        self,
+        droplet_classification: DropletClassification,
     ) -> None:
         """Returns information about the status of der retrieval."""
         is_retrieved = ~self.data["der"][:].mask
         is_mixed = droplet_classification.is_mixed
         is_ice = droplet_classification.is_ice
-        is_rain = np.tile(self.is_rain, (is_retrieved.shape[1], 1)).T
+        is_rain = np.tile(self.is_rain == 1, (is_retrieved.shape[1], 1)).T
 
         retrieval_status = np.zeros(is_retrieved.shape, dtype=int)
         retrieval_status[is_ice] = 4
@@ -214,17 +237,18 @@ class DerSource(DataSource):
 
 
 DEFINITIONS = {
-    "der_retrieval_status": (
-        "\n"
-        "Value 0: No data: No cloud observed.\n"
-        "Value 1: Reliable retrieval.\n"
-        "Value 2: Mix of drops and ice: Droplets and ice crystals coexist within\n"
-        "pixel. Z may be biased by large crystals.\n"
-        "Value 3: Precipitation in profile: Drizzle and rain affects LWP retrieval\n"
-        "         of MWR but also the target reflectivity.\n"
-        "Value 4: Surrounding ice: Less crucial! Ice crystals in the vicinity of a\n"
-        "         droplet pixel may also bias its reflectivity.\n"
-    )
+    "der_retrieval_status": utils.status_field_definition(
+        {
+            0: """No data: No cloud observed.""",
+            1: """Reliable retrieval.""",
+            2: """Mix of drops and ice: Droplets and ice crystals coexist within
+                  pixel. Z may be biased by large crystals.""",
+            3: """Precipitation in profile: Drizzle and rain affects LWP
+                  retrieval of MWR but also the target reflectivity.""",
+            4: """Surrounding ice: Less crucial! Ice crystals in the vicinity of
+                  a droplet pixel may also bias its reflectivity.""",
+        }
+    ),
 }
 
 
@@ -271,7 +295,7 @@ def _add_der_error_comment(attributes: dict, der_source: DerSource) -> dict:
         comment="This variable is an estimate of the random error in effective\n"
         f"radius assuming an error in Z of ddBZ = {params.ddBZ} in N of\n"
         f"dN = {params.dN} and in the spectral width dsigma_x = {params.dsigma_x}\n"
-        f"and in the LWP Q of {params.dQ} kg m-3."
+        f"and in the LWP Q of {params.dQ} kg m-3.",
     )
     return attributes
 
@@ -283,32 +307,38 @@ REFF_ATTRIBUTES = {
         units="m",
         ancillary_variables="der_error",
         comment=COMMENTS["der"],
+        dimensions=("time", "height"),
     ),
     "der_error": MetaData(
         long_name="Absolute error in droplet effective radius",
         units="m",
         comment="",
+        dimensions=("time", "height"),
     ),
     "der_scaled": MetaData(
         long_name="Droplet effective radius (scaled to LWP)",
         units="m",
         ancillary_variables="der_scaled_error",
         comment=COMMENTS["der_scaled"],
+        dimensions=("time", "height"),
     ),
     "der_scaled_error": MetaData(
         long_name="Absolute error in droplet effective radius (scaled to LWP)",
         units="m",
         comment=COMMENTS["der_scaled_error"],
+        dimensions=("time", "height"),
     ),
     "N_scaled": MetaData(
         long_name="Cloud droplet number concentration",
         units="1",
         ancillary_variables="der_error der_scaled der_scaled_error",
         comment=COMMENTS["N_scaled"],
+        dimensions=("time", "height"),
     ),
     "der_retrieval_status": MetaData(
         long_name="Droplet effective radius retrieval status",
         definition=DEFINITIONS["der_retrieval_status"],
         units="1",
+        dimensions=("time", "height"),
     ),
 }

cloudnetpy/products/drizzle.py

@@ -1,6 +1,10 @@
-"""Module for creating Cloudnet drizzle product.
-"""
+"""Module for creating Cloudnet drizzle product."""
+
+from os import PathLike
+from uuid import UUID
+
 import numpy as np
+import numpy.typing as npt
 from numpy import ma
 from scipy.special import gamma
 
@@ -16,8 +20,10 @@ from cloudnetpy.products.drizzle_tools import (
 
 
 def generate_drizzle(
-    categorize_file: str, output_file: str, uuid: str | None = None
-) -> str:
+    categorize_file: str | PathLike,
+    output_file: str | PathLike,
+    uuid: str | UUID | None = None,
+) -> UUID:
     """Generates Cloudnet drizzle product.
 
     This function calculates different drizzle properties from
@@ -41,6 +47,7 @@ def generate_drizzle(
         J. Appl. Meteor., 44, 14–27, https://doi.org/10.1175/JAM-2181.1
 
     """
+    uuid = utils.get_uuid(uuid)
     with DrizzleSource(categorize_file) as drizzle_source:
         drizzle_class = DrizzleClassification(categorize_file)
         spectral_width = SpectralWidth(categorize_file)
@@ -59,8 +66,8 @@ def generate_drizzle(
         date = drizzle_source.get_date()
         attributes = output.add_time_attribute(DRIZZLE_ATTRIBUTES, date)
         output.update_attributes(drizzle_source.data, attributes)
-        uuid = output.save_product_file("drizzle", drizzle_source, output_file, uuid)
-    return uuid
+        output.save_product_file("drizzle", drizzle_source, output_file, uuid)
+        return uuid
 
 
 class DrizzleProducts:
@@ -76,13 +83,15 @@ class DrizzleProducts:
 
     """
 
-    def __init__(self, drizzle_source: DrizzleSource, drizzle_solver: DrizzleSolver):
+    def __init__(
+        self, drizzle_source: DrizzleSource, drizzle_solver: DrizzleSolver
+    ) -> None:
         self._data = drizzle_source
         self._params = drizzle_solver.params
         self._ind_drizzle, self._ind_lut = self._find_indices()
         self.derived_products = self._calc_derived_products()
 
-    def _find_indices(self):
+    def _find_indices(self) -> tuple:
         drizzle_ind = np.where(self._params["Do"])
         ind_mu = np.searchsorted(self._data.mie["mu"], self._params["mu"][drizzle_ind])
         ind_dia = np.searchsorted(self._data.mie["Do"], self._params["Do"][drizzle_ind])
@@ -91,7 +100,7 @@ class DrizzleProducts:
         ind_dia[ind_dia >= n_dia] = n_dia - 1
         return drizzle_ind, (ind_mu, ind_dia)
 
-    def _calc_derived_products(self):
+    def _calc_derived_products(self) -> dict:
         density = self._calc_density()
         lwc = self._calc_lwc()
         lwf = self._calc_lwf(lwc)
@@ -105,23 +114,23 @@ class DrizzleProducts:
             "v_air": v_air,
         }
 
-    def _calc_density(self):
+    def _calc_density(self) -> npt.NDArray:
         """Calculates drizzle number density (m-3)."""
         a = self._data.z * 3.67**6
         b = self._params["Do"] ** 6
         return np.divide(a, b, out=np.zeros_like(a), where=b != 0)
 
-    def _calc_lwc(self):
-        """Calculates drizzle liquid water content (kg m-3)"""
+    def _calc_lwc(self) -> npt.NDArray:
+        """Calculates drizzle liquid water content (kg m-3)."""
         rho_water = 1000
-        dia, mu, s = [self._params.get(key) for key in ("Do", "mu", "S")]
-        assert isinstance(mu, np.ndarray)
-        assert isinstance(s, np.ndarray)
-        assert isinstance(dia, np.ndarray)
+        dia, mu, s = (self._params.get(key) for key in ("Do", "mu", "S"))
+        dia = ma.array(dia)
+        mu = ma.array(mu)
+        s = ma.array(s)
         gamma_ratio = gamma(4 + mu) / gamma(3 + mu) / (3.67 + mu)
         return rho_water / 3 * self._data.beta * s * dia * gamma_ratio
 
-    def _calc_lwf(self, lwc_in):
+    def _calc_lwf(self, lwc_in: npt.NDArray) -> npt.NDArray:
         """Calculates drizzle liquid water flux."""
         flux = ma.copy(lwc_in)
         flux[self._ind_drizzle] *= (
@@ -130,13 +139,13 @@ class DrizzleProducts:
         )
         return flux
 
-    def _calc_fall_velocity(self):
+    def _calc_fall_velocity(self) -> npt.NDArray:
         """Calculates drizzle droplet fall velocity (m s-1)."""
         velocity = np.zeros_like(self._params["Do"])
         velocity[self._ind_drizzle] = -self._data.mie["v"][self._ind_lut]
         return velocity
 
-    def _calc_v_air(self, droplet_velocity):
+    def _calc_v_air(self, droplet_velocity: npt.NDArray) -> npt.NDArray:
         """Calculates vertical air velocity."""
         velocity = -np.copy(droplet_velocity)
         velocity[self._ind_drizzle] += self._data.v[self._ind_drizzle]
@@ -155,42 +164,44 @@ class RetrievalStatus:
             status information.
     """
 
-    def __init__(self, drizzle_class: DrizzleClassification):
+    def __init__(self, drizzle_class: DrizzleClassification) -> None:
         self.drizzle_class = drizzle_class
-        self.retrieval_status: np.ndarray = np.array([])
+        self.retrieval_status: npt.NDArray = np.array([])
         self._get_retrieval_status()
 
-    def _get_retrieval_status(self):
+    def _get_retrieval_status(self) -> None:
         self.retrieval_status = np.copy(self.drizzle_class.drizzle).astype(int)
         self._find_retrieval_below_melting()
         self.retrieval_status[self.drizzle_class.would_be_drizzle == 1] = 3
         self._find_retrieval_in_warm_liquid()
         self.retrieval_status[self.drizzle_class.is_rain == 1, :] = 5
 
-    def _find_retrieval_below_melting(self):
+    def _find_retrieval_below_melting(self) -> None:
         cold_rain = utils.transpose(self.drizzle_class.cold_rain)
         below_melting = cold_rain * self.drizzle_class.drizzle
         self.retrieval_status[below_melting == 1] = 2
 
-    def _find_retrieval_in_warm_liquid(self):
+    def _find_retrieval_in_warm_liquid(self) -> None:
         in_warm_liquid = (self.retrieval_status == 0) * self.drizzle_class.warm_liquid
         self.retrieval_status[in_warm_liquid == 1] = 4
 
 
-def _screen_rain(results: dict, classification: DrizzleClassification):
+def _screen_rain(results: dict, classification: DrizzleClassification) -> dict:
     """Removes rainy profiles from drizzle variables.."""
-    for key in results.keys():
+    for key in results:
         if not utils.isscalar(results[key]):
             results[key][classification.is_rain, :] = 0
     return results
 
 
-def _append_data(drizzle_data: DrizzleSource, results: dict):
+def _append_data(drizzle_data: DrizzleSource, results: dict) -> None:
     """Save retrieved fields to the drizzle_data object."""
     for key, value in results.items():
         if key != "drizzle_retrieval_status":
-            value = ma.masked_where(value == 0, value)
-        drizzle_data.append_data(value, key)
+            arr = ma.masked_where(value == 0, value)
+        else:
+            arr = value
+        drizzle_data.append_data(arr, key)
 
 
 DRIZZLE_ATTRIBUTES = {
@@ -198,95 +209,127 @@ DRIZZLE_ATTRIBUTES = {
         long_name="Drizzle number concentration",
         units="m-3",
         ancillary_variables="drizzle_N_error drizzle_N_bias",
+        dimensions=("time", "height"),
     ),
     "drizzle_N_error": MetaData(
-        long_name="Random error in drizzle number concentration", units="dB"
+        long_name="Random error in drizzle number concentration",
+        units="dB",
+        dimensions=("time", "height"),
     ),
     "drizzle_N_bias": MetaData(
         long_name="Possible bias in drizzle number concentration",
         units="dB",
+        dimensions=None,
     ),
     "drizzle_lwc": MetaData(
         long_name="Drizzle liquid water content",
         units="kg m-3",
         ancillary_variables="drizzle_lwc_error drizzle_lwc_bias",
+        standard_name="mass_concentration_of_drizzle_in_air",
+        dimensions=("time", "height"),
     ),
     "drizzle_lwc_error": MetaData(
         long_name="Random error in drizzle liquid water content",
         units="dB",
+        dimensions=("time", "height"),
     ),
     "drizzle_lwc_bias": MetaData(
         long_name="Possible bias in drizzle liquid water content",
         units="dB",
+        dimensions=None,
     ),
     "drizzle_lwf": MetaData(
         long_name="Drizzle liquid water flux",
         units="kg m-2 s-1",
         ancillary_variables="drizzle_lwf_error drizzle_lwf_bias",
+        dimensions=("time", "height"),
     ),
     "drizzle_lwf_error": MetaData(
         long_name="Random error in drizzle liquid water flux",
         units="dB",
+        dimensions=("time", "height"),
     ),
     "drizzle_lwf_bias": MetaData(
         long_name="Possible bias in drizzle liquid water flux",
         units="dB",
+        dimensions=None,
     ),
     "v_drizzle": MetaData(
         long_name="Drizzle droplet fall velocity",  # TODO: should include 'terminal' ?
         units="m s-1",
         ancillary_variables="v_drizzle_error v_drizzle_bias",
         comment="Positive values are towards the ground.",
+        dimensions=("time", "height"),
     ),
     "v_drizzle_error": MetaData(
-        long_name="Random error in drizzle droplet fall velocity", units="dB"
+        long_name="Random error in drizzle droplet fall velocity",
+        units="dB",
+        dimensions=("time", "height"),
     ),
     "v_drizzle_bias": MetaData(
         long_name="Possible bias in drizzle droplet fall velocity",
         units="dB",
+        dimensions=None,
     ),
     "v_air": MetaData(
         long_name="Vertical air velocity",
         units="m s-1",
         ancillary_variables="v_air_error",
         comment="Positive values are towards the sky.",
+        standard_name="upward_air_velocity",
+        dimensions=("time", "height"),
     ),
     "v_air_error": MetaData(
-        long_name="Random error in vertical air velocity", units="dB"
+        long_name="Random error in vertical air velocity",
+        units="dB",
+        dimensions=("time", "height"),
     ),
     "Do": MetaData(
         long_name="Drizzle median diameter",
         units="m",
         ancillary_variables="Do_error Do_bias",
+        dimensions=("time", "height"),
     ),
     "Do_error": MetaData(
         long_name="Random error in drizzle median diameter",
         units="dB",
+        dimensions=("time", "height"),
     ),
     "Do_bias": MetaData(
         long_name="Possible bias in drizzle median diameter",
         units="dB",
+        dimensions=None,
     ),
     "mu": MetaData(
         long_name="Drizzle droplet size distribution shape parameter",
         ancillary_variables="mu_error",
         units="1",
+        dimensions=("time", "height"),
     ),
     "mu_error": MetaData(
         long_name="Random error in drizzle droplet size distribution shape parameter",
         units="dB",
+        dimensions=("time", "height"),
     ),
     "S": MetaData(
         long_name="Lidar backscatter-to-extinction ratio",
         ancillary_variables="S_error",
         units="sr",
+        dimensions=("time", "height"),
     ),
     "S_error": MetaData(
         long_name="Random error in lidar backscatter-to-extinction ratio",
         units="dB",
+        dimensions=("time", "height"),
+    ),
+    "beta_corr": MetaData(
+        long_name="Lidar backscatter correction factor",
+        units="1",
+        dimensions=("time", "height"),
     ),
-    "beta_corr": MetaData(long_name="Lidar backscatter correction factor", units="1"),
     "drizzle_retrieval_status": MetaData(
-        long_name="Drizzle parameter retrieval status", units="1"
+        long_name="Drizzle parameter retrieval status",
+        units="1",
+        dimensions=("time", "height"),
     ),
 }