cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/products/drizzle_error.py

@@ -1,4 +1,5 @@
 import numpy as np
+import numpy.typing as npt
 from numpy import ma
 
 from cloudnetpy import utils
@@ -9,7 +10,8 @@ MU_ERROR_SMALL = 0.25
 
 
 def get_drizzle_error(
-    categorize: DrizzleSource, drizzle_parameters: DrizzleSolver
+    categorize: DrizzleSource,
+    drizzle_parameters: DrizzleSolver,
 ) -> dict:
     """Estimates error and bias for drizzle classification.
 
@@ -29,11 +31,10 @@ def get_drizzle_error(
         z_error = np.full(categorize.z.shape, z_error)
         error_input = z_error, bias_error
     bias_input = _read_input_uncertainty(categorize, "bias")
-    errors = _calc_errors(drizzle_indices, error_input, bias_input)
-    return errors
+    return _calc_errors(drizzle_indices, error_input, bias_input)
 
 
-def _get_drizzle_indices(diameter: np.ndarray) -> dict:
+def _get_drizzle_indices(diameter: npt.NDArray) -> dict:
     return {
         "drizzle": diameter > 0,
         "small": np.logical_and(diameter <= 1e-4, diameter > 1e-5),
@@ -100,7 +101,9 @@ def _calc_parameter_biases(bias_input: tuple) -> dict:
 
 
 def _add_supplementary_errors(
-    results: dict, drizzle_indices: dict, error_input: tuple
+    results: dict,
+    drizzle_indices: dict,
+    error_input: tuple,
 ) -> dict:
     def _calc_n_error() -> ma.MaskedArray:
         z_error = error_input[0]
@@ -139,6 +142,7 @@ def _calc_error(
     scale: float,
     weights: tuple,
     error_input: tuple,
+    *,
     add_mu: bool = False,
     add_mu_small: bool = False,
 ) -> ma.MaskedArray:
@@ -151,12 +155,12 @@ def _calc_error(
 
 
 def _stack_errors(
-    error_in: np.ndarray,
+    error_in: npt.NDArray,
     drizzle_indices: dict,
-    error_small=None,
-    error_tiny=None,
+    error_small: npt.NDArray | None = None,
+    error_tiny: npt.NDArray | None = None,
 ) -> ma.MaskedArray:
-    def _add_error_component(source: np.ndarray, ind: tuple):
+    def _add_error_component(source: npt.NDArray, ind: tuple) -> None:
         error[ind] = source[ind]
 
     error = ma.zeros(error_in.shape)
@@ -171,14 +175,14 @@ def _stack_errors(
 COR = 10 / np.log(10)
 
 
-def db2lin(x_in: np.ndarray) -> ma.MaskedArray:
+def db2lin(x_in: npt.NDArray) -> ma.MaskedArray:
     x = ma.copy(x_in)
     threshold = 100
     x[x > threshold] = threshold
     return ma.exp(x / COR) - 1
 
 
-def lin2db(x_in) -> ma.MaskedArray:
+def lin2db(x_in: npt.NDArray) -> ma.MaskedArray:
     x = ma.copy(x_in)
     threshold = -0.9
     x[x < threshold] = threshold

cloudnetpy/products/drizzle_tools.py

@@ -1,9 +1,12 @@
 import logging
 import os
 from bisect import bisect_left
+from os import PathLike
 
 import netCDF4
 import numpy as np
+import numpy.typing as npt
+from numpy import ma
 from scipy.special import gamma
 
 from cloudnetpy import utils
@@ -20,28 +23,27 @@ class DrizzleSource(DataSource):
 
     Attributes:
         mie (dict): Mie look-up table data.
-        dheight (float): Median difference of height array.
         z (ndarray): 2D radar echo (linear units).
         beta (ndarray): 2D lidar backscatter.
         v (ndarray): 2D doppler velocity.
 
     """
 
-    def __init__(self, categorize_file: str):
+    def __init__(self, categorize_file: str | PathLike) -> None:
         super().__init__(categorize_file)
         self.mie = self._read_mie_lut()
-        self.dheight = utils.mdiff(self.getvar("height"))
         self.z = self._convert_z_units()
         self.beta = self.getvar("beta")
         self.v = self.getvar("v")
+        self.height_agl: npt.NDArray
 
-    def _convert_z_units(self):
+    def _convert_z_units(self) -> npt.NDArray:
         """Converts reflectivity factor to SI units."""
         z = self.getvar("Z") - 180
         z[z > 0.0] = 0.0
         return utils.db2lin(z)
 
-    def _read_mie_lut(self):
+    def _read_mie_lut(self) -> dict:
         """Reads mie scattering look-up table."""
         mie_file = self._get_mie_file()
         with netCDF4.Dataset(mie_file) as nc:
@@ -59,20 +61,25 @@ class DrizzleSource(DataSource):
                     "width": mie[f"lu_width_{band}"][:],
                     "ray": mie[f"lu_mie_ray_{band}"][:],
                     "v": mie[f"lu_v_{band}"][:],
-                }
+                },
             )
         return lut
 
     @staticmethod
-    def _get_mie_file():
+    def _get_mie_file() -> str:
         module_path = os.path.dirname(os.path.abspath(__file__))
-        return "/".join((module_path, "mie_lu_tables.nc"))
+        return f"{module_path}/mie_lu_tables.nc"
 
-    def _get_wl_band(self):
+    def _get_wl_band(self) -> str:
         """Returns string corresponding the radar frequency."""
         radar_frequency = float(self.getvar("radar_frequency"))
         wl_band = utils.get_wl_band(radar_frequency)
-        return "35" if wl_band == 0 else "94"
+        if wl_band == "Ka":
+            return "35"
+        if wl_band == "W":
+            return "94"
+        msg = f"Unsupported band: {wl_band}"
+        raise ValueError(msg)
 
 
 class DrizzleClassification(ProductClassification):
@@ -91,7 +98,7 @@ class DrizzleClassification(ProductClassification):
 
     """
 
-    def __init__(self, categorize_file: str):
+    def __init__(self, categorize_file: str | PathLike) -> None:
         super().__init__(categorize_file)
         self.is_v_sigma = self._find_v_sigma(categorize_file)
         self.warm_liquid = self._find_warm_liquid()
@@ -100,43 +107,44 @@ class DrizzleClassification(ProductClassification):
         self.cold_rain = self._find_cold_rain()
 
     @staticmethod
-    def _find_v_sigma(cat_file: str):
-        v_sigma = product_tools.read_nc_fields(cat_file, "v_sigma")
+    def _find_v_sigma(cat_file: str | PathLike) -> npt.NDArray:
+        v_sigma = product_tools.read_nc_field(cat_file, "v_sigma")
         return np.isfinite(v_sigma)
 
-    def _find_warm_liquid(self):
-        return self.category_bits["droplet"] & ~self.category_bits["cold"]
+    def _find_warm_liquid(self) -> npt.NDArray:
+        return self.category_bits.droplet & ~self.category_bits.freezing
 
-    def _find_drizzle(self):
+    def _find_drizzle(self) -> npt.NDArray:
         return (
             ~utils.transpose(self.is_rain)
-            & self.category_bits["falling"]
-            & ~self.category_bits["droplet"]
-            & ~self.category_bits["cold"]
-            & ~self.category_bits["melting"]
-            & ~self.category_bits["insect"]
-            & self.quality_bits["radar"]
-            & self.quality_bits["lidar"]
-            & ~self.quality_bits["clutter"]
-            & ~self.quality_bits["molecular"]
-            & ~self.quality_bits["attenuated"]
+            & self.category_bits.falling
+            & ~self.category_bits.droplet
+            & ~self.category_bits.freezing
+            & ~self.category_bits.melting
+            & ~self.category_bits.insect
+            & self.quality_bits.radar
+            & self.quality_bits.lidar
+            & ~self.quality_bits.clutter
+            & ~self.quality_bits.molecular
+            & ~self.quality_bits.attenuated_liquid
+            & ~self.quality_bits.attenuated_rain
             & self.is_v_sigma
         )
 
-    def _find_would_be_drizzle(self):
+    def _find_would_be_drizzle(self) -> npt.NDArray:
         return (
             ~utils.transpose(self.is_rain)
             & self.warm_liquid
-            & self.category_bits["falling"]
-            & ~self.category_bits["melting"]
-            & ~self.category_bits["insect"]
-            & self.quality_bits["radar"]
-            & ~self.quality_bits["clutter"]
-            & ~self.quality_bits["molecular"]
+            & self.category_bits.falling
+            & ~self.category_bits.melting
+            & ~self.category_bits.insect
+            & self.quality_bits.radar
+            & ~self.quality_bits.clutter
+            & ~self.quality_bits.molecular
         )
 
-    def _find_cold_rain(self):
-        return np.any(self.category_bits["melting"], axis=1)
+    def _find_cold_rain(self) -> npt.NDArray:
+        return np.any(self.category_bits.melting, axis=1)
 
 
 class SpectralWidth:
@@ -155,33 +163,35 @@ class SpectralWidth:
 
     """
 
-    def __init__(self, categorize_file: str):
+    def __init__(self, categorize_file: str | PathLike) -> None:
         self.cat_file = categorize_file
         self.width_ht = self._calculate_spectral_width()
 
-    def _calculate_spectral_width(self):
-        v_sigma = product_tools.read_nc_fields(self.cat_file, "v_sigma")
+    def _calculate_spectral_width(self) -> npt.NDArray:
+        v_sigma = product_tools.read_nc_field(self.cat_file, "v_sigma")
         try:
-            width = product_tools.read_nc_fields(self.cat_file, "width")
+            width = product_tools.read_nc_field(self.cat_file, "width")
         except KeyError:
-            width = [0]
-            logging.warning(f"No spectral width, assuming width = {width[0]}")
+            width = ma.array([0])
+            logging.warning("No spectral width, assuming width = %s", width[0])
         sigma_factor = self._calc_v_sigma_factor()
         return width - sigma_factor * v_sigma
 
-    def _calc_v_sigma_factor(self):
+    def _calc_v_sigma_factor(self) -> npt.NDArray:
         beam_divergence = self._calc_beam_divergence()
         wind = self._calc_horizontal_wind()
         actual_wind = (wind + beam_divergence) ** (2 / 3)
         scaled_wind = (30 * wind + beam_divergence) ** (2 / 3)
         return actual_wind / (scaled_wind - actual_wind)
 
-    def _calc_beam_divergence(self):
+    def _calc_beam_divergence(self) -> npt.NDArray:
         beam_width = 0.5
-        height = product_tools.read_nc_fields(self.cat_file, "height")
-        return height * np.deg2rad(beam_width)
+        height = product_tools.read_nc_field(self.cat_file, "height")
+        altitude = np.mean(product_tools.read_nc_field(self.cat_file, "altitude"))
+        height_agl = height - altitude
+        return height_agl * np.deg2rad(beam_width)
 
-    def _calc_horizontal_wind(self):
+    def _calc_horizontal_wind(self) -> npt.NDArray:
         """Calculates magnitude of horizontal wind.
 
         Returns:
@@ -213,7 +223,7 @@ class DrizzleSolver:
         drizzle_source: DrizzleSource,
         drizzle_class: DrizzleClassification,
         spectral_width: SpectralWidth,
-    ):
+    ) -> None:
         self._data = drizzle_source
         self._drizzle_class = drizzle_class
         self._width_ht = spectral_width.width_ht
@@ -222,7 +232,7 @@ class DrizzleSolver:
         self._beta_z_ratio = self._calc_beta_z_ratio()
         self._solve_drizzle(self._dia_init)
 
-    def _init_variables(self) -> tuple[dict, np.ndarray]:
+    def _init_variables(self) -> tuple[dict, npt.NDArray]:
         shape = self._data.z.shape
         res = {
             "Do": np.zeros(shape),
@@ -232,22 +242,27 @@ class DrizzleSolver:
         }
         return res, np.zeros(shape)
 
-    def _calc_beta_z_ratio(self) -> np.ndarray:
+    def _calc_beta_z_ratio(self) -> npt.NDArray:
         return 2 / np.pi * self._data.beta / self._data.z
 
-    def _find_lut_indices(self, ind, dia_init, n_dia, n_widths) -> tuple[int, int]:
+    def _find_lut_indices(
+        self, ind: tuple[int, ...], dia_init: npt.NDArray, n_dia: int, n_widths: int
+    ) -> tuple[int, int]:
         ind_dia = bisect_left(self._data.mie["Do"], dia_init[ind], hi=n_dia - 1)
         ind_width = bisect_left(
-            self._width_lut[:, ind_dia], -self._width_ht[ind], hi=n_widths - 1
+            self._width_lut[:, ind_dia],
+            -self._width_ht[ind],
+            hi=n_widths - 1,
         )
         return ind_width, ind_dia
 
-    def _solve_drizzle(self, dia_init: np.ndarray):
+    def _solve_drizzle(self, dia_init: npt.NDArray) -> None:
         drizzle_ind = np.where(self._drizzle_class.drizzle == 1)
         dia_init[drizzle_ind] = self._calc_dia(self._beta_z_ratio[drizzle_ind], k=18.8)
         n_widths, n_dia = self._width_lut.shape[0], len(self._data.mie["Do"])
         max_ite = 10
-        for ind in zip(*drizzle_ind):
+        path_lengths = utils.path_lengths_from_ground(self._data.height_agl)
+        for ind in zip(*drizzle_ind, strict=True):
             for _ in range(max_ite):
                 lut_ind = self._find_lut_indices(ind, dia_init, n_dia, n_widths)
                 dia = self._calc_dia(
@@ -261,24 +276,27 @@ class DrizzleSolver:
                     break
                 self._dia_init[ind] = dia
             beta_factor = np.exp(
-                2 * self.params["S"][ind] * self._data.beta[ind] * self._data.dheight
+                2 * self.params["S"][ind] * self._data.beta[ind] * path_lengths[ind[-1]]
             )
             self.params["beta_corr"][ind[0], (ind[-1] + 1) :] *= beta_factor
 
     def _update_result_tables(
-        self, ind: tuple, dia: np.ndarray | float, lut_ind: tuple
-    ):
+        self,
+        ind: tuple,
+        dia: npt.NDArray | float,
+        lut_ind: tuple,
+    ) -> None:
         self.params["Do"][ind] = dia
         self.params["mu"][ind] = self._data.mie["mu"][lut_ind[0]]
         self.params["S"][ind] = self._data.mie["S"][lut_ind]
 
     @staticmethod
     def _calc_dia(
-        beta_z_ratio: np.ndarray | float,
+        beta_z_ratio: npt.NDArray | float,
         mu: float = 0.0,
         ray: float = 1.0,
         k: float = 1.0,
-    ) -> np.ndarray | float:
+    ) -> npt.NDArray | float:
         """Drizzle diameter calculation.
 
         Args:
@@ -299,7 +317,9 @@ class DrizzleSolver:
 
     @staticmethod
     def _is_converged(
-        ind: tuple, dia: np.ndarray | float, dia_init: np.ndarray
+        ind: tuple,
+        dia: npt.NDArray | float,
+        dia_init: npt.NDArray,
     ) -> bool:
         threshold = 1e-3
         return abs((dia - dia_init[ind]) / dia_init[ind]) < threshold

cloudnetpy/products/epsilon.py

@@ -0,0 +1,211 @@
+from os import PathLike
+from pathlib import Path
+from uuid import UUID
+
+import doppy
+import doppy.netcdf
+import netCDF4
+import numpy as np
+import numpy.typing as npt
+import scipy.constants
+from doppy.product.turbulence import HorizontalWind, Options, Turbulence, VerticalWind
+from scipy.interpolate import LinearNDInterpolator, NearestNDInterpolator
+
+import cloudnetpy
+from cloudnetpy.exceptions import ValidTimeStampError
+from cloudnetpy.output import copy_variables
+from cloudnetpy.utils import get_time, get_uuid
+
+
+def generate_epsilon_from_lidar(
+    doppler_lidar_file: str | PathLike,
+    doppler_lidar_wind_file: str | PathLike,
+    output_file: str | PathLike,
+    uuid: str | UUID | None = None,
+) -> UUID:
+    sliding_window_in_seconds = 3 * 60
+    uuid = get_uuid(uuid)
+    opts = _get_options(doppler_lidar_file)
+    opts.period = sliding_window_in_seconds
+    vert = _vertical_wind_from_doppler_lidar_file(doppler_lidar_file)
+    hori = _horizontal_wind_from_doppler_lidar_file(doppler_lidar_wind_file)
+    turb = Turbulence.from_winds(vert, hori, opts)
+
+    with (
+        netCDF4.Dataset(Path(doppler_lidar_file), "r") as nc_src,
+        doppy.netcdf.Dataset(Path(output_file), format="NETCDF4_CLASSIC") as nc,
+    ):
+        nc.add_dimension("time")
+        nc.add_dimension("height", size=len(turb.height))
+        nc.add_time(
+            name="time",
+            dimensions=("time",),
+            standard_name="time",
+            long_name="Time UTC",
+            data=turb.time,
+            dtype="f8",
+        )
+        nc.add_variable(
+            name="height",
+            dimensions=("height",),
+            units="m",
+            data=turb.height,
+            standard_name=nc_src["height"].standard_name,
+            long_name=nc_src["height"].long_name,
+            dtype="f4",
+        )
+        nc.add_variable(
+            name="epsilon",
+            dimensions=("time", "height"),
+            units="m2 s-3",
+            data=turb.turbulent_kinetic_energy_dissipation_rate,
+            mask=turb.mask,
+            dtype="f4",
+            long_name="Dissipation rate of turbulent kinetic energy",
+        )
+        nc.add_scalar_variable(
+            name="ray_accumulation_time",
+            units="s",
+            long_name="Ray accumulation time",
+            data=opts.ray_accumulation_time,
+            dtype="f4",
+        )
+        nc.add_scalar_variable(
+            name="rolling_window_period",
+            units="s",
+            long_name="Rolling window period",
+            data=opts.period,
+            dtype="f4",
+        )
+
+        nc.add_attribute("file_uuid", str(uuid))
+        nc.add_attribute("cloudnet_file_type", "epsilon-lidar")
+        nc.add_attribute("doppy_version", doppy.__version__)
+        nc.add_attribute("cloudnetpy_version", cloudnetpy.__version__)
+        nc.add_attribute(
+            "title",
+            "Dissipation rate of turbulent kinetic energy (lidar) "
+            f"from {nc_src.location}",
+        )
+
+    copy_attributes_from_src(doppler_lidar_file, output_file)
+
+    with (
+        netCDF4.Dataset(output_file, "r+") as nc_out,
+        netCDF4.Dataset(doppler_lidar_file, "r") as nc_src_stare,
+        netCDF4.Dataset(doppler_lidar_wind_file, "r") as nc_src_wind,
+    ):
+        copy_variables(
+            nc_src_stare, nc_out, ("latitude", "longitude", "altitude", "source")
+        )
+        nc_out.source_file_uuids = f"{nc_src_stare.file_uuid}, {nc_src_wind.file_uuid}"
+        sources = {nc_src_stare.source, nc_src_wind.source}
+        nc_out.source = ", ".join(sources)
+        history = (
+            f"{get_time()} - epsilon-lidar file created using doppy "
+            f"v{doppy.__version__} and cloudnetpy v{cloudnetpy.__version__}\n"
+            f"{nc_src_stare.history}\n"
+            f"{nc_src_wind.history}"
+        )
+        history = "\n".join(
+            line.strip() for line in history.splitlines() if line.strip()
+        )
+        nc_out.history = history
+        nc_out.references = "https://doi.org/10.1175/2010JTECHA1455.1"
+    return uuid
+
+
+def copy_attributes_from_src(src: str | PathLike, trg: str | PathLike) -> None:
+    with netCDF4.Dataset(src, "r") as nc_src, netCDF4.Dataset(trg, "a") as nc_trg:
+        for attr in ("year", "month", "day", "location", "Conventions"):
+            nc_trg.setncattr(attr, nc_src.getncattr(attr))
+
+
+def _horizontal_wind_from_doppler_lidar_file(
+    doppler_lidar_wind_file: str | PathLike,
+) -> HorizontalWind:
+    with netCDF4.Dataset(doppler_lidar_wind_file, "r") as nc:
+        time = _datetime64_from_nc_var(nc["time"])
+        height = np.array(nc["height"][:].data, dtype=np.float64)
+        uwind = np.array(nc["uwind"][:].data, dtype=np.float64)
+        vwind = np.array(nc["vwind"][:].data, dtype=np.float64)
+        umask = np.array(nc["uwind"][:].mask, dtype=np.bool_)
+        vmask = np.array(nc["vwind"][:].mask, dtype=np.bool_)
+        V = np.sqrt(uwind**2 + vwind**2)
+        mask = umask | vmask
+        if np.all(mask):
+            raise ValidTimeStampError
+        t = np.broadcast_to(time[:, None], mask.shape)[~mask]
+        h = np.broadcast_to(height[None, :], mask.shape)[~mask]
+
+        if len(np.unique(t)) < 3 or len(np.unique(h)) < 3:
+            msg = "Not enough unique values for interpolation"
+            raise ValidTimeStampError(msg)
+
+        interp_linear = LinearNDInterpolator(list(zip(t, h, strict=True)), V[~mask])
+        interp_nearest = NearestNDInterpolator(list(zip(t, h, strict=True)), V[~mask])
+        T, H = np.meshgrid(time, height, indexing="ij")
+        V_linear = interp_linear(T, H)
+        V_nearest = interp_nearest(T, H)
+        isnan = np.isnan(V_linear)
+        V_interp = V_linear
+        V_interp[isnan] = V_nearest[isnan]
+        if np.isnan(V_interp).any():
+            msg = "Unexpected nans"
+            raise ValueError(msg)
+        return HorizontalWind(time=time, height=height, V=V_interp)
+
+
+def _get_options(doppler_lidar_file: str | PathLike) -> Options:
+    with netCDF4.Dataset(doppler_lidar_file, "r") as nc:
+        if "ray_accumulation_time" in nc.variables:
+            return Options(ray_accumulation_time=nc["ray_accumulation_time"][:])
+        if "pulses_per_ray" in nc.variables:
+            prf = _infer_pulse_repetition_frequency(
+                np.array(nc["range"][:].data, dtype=np.float64)
+            )
+            return Options(ray_accumulation_time=float(nc["pulses_per_ray"][:] / prf))
+        msg = "Missing ray info"
+        raise ValueError(msg)
+
+
+def _infer_pulse_repetition_frequency(range_: npt.NDArray[np.float64]) -> float:
+    c = scipy.constants.c
+    dist = range_.max() - range_.min()
+    round_trip_time = 2 * dist / c
+
+    T_LOW = 1 / 10_000  # Halo XR instruments operate on lower frequency
+    T_HIGH = 1 / 15_000  # Rest should operate on higher frequency
+    if round_trip_time / T_HIGH < 1:
+        return 15e3
+    if round_trip_time / T_LOW < 1:
+        return 10e3
+    msg = f"Suspiciously large range ({dist}m). Cannot infer pulse repetition rate"
+    raise ValueError(msg)
+
+
+def _vertical_wind_from_doppler_lidar_file(
+    doppler_lidar_file: str | PathLike,
+) -> VerticalWind:
+    with netCDF4.Dataset(doppler_lidar_file, "r") as nc:
+        time = _datetime64_from_nc_var(nc["time"])
+        height = np.array(nc["height"][:].data, dtype=np.float64)
+        w = np.array(nc["v"][:].data, dtype=np.float64)
+        mask = np.array(nc["v"][:].mask, dtype=np.bool_)
+    if isinstance(mask, np.ndarray) and mask.any():
+        w[mask] = np.nan
+
+    return VerticalWind(time=time, height=height, w=w, mask=mask)
+
+
+def _datetime64_from_nc_var(var: netCDF4.Variable) -> npt.NDArray[np.datetime64]:
+    return np.array(
+        netCDF4.num2date(
+            var[:].data,
+            units=var.units,
+            calendar=var.calendar,
+            only_use_cftime_datetimes=False,
+            only_use_python_datetimes=True,
+        ),
+        dtype="datetime64[us]",
+    )