cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/categorize/droplet.py

@@ -1,20 +1,21 @@
-""" This module has functions for liquid layer detection.
-"""
+"""This module has functions for liquid layer detection."""
+
 import numpy as np
+import numpy.typing as npt
 import scipy.signal
 from numpy import ma
 
-import cloudnetpy.categorize.atmos
 from cloudnetpy import utils
+from cloudnetpy.categorize import atmos_utils
 from cloudnetpy.categorize.containers import ClassData
 
 
 def correct_liquid_top(
     obs: ClassData,
-    is_liquid: np.ndarray,
-    is_freezing: np.ndarray,
+    is_liquid: npt.NDArray,
+    is_freezing: npt.NDArray,
     limit: float = 200,
-) -> np.ndarray:
+) -> npt.NDArray:
     """Corrects lidar detected liquid cloud top using radar data.
 
     Args:
@@ -32,9 +33,9 @@ def correct_liquid_top(
 
     """
     is_liquid_corrected = np.copy(is_liquid)
-    liquid_tops = cloudnetpy.categorize.atmos.find_cloud_tops(is_liquid)
+    liquid_tops = atmos_utils.find_cloud_tops(is_liquid)
     top_above = utils.n_elements(obs.height, limit)
-    for prof, top in zip(*np.where(liquid_tops)):
+    for prof, top in zip(*np.where(liquid_tops), strict=True):
         ind = _find_ind_above_top(is_freezing[prof, top:], top_above)
         rad = obs.z[prof, top : top + ind + 1]
         if not (rad.mask.all() or ~rad.mask.any()):
@@ -43,7 +44,7 @@ def correct_liquid_top(
     return is_liquid_corrected
 
 
-def _find_ind_above_top(is_freezing_from_peak: np.ndarray, top_above: int) -> int:
+def _find_ind_above_top(is_freezing_from_peak: npt.NDArray, top_above: int) -> int:
     first_point_below_zero = np.where(is_freezing_from_peak)[0][0]
     ind = first_point_below_zero + top_above
     return min(len(is_freezing_from_peak) - 1, ind)
@@ -57,7 +58,7 @@ def find_liquid(
     min_top_der: float = 1e-7,
     min_lwp: float = 0,
     min_alt: float = 100,
-) -> np.ndarray:
+) -> npt.NDArray:
     """Estimate liquid layers from SNR-screened attenuated backscatter.
 
     Args:
@@ -80,7 +81,7 @@ def find_liquid(
 
     """
 
-    def _is_proper_peak():
+    def _is_proper_peak() -> bool:
         conditions = (
             npoints >= min_points,
             peak_width < max_width,
@@ -97,13 +98,12 @@ def find_liquid(
     is_liquid = np.zeros(beta.shape, dtype=bool)
     base_below_peak = utils.n_elements(height, 200)
     top_above_peak = utils.n_elements(height, 150)
-    difference = np.diff(beta, axis=1)
-    assert isinstance(difference, ma.MaskedArray)
+    difference = ma.array(np.diff(beta, axis=1))
     beta_diff = difference.filled(0)
     beta = beta.filled(0)
     peak_indices = _find_strong_peaks(beta, peak_amp)
 
-    for n, peak in zip(*peak_indices):
+    for n, peak in zip(*peak_indices, strict=True):
         lprof = beta[n, :]
         dprof = beta_diff[n, :]
         try:
@@ -115,14 +115,14 @@ def find_liquid(
         peak_width = height[top] - height[base]
         peak_alt = height[peak] - height[0]
         top_der = (lprof[peak] - lprof[top]) / (height[top] - height[peak])
-        is_positive_lwp = lwp_int[n] > min_lwp
+        is_positive_lwp = lwp_int[n] >= min_lwp
         if _is_proper_peak():
             is_liquid[n, base : top + 1] = True
 
     return is_liquid
 
 
-def ind_base(dprof: np.ndarray, ind_peak: int, dist: int, lim: float) -> int:
+def ind_base(dprof: npt.NDArray, ind_peak: int, dist: int, lim: float) -> int:
     """Finds base index of a peak in profile.
 
     Return the lowermost index of profile where 1st order differences
@@ -177,7 +177,7 @@ def ind_base(dprof: np.ndarray, ind_peak: int, dist: int, lim: float) -> int:
         >>> ind_base(dx, 5, 4, 10)
             1
 
-    See also:
+    See Also:
         droplet.ind_top()
 
     """
@@ -187,7 +187,9 @@ def ind_base(dprof: np.ndarray, ind_peak: int, dist: int, lim: float) -> int:
     return start + np.where(diffs > diffs[mind] / lim)[0][0]
 
 
-def ind_top(dprof: np.ndarray, ind_peak: int, nprof: int, dist: int, lim: float) -> int:
+def ind_top(
+    dprof: npt.NDArray, ind_peak: int, nprof: int, dist: int, lim: float
+) -> int:
     """Finds top index of a peak in profile.
 
     Return the uppermost index of profile where 1st order differences
@@ -213,7 +215,7 @@ def ind_top(dprof: np.ndarray, ind_peak: int, nprof: int, dist: int, lim: float)
         IndexError: Can not find proper top index (probably too many masked
             values in the profile).
 
-    See also:
+    See Also:
         droplet.ind_base()
 
     """
@@ -223,7 +225,7 @@ def ind_top(dprof: np.ndarray, ind_peak: int, nprof: int, dist: int, lim: float)
     return ind_peak + np.where(diffs < diffs[mind] / lim)[0][-1] + 1
 
 
-def interpolate_lwp(obs: ClassData) -> np.ndarray:
+def interpolate_lwp(obs: ClassData) -> npt.NDArray:
     """Linear interpolation of liquid water path to fill masked values.
 
     Args:
@@ -239,7 +241,7 @@ def interpolate_lwp(obs: ClassData) -> np.ndarray:
     return np.interp(obs.time, obs.time[ind], obs.lwp[ind])
 
 
-def _find_strong_peaks(data: np.ndarray, threshold: float) -> tuple:
+def _find_strong_peaks(data: npt.NDArray, threshold: float) -> tuple:
     """Finds local maximums from data (greater than *threshold*)."""
     peaks = scipy.signal.argrelextrema(data, np.greater, order=4, axis=1)
     strong_peaks = np.where(data[peaks] > threshold)

cloudnetpy/categorize/falling.py

@@ -1,15 +1,19 @@
 """Module to find falling hydrometeors from data."""
+
 import numpy as np
+import numpy.typing as npt
 from numpy import ma
 
-from cloudnetpy.categorize import atmos
+from cloudnetpy.categorize import atmos_utils
 from cloudnetpy.categorize.containers import ClassData
 from cloudnetpy.constants import T0
 
 
 def find_falling_hydrometeors(
-    obs: ClassData, is_liquid: np.ndarray, is_insects: np.ndarray
-) -> np.ndarray:
+    obs: ClassData,
+    is_liquid: npt.NDArray,
+    is_insects: npt.NDArray,
+) -> npt.NDArray:
     """Finds falling hydrometeors.
 
     Falling hydrometeors are radar signals that are
@@ -30,56 +34,80 @@ def find_falling_hydrometeors(
         Hogan R. and O'Connor E., 2004, https://bit.ly/2Yjz9DZ.
 
     """
-
     falling_from_radar = _find_falling_from_radar(obs, is_insects)
     falling_from_radar_fixed = _fix_liquid_dominated_radar(
-        obs, falling_from_radar, is_liquid
+        obs,
+        falling_from_radar,
+        is_liquid,
     )
     cold_aerosols = _find_cold_aerosols(obs, is_liquid)
     return falling_from_radar_fixed | cold_aerosols
 
 
-def _find_falling_from_radar(obs: ClassData, is_insects: np.ndarray) -> np.ndarray:
+def _find_falling_from_radar(obs: ClassData, is_insects: npt.NDArray) -> npt.NDArray:
     is_z = ~obs.z.mask
     no_clutter = ~obs.is_clutter
     no_insects = ~is_insects
     return is_z & no_clutter & no_insects
 
 
-def _find_cold_aerosols(obs: ClassData, is_liquid: np.ndarray) -> np.ndarray:
+def _find_cold_aerosols(obs: ClassData, is_liquid: npt.NDArray) -> npt.NDArray:
     """Lidar signals which are in colder than the threshold temperature
-    and threshold altitude from the ground are assumed ice. This method
-    should be improved in the future if possible.
+    and threshold altitude from the ground are assumed ice.
+
+    These pixels are easily mixed with aerosols at lower altitudes,
+    and at higher altitudes they could be supercooled liquid, actually.
+    This should be investigated and fixed in the future.
     """
     cold_aerosols = np.zeros(is_liquid.shape, dtype=bool)
-    temperature_limit = T0 - 15
-    range_limit = 2000
+    lidar_range = obs.height - obs.altitude
+    cold_aerosol_temperature_limit = T0 - 15
+    cold_aerosol_min_altitude = 2000
     is_beta = ~obs.beta.mask
     lidar_ice_indices = np.where(
-        (obs.tw.data < temperature_limit) & is_beta & ~is_liquid
+        (obs.tw.data < cold_aerosol_temperature_limit) & is_beta & ~is_liquid,
     )
     cold_aerosols[lidar_ice_indices] = True
-    low_range_indices = np.where(obs.height + obs.altitude < range_limit)
+    low_range_indices = np.where(lidar_range < cold_aerosol_min_altitude)
     if low_range_indices:
         cold_aerosols[:, low_range_indices] = False
+
+    # Further investigate range gates between 2000 and 4000 m
+    # to avoid abrupt transitions from aerosol to ice.
+    altitude_limit = 4000
+    window_size = 6
+    n_beta_in_window = 2
+    for time_ind, profile in enumerate(cold_aerosols):
+        for alt_ind, is_cold_aerosol in enumerate(profile):
+            if is_cold_aerosol and lidar_range[alt_ind] < altitude_limit:
+                start_ind = max(0, alt_ind - window_size + 1)
+                end_ind = alt_ind + 1
+                n_beta_below = np.sum(is_beta[time_ind, start_ind:end_ind])
+                if n_beta_below > n_beta_in_window:
+                    cold_aerosols[time_ind, alt_ind] = False
+
     return cold_aerosols
 
 
 def _fix_liquid_dominated_radar(
-    obs: ClassData, falling_from_radar: np.ndarray, is_liquid: np.ndarray
-) -> np.ndarray:
+    obs: ClassData,
+    falling_from_radar: npt.NDArray,
+    is_liquid: npt.NDArray,
+) -> npt.NDArray:
     """Radar signals inside liquid clouds are NOT ice if Z is
     increasing in height inside the cloud.
     """
-    liquid_bases = atmos.find_cloud_bases(is_liquid)
-    liquid_tops = atmos.find_cloud_tops(is_liquid)
+    liquid_bases = atmos_utils.find_cloud_bases(is_liquid)
+    liquid_tops = atmos_utils.find_cloud_tops(is_liquid)
     base_indices = np.where(liquid_bases)
     top_indices = np.where(liquid_tops)
 
-    for n, base, _, top in zip(*base_indices, *top_indices):
+    for n, base, _, top in zip(*base_indices, *top_indices, strict=True):
         z_prof = obs.z[n, :]
         if _is_z_missing_above_liquid(z_prof, top) and _is_z_increasing(
-            z_prof, base, top
+            z_prof,
+            base,
+            top,
         ):
             falling_from_radar[n, base : top + 1] = False
 
@@ -87,15 +115,16 @@ def _fix_liquid_dominated_radar(
 
 
 def _is_z_missing_above_liquid(z: ma.MaskedArray, ind_top: int) -> bool:
-    """Checks is z is masked right above the liquid layer top."""
+    """Checks if z is masked right above the liquid layer top."""
     if ind_top == len(z) - 1:
         return False
-    return z.mask[ind_top + 1]
+    mask = ma.getmaskarray(z)
+    return bool(mask[ind_top + 1])
 
 
 def _is_z_increasing(z: ma.MaskedArray, ind_base: int, ind_top: int) -> bool:
     """Checks is z is increasing inside the liquid cloud."""
-    z = z[ind_base : ind_top + 1].compressed()
-    if len(z) > 1:
-        return z[-1] > z[0]
+    z_valid = z[ind_base : ind_top + 1].compressed()
+    if len(z_valid) > 1:
+        return z_valid[-1] > z_valid[0]
     return False

cloudnetpy/categorize/freezing.py

@@ -1,7 +1,9 @@
 """Module to find freezing region from data."""
+
 import logging
 
 import numpy as np
+import numpy.typing as npt
 from numpy import ma
 from scipy.interpolate import interp1d
 
@@ -10,7 +12,7 @@ from cloudnetpy.categorize.containers import ClassData
 from cloudnetpy.constants import T0
 
 
-def find_freezing_region(obs: ClassData, melting_layer: np.ndarray) -> np.ndarray:
+def find_freezing_region(obs: ClassData, melting_layer: npt.NDArray) -> npt.NDArray:
     """Finds freezing region using the model temperature and melting layer.
 
     Every profile that contains melting layer, subzero region starts from
@@ -34,7 +36,7 @@ def find_freezing_region(obs: ClassData, melting_layer: np.ndarray) -> np.ndarra
 
     """
     is_freezing = np.zeros(obs.tw.shape, dtype=bool)
-    t0_alt = _find_t0_alt(obs.tw, obs.height)
+    t0_alt = find_t0_alt(obs.tw, obs.height)
     mean_melting_alt = _find_mean_melting_alt(obs, melting_layer)
 
     if _is_all_freezing(mean_melting_alt, t0_alt, obs.height):
@@ -50,8 +52,8 @@ def find_freezing_region(obs: ClassData, melting_layer: np.ndarray) -> np.ndarra
     for n in range(len(obs.time) - win):
         if mean_melting_alt[n : n + win].mask.all():
             freezing_alt[n + mid_win] = t0_alt[n + mid_win]
-    ind = ~freezing_alt.mask
-    f = interp1d(obs.time[ind], freezing_alt[ind])
+    ind_valid = ~freezing_alt.mask
+    f = interp1d(obs.time[ind_valid], freezing_alt[ind_valid])
     freezing_alt_interpolated = f(obs.time) - 1
     for ii, alt in enumerate(freezing_alt_interpolated):
         is_freezing[ii, obs.height > alt] = True
@@ -59,21 +61,27 @@ def find_freezing_region(obs: ClassData, melting_layer: np.ndarray) -> np.ndarra
 
 
 def _is_all_freezing(
-    mean_melting_alt: np.ndarray, t0_alt: np.ndarray, height: np.ndarray
+    mean_melting_alt: npt.NDArray,
+    t0_alt: npt.NDArray,
+    height: npt.NDArray,
 ) -> bool:
     no_detected_melting = mean_melting_alt.all() is ma.masked
     all_temperatures_below_freezing = (t0_alt <= height[0]).all()
     return no_detected_melting and all_temperatures_below_freezing
 
 
-def _find_mean_melting_alt(obs: ClassData, melting_layer: np.ndarray) -> ma.MaskedArray:
-    assert melting_layer.dtype == bool
+def _find_mean_melting_alt(
+    obs: ClassData, melting_layer: npt.NDArray
+) -> ma.MaskedArray:
+    if melting_layer.dtype != bool:
+        msg = "melting_layer data type should be boolean"
+        raise ValueError(msg)
     alt_array = np.tile(obs.height, (len(obs.time), 1))
     melting_alts = ma.array(alt_array, mask=~melting_layer)
     return ma.median(melting_alts, axis=1)
 
 
-def _find_t0_alt(temperature: np.ndarray, height: np.ndarray) -> np.ndarray:
+def find_t0_alt(temperature: npt.NDArray, height: npt.NDArray) -> npt.NDArray:
     """Interpolates altitudes where temperature goes below freezing.
 
     Args:
@@ -84,14 +92,19 @@ def _find_t0_alt(temperature: np.ndarray, height: np.ndarray) -> np.ndarray:
         1-D array denoting altitudes where the temperature drops below 0 deg C.
 
     """
-    alt: np.ndarray = np.array([])
+    alt: npt.NDArray = np.array([])
     for prof in temperature:
         ind = np.where(prof < T0)[0][0]
         if ind == 0:
             alt = np.append(alt, height[0])
         else:
-            x = prof[ind - 1 : ind + 1]
-            y = height[ind - 1 : ind + 1]
-            x, y = zip(*sorted(zip(x, y)))
+            x, y = zip(
+                *sorted(
+                    zip(
+                        prof[ind - 1 : ind + 1], height[ind - 1 : ind + 1], strict=True
+                    ),
+                ),
+                strict=True,
+            )
             alt = np.append(alt, np.interp(T0, x, y))
     return alt

cloudnetpy/categorize/insects.py

@@ -1,5 +1,7 @@
 """Module to find insects from data."""
+
 import numpy as np
+import numpy.typing as npt
 from numpy import ma
 from scipy.ndimage import gaussian_filter
 
@@ -10,10 +12,10 @@ from cloudnetpy.categorize.containers import ClassData
 
 def find_insects(
     obs: ClassData,
-    melting_layer: np.ndarray,
-    liquid_layers: np.ndarray,
+    melting_layer: npt.NDArray,
+    liquid_layers: npt.NDArray,
     prob_lim: float = 0.8,
-) -> tuple[np.ndarray, np.ndarray]:
+) -> tuple[npt.NDArray, npt.NDArray]:
     """Returns insect probability and boolean array of insect presence.
 
     Insects are classified by estimating heuristic probability
@@ -51,7 +53,7 @@ def find_insects(
     return is_insects, ma.masked_where(insect_prob == 0, insect_prob)
 
 
-def _insect_probability(obs: ClassData) -> tuple[np.ndarray, np.ndarray]:
+def _insect_probability(obs: ClassData) -> tuple[npt.NDArray, npt.NDArray]:
     prob = _get_probabilities(obs)
     prob_from_ldr = _calc_prob_from_ldr(prob)
     prob_from_others = _calc_prob_from_all(prob)
@@ -65,9 +67,9 @@ def _get_probabilities(obs: ClassData) -> dict:
     lwp_interp = droplet.interpolate_lwp(obs)
     fun = utils.array_to_probability
     return {
-        "width": fun(obs.width, 1, 0.3, True) if hasattr(obs, "width") else 1,
-        "z_strong": fun(obs.z, 0, 8, True),
-        "z_weak": fun(obs.z, -20, 8, True),
+        "width": fun(obs.width, 1, 0.3, invert=True) if hasattr(obs, "width") else 1,
+        "z_strong": fun(obs.z, 0, 8, invert=True),
+        "z_weak": fun(obs.z, -20, 8, invert=True),
         "ldr": fun(obs.ldr, -25, 5) if hasattr(obs, "ldr") else None,
         "sldr": fun(obs.sldr, -25, 5) if hasattr(obs, "sldr") else None,
         "temp_loose": fun(obs.tw, 268, 2),
@@ -79,14 +81,14 @@ def _get_probabilities(obs: ClassData) -> dict:
 
 
 def _get_smoothed_v(
-    obs: ClassData, sigma: tuple[float, float] = (5, 5)
+    obs: ClassData,
+    sigma: tuple[float, float] = (5, 5),
 ) -> ma.MaskedArray:
-    smoothed_v = gaussian_filter(obs.v, sigma)
-    smoothed_v = ma.masked_where(obs.v.mask, smoothed_v)
-    return smoothed_v
+    smoothed_v = gaussian_filter(obs.v.filled(0), sigma)
+    return ma.masked_where(obs.v.mask, smoothed_v)
 
 
-def _calc_prob_from_ldr(prob: dict) -> np.ndarray:
+def _calc_prob_from_ldr(prob: dict) -> npt.NDArray:
     """This is the most reliable proxy for insects."""
     if prob["ldr"] is not None:
         return prob["ldr"] * prob["temp_loose"]
@@ -99,16 +101,19 @@ def _calc_prob_from_ldr(prob: dict) -> np.ndarray:
     return np.zeros(prob["z_strong"].shape)
 
 
-def _calc_prob_from_all(prob: dict) -> np.ndarray:
+def _calc_prob_from_all(prob: dict) -> npt.NDArray:
     """This can be tried when LDR is not available. To detect insects without LDR
     unambiguously is difficult and might result in many false positives and/or false
-    negatives."""
+    negatives.
+    """
     return prob["z_weak"] * prob["temp_strict"] * prob["width"] * prob["v"]
 
 
 def _adjust_for_radar(
-    obs: ClassData, prob: dict, prob_from_others: np.ndarray
-) -> np.ndarray:
+    obs: ClassData,
+    prob: dict,
+    prob_from_others: npt.NDArray,
+) -> npt.NDArray:
     """Adds radar-specific weighting to insect probabilities."""
     if "mira" in obs.radar_type.lower():
         prob_from_others *= prob["lwp"]
@@ -116,27 +121,34 @@ def _adjust_for_radar(
 
 
 def _fill_missing_pixels(
-    prob_from_ldr: np.ndarray, prob_from_others: np.ndarray
-) -> np.ndarray:
+    prob_from_ldr: npt.NDArray,
+    prob_from_others: npt.NDArray,
+) -> npt.NDArray:
     prob_combined = np.copy(prob_from_ldr)
     no_ldr = np.where(prob_from_ldr == 0)
     prob_combined[no_ldr] = prob_from_others[no_ldr]
     return prob_combined
 
 
-def _screen_insects(insect_prob, insect_prob_no_ldr, melting_layer, liquid_layers, obs):
-    def _screen_liquid_layers():
+def _screen_insects(
+    insect_prob: npt.NDArray,
+    insect_prob_no_ldr: npt.NDArray,
+    melting_layer: npt.NDArray,
+    liquid_layers: npt.NDArray,
+    obs: ClassData,
+) -> npt.NDArray:
+    def _screen_liquid_layers() -> None:
         prob[liquid_layers == 1] = 0
 
-    def _screen_above_melting():
+    def _screen_above_melting() -> None:
         above_melting = utils.ffill(melting_layer)
         prob[above_melting == 1] = 0
 
-    def _screen_above_liquid():
+    def _screen_above_liquid() -> None:
         above_liquid = utils.ffill(liquid_layers)
         prob[(above_liquid == 1) & (insect_prob_no_ldr > 0)] = 0
 
-    def _screen_rainy_profiles():
+    def _screen_rainy_profiles() -> None:
         prob[obs.is_rain == 1, :] = 0
 
     prob = np.copy(insect_prob)