cloudnetpy 1.89.0__tar.gz → 1.89.2__tar.gz

{cloudnetpy-1.89.0/cloudnetpy.egg-info → cloudnetpy-1.89.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnetpy
-Version: 1.89.0
+Version: 1.89.2
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

{cloudnetpy-1.89.0 → cloudnetpy-1.89.2}/cloudnetpy/categorize/classify.py

@@ -1,6 +1,7 @@
 import numpy as np
 import numpy.typing as npt
 from numpy import ma
+from scipy import ndimage
 
 from cloudnetpy import utils
 from cloudnetpy.categorize import (
@@ -84,19 +85,21 @@ def classify_measurements(data: Observations) -> ClassificationResult:
     bits.droplet = droplet.correct_liquid_top(obs, is_liquid, bits.freezing, limit=500)
     bits.insect, insect_prob = insects.find_insects(obs, bits.melting, bits.droplet)
     bits.falling = falling.find_falling_hydrometeors(obs, bits.droplet, bits.insect)
-    filtered_ice = _filter_falling(bits)
     for _ in range(5):
         _fix_undetected_melting_layer(bits)
         _filter_insects(bits)
     bits.aerosol = _find_aerosols(obs, bits)
-    bits.aerosol[filtered_ice] = False
     _fix_super_cold_liquid(obs, bits)
     _reclassify_ice_with_high_ldr(obs, bits)
+    _filter_insects_in_falling(bits)
+    _filter_falling(bits, obs)
+
+    is_clutter = _remove_clutter_in_liquid(obs.is_clutter, bits)
 
     return ClassificationResult(
         category_bits=bits,
         is_rain=obs.is_rain,
-        is_clutter=obs.is_clutter,
+        is_clutter=is_clutter,
         insect_prob=insect_prob,
         liquid_prob=liquid_prob,
     )
@@ -137,13 +140,50 @@ def _reclassify_ice_with_high_ldr(obs: ClassData, bits: CategoryBits) -> None:
     is_ice = bits.falling & bits.freezing & ~bits.droplet & ~bits.melting
     high_ldr = ~ma.getmaskarray(obs.ldr) & (obs.ldr > ldr_limit)
     warm_enough = obs.tw > temp_limit
-    reclassify = is_ice & high_ldr & warm_enough
+    above_melting = utils.ffill(bits.melting)
+    reclassify = is_ice & high_ldr & warm_enough & ~above_melting
     bits.insect[reclassify] = True
     bits.falling[reclassify] = False
     has_lidar = ~obs.beta.mask
     bits.aerosol[reclassify & has_lidar] = True
 
 
+def _filter_insects_in_falling(
+    bits: CategoryBits,
+    min_falling_size: int = 100,
+    dilation: int = 2,
+) -> None:
+    """Reclassifies insect pixels in freezing regions near large falling ice.
+
+    Insect pixels in freezing temperatures near sufficiently large falling
+    hydrometeor regions are almost certainly false positives. Real insects
+    cannot survive in freezing conditions, so the freezing constraint
+    protects legitimate warm-region insects from being reclassified.
+
+    Args:
+        bits: A :class:`CategoryBits` instance.
+        min_falling_size: Minimum size of nearby falling regions required
+            to trigger reclassification.
+        dilation: Number of pixels to dilate around falling regions.
+    """
+    structure = ndimage.generate_binary_structure(2, 1)
+    falling_labels, n_falling = ndimage.label(bits.falling, structure=structure)
+    if n_falling == 0:
+        return
+    falling_ids = np.arange(1, n_falling + 1)
+    falling_sizes = ndimage.sum(bits.falling, falling_labels, falling_ids)
+    large_falling = np.isin(
+        falling_labels, falling_ids[falling_sizes >= min_falling_size]
+    )
+    near_large_falling = ndimage.binary_dilation(
+        large_falling, structure, iterations=dilation
+    )
+    reclassify = bits.insect & bits.freezing & near_large_falling
+    bits.falling[reclassify] = True
+    bits.insect[reclassify] = False
+    bits.aerosol[reclassify] = False
+
+
 def _fix_super_cold_liquid(obs: ClassData, bits: CategoryBits) -> None:
     """Supercooled liquid droplets do not exist in atmosphere below around -38 C."""
     t_limit = T0 - 38
@@ -246,21 +286,44 @@ def _filter_insects(bits: CategoryBits) -> None:
     bits.insect = is_insects
 
 
-def _filter_falling(bits: CategoryBits) -> tuple:
+def _remove_clutter_in_liquid(
+    is_clutter: npt.NDArray,
+    bits: CategoryBits,
+) -> npt.NDArray:
+    """Removes clutter that is inside liquid or falling hydrometeor layers.
+
+    Clutter detection based on near-zero velocity can produce false positives
+    inside cloud and drizzle layers. This function removes clutter flags from
+    pixels that are adjacent to droplet or falling hydrometeor pixels, and
+    reclassifies them as falling hydrometeors.
+    """
+    is_clutter = is_clutter.copy()
+    liquid_or_falling = bits.droplet | bits.falling
+    structure = ndimage.generate_binary_structure(2, 1)
+    near_liquid = ndimage.binary_dilation(liquid_or_falling, structure)
+    false_clutter = is_clutter & near_liquid
+    is_clutter[false_clutter] = False
+    bits.falling[false_clutter] = True
+    return is_clutter
+
+
+def _filter_falling(bits: CategoryBits, obs: ClassData) -> None:
     # filter falling ice speckle noise
+    is_beta = ~obs.beta.mask
     is_freezing = bits.freezing
     is_falling = bits.falling
-    is_falling_filtered = utils.remove_small_objects(
-        is_falling,
-        max_size=10,
-        connectivity=1,
+    filtered_out = is_falling & ~np.asarray(
+        utils.remove_small_objects(
+            is_falling,
+            max_size=10,
+            connectivity=1,
+        ),
+        dtype=bool,
     )
-    is_filtered = is_falling & ~np.array(is_falling_filtered)
-    ice_ind = np.where(is_freezing & is_filtered)
-    is_falling[ice_ind] = False
-    # in warm these are (probably) insects
-    insect_ind = np.where(~is_freezing & is_filtered)
-    is_falling[insect_ind] = False
+    is_falling[filtered_out] = False
+    # In warm conditions, these are likely insects
+    bits.insect[filtered_out & ~is_freezing] = True
+    bits.aerosol[filtered_out & ~is_freezing & is_beta] = True
+    # In cold conditions, classify as aerosol if lidar signal is present
+    bits.aerosol[filtered_out & is_freezing & is_beta] = True
     bits.falling = is_falling
-    bits.insect[insect_ind] = True
-    return ice_ind

{cloudnetpy-1.89.0 → cloudnetpy-1.89.2}/cloudnetpy/categorize/insects.py

@@ -66,7 +66,7 @@ def _get_probabilities(obs: ClassData) -> dict:
     smooth_v = _get_smoothed_v(obs)
     lwp_interp = droplet.interpolate_lwp(obs)
     fun = utils.array_to_probability
-    return {
+    prob: dict = {
         "width": fun(obs.width, 1, 0.3, invert=True) if hasattr(obs, "width") else 1,
         "z_strong": fun(obs.z, 0, 8, invert=True),
         "z_weak": fun(obs.z, -20, 8, invert=True),
@@ -78,6 +78,19 @@ def _get_probabilities(obs: ClassData) -> dict:
         "lwp": utils.transpose(fun(lwp_interp, 0.15, 0.05, invert=True)),
         "v_sigma": fun(obs.v_sigma, 0.01, 0.1),
     }
+    if hasattr(obs, "sldr"):
+        prob.update(
+            {
+                "width_narrow": fun(obs.width, 0.3, 0.1, invert=True)
+                if hasattr(obs, "width")
+                else 1,
+                "z_insect": fun(obs.z, -30, 5, invert=True),
+                "temp_sldr": fun(obs.tw, 265, 3),
+                "v_sigma_sldr": fun(obs.v_sigma, 0.2, 0.15),
+                "v_sigma_mask": ma.getmaskarray(obs.v_sigma),
+            }
+        )
+    return prob
 
 
 def _get_smoothed_v(
@@ -92,12 +105,26 @@ def _calc_prob_from_ldr(prob: dict) -> npt.NDArray:
     """This is the most reliable proxy for insects."""
     if prob["ldr"] is not None:
         return prob["ldr"] * prob["temp_loose"]
-    if (
-        prob["sldr"] is not None
-    ):  # Strong SLDR values are probably insects, weak CAN be but not necessarily
-        p = prob["sldr"]
-        p[p < 0.9] = ma.masked
-        return p * prob["temp_loose"]
+    if prob["sldr"] is not None:
+        # SLDR alone is unreliable at low SNR. Use v_sigma * width as
+        # alternative proxy (insects show erratic motion + narrow width).
+        # When v_sigma is also unavailable, fall back to weak echo * narrow
+        # width -- temperature provides the main discrimination.
+        p_sldr = prob["sldr"]
+        p_vsig = prob["v_sigma_sldr"]
+        p_wn = prob["width_narrow"]
+        p_zi = prob["z_insect"]
+        # Echo fallback only where v_sigma is unavailable -- if v_sigma
+        # exists and is low, that's evidence against insects.
+        p_echo = np.where(prob["v_sigma_mask"], p_zi, 0)
+        p_proxy = np.maximum(p_sldr, np.maximum(p_vsig, p_echo))
+        p_total = p_proxy * p_wn * prob["temp_sldr"]
+        # High SLDR at warm temperatures are reliable as is
+        p_high_sldr = p_sldr * prob["temp_loose"]
+        high = p_high_sldr > 0.9
+        p_total[high] = p_high_sldr[high]
+        return p_total
+
     return np.zeros(prob["z_strong"].shape)
 
 

{cloudnetpy-1.89.0 → cloudnetpy-1.89.2}/cloudnetpy/cli.py

@@ -1,5 +1,6 @@
 import argparse
 import base64
+import concurrent.futures
 import datetime
 import gzip
 import hashlib
@@ -88,21 +89,51 @@ def _process_categorize(
     input_files: dict, args: argparse.Namespace, client: APIClient
 ) -> str | None:
     cat_filepath = _create_categorize_filepath(args)
+    instrument_prefs = _parse_instrument_preferences(args.instrument)
+
+    def fetch_instrument(product: str) -> tuple[str, str | None]:
+        source = instrument_prefs.get(product)
+        filepath = _fetch_product(args, product, client, source=source)
+        if filepath is None and source:
+            logging.info(
+                "Preferred instrument '%s' not found for %s, using available",
+                source,
+                product,
+            )
+            filepath = _fetch_product(args, product, client)
+        return product, filepath
 
-    input_files["model"] = _fetch_model(args, client)
-    if input_files["model"] is None:
+    def fetch_mwr() -> tuple[str, str | None]:
+        try:
+            return "mwr", _fetch_mwr(args, client)
+        except (CloudnetAPIError, KeyError, ValueError):
+            logging.info("LWP data not available, continuing without it")
+            return "mwr", None
+
+    def fetch_model() -> tuple[str, str | None]:
+        return "model", _fetch_model(args, client)
+
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = [executor.submit(fetch_model)]
+        for product in ("radar", "lidar", "disdrometer"):
+            if product not in input_files:
+                futures.append(executor.submit(fetch_instrument, product))
+
+        for future in concurrent.futures.as_completed(futures):
+            product, filepath = future.result()
+            if filepath:
+                input_files[product] = filepath
+
+    # MWR runs after other downloads because its fallback chain may
+    # try to download the same radar file fetched above.
+    mwr_key, mwr_path = fetch_mwr()
+    if mwr_path:
+        input_files[mwr_key] = mwr_path
+
+    if "model" not in input_files:
         logging.info("No model data available for this date.")
         return None
 
-    for product in ("radar", "lidar", "disdrometer"):
-        if product not in input_files and (
-            filepath := _fetch_product(args, product, client)
-        ):
-            input_files[product] = filepath
-
-    if mwr := _fetch_mwr(args, client):
-        input_files["mwr"] = mwr
-
     try:
         logging.info("Processing categorize...")
         generate_categorize(cast("CategorizeInput", input_files), cat_filepath)
@@ -338,6 +369,20 @@ def _get_product_sources(
     return source_products
 
 
+def _parse_instrument_preferences(args: list[str] | None) -> dict[str, str]:
+    """Parses instrument preferences like 'radar:mira-35' into a dict."""
+    if not args:
+        return {}
+    prefs = {}
+    for arg in args:
+        if ":" not in arg:
+            msg = f"Invalid instrument format '{arg}', expected 'product:instrument_id'"
+            raise argparse.ArgumentTypeError(msg)
+        product, instrument_id = arg.split(":", 1)
+        prefs[product] = instrument_id
+    return prefs
+
+
 def _parse_instrument(s: str) -> tuple[str, str | None]:
     if "[" in s and s.endswith("]"):
         name = s[: s.index("[")]
@@ -382,7 +427,6 @@ def _fetch_product(
             if m.instrument is not None and m.instrument.instrument_id == source
         ]
     if not meta:
-        logging.info("No data available for %s", product)
         return None
     if len(meta) > 1:
         logging.info(
@@ -560,6 +604,17 @@ def main() -> None:
         type=lambda arg: _parse_model(arg, client),
         help="Model to use in categorize.",
     )
+    parser.add_argument(
+        "-i",
+        "--instrument",
+        type=str,
+        action="append",
+        help=(
+            "Preferred instrument for categorize input, e.g. "
+            "'radar:mira-35' or 'lidar:cl61d'. Can be specified multiple times."
+        ),
+        default=None,
+    )
     parser.add_argument("--input", type=Path, help="Input path", default="input/")
     parser.add_argument("--output", type=Path, help="Output path", default="output/")
     parser.add_argument(

{cloudnetpy-1.89.0 → cloudnetpy-1.89.2}/cloudnetpy/version.py

@@ -1,4 +1,4 @@
 MAJOR = 1
 MINOR = 89
-PATCH = 0
+PATCH = 2
 __version__ = f"{MAJOR}.{MINOR}.{PATCH}"

{cloudnetpy-1.89.0 → cloudnetpy-1.89.2/cloudnetpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnetpy
-Version: 1.89.0
+Version: 1.89.2
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License