cloudnetpy 1.88.3__tar.gz → 1.89.1__tar.gz

{cloudnetpy-1.88.3/cloudnetpy.egg-info → cloudnetpy-1.89.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnetpy
-Version: 1.88.3
+Version: 1.89.1
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

{cloudnetpy-1.88.3 → cloudnetpy-1.89.1}/cloudnetpy/categorize/classify.py

@@ -1,6 +1,7 @@
 import numpy as np
 import numpy.typing as npt
 from numpy import ma
+from scipy import ndimage
 
 from cloudnetpy import utils
 from cloudnetpy.categorize import (
@@ -84,18 +85,21 @@ def classify_measurements(data: Observations) -> ClassificationResult:
     bits.droplet = droplet.correct_liquid_top(obs, is_liquid, bits.freezing, limit=500)
     bits.insect, insect_prob = insects.find_insects(obs, bits.melting, bits.droplet)
     bits.falling = falling.find_falling_hydrometeors(obs, bits.droplet, bits.insect)
-    filtered_ice = _filter_falling(bits)
     for _ in range(5):
         _fix_undetected_melting_layer(bits)
         _filter_insects(bits)
     bits.aerosol = _find_aerosols(obs, bits)
-    bits.aerosol[filtered_ice] = False
     _fix_super_cold_liquid(obs, bits)
+    _reclassify_ice_with_high_ldr(obs, bits)
+    _filter_insects_in_falling(bits)
+    _filter_falling(bits, obs)
+
+    is_clutter = _remove_clutter_in_liquid(obs.is_clutter, bits)
 
     return ClassificationResult(
         category_bits=bits,
         is_rain=obs.is_rain,
-        is_clutter=obs.is_clutter,
+        is_clutter=is_clutter,
         insect_prob=insect_prob,
         liquid_prob=liquid_prob,
     )
@@ -122,6 +126,64 @@ def fetch_quality(
     )
 
 
+def _reclassify_ice_with_high_ldr(obs: ClassData, bits: CategoryBits) -> None:
+    """Reclassifies ice pixels with high LDR as insects.
+
+    Ice particles typically have LDR below -13 dB. Higher values in the
+    freezing region (excluding the melting layer) indicate insects, which
+    can be present in temperatures down to about -10 C.
+    """
+    if not hasattr(obs, "ldr"):
+        return
+    ldr_limit = -13
+    temp_limit = T0 - 10
+    is_ice = bits.falling & bits.freezing & ~bits.droplet & ~bits.melting
+    high_ldr = ~ma.getmaskarray(obs.ldr) & (obs.ldr > ldr_limit)
+    warm_enough = obs.tw > temp_limit
+    above_melting = utils.ffill(bits.melting)
+    reclassify = is_ice & high_ldr & warm_enough & ~above_melting
+    bits.insect[reclassify] = True
+    bits.falling[reclassify] = False
+    has_lidar = ~obs.beta.mask
+    bits.aerosol[reclassify & has_lidar] = True
+
+
+def _filter_insects_in_falling(
+    bits: CategoryBits,
+    min_falling_size: int = 100,
+    dilation: int = 2,
+) -> None:
+    """Reclassifies insect pixels in freezing regions near large falling ice.
+
+    Insect pixels in freezing temperatures near sufficiently large falling
+    hydrometeor regions are almost certainly false positives. Real insects
+    cannot survive in freezing conditions, so the freezing constraint
+    protects legitimate warm-region insects from being reclassified.
+
+    Args:
+        bits: A :class:`CategoryBits` instance.
+        min_falling_size: Minimum size of nearby falling regions required
+            to trigger reclassification.
+        dilation: Number of pixels to dilate around falling regions.
+    """
+    structure = ndimage.generate_binary_structure(2, 1)
+    falling_labels, n_falling = ndimage.label(bits.falling, structure=structure)
+    if n_falling == 0:
+        return
+    falling_ids = np.arange(1, n_falling + 1)
+    falling_sizes = ndimage.sum(bits.falling, falling_labels, falling_ids)
+    large_falling = np.isin(
+        falling_labels, falling_ids[falling_sizes >= min_falling_size]
+    )
+    near_large_falling = ndimage.binary_dilation(
+        large_falling, structure, iterations=dilation
+    )
+    reclassify = bits.insect & bits.freezing & near_large_falling
+    bits.falling[reclassify] = True
+    bits.insect[reclassify] = False
+    bits.aerosol[reclassify] = False
+
+
 def _fix_super_cold_liquid(obs: ClassData, bits: CategoryBits) -> None:
     """Supercooled liquid droplets do not exist in atmosphere below around -38 C."""
     t_limit = T0 - 38
@@ -224,21 +286,44 @@ def _filter_insects(bits: CategoryBits) -> None:
     bits.insect = is_insects
 
 
-def _filter_falling(bits: CategoryBits) -> tuple:
+def _remove_clutter_in_liquid(
+    is_clutter: npt.NDArray,
+    bits: CategoryBits,
+) -> npt.NDArray:
+    """Removes clutter that is inside liquid or falling hydrometeor layers.
+
+    Clutter detection based on near-zero velocity can produce false positives
+    inside cloud and drizzle layers. This function removes clutter flags from
+    pixels that are adjacent to droplet or falling hydrometeor pixels, and
+    reclassifies them as falling hydrometeors.
+    """
+    is_clutter = is_clutter.copy()
+    liquid_or_falling = bits.droplet | bits.falling
+    structure = ndimage.generate_binary_structure(2, 1)
+    near_liquid = ndimage.binary_dilation(liquid_or_falling, structure)
+    false_clutter = is_clutter & near_liquid
+    is_clutter[false_clutter] = False
+    bits.falling[false_clutter] = True
+    return is_clutter
+
+
+def _filter_falling(bits: CategoryBits, obs: ClassData) -> None:
     # filter falling ice speckle noise
+    is_beta = ~obs.beta.mask
     is_freezing = bits.freezing
     is_falling = bits.falling
-    is_falling_filtered = utils.remove_small_objects(
-        is_falling,
-        max_size=10,
-        connectivity=1,
+    filtered_out = is_falling & ~np.asarray(
+        utils.remove_small_objects(
+            is_falling,
+            max_size=10,
+            connectivity=1,
+        ),
+        dtype=bool,
     )
-    is_filtered = is_falling & ~np.array(is_falling_filtered)
-    ice_ind = np.where(is_freezing & is_filtered)
-    is_falling[ice_ind] = False
-    # in warm these are (probably) insects
-    insect_ind = np.where(~is_freezing & is_filtered)
-    is_falling[insect_ind] = False
+    is_falling[filtered_out] = False
+    # In warm conditions, these are likely insects
+    bits.insect[filtered_out & ~is_freezing] = True
+    bits.aerosol[filtered_out & ~is_freezing & is_beta] = True
+    # In cold conditions, classify as aerosol if lidar signal is present
+    bits.aerosol[filtered_out & is_freezing & is_beta] = True
     bits.falling = is_falling
-    bits.insect[insect_ind] = True
-    return ice_ind

{cloudnetpy-1.88.3 → cloudnetpy-1.89.1}/cloudnetpy/categorize/model.py

@@ -16,6 +16,7 @@ from cloudnetpy.categorize.itu import (
     calc_saturation_vapor_pressure,
 )
 from cloudnetpy.cloudnetarray import CloudnetArray
+from cloudnetpy.constants import G
 from cloudnetpy.datasource import DataSource
 from cloudnetpy.exceptions import ModelDataError
 
@@ -115,6 +116,7 @@ class Model(DataSource):
                 array,
                 time_grid,
                 half_height if "atten" in key else height_grid,
+                extrapolate=True,
             )
         self.height = height_grid
         return utils.find_masked_profiles_indices(self.data_dense["temperature"])
@@ -144,7 +146,26 @@ class Model(DataSource):
         except KeyError as err:
             msg = "No 'height' variable in the model file."
             raise ModelDataError(msg) from err
-        return self.to_m(model_heights) + alt_site
+        surface_altitude = self._get_model_surface_altitude(alt_site)
+        return self.to_m(model_heights) + surface_altitude
+
+    def _get_model_surface_altitude(self, alt_site: float) -> float | npt.NDArray:
+        """Returns model surface altitude from geopotential if available.
+
+        For sites in complex terrain (e.g. mountains), the model grid cell
+        surface height can differ significantly from the actual site altitude.
+        Using the model's own surface height ensures that thermodynamic fields
+        are placed at their physically correct absolute heights.
+
+        Note: Model surface altitude might be higher than the site altitude.
+        """
+        if "sfc_height" in self.dataset.variables:
+            sfc_height = self.dataset.variables["sfc_height"][:]
+            return sfc_height[:, np.newaxis]
+        if "sfc_geopotential" in self.dataset.variables:
+            geopotential = self.dataset.variables["sfc_geopotential"][:]
+            return geopotential[:, np.newaxis] / G
+        return alt_site
 
     def calc_attenuations(self, frequency: float) -> None:
         temperature = self.getvar("temperature")

{cloudnetpy-1.88.3 → cloudnetpy-1.89.1}/cloudnetpy/categorize/radar.py

@@ -123,26 +123,12 @@ class Radar(DataSource):
         if variable not in self.data:
             return
         data = ma.copy(self.data[variable][:])
-        n_points_in_profiles = ma.count(data, axis=1)
-        n_profiles_with_data = np.count_nonzero(n_points_in_profiles)
+        n_profiles_with_data = np.count_nonzero(ma.count(data, axis=1))
         if n_profiles_with_data < 300:
             return
-        n_vertical = self._filter(
-            data,
-            axis=1,
-            min_coverage=0.5,
-            z_limit=10,
-            distance=4,
-            n_blocks=100,
-        )
-        n_horizontal = self._filter(
-            data,
-            axis=0,
-            min_coverage=0.3,
-            z_limit=-30,
-            distance=3,
-            n_blocks=20,
-        )
+
+        n_vertical = self._filter_stripes_axis(data, variable, axis=1)
+        n_horizontal = self._filter_stripes_axis(data, variable, axis=0)
         if n_vertical > 0 or n_horizontal > 0:
             logging.debug(
                 "Filtered %s vertical and %s horizontal stripes "
@@ -152,57 +138,74 @@ class Radar(DataSource):
                 variable,
             )
 
-    def _filter(
+    def _filter_stripes_axis(
         self,
         data: npt.NDArray,
+        variable: str,
         axis: int,
-        min_coverage: float,
-        z_limit: float,
-        distance: float,
-        n_blocks: int,
     ) -> int:
-        if axis == 0:
-            data = data.T
-            echo = self.data["Z"][:].T
-        else:
-            echo = self.data["Z"][:]
-
-        len_block = int(np.floor(data.shape[0] / n_blocks))
-        block_indices = np.arange(len_block)
-        n_removed_total = 0
-
-        for block_number in range(n_blocks):
-            data_block = data[block_indices, :]
-            n_values = ma.count(data_block, axis=1)
-            try:
-                q1 = np.quantile(n_values, 0.25)
-                q3 = np.quantile(n_values, 0.75)
-            except IndexError:
-                continue
-
-            if q1 == q3:
-                continue
-
-            threshold = distance * (q3 - q1) + q3
-
-            indices = np.where(
-                (n_values > threshold) & (n_values > (min_coverage * data.shape[1])),
-            )[0]
-            true_ind = [int(x) for x in (block_number * len_block + indices)]
-            n_removed = len(indices)
-
-            if n_removed > 5:
+        n_values = ma.count(data, axis=axis)
+        n = len(n_values)
+        margin = 5
+
+        spike_indices: set[int] = set()
+        for i in range(margin, n - margin):
+            window = [n_values[i + j] for j in range(-margin, margin + 1) if j != 0]
+            q1, q3 = np.percentile(window, [25, 75])
+            iqr = q3 - q1
+            if iqr == 0 and q3 > 0:
                 continue
+            threshold = q3 + 3 * max(iqr, 1)
+            if n_values[i] > max(threshold, 20):
+                spike_indices.add(i)
+
+        for ind in sorted(spike_indices):
+            below = ind - 1
+            while below >= 0 and below in spike_indices:
+                below -= 1
+            above = ind + 1
+            while above < n and above in spike_indices:
+                above += 1
+            self._mask_stripe_pixels(variable, ind, axis, below, above, n)
+
+        return len(spike_indices)
+
+    def _mask_stripe_pixels(
+        self,
+        variable: str,
+        ind: int,
+        axis: int,
+        below: int,
+        above: int,
+        dim_size: int,
+    ) -> None:
+        self._mask_isolated("v", "Z", ind, axis, below, above, dim_size)
+        if variable != "v" and variable in self.data:
+            self._mask_isolated(variable, variable, ind, axis, below, above, dim_size)
 
-            if n_removed > 0:
-                n_removed_total += n_removed
-                for ind in true_ind:
-                    ind2 = np.where(echo[ind, :] < z_limit)
-                    bad_indices = (ind, ind2) if axis == 1 else (ind2, ind)
-                    self.data["v"][:][bad_indices] = ma.masked
-            block_indices += len_block
-
-        return n_removed_total
+    def _mask_isolated(
+        self,
+        target: str,
+        source: str,
+        ind: int,
+        axis: int,
+        below: int,
+        above: int,
+        dim_size: int,
+    ) -> None:
+        mask = ma.getmaskarray(self.data[source][:])
+        has = ~mask[:, ind] if axis == 0 else ~mask[ind, :]
+        lo = (mask[:, below] if axis == 0 else mask[below, :]) if below >= 0 else True
+        hi = (
+            (mask[:, above] if axis == 0 else mask[above, :])
+            if above < dim_size
+            else True
+        )
+        isolated = has & lo & hi
+        if axis == 0:
+            self.data[target][:][isolated, ind] = ma.masked
+        else:
+            self.data[target][:][ind, isolated] = ma.masked
 
     def correct_atten(self, attenuations: RadarAttenuation) -> None:
         """Corrects radar echo for liquid and gas attenuation.

{cloudnetpy-1.88.3 → cloudnetpy-1.89.1}/cloudnetpy/cli.py

@@ -1,5 +1,6 @@
 import argparse
 import base64
+import concurrent.futures
 import datetime
 import gzip
 import hashlib
@@ -88,21 +89,51 @@ def _process_categorize(
     input_files: dict, args: argparse.Namespace, client: APIClient
 ) -> str | None:
     cat_filepath = _create_categorize_filepath(args)
+    instrument_prefs = _parse_instrument_preferences(args.instrument)
+
+    def fetch_instrument(product: str) -> tuple[str, str | None]:
+        source = instrument_prefs.get(product)
+        filepath = _fetch_product(args, product, client, source=source)
+        if filepath is None and source:
+            logging.info(
+                "Preferred instrument '%s' not found for %s, using available",
+                source,
+                product,
+            )
+            filepath = _fetch_product(args, product, client)
+        return product, filepath
 
-    input_files["model"] = _fetch_model(args, client)
-    if input_files["model"] is None:
+    def fetch_mwr() -> tuple[str, str | None]:
+        try:
+            return "mwr", _fetch_mwr(args, client)
+        except (CloudnetAPIError, KeyError, ValueError):
+            logging.info("LWP data not available, continuing without it")
+            return "mwr", None
+
+    def fetch_model() -> tuple[str, str | None]:
+        return "model", _fetch_model(args, client)
+
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = [executor.submit(fetch_model)]
+        for product in ("radar", "lidar", "disdrometer"):
+            if product not in input_files:
+                futures.append(executor.submit(fetch_instrument, product))
+
+        for future in concurrent.futures.as_completed(futures):
+            product, filepath = future.result()
+            if filepath:
+                input_files[product] = filepath
+
+    # MWR runs after other downloads because its fallback chain may
+    # try to download the same radar file fetched above.
+    mwr_key, mwr_path = fetch_mwr()
+    if mwr_path:
+        input_files[mwr_key] = mwr_path
+
+    if "model" not in input_files:
         logging.info("No model data available for this date.")
         return None
 
-    for product in ("radar", "lidar", "disdrometer"):
-        if product not in input_files and (
-            filepath := _fetch_product(args, product, client)
-        ):
-            input_files[product] = filepath
-
-    if mwr := _fetch_mwr(args, client):
-        input_files["mwr"] = mwr
-
     try:
         logging.info("Processing categorize...")
         generate_categorize(cast("CategorizeInput", input_files), cat_filepath)
@@ -153,6 +184,9 @@ def _process_instrument_product(
     fun: Callable
     match (product, instrument.instrument_id):
         case ("radar", _id) if "mira" in _id:
+            site_meta["range_correction_factor"] = calibration.get(
+                "range_correction_factor"
+            )
             fun = instruments.mira2nc
         case ("radar", _id) if "rpg" in _id:
             fun = instruments.rpg2nc
@@ -335,6 +369,20 @@ def _get_product_sources(
     return source_products
 
 
+def _parse_instrument_preferences(args: list[str] | None) -> dict[str, str]:
+    """Parses instrument preferences like 'radar:mira-35' into a dict."""
+    if not args:
+        return {}
+    prefs = {}
+    for arg in args:
+        if ":" not in arg:
+            msg = f"Invalid instrument format '{arg}', expected 'product:instrument_id'"
+            raise argparse.ArgumentTypeError(msg)
+        product, instrument_id = arg.split(":", 1)
+        prefs[product] = instrument_id
+    return prefs
+
+
 def _parse_instrument(s: str) -> tuple[str, str | None]:
     if "[" in s and s.endswith("]"):
         name = s[: s.index("[")]
@@ -379,7 +427,6 @@ def _fetch_product(
             if m.instrument is not None and m.instrument.instrument_id == source
         ]
     if not meta:
-        logging.info("No data available for %s", product)
         return None
     if len(meta) > 1:
         logging.info(
@@ -388,16 +435,18 @@ def _fetch_product(
         meta = [meta[0]]
     suffix = "geophysical" if "geophysical" in meta[0].product.type else "instrument"
     folder = _create_output_folder(suffix, args)
-    return _download_product_file(meta, folder, client)
+    return _download_product_file(meta, folder, client, force=args.force_download)
 
 
 def _fetch_model(args: argparse.Namespace, client: APIClient) -> str | None:
-    files = client.files(product_id="model", date=args.date, site_id=args.site)
+    files = client.files(
+        product_id="model", model_id=args.model, date=args.date, site_id=args.site
+    )
     if not files:
         logging.info("No model data available for this date")
         return None
     folder = _create_output_folder("instrument", args)
-    return _download_product_file(files, folder, client)
+    return _download_product_file(files, folder, client, force=args.force_download)
 
 
 def _fetch_raw(
@@ -406,19 +455,32 @@ def _fetch_raw(
     pid = _shorten_pid(metadata[0].instrument.pid)
     instrument = f"{metadata[0].instrument.instrument_id}_{pid}"
     folder = _create_input_folder(instrument, args)
+    if args.force_download:
+        for meta in metadata:
+            filepath = folder / meta.download_url.split("/")[-1]
+            if filepath.exists():
+                logging.info("Removing existing file: %s", filepath)
+                filepath.unlink()
     return client.download(metadata, output_directory=folder)
 
 
 def _download_product_file(
-    meta: list[ProductMetadata], folder: Path, client: APIClient
+    meta: list[ProductMetadata],
+    folder: Path,
+    client: APIClient,
+    *,
+    force: bool = False,
 ) -> str:
     if len(meta) > 1:
         msg = "Multiple product files found"
         raise ValueError(msg)
     filepath = folder / meta[0].filename
-    if filepath.exists():
+    if filepath.exists() and not force:
         logging.info("Existing file found: %s", filepath)
         return str(filepath)
+    if filepath.exists():
+        logging.info("Removing existing file: %s", filepath)
+        filepath.unlink()
     logging.info("Downloading file: %s", filepath)
     return str(client.download(meta, output_directory=folder)[0])
 
@@ -438,10 +500,13 @@ def _plot(
 ) -> None:
     if filepath is None or (not args.plot and not args.show):
         return
-    res = requests.get(f"{cloudnet_api_url}products/variables", timeout=60)
-    res.raise_for_status()
-    variables = next(var["variables"] for var in res.json() if var["id"] == product)
-    variables = [var["id"].split("-")[-1] for var in variables]
+    if args.variables is not None:
+        variables = args.variables.split(",")
+    else:
+        res = requests.get(f"{cloudnet_api_url}products/variables", timeout=60)
+        res.raise_for_status()
+        variables = next(var["variables"] for var in res.json() if var["id"] == product)
+        variables = [var["id"].split("-")[-1] for var in variables]
     image_name = str(filepath).replace(".nc", ".png") if args.plot else None
     try:
         generate_figure(
@@ -486,6 +551,21 @@ def _parse_products(product_argument: str, client: APIClient) -> list[str]:
     return valid_products
 
 
+def _parse_model(model_id: str, client: APIClient) -> str:
+    models = client.models()
+    found = False
+    for model in models:
+        if model_id == model.source_model_id and model.id != model.source_model_id:
+            msg = f"Use {model.id} or similar instead of {model_id}"
+            raise argparse.ArgumentTypeError(msg)
+        if model_id == model.id:
+            found = True
+    if not found:
+        msg = f"Invalid model: {model_id}"
+        raise argparse.ArgumentTypeError(msg)
+    return model_id
+
+
 def main() -> None:
     client = APIClient()
     parser = argparse.ArgumentParser(
@@ -501,7 +581,11 @@ def main() -> None:
         metavar="SITE",
     )
     parser.add_argument(
-        "-d", "--date", type=str, help="Date in YYYY-MM-DD", required=True
+        "-d",
+        "--date",
+        type=str,
+        help="Date in YYYY-MM-DD (default: today)",
+        default=datetime.datetime.now(tz=datetime.timezone.utc).date().isoformat(),
     )
     parser.add_argument(
         "-p",
@@ -514,6 +598,23 @@ def main() -> None:
         ),
         required=True,
     )
+    parser.add_argument(
+        "-m",
+        "--model",
+        type=lambda arg: _parse_model(arg, client),
+        help="Model to use in categorize.",
+    )
+    parser.add_argument(
+        "-i",
+        "--instrument",
+        type=str,
+        action="append",
+        help=(
+            "Preferred instrument for categorize input, e.g. "
+            "'radar:mira-35' or 'lidar:cl61d'. Can be specified multiple times."
+        ),
+        default=None,
+    )
     parser.add_argument("--input", type=Path, help="Input path", default="input/")
     parser.add_argument("--output", type=Path, help="Output path", default="output/")
     parser.add_argument(
@@ -534,6 +635,19 @@ def main() -> None:
         default=False,
         action=argparse.BooleanOptionalAction,
     )
+    parser.add_argument(
+        "--force-download",
+        help="Force re-download of input files even if they exist locally",
+        default=False,
+        action=argparse.BooleanOptionalAction,
+    )
+    parser.add_argument(
+        "-v",
+        "--variables",
+        type=str,
+        help="Variables to plot (comma-separated), e.g. 'target_classification'",
+        default=None,
+    )
     args = parser.parse_args()
 
     logger = logging.getLogger()

{cloudnetpy-1.88.3 → cloudnetpy-1.89.1}/cloudnetpy/instruments/mira.py

@@ -156,6 +156,7 @@ class Mira(NcRadar):
         if "tpow" not in self.data:
             logging.warning("Variable tpow is missing")
             return
+        self._correct_fzk_tpow()
         tpow = self.data["tpow"][:]
         # Threshold for abnormally low power e.g. Limassol 2024-10-20. Average
         # power should 30 to 60 W according to MIRA-35 data sheet. Based on a
@@ -169,6 +170,27 @@ class Mira(NcRadar):
             )
             self.screen_time_indices(~is_low)
 
+    def _correct_fzk_tpow(self) -> None:
+        """Corrects tpow for old FZK-serial instruments missing the FZK100 fix.
+
+        Some MIRA-35 instruments with serial number 'fzk' had a firmware bug
+        where tpow was reported 100x too small. Files produced after the
+        firmware fix contain 'FZK100' in the 'hrd' global attribute. Files
+        without this field need tpow multiplied by 100.
+        """
+        hrd = getattr(self.dataset, "hrd", "")
+        if "FZK100" in hrd:
+            return
+        sn_lines = [line for line in hrd.split("\n") if line.strip().startswith("SN:")]
+        if not sn_lines:
+            return
+        sn = sn_lines[0].split(":", 1)[1].strip()
+        if sn != "fzk":
+            return
+        logging.info("Correcting tpow by factor 100 for old FZK instrument firmware")
+        self.data["tpow"].data[:] *= 100
+        self.data["tpow"].correction_factor = 100
+
     def screen_invalid_ldr(self) -> None:
         """Masks LDR in MIRA STSR mode data.
         Is there a better way to identify this mode?