cloudnetpy 1.86.0__tar.gz → 1.86.2__tar.gz

{cloudnetpy-1.86.0/cloudnetpy.egg-info → cloudnetpy-1.86.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnetpy
-Version: 1.86.0
+Version: 1.86.2
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

{cloudnetpy-1.86.0 → cloudnetpy-1.86.2}/cloudnetpy/categorize/radar.py

@@ -401,7 +401,6 @@ class Radar(DataSource):
 
         if fvel.shape == (n_time, n_height):
             # Folding velocity is already expanded in radar file
-            # Not yet in current files
             return fvel
         if utils.isscalar(fvel):
             # e.g. MIRA

{cloudnetpy-1.86.0 → cloudnetpy-1.86.2}/cloudnetpy/instruments/bowtie.py

@@ -127,4 +127,9 @@ ATTRIBUTES = RPG_ATTRIBUTES | {
         ),
         dimensions=("time", "range"),
     ),
+    "nyquist_velocity": MetaData(
+        long_name="Nyquist velocity",
+        units="m s-1",
+        dimensions=("time", "chirp_sequence"),
+    ),
 }

{cloudnetpy-1.86.0 → cloudnetpy-1.86.2}/cloudnetpy/instruments/mira.py

@@ -3,6 +3,7 @@
 import datetime
 import logging
 import os
+import re
 from collections import OrderedDict
 from collections.abc import Sequence
 from os import PathLike
@@ -224,7 +225,7 @@ def _parse_input_files(
                 )
                 raise FileNotFoundError(msg)
 
-            filetypes = list({f.split(".")[-1].lower() for f in valid_files})
+            filetypes = list({_get_suffix(f) for f in valid_files})
 
             if len(filetypes) > 1:
                 err_msg = "Mixed mmclx and znc files. Please use only one filetype."
@@ -241,7 +242,7 @@ def _parse_input_files(
             )
     else:
         input_filename = input_files
-        keymap = _get_keymap(str(input_filename).split(".")[-1])
+        keymap = _get_keymap(_get_suffix(input_filename))
 
     return input_filename, keymap
 
@@ -258,6 +259,13 @@ def _get_ignored_variables(filetype: str) -> list | None:
     return keymaps.get(filetype.lower(), keymaps.get("mmclx"))
 
 
+def _get_suffix(filename: str | PathLike) -> str:
+    m = re.search(r"\.(\w+)(\.\d+)?$", str(filename))
+    if m is None:
+        return ""
+    return m[1].lower()
+
+
 def _get_keymap(filetype: str) -> dict[str, str]:
     """Returns a dictionary mapping the variables in the raw data to the processed
     Cloudnet file.
@@ -284,6 +292,7 @@ def _get_keymap(filetype: str) -> dict[str, str]:
                 ("nave", "nave"),
                 ("prf", "prf"),
                 ("rg0", "rg0"),
+                ("tpow", "tpow"),
             ],
         ),
         "mmclx": OrderedDict(

{cloudnetpy-1.86.0 → cloudnetpy-1.86.2}/cloudnetpy/instruments/rpg.py

@@ -1,9 +1,7 @@
-"""This module contains RPG Cloud Radar related functions."""
-
 import datetime
 import logging
 import math
-from collections.abc import Callable, Sequence
+from collections.abc import Sequence
 from os import PathLike
 from uuid import UUID
 
@@ -15,7 +13,7 @@ from rpgpy import RPGFileError
 from cloudnetpy import output, utils
 from cloudnetpy.cloudnetarray import CloudnetArray
 from cloudnetpy.constants import G_TO_KG, HPA_TO_PA, KM_H_TO_M_S, MM_H_TO_M_S
-from cloudnetpy.exceptions import InconsistentDataError, ValidTimeStampError
+from cloudnetpy.exceptions import ValidTimeStampError
 from cloudnetpy.instruments import instruments
 from cloudnetpy.instruments.cloudnet_instrument import CloudnetInstrument
 from cloudnetpy.instruments.instruments import Instrument
@@ -68,9 +66,8 @@ def rpg2nc(
     l1_files = utils.get_sorted_filenames(path_to_l1_files, ".LV1")
     fmcw94_objects, valid_files = _get_fmcw94_objects(l1_files, date)
     one_day_of_data = create_one_day_data_record(fmcw94_objects)
-    if not valid_files:
-        return uuid, []
-    print_info(one_day_of_data)
+    one_day_of_data["nyquist_velocity"] = _expand_nyquist(one_day_of_data)
+    _print_info(one_day_of_data)
     fmcw = Fmcw(one_day_of_data, site_meta)
     fmcw.convert_time_to_fraction_hour()
     fmcw.mask_invalid_ldr()
@@ -93,7 +90,7 @@ def rpg2nc(
     return uuid, valid_files
 
 
-def print_info(data: dict) -> None:
+def _print_info(data: dict) -> None:
     dual_pol = data["dual_polarization"]
     if dual_pol == 0:
         mode = "single polarisation"
@@ -108,57 +105,66 @@ RpgObjects = Sequence[Fmcw94Bin] | Sequence[HatproBinCombined]
 
 
 def create_one_day_data_record(rpg_objects: RpgObjects) -> dict:
-    """Concatenates all RPG data from one day."""
+    """Concatenates all RPG FMCW / HATPRO data from one day."""
     rpg_raw_data, rpg_header = _stack_rpg_data(rpg_objects)
-    if len(rpg_objects) > 1:
-        rpg_header = _reduce_header(rpg_header)
+    if "range" in rpg_header:
+        rpg_header["range"] = rpg_objects[0].header["range"]
+    should_be_constant = [
+        "model_number",
+        "dual_polarization",
+        "antenna_separation",
+        "antenna_diameter",
+        "antenna_gain",
+        "half_power_beam_width",
+        "radar_frequency",
+    ]
+    for key in should_be_constant:
+        if key not in rpg_header:
+            continue
+        unique_values = np.unique(rpg_header[key])
+        if len(unique_values) > 1:
+            msg = f"More than one value for {key} found: {unique_values}"
+            raise ValueError(msg)
+        rpg_header[key] = unique_values[0]
+
     rpg_raw_data = _mask_invalid_data(rpg_raw_data)
     return {**rpg_header, **rpg_raw_data}
 
 
-def _stack_rpg_data(rpg_objects: RpgObjects) -> tuple[dict, dict]:
-    """Combines data from hourly RPG objects.
-
-    Notes:
-        Ignores variable names starting with an underscore.
+def _expand_nyquist(data: dict) -> npt.NDArray:
+    """Expands Nyquist velocity from time X chirp => time X range."""
+    nyquist_velocity = ma.array(data["nyquist_velocity"])
+    chirp_start_indices = ma.array(data["chirp_start_indices"])
+    n_time = chirp_start_indices.shape[0]
+    n_range = len(data["range"])
+    expanded_nyquist = np.empty((n_time, n_range))
+    for t in range(n_time):
+        starts = chirp_start_indices[t].compressed()
+        v_nyq = nyquist_velocity[t].compressed()
+        ends = np.r_[starts[1:], n_range]
+        seg_lengths = ends - starts
+        expanded_nyquist[t, :] = np.repeat(v_nyq, seg_lengths)
+    return expanded_nyquist
 
-    """
-
-    def _stack(source: dict, target: dict, fun: Callable) -> None:
-        for name, value in source.items():
-            if not name.startswith("_"):
-                target[name] = fun((target[name], value)) if name in target else value
 
+def _stack_rpg_data(rpg_objects: RpgObjects) -> tuple[dict, dict]:
     data: dict = {}
     header: dict = {}
     for rpg in rpg_objects:
-        _stack(rpg.data, data, ma.concatenate)
-        _stack(rpg.header, header, ma.vstack)
+        for src, dst in ((rpg.data, data), (rpg.header, header)):
+            for name, value in src.items():
+                if name.startswith("_"):
+                    continue
+                arr = dst.get(name)
+                fun = (
+                    ma.concatenate
+                    if any(isinstance(x, ma.MaskedArray) for x in (value, arr))
+                    else np.concatenate
+                )
+                dst[name] = fun((arr, value)) if arr is not None else value
     return data, header
 
 
-def _reduce_header(header: dict) -> dict:
-    """Removes duplicate header data. Otherwise, we would need n_files dimension."""
-    reduced_header = {}
-    for key, data in header.items():
-        # Handle outliers in latitude and longitude (e.g. Galati 2024-02-11):
-        if key in ("latitude", "longitude"):
-            reduced_header[key] = ma.median(data)
-            continue
-        first_profile_value = data[0]
-        is_identical_value = bool(
-            np.isclose(data, first_profile_value, rtol=1e-2).all(),
-        )
-        if is_identical_value is False:
-            msg = f"Inconsistent header: {key}: {data}"
-            if key in ("sample_duration", "calibration_interval", "noise_threshold"):
-                logging.warning(msg)
-            else:
-                raise InconsistentDataError(msg)
-        reduced_header[key] = first_profile_value
-    return reduced_header
-
-
 def _mask_invalid_data(data_in: dict) -> dict:
     """Masks zeros and other fill values from data."""
     data = data_in.copy()
@@ -193,33 +199,68 @@ def _get_fmcw94_objects(
             continue
         objects.append(obj)
         valid_files.append(file)
-    if objects:
-        objects, valid_files = _remove_files_with_bad_height(objects, valid_files)
-    if not valid_files:
-        raise ValidTimeStampError
+    if not objects:
+        msg = "No valid files found"
+        raise ValidTimeStampError(msg)
+    objects = _interpolate_to_common_height(objects)
+    objects = _pad_chirp_related_fields(objects)
+    objects = _expand_time_related_fields(objects)
     return objects, valid_files
 
 
-def _remove_files_with_bad_height(objects: list, files: list) -> tuple[list, list]:
-    lengths = [obj.data["Zh"].shape[1] for obj in objects]
-    most_common = np.bincount(lengths).argmax()
-    files = [
-        file
-        for file, obj, length in zip(files, objects, lengths, strict=True)
-        if length == most_common
-    ]
-    objects = [
-        obj
-        for obj, length in zip(objects, lengths, strict=True)
-        if length == most_common
-    ]
-    n_removed = len(lengths) - len(files)
-    if n_removed > 0:
-        logging.warning(
-            "Removed %s RPG-FMCW-94 files due to inconsistent height vector",
-            n_removed,
-        )
-    return objects, files
+def _interpolate_to_common_height(objects: list[Fmcw94Bin]) -> list[Fmcw94Bin]:
+    range_arrays = [obj.header["range"] for obj in objects]
+    if all(np.array_equal(range_arrays[0], r) for r in range_arrays[1:]):
+        return objects
+    # Use range with the highest range gate for interpolation
+    target_height = max(range_arrays, key=lambda r: r[-1])
+    for obj in objects:
+        src_range = obj.header["range"]
+        if np.array_equal(src_range, target_height):
+            continue
+        for key, arr in obj.data.items():
+            if arr.ndim == 2 and arr.shape[1] == src_range.size:
+                obj.data[key] = utils.interpolate_2D_along_y(
+                    src_range, arr, target_height
+                )
+        obj.header["range"] = target_height
+    return objects
+
+
+def _pad_chirp_related_fields(objects: list[Fmcw94Bin]) -> list[Fmcw94Bin]:
+    """Pads chirp-related header fields with masked values to have the same length."""
+    chirp_lens = [len(obj.header["chirp_start_indices"]) for obj in objects]
+    if all(chirp_lens[0] == length for length in chirp_lens[1:]):
+        return objects
+    max_chirp_len = max(chirp_lens)
+    for obj in objects:
+        n_chirps = len(obj.header["chirp_start_indices"])
+        if n_chirps == max_chirp_len:
+            continue
+        for key, arr in obj.header.items():
+            if not isinstance(arr, str) and arr.ndim == 1 and arr.size == n_chirps:
+                pad_len = max_chirp_len - n_chirps
+                masked_arr = ma.array(arr, dtype=arr.dtype)
+                pad = ma.masked_all(pad_len, dtype=arr.dtype)
+                obj.header[key] = ma.concatenate([masked_arr, pad])
+    return objects
+
+
+def _expand_time_related_fields(objects: list[Fmcw94Bin]) -> list[Fmcw94Bin]:
+    for obj in objects:
+        n_time = obj.data["time"].size
+        for key in obj.header:
+            if key in ("range", "time") or key.startswith("_"):
+                continue
+            arr = obj.header[key]
+            # Handle outliers in latitude and longitude (e.g. Galati 2024-02-11):
+            if key in ("latitude", "longitude"):
+                arr = ma.median(arr)
+            if utils.isscalar(arr):
+                obj.header[key] = np.repeat(arr, n_time)
+            else:
+                obj.header[key] = np.tile(arr, (n_time, 1))
+    return objects
 
 
 def _validate_date(obj: Fmcw94Bin, expected_date: datetime.date) -> None:
@@ -443,9 +484,11 @@ RPG_ATTRIBUTES = {
         long_name="File code",
         units="1",
         comment="Indicates the RPG software version.",
-        dimensions=None,
+        dimensions=("time",),
+    ),
+    "program_number": MetaData(
+        long_name="Program number", units="1", dimensions=("time",)
     ),
-    "program_number": MetaData(long_name="Program number", units="1", dimensions=None),
     "model_number": MetaData(
         long_name="Model number",
         units="1",
@@ -469,54 +512,51 @@ RPG_ATTRIBUTES = {
         dimensions=None,
     ),
     "sample_duration": MetaData(
-        long_name="Sample duration", units="s", dimensions=None
+        long_name="Sample duration", units="s", dimensions=("time",)
     ),
     "calibration_interval": MetaData(
-        long_name="Calibration interval in samples", units="1", dimensions=None
+        long_name="Calibration interval in samples", units="1", dimensions=("time",)
     ),
     "number_of_spectral_samples": MetaData(
         long_name="Number of spectral samples in each chirp sequence",
         units="1",
-        dimensions=("chirp_sequence",),
-    ),
-    "nyquist_velocity": MetaData(
-        long_name="Nyquist velocity", units="m s-1", dimensions=("chirp_sequence",)
+        dimensions=("time", "chirp_sequence"),
     ),
     "number_of_averaged_chirps": MetaData(
         long_name="Number of averaged chirps in sequence",
         units="1",
-        dimensions=("chirp_sequence",),
+        dimensions=("time", "chirp_sequence"),
     ),
     "chirp_start_indices": MetaData(
         long_name="Chirp sequences start indices",
         units="1",
-        dimensions=("chirp_sequence",),
+        dimensions=("time", "chirp_sequence"),
     ),
     "integration_time": MetaData(
         long_name="Integration time",
         units="s",
         comment="Effective integration time of chirp sequence",
-        dimensions=("chirp_sequence",),
+        dimensions=("time", "chirp_sequence"),
     ),
     "range_resolution": MetaData(
         long_name="Vertical resolution of range",
         units="m",
-        dimensions=("chirp_sequence",),
+        dimensions=("time", "chirp_sequence"),
     ),
     "FFT_window": MetaData(
         long_name="FFT window type",
         units="1",
         definition=DEFINITIONS["FFT_window"],
-        dimensions=None,
+        dimensions=("time",),
     ),
     "input_voltage_range": MetaData(
-        long_name="ADC input voltage range (+/-)", units="mV", dimensions=None
+        long_name="ADC input voltage range (+/-)", units="mV", dimensions=("time",)
     ),
     "noise_threshold": MetaData(
         long_name="Noise filter threshold factor",
         units="1",
         comment="Multiple of the standard deviation of Doppler spectra.",
-        dimensions=None,
+        dimensions=("time",),
     ),
     "time_ms": MetaData(long_name="Time ms", units="ms", dimensions=("time",)),
     "quality_flag": MetaData(
@@ -545,7 +585,4 @@ RPG_ATTRIBUTES = {
     "pc_temperature": MetaData(
         long_name="PC temperature", units="K", dimensions=("time",)
     ),
-    "correlation_coefficient": MetaData(
-        long_name="Correlation coefficient", units="1", dimensions=None
-    ),
 }

{cloudnetpy-1.86.0 → cloudnetpy-1.86.2}/cloudnetpy/output.py

@@ -391,7 +391,6 @@ def _write_vars2nc(nc: netCDF4.Dataset, cloudnet_variables: dict) -> None:
         else:
             fill_value = False
         size = obj.dimensions if obj.dimensions is not None else ()
-
         nc_variable = nc.createVariable(
             obj.name,
             obj.data_type,
@@ -399,7 +398,11 @@ def _write_vars2nc(nc: netCDF4.Dataset, cloudnet_variables: dict) -> None:
             zlib=True,
             fill_value=fill_value,
         )
-        nc_variable[:] = obj.data
+        try:
+            nc_variable[:] = obj.data
+        except IndexError as err:
+            msg = f"Unable to write variable {obj.name} to file: {err}"
+            raise IndexError(msg) from err
         for attr in obj.fetch_attributes():
             setattr(nc_variable, attr, getattr(obj, attr))
 

{cloudnetpy-1.86.0 → cloudnetpy-1.86.2}/cloudnetpy/utils.py

@@ -387,7 +387,7 @@ def interpolate_2d_mask(
 def interpolate_2d_nearest(
     x: npt.NDArray,
     y: npt.NDArray,
-    z: npt.NDArray,
+    z: ma.MaskedArray,
     x_new: npt.NDArray,
     y_new: npt.NDArray,
 ) -> ma.MaskedArray:
@@ -419,18 +419,53 @@ def interpolate_2d_nearest(
     return fun((xx, yy)).T
 
 
+def interpolate_2D_along_y(
+    y: npt.NDArray,
+    z: npt.NDArray | ma.MaskedArray,
+    y_new: npt.NDArray,
+) -> ma.MaskedArray:
+    """Fast 1D nearest-neighbor interpolation along y for each x.
+
+    Args:
+        y: 1D numpy array of y-coordinates (length M).
+        z: 2D array of shape (N, M).
+        y_new: 1D numpy array of new y-coordinates.
+
+    Returns:
+        Masked 2D masked array interpolated along y.
+
+    Notes:
+        Only interpolates along y. Points outside range are masked.
+    """
+    idx = np.searchsorted(y, y_new, side="left")
+    idx = np.clip(idx, 0, len(y) - 1)
+    left = np.maximum(idx - 1, 0)
+    choose_right = (idx == 0) | (
+        (idx < len(y)) & (np.abs(y[idx] - y_new) < np.abs(y_new - y[left]))
+    )
+    idx[~choose_right] = left[~choose_right]
+    z_interp = ma.array(z[:, idx])
+    mask = (y_new < y.min()) | (y_new > y.max())
+    if z_interp.mask is ma.nomask:
+        z_mask = np.zeros(z_interp.shape, dtype=bool)
+    else:
+        z_mask = z_interp.mask.copy()
+    z_mask[:, mask] = True
+    return ma.MaskedArray(z_interp, mask=z_mask)
+
+
 def interpolate_1d(
     time: npt.NDArray,
     y: ma.MaskedArray,
     time_new: npt.NDArray,
     max_time: float,
     method: str = "linear",
-) -> npt.NDArray:
+) -> ma.MaskedArray:
     """1D linear interpolation preserving the mask.
 
     Args:
         time: 1D array in fraction hour.
-        y: 1D masked array, data values.
+        y: 1D array, data values.
         time_new: 1D array, new time coordinates.
         max_time: Maximum allowed gap in minutes. Values outside this gap will
             be masked.

{cloudnetpy-1.86.0 → cloudnetpy-1.86.2}/cloudnetpy/version.py

@@ -1,4 +1,4 @@
 MAJOR = 1
 MINOR = 86
-PATCH = 0
+PATCH = 2
 __version__ = f"{MAJOR}.{MINOR}.{PATCH}"

{cloudnetpy-1.86.0 → cloudnetpy-1.86.2/cloudnetpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnetpy
-Version: 1.86.0
+Version: 1.86.2
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License