cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/categorize/radar.py

@@ -1,13 +1,17 @@
 """Radar module, containing the :class:`Radar` class."""
+
 import logging
 import math
+from os import PathLike
 
 import numpy as np
+import numpy.typing as npt
 from numpy import ma
 from scipy import constants
 
 from cloudnetpy import utils
-from cloudnetpy.categorize.classify import ClassificationResult
+from cloudnetpy.categorize.attenuations import RadarAttenuation
+from cloudnetpy.constants import GHZ_TO_HZ, SEC_IN_HOUR, SPEED_OF_LIGHT
 from cloudnetpy.datasource import DataSource
 
 
@@ -22,9 +26,7 @@ class Radar(DataSource):
         folding_velocity (float): Radar's folding velocity (m/s).
         location (str): Location of the radar, copied from the global attribute
             `location` of the input file.
-        sequence_indices (list): Indices denoting the different altitude
-            regimes of the radar.
-        type (str): Type of the radar, copied from the global attribute
+        source_type (str): Type of the radar, copied from the global attribute
             `source` of the *radar_file*. Can be free form string but must
             include either 'rpg' or 'mira' denoting one of the two supported
             radars.
@@ -34,22 +36,21 @@ class Radar(DataSource):
 
     """
 
-    def __init__(self, full_path: str):
+    def __init__(self, full_path: str | PathLike) -> None:
         super().__init__(full_path, radar=True)
         self.radar_frequency = float(self.getvar("radar_frequency"))
-        self.folding_velocity = self._get_folding_velocity()
-        self.sequence_indices = self._get_sequence_indices()
         self.location = getattr(self.dataset, "location", "")
-        self.type = getattr(self.dataset, "source", "")
+        self.source_type = getattr(self.dataset, "source", "")
+        self.height: npt.NDArray
+        self.height_agl: npt.NDArray
+        self.altitude: float
         self._init_data()
-        self._init_sigma_v()
-        self._get_folding_velocity_full()
 
-    def rebin_to_grid(self, time_new: np.ndarray) -> list:
-        """Rebins radar data in time using mean.
+    def rebin_to_grid(self, time_new: npt.NDArray) -> list:
+        """Rebins radar data in time.
 
         Args:
-            time_new: Target time array as fraction hour. Updates *time* attribute.
+            time_new: Target time array as fraction hour.
 
         """
         bad_time_indices = []
@@ -60,19 +61,25 @@ class Radar(DataSource):
                     bad_time_indices = array.rebin_data(self.time, time_new)
                     array.lin2db()
                 case "v":
-                    array.rebin_velocity(
-                        self.time,
+                    array.data = self._rebin_velocity(
+                        array.data,
                         time_new,
-                        self.folding_velocity,
-                        self.sequence_indices,
                     )
                 case "v_sigma":
-                    array.calc_linear_std(self.time, time_new)
-                case "width" | "rainfall_rate":
+                    array.data, _ = utils.rebin_2d(
+                        self.time,
+                        array.data,
+                        time_new,
+                        "std",
+                        mask_zeros=True,
+                    )
+                case "width":
+                    array.rebin_data(self.time, time_new)
+                case "rainfall_rate":
                     array.rebin_data(self.time, time_new)
                 case _:
                     continue
-        return bad_time_indices
+        return list(bad_time_indices)
 
     def remove_incomplete_pixels(self) -> None:
         """Mask radar pixels where one or more required quantities are missing.
@@ -83,7 +90,7 @@ class Radar(DataSource):
 
         """
         good_ind = ~ma.getmaskarray(self.data["Z"][:]) & ~ma.getmaskarray(
-            self.data["v"][:]
+            self.data["v"][:],
         )
 
         if "width" in self.data:
@@ -121,20 +128,33 @@ class Radar(DataSource):
         if n_profiles_with_data < 300:
             return
         n_vertical = self._filter(
-            data, 1, min_coverage=0.5, z_limit=10, distance=4, n_blocks=100
+            data,
+            axis=1,
+            min_coverage=0.5,
+            z_limit=10,
+            distance=4,
+            n_blocks=100,
         )
         n_horizontal = self._filter(
-            data, 0, min_coverage=0.3, z_limit=-30, distance=3, n_blocks=20
+            data,
+            axis=0,
+            min_coverage=0.3,
+            z_limit=-30,
+            distance=3,
+            n_blocks=20,
         )
         if n_vertical > 0 or n_horizontal > 0:
             logging.debug(
-                f"Filtered {n_vertical} vertical and {n_horizontal} horizontal stripes "
-                f"from radar data using {variable}"
+                "Filtered %s vertical and %s horizontal stripes "
+                "from radar data using %s",
+                n_vertical,
+                n_horizontal,
+                variable,
             )
 
     def _filter(
         self,
-        data: np.ndarray,
+        data: npt.NDArray,
         axis: int,
         min_coverage: float,
         z_limit: float,
@@ -159,10 +179,14 @@ class Radar(DataSource):
                 q3 = np.quantile(n_values, 0.75)
             except IndexError:
                 continue
+
+            if q1 == q3:
+                continue
+
             threshold = distance * (q3 - q1) + q3
 
             indices = np.where(
-                (n_values > threshold) & (n_values > (min_coverage * data.shape[1]))
+                (n_values > threshold) & (n_values > (min_coverage * data.shape[1])),
             )[0]
             true_ind = [int(x) for x in (block_number * len_block + indices)]
             n_removed = len(indices)
@@ -180,25 +204,30 @@ class Radar(DataSource):
 
         return n_removed_total
 
-    def correct_atten(self, attenuations: dict) -> None:
+    def correct_atten(self, attenuations: RadarAttenuation) -> None:
         """Corrects radar echo for liquid and gas attenuation.
 
         Args:
-            attenuations: 2-D attenuations due to atmospheric gases and liquid:
-                `radar_gas_atten`, `radar_liquid_atten`.
+            attenuations: Radar attenuation object.
 
         References:
             The method is based on Hogan R. and O'Connor E., 2004,
             https://bit.ly/2Yjz9DZ and the original Cloudnet Matlab implementation.
 
         """
-        z_corrected = self.data["Z"][:] + attenuations["radar_gas_atten"]
-        ind = ma.where(attenuations["radar_liquid_atten"])
-        z_corrected[ind] += attenuations["radar_liquid_atten"][ind]
+        z_corrected = self.data["Z"][:] + attenuations.gas.amount
+        ind = ma.where(attenuations.liquid.amount)
+        z_corrected[ind] += attenuations.liquid.amount[ind]
+        ind = ma.where(attenuations.rain.amount)
+        z_corrected[ind] += attenuations.rain.amount[ind]
+        ind = ma.where(attenuations.melting.amount)
+        z_corrected[ind] += attenuations.melting.amount[ind]
         self.append_data(z_corrected, "Z")
 
     def calc_errors(
-        self, attenuations: dict, classification: ClassificationResult
+        self,
+        attenuations: RadarAttenuation,
+        is_clutter: npt.NDArray,
     ) -> None:
         """Calculates uncertainties of radar echo.
 
@@ -207,7 +236,7 @@ class Radar(DataSource):
 
         Args:
             attenuations: 2-D attenuations due to atmospheric gases.
-            classification: The :class:`ClassificationResult` instance.
+            is_clutter: 2-D boolean array denoting pixels contaminated by clutter.
 
         References:
             The method is based on Hogan R. and O'Connor E., 2004,
@@ -215,40 +244,62 @@ class Radar(DataSource):
 
         """
 
-        def _calc_sensitivity() -> np.ndarray:
+        def _calc_sensitivity() -> npt.NDArray:
             """Returns sensitivity of radar as function of altitude."""
-            mean_gas_atten = ma.mean(attenuations["radar_gas_atten"], axis=0)
+            mean_gas_atten = ma.mean(attenuations.gas.amount, axis=0)
             z_sensitivity = z_power_min + log_range + mean_gas_atten
-            zc = ma.median(ma.array(z, mask=~classification.is_clutter), axis=0)
+            zc = ma.median(ma.array(z, mask=~is_clutter), axis=0)
             valid_values = np.logical_not(zc.mask)
             z_sensitivity[valid_values] = zc[valid_values]
             return z_sensitivity
 
-        def _calc_error() -> np.ndarray | float:
-            if "width" not in self.data:
-                return 0.3
-            z_precision = 4.343 * (
-                1 / np.sqrt(_number_of_independent_pulses())
-                + utils.db2lin(z_power_min - z_power) / 3
+        def _calc_error() -> npt.NDArray | float:
+            """Returns error of radar as function of altitude.
+
+            References:
+                Hogan, R. J., 1998: Dual-wavelength radar studies of clouds.
+                PhD Thesis, University of Reading, UK.
+
+            """
+            noise_threshold = 3
+            n_pulses = _number_of_independent_pulses()
+            ln_to_log10 = 10 / np.log(10)
+            z_precision = ma.divide(ln_to_log10, np.sqrt(n_pulses)) * (
+                1 + (utils.db2lin(z_power_min - z_power) / noise_threshold)
+            )
+
+            z_error = utils.l2norm(
+                z_precision,
+                attenuations.liquid.error.filled(0),
+                attenuations.rain.error.filled(0),
+                attenuations.melting.error.filled(0),
             )
-            gas_error = attenuations["radar_gas_atten"] * 0.1
-            liq_error = attenuations["liquid_atten_err"].filled(0)
-            z_error = utils.l2norm(gas_error, liq_error, z_precision)
-            z_error[attenuations["liquid_uncorrected"]] = ma.masked
+
+            z_error[
+                attenuations.liquid.uncorrected
+                | attenuations.rain.uncorrected
+                | attenuations.melting.uncorrected
+            ] = ma.masked
+
             return z_error
 
         def _number_of_independent_pulses() -> float:
-            seconds_in_hour = 3600
-            dwell_time = utils.mdiff(self.time) * seconds_in_hour
-            return (
-                dwell_time
-                * self.radar_frequency
-                * 1e9
-                * 4
-                * np.sqrt(math.pi)
-                * self.data["width"][:]
-                / 3e8
-            )
+            """Returns number of independent pulses.
+
+            References:
+                Atlas, D., 1964: Advances in radar meteorology.
+                Advances in Geophys., 10, 318-478.
+
+            """
+            if "width" not in self.data:
+                default_width = 0.3
+                width = np.zeros_like(self.data["Z"][:])
+                width[~width.mask] = default_width
+            else:
+                width = self.data["width"][:]
+            dwell_time = utils.mdiff(self.time) * SEC_IN_HOUR
+            wl = SPEED_OF_LIGHT / (self.radar_frequency * GHZ_TO_HZ)
+            return 4 * np.sqrt(math.pi) * dwell_time * width / wl
 
         def _calc_z_power_min() -> float:
             if ma.all(z_power.mask):
@@ -256,7 +307,7 @@ class Radar(DataSource):
             return np.percentile(z_power.compressed(), 0.1)
 
         z = self.data["Z"][:]
-        radar_range = self.km2m(self.dataset.variables["range"])
+        radar_range = self.to_m(self.dataset.variables["range"])
         log_range = utils.lin2db(radar_range, scale=20)
         z_power = z - log_range
         z_power_min = _calc_z_power_min()
@@ -266,57 +317,119 @@ class Radar(DataSource):
 
     def add_meta(self) -> None:
         """Copies misc. metadata from the input file."""
-        for key in ("latitude", "longitude", "altitude"):
-            self.append_data(np.array(self.getvar(key)), key)
         for key in ("time", "height", "radar_frequency"):
             self.append_data(np.array(getattr(self, key)), key)
 
-    def _init_data(self):
+    def add_location(self, time_new: npt.NDArray) -> None:
+        """Add latitude, longitude and altitude from nearest timestamp."""
+        idx = np.searchsorted(self.time, time_new)
+        idx_left = np.clip(idx - 1, 0, len(self.time) - 1)
+        idx_right = np.clip(idx, 0, len(self.time) - 1)
+        diff_left = np.abs(time_new - self.time[idx_left])
+        diff_right = np.abs(time_new - self.time[idx_right])
+        idx_closest = np.where(diff_left < diff_right, idx_left, idx_right)
+        for key in ("latitude", "longitude", "altitude"):
+            data = self.getvar(key)
+            if not utils.isscalar(data):
+                data = data[idx_closest]
+            if not np.any(ma.getmaskarray(data)):
+                data = np.array(data)
+            self.append_data(data, key)
+
+    def _init_data(self) -> None:
         self.append_data(self.getvar("Zh"), "Z", units="dBZ")
-        for key in ("v", "ldr", "width", "sldr", "rainfall_rate"):
-            try:
-                self._variables_to_cloudnet_arrays((key,))
-            except KeyError:
-                continue
-
-    def _init_sigma_v(self) -> None:
-        """Initializes std of the velocity field. The std will be calculated
-        later when re-binning the data."""
         self.append_data(self.getvar("v"), "v_sigma")
+        for key in ("v", "ldr", "width", "sldr", "rainfall_rate", "nyquist_velocity"):
+            if key in self.dataset.variables:
+                data = self.dataset.variables[key]
+                self.append_data(data, key)
 
-    def _get_sequence_indices(self) -> list:
-        """Mira has only one sequence and one folding velocity. RPG has
-        several sequences with different folding velocities."""
-        assert self.height is not None
-        all_indices = np.arange(len(self.height))
-        if not utils.isscalar(self.folding_velocity):
-            starting_indices = self.getvar("chirp_start_indices")
-            return np.split(all_indices, starting_indices[1:])
-        return [all_indices]
-
-    def _get_folding_velocity(self) -> np.ndarray | float:
+    def _rebin_velocity(
+        self,
+        data: npt.NDArray,
+        time_new: npt.NDArray,
+    ) -> npt.NDArray:
+        """Rebins Doppler velocity in polar coordinates."""
+        folding_velocity = self._get_expanded_folding_velocity()
+        # with the new shape (maximum value in every bin)
+        max_folding_binned, _ = utils.rebin_2d(
+            self.time,
+            folding_velocity,
+            time_new,
+            "max",
+        )
+        # store this in the file
+        self.append_data(max_folding_binned, "nyquist_velocity")
+
+        if "correction_bits" in self.dataset.variables:
+            bits = self.dataset.variables["correction_bits"][:]
+            dealiasing_bit = utils.isbit(bits, 0)
+            if ma.all(dealiasing_bit):
+                return utils.rebin_2d(self.time, data, time_new, "mean")[0]
+            if ma.any(dealiasing_bit):
+                msg = "Data are only party dealiased. Deal with this later."
+                raise NotImplementedError(msg)
+
+        # with original shape (repeat maximum value for each point in every bin)
+        max_folding_full, _ = utils.rebin_2d(
+            self.time,
+            folding_velocity,
+            time_new,
+            "max",
+            keepdim=True,
+        )
+
+        data_scaled = data * (np.pi / max_folding_full)
+        vel_x = ma.cos(data_scaled)
+        vel_y = ma.sin(data_scaled)
+        vel_x_mean, _ = utils.rebin_2d(self.time, vel_x, time_new)
+        vel_y_mean, _ = utils.rebin_2d(self.time, vel_y, time_new)
+        vel_scaled = ma.arctan2(vel_y_mean, vel_x_mean)
+        return vel_scaled / (np.pi / max_folding_binned)
+
+    def _get_expanded_folding_velocity(self) -> npt.NDArray:
         if "nyquist_velocity" in self.dataset.variables:
-            return self.getvar("nyquist_velocity")
-        if "prf" in self.dataset.variables:
+            fvel = self.getvar("nyquist_velocity")
+        elif "prf" in self.dataset.variables:
             prf = self.getvar("prf")
-            return _prf_to_folding_velocity(prf, self.radar_frequency)
-        raise RuntimeError("Unable to determine folding velocity")
-
-    def _get_folding_velocity_full(self):
-        folding_velocity: list | np.ndarray = []
-        if utils.isscalar(self.folding_velocity):
-            folding_velocity = np.repeat(
-                self.folding_velocity, len(self.sequence_indices[0])
-            )
+            fvel = _prf_to_folding_velocity(prf, self.radar_frequency)
         else:
-            assert isinstance(folding_velocity, list)
-            assert isinstance(self.folding_velocity, np.ndarray)
-            for indices, velocity in zip(self.sequence_indices, self.folding_velocity):
-                folding_velocity.append(np.repeat(velocity, len(indices)))
-            folding_velocity = np.hstack(folding_velocity)
-        self.append_data(folding_velocity, "nyquist_velocity")
+            msg = "Unable to determine folding velocity"
+            raise RuntimeError(msg)
+
+        n_time = self.getvar("time").size
+        n_height = self.height.size
+
+        if fvel.shape == (n_time, n_height):
+            # Folding velocity is already expanded in radar file
+            return fvel
+        if utils.isscalar(fvel):
+            # e.g. MIRA
+            return np.broadcast_to(fvel, (n_time, n_height))
+
+        # RPG radars have chirp segments
+        starts = self.getvar("chirp_start_indices")
+        n_seg = starts.size if starts.ndim == 1 else starts.shape[1]
+
+        starts = np.broadcast_to(starts, (n_time, n_seg))
+        fvel = np.broadcast_to(fvel, (n_time, n_seg))
+
+        # Indices should start from zero (first range gate)
+        # In pre-processed RV Meteor files the first index is 1, so normalize:
+        # Normalize starts so indices begin from zero
+        first_values = starts[:, [0]]
+        if not np.all(np.isin(first_values, [0, 1])):
+            msg = "First value of chirp_start_indices must be 0 or 1"
+            raise ValueError(msg)
+        starts = starts - first_values
+
+        chirp_size = np.diff(starts, append=n_height)
+        return np.repeat(fvel.ravel(), chirp_size.ravel()).reshape((n_time, n_height))
 
 
-def _prf_to_folding_velocity(prf: np.ndarray, radar_frequency: float) -> float:
+def _prf_to_folding_velocity(prf: npt.NDArray, radar_frequency: float) -> npt.NDArray:
     ghz_to_hz = 1e9
-    return float(prf * constants.c / (4 * radar_frequency * ghz_to_hz))
+    if len(prf) != 1:
+        msg = "Unable to determine folding velocity"
+        raise RuntimeError(msg)
+    return prf[0] * constants.c / (4 * radar_frequency * ghz_to_hz)