cloudnetpy 1.65.7__py3-none-any.whl → 1.66.0__py3-none-any.whl

cloudnetpy/categorize/attenuations/rain_attenuation.py

@@ -0,0 +1,84 @@
+import numpy as np
+from numpy import ma
+from numpy.typing import NDArray
+
+import cloudnetpy.constants as con
+from cloudnetpy import utils
+from cloudnetpy.categorize.attenuations import (
+    Attenuation,
+    calc_two_way_attenuation,
+)
+from cloudnetpy.categorize.containers import ClassificationResult, Observations
+
+
+def calc_rain_attenuation(
+    data: Observations, classification: ClassificationResult
+) -> Attenuation:
+    affected_region, inducing_region = _find_regions(classification)
+    shape = affected_region.shape
+
+    if data.disdrometer is None:
+        return Attenuation(
+            amount=ma.masked_all(shape),
+            error=ma.masked_all(shape),
+            attenuated=affected_region,
+            uncorrected=affected_region,
+        )
+
+    rainfall_rate = data.disdrometer.data["rainfall_rate"][:].copy()
+    rainfall_rate[classification.is_rain == 0] = ma.masked
+    frequency = data.radar.radar_frequency
+
+    specific_attenuation_array = _calc_rain_specific_attenuation(
+        rainfall_rate, frequency
+    )
+
+    specific_attenuation = utils.transpose(specific_attenuation_array) * ma.ones(shape)
+
+    two_way_attenuation = calc_two_way_attenuation(
+        data.radar.height, specific_attenuation
+    )
+
+    two_way_attenuation[~inducing_region] = 0
+    two_way_attenuation = ma.array(utils.ffill(two_way_attenuation.data))
+    two_way_attenuation[two_way_attenuation == 0] = ma.masked
+
+    return Attenuation(
+        amount=two_way_attenuation,
+        error=two_way_attenuation * 0.2,
+        attenuated=affected_region,
+        uncorrected=np.zeros_like(affected_region, dtype=bool),
+    )
+
+
+def _find_regions(
+    classification: ClassificationResult,
+) -> tuple[NDArray[np.bool_], NDArray[np.bool_]]:
+    """Finds regions where rain attenuation is present and can be corrected or not."""
+    warm_region = ~classification.category_bits.freezing
+    is_rain = utils.transpose(classification.is_rain).astype(bool)
+    affected_region = np.ones_like(warm_region, dtype=bool) * is_rain
+    inducing_region = warm_region * is_rain
+    return affected_region, inducing_region
+
+
+def _calc_rain_specific_attenuation(
+    rainfall_rate: np.ndarray, frequency: float
+) -> np.ndarray:
+    """Calculates specific attenuation due to rain (dB km-1).
+
+    References:
+        Crane, R. (1980). Prediction of Attenuation by Rain.
+        IEEE Transactions on Communications, 28(9), 1717–1733.
+        doi:10.1109/tcom.1980.1094844
+    """
+    if frequency > 8 and frequency < 12:
+        alpha, beta = 0.0125, 1.18
+    if frequency > 34 and frequency < 37:
+        alpha, beta = 0.242, 1.04
+    elif frequency > 93 and frequency < 96:
+        alpha, beta = 0.95, 0.72
+    else:
+        msg = "Radar frequency not supported"
+        raise ValueError(msg)
+    return alpha * (rainfall_rate * con.M_S_TO_MM_H) ** beta

cloudnetpy/categorize/categorize.py

@@ -1,9 +1,15 @@
 """Module that generates Cloudnet categorize file."""
 
+import dataclasses
 import logging
+from dataclasses import fields
+
+import numpy as np
+from numpy.typing import NDArray
 
 from cloudnetpy import output, utils
-from cloudnetpy.categorize import atmos, classify
+from cloudnetpy.categorize import attenuation, classify
+from cloudnetpy.categorize.containers import Observations
 from cloudnetpy.categorize.disdrometer import Disdrometer
 from cloudnetpy.categorize.lidar import Lidar
 from cloudnetpy.categorize.model import Model
@@ -12,6 +18,7 @@ from cloudnetpy.categorize.radar import Radar
 from cloudnetpy.datasource import DataSource
 from cloudnetpy.exceptions import DisdrometerDataError, ValidTimeStampError
 from cloudnetpy.metadata import MetaData
+from cloudnetpy.products.product_tools import CategoryBits, QualityBits
 
 
 def generate_categorize(
@@ -66,24 +73,25 @@ def generate_categorize(
         >>> generate_categorize(input_files, 'output.nc')  # Use the Voodoo method
     """
 
-    def _interpolate_to_cloudnet_grid() -> list:
-        wl_band = utils.get_wl_band(data["radar"].radar_frequency)
-        if data["disdrometer"] is not None:
-            data["disdrometer"].interpolate_to_grid(time)
-        if data["mwr"] is not None:
-            data["mwr"].rebin_to_grid(time)
-        data["model"].interpolate_to_common_height(wl_band)
-        model_gap_ind = data["model"].interpolate_to_grid(time, height)
-        radar_gap_ind = data["radar"].rebin_to_grid(time)
-        lidar_gap_ind = data["lidar"].interpolate_to_grid(time, height)
+    def _interpolate_to_cloudnet_grid() -> list[int]:
+        if data.disdrometer is not None:
+            data.disdrometer.interpolate_to_grid(time)
+        if data.mwr is not None:
+            data.mwr.rebin_to_grid(time)
+        data.model.calc_attenuations(data.radar.radar_frequency)
+        data.model.interpolate_to_common_height()
+        model_gap_ind = data.model.interpolate_to_grid(time, height)
+        radar_gap_ind = data.radar.rebin_to_grid(time)
+        lidar_gap_ind = data.lidar.interpolate_to_grid(time, height)
         gap_indices = set(radar_gap_ind + lidar_gap_ind + model_gap_ind)
         return [ind for ind in range(len(time)) if ind not in gap_indices]
 
     def _screen_bad_time_indices(valid_indices: list) -> None:
         n_time_full = len(time)
-        data["radar"].time = time[valid_indices]
-        for data_key, obj in data.items():
-            if obj is None or data_key == "lv0_files":
+        data.radar.time = time[valid_indices]
+        for field in fields(data):
+            obj = getattr(data, field.name)
+            if not hasattr(obj, "data"):
                 continue
             for key, item in obj.data.items():
                 if utils.isscalar(item.data):
@@ -97,79 +105,92 @@ def generate_categorize(
                     else:
                         continue
                     obj.data[key].data = array
-        for key, item in data["model"].data_dense.items():
-            data["model"].data_dense[key] = item[valid_indices, :]
+        for key, item in data.model.data_dense.items():
+            data.model.data_dense[key] = item[valid_indices, :]
 
     def _prepare_output() -> dict:
-        data["radar"].add_meta()
-        data["model"].screen_sparse_fields()
-        if data["disdrometer"] is not None:
-            data["radar"].data.pop("rainfall_rate", None)
-            data["disdrometer"].data.pop("n_particles", None)
-        for key in ("category_bits", "insect_prob"):
-            data["radar"].append_data(getattr(classification, key), key)
+        data.radar.add_meta()
+        data.model.screen_sparse_fields()
+
+        if data.disdrometer is not None:
+            data.radar.data.pop("rainfall_rate", None)
+            data.disdrometer.data.pop("n_particles", None)
+
+        data.radar.append_data(attenuations.gas.amount, "radar_gas_atten")
+        data.radar.append_data(attenuations.liquid.amount, "radar_liquid_atten")
+        data.radar.append_data(attenuations.rain.amount, "radar_rain_atten")
+        data.radar.append_data(attenuations.melting.amount, "radar_melting_atten")
+
+        data.radar.append_data(_classes_to_bits(quality), "quality_bits")
+
+        data.radar.append_data(classification.insect_prob, "insect_prob")
+        data.radar.append_data(classification.is_rain, "rain_detected")
+        data.radar.append_data(
+            _classes_to_bits(classification.category_bits), "category_bits"
+        )
+
         if classification.liquid_prob is not None:
-            data["radar"].append_data(classification.liquid_prob, "liquid_prob")
-        for key in ("radar_liquid_atten", "radar_gas_atten"):
-            data["radar"].append_data(attenuations[key], key)
-        data["radar"].append_data(quality["quality_bits"], "quality_bits")
-        data["radar"].append_data(classification.is_rain, "rain_detected")
+            data.radar.append_data(classification.liquid_prob, "liquid_prob")
+
         return {
-            **data["radar"].data,
-            **data["lidar"].data,
-            **data["model"].data,
-            **data["model"].data_sparse,
-            **(data["mwr"].data if data["mwr"] is not None else {}),
-            **(data["disdrometer"].data if data["disdrometer"] is not None else {}),
+            **data.radar.data,
+            **data.lidar.data,
+            **data.model.data,
+            **data.model.data_sparse,
+            **(data.mwr.data if data.mwr is not None else {}),
+            **(data.disdrometer.data if data.disdrometer is not None else {}),
         }
 
     def _define_dense_grid() -> tuple:
-        return utils.time_grid(), data["radar"].height
+        return utils.time_grid(), data.radar.height
 
     def _close_all() -> None:
-        for obj in data.values():
+        for field in fields(data):
+            obj = getattr(data, field.name)
             if isinstance(obj, DataSource):
                 obj.close()
 
     try:
-        data: dict = {
-            "radar": Radar(input_files["radar"]),
-            "lidar": Lidar(input_files["lidar"]),
-            "lv0_files": input_files.get("lv0_files"),
-            "mwr": None,
-            "disdrometer": None,
-        }
+        radar = Radar(input_files["radar"])
+        data = Observations(
+            radar=radar,
+            lidar=Lidar(input_files["lidar"]),
+            model=Model(input_files["model"], radar.altitude, options),
+            lv0_files=input_files.get("lv0_files"),
+        )
         if "mwr" in input_files:
-            data["mwr"] = Mwr(input_files["mwr"])
+            data.mwr = Mwr(input_files["mwr"])
         if "disdrometer" in input_files:
             try:
-                data["disdrometer"] = Disdrometer(input_files["disdrometer"])
+                data.disdrometer = Disdrometer(input_files["disdrometer"])
             except DisdrometerDataError as err:
                 logging.warning("Unable to use disdrometer: %s", err)
-        data["model"] = Model(input_files["model"], data["radar"].altitude, options)
         time, height = _define_dense_grid()
         valid_ind = _interpolate_to_cloudnet_grid()
         if not valid_ind:
             msg = "No overlapping radar and lidar timestamps found"
             raise ValidTimeStampError(msg)
         _screen_bad_time_indices(valid_ind)
-        if (
-            "rpg" in data["radar"].source_type.lower()
-            or "basta" in data["radar"].source_type.lower()
-        ):
-            data["radar"].filter_speckle_noise()
-            data["radar"].filter_1st_gate_artifact()
+        if any(source in data.radar.source_type.lower() for source in ("rpg", "basta")):
+            data.radar.filter_speckle_noise()
+            data.radar.filter_1st_gate_artifact()
         for variable in ("v", "v_sigma", "ldr"):
-            data["radar"].filter_stripes(variable)
-        data["radar"].remove_incomplete_pixels()
-        data["model"].calc_wet_bulb()
+            data.radar.filter_stripes(variable)
+        data.radar.remove_incomplete_pixels()
+        data.model.calc_wet_bulb()
+
         classification = classify.classify_measurements(data)
-        attenuations = atmos.get_attenuations(data, classification)
-        data["radar"].correct_atten(attenuations)
-        data["radar"].calc_errors(attenuations, classification.is_clutter)
+
+        attenuations = attenuation.get_attenuations(data, classification)
+
+        data.radar.correct_atten(attenuations)
+        data.radar.calc_errors(attenuations, classification.is_clutter)
+
         quality = classify.fetch_quality(data, classification, attenuations)
+
         cloudnet_arrays = _prepare_output()
-        date = data["radar"].get_date()
+
+        date = data.radar.get_date()
         attributes = output.add_time_attribute(CATEGORIZE_ATTRIBUTES, date)
         attributes = output.add_time_attribute(attributes, date, "model_time")
         attributes = output.add_source_attribute(attributes, data)
@@ -181,16 +202,16 @@ def generate_categorize(
 
 def _save_cat(
     full_path: str,
-    data_obs: dict,
+    data_obs: Observations,
     cloudnet_arrays: dict,
     uuid: str | None,
 ) -> str:
     """Creates a categorize netCDF4 file and saves all data into it."""
     dims = {
-        "time": len(data_obs["radar"].time),
-        "height": len(data_obs["radar"].height),
-        "model_time": len(data_obs["model"].time),
-        "model_height": len(data_obs["model"].mean_height),
+        "time": len(data_obs.radar.time),
+        "height": len(data_obs.radar.height),
+        "model_time": len(data_obs.model.time),
+        "model_height": len(data_obs.model.mean_height),
     }
 
     file_type = "categorize"
@@ -198,12 +219,12 @@ def _save_cat(
         uuid_out = nc.file_uuid
         nc.cloudnet_file_type = file_type
         output.copy_global(
-            data_obs["radar"].dataset,
+            data_obs.radar.dataset,
             nc,
             ("year", "month", "day", "location"),
         )
-        nc.title = f"Cloud categorization products from {data_obs['radar'].location}"
-        nc.source_file_uuids = output.get_source_uuids(*data_obs.values())
+        nc.title = f"Cloud categorization products from {data_obs.radar.location}"
+        nc.source_file_uuids = output.get_source_uuids(data_obs)
         is_voodoo = "liquid_prob" in cloudnet_arrays
         extra_references = (
             ["https://doi.org/10.5194/amt-15-5343-2022"] if is_voodoo else None
@@ -221,6 +242,14 @@ def _save_cat(
     return uuid_out
 
 
+def _classes_to_bits(data: QualityBits | CategoryBits) -> NDArray[np.int_]:
+    shape = data.radar.shape if hasattr(data, "radar") else data.droplet.shape
+    quality = np.zeros(shape, dtype=np.int64)
+    for i, field in enumerate(dataclasses.fields(data)):
+        quality |= (1 << i) * getattr(data, field.name)
+    return quality
+
+
 COMMENTS = {
     "category_bits": (
         "This variable contains information on the nature of the targets\n"
@@ -255,14 +284,17 @@ COMMENTS = {
         "humidity, but forcing pixels containing liquid cloud to saturation with\n"
         "respect to liquid water. It has been used to correct Z."
     ),
-    "Tw": (
-        "This variable was derived from model temperature, pressure and relative\n"
-        "humidity."
+    "radar_rain_atten": (
+        "This variable was calculated from the disdrometer rainfall rate."
     ),
+    "radar_melting_atten": (
+        "This variable was calculated from the disdrometer rainfall rate."
+    ),
+    "Tw": "This variable was derived from model temperature, pressure and humidity.",
     "Z_sensitivity": (
         "This variable is an estimate of the radar sensitivity, i.e. the minimum\n"
         "detectable radar reflectivity, as a function of height. It includes the\n"
-        "effect of ground clutter and gas attenuation but not liquid attenuation."
+        "effect of ground clutter and gas attenuation but not other attenuations."
     ),
     "Z_error": (
         "This variable is an estimate of the one-standard-deviation random error\n"
@@ -272,18 +304,19 @@ COMMENTS = {
         "   finite number of pulses\n"
         "2) 10% uncertainty in gaseous attenuation correction (mainly due to error\n"
         "   in model humidity field)\n"
-        "3) Error in liquid water path (given by the variable lwp_error) and its\n"
+        "3) 20% uncertainty in rain attenuation correction (mainly due to error\n"
+        "   in disdrometer rainfall rate)\n"
+        "4) 10%-20% uncertainty in melting layer attenuation correction (mainly due\n"
+        "   to error in disdrometer rainfall rate)\n"
+        "5) Error in liquid water path (given by the variable lwp_error) and its\n"
         "   partitioning with height)."
     ),
     "Z": (
-        "This variable has been corrected for attenuation by gaseous attenuation\n"
-        "(using the thermodynamic variables from a forecast model; see the\n"
-        "radar_gas_atten variable) and liquid attenuation (using liquid water path\n"
-        "from a microwave radiometer; see the radar_liquid_atten variable) but rain\n"
-        "and melting-layer attenuation has not been corrected.\n"
-        "Calibration convention: in the absence of attenuation, a cloud at 273 K\n"
-        "containing one million 100-micron droplets per cubic metre will have\n"
-        "a reflectivity of 0 dBZ at all frequencies."
+        "This variable has been corrected for attenuation by gaseous attenuation,\n"
+        "and possibly liquid water, rain and melting layer (see quality_bits\n"
+        "variable). Calibration convention: in the absence of attenuation, a cloud\n"
+        "at 273 K containing one million 100-micron droplets per cubic metre will\n"
+        "have\n a reflectivity of 0 dBZ at all frequencies."
     ),
     "bias": (
         "This variable is an estimate of the one-standard-deviation\n"
@@ -329,6 +362,19 @@ DEFINITIONS = {
                   liquid water path and the lidar estimation of the location of
                   liquid water cloud; be aware that errors in reflectivity may
                   result.""",
+            6: """Rain has caused radar attenuation; if bit 7 is set then a
+                  correction for the radar attenuation has been performed;
+                  otherwise do not trust the absolute values of reflectivity
+                  factor. No correction is performed for lidar attenuation.""",
+            7: """Radar reflectivity has been corrected for rain attenuation
+                  using rainfall rate from a disdrometer; be aware that errors
+                  in reflectivity may result.""",
+            8: """Melting layer has caused radar attenuation; if bit 9 is set then a
+                  correction for the radar attenuation has been performed;
+                  otherwise do not trust the absolute values of reflectivity
+                  factor. No correction is performed for lidar attenuation.""",
+            9: """Radar reflectivity has been corrected for melting layer
+                  attenuation; be aware that errors in reflectivity may result.""",
         }
     ),
 }
@@ -416,12 +462,25 @@ CATEGORIZE_ATTRIBUTES = {
         long_name="Two-way radar attenuation due to liquid water",
         units="dB",
         comment=COMMENTS["radar_liquid_atten"],
+        references="ITU-R P.840-9",
+    ),
+    "radar_rain_atten": MetaData(
+        long_name="Two-way radar attenuation due to rain",
+        units="dB",
+        references="Crane, R. (1980)",
+        comment=COMMENTS["radar_rain_atten"],
+    ),
+    "radar_melting_atten": MetaData(
+        long_name="Two-way radar attenuation due to melting ice",
+        units="dB",
+        references="Li, H., & Moisseev, D. (2019)",
+        comment=COMMENTS["radar_melting_atten"],
     ),
     "radar_gas_atten": MetaData(
         long_name="Two-way radar attenuation due to atmospheric gases",
         units="dB",
         comment=COMMENTS["radar_gas_atten"],
-        references="Liebe (1985, Radio Sci. 20(5), 1069-1089)",
+        references="ITU-R P.676-13",
     ),
     "insect_prob": MetaData(
         long_name="Insect probability",