cloudnetpy 1.65.8__py3-none-any.whl → 1.66.1__py3-none-any.whl

cloudnetpy/products/iwc.py

@@ -43,8 +43,7 @@ def generate_iwc(
     product = "iwc"
     with IwcSource(categorize_file, product) as iwc_source:
         ice_classification = IceClassification(categorize_file)
-        iwc_source.append_main_variable_including_rain(ice_classification)
-        iwc_source.append_main_variable(ice_classification)
+        iwc_source.append_icy_data(ice_classification)
         iwc_source.append_bias()
         iwc_source.append_sensitivity()
         lwp_prior, bias = iwc_source.append_error(ice_classification)
@@ -92,7 +91,7 @@ class IwcSource(IceSource):
             retrieval_uncertainty,
             error_uncorrected,
         )
-        iwc_error[(~ice_classification.is_ice | ice_classification.ice_above_rain)] = (
+        iwc_error[~ice_classification.is_ice | ice_classification.uncorrected_ice] = (
             ma.masked
         )
         self.append_data(iwc_error, f"{self.product}_error")
@@ -162,12 +161,6 @@ COMMENTS = {
         "This variable describes whether a retrieval was performed\n"
         "for each pixel, and its associated quality."
     ),
-    "iwc_inc_rain": (
-        "This variable is the same as iwc but it also contains iwc values\n"
-        "above rain. The iwc values above rain have been severely affected\n"
-        "by attenuation and should be used when the effect of attenuation\n"
-        "is being studied."
-    ),
 }
 
 DEFINITIONS = {
@@ -175,15 +168,12 @@ DEFINITIONS = {
         {
             0: """No ice present.""",
             1: """Reliable retrieval.""",
-            2: """Unreliable retrieval due to uncorrected attenuation from
-                  liquid water below the ice (no liquid water path measurement
-                  available).""",
-            3: """Retrieval performed but radar corrected for liquid attenuation
-                  using radiometer liquid water path which is not always
-                  accurate.""",
+            2: """Unreliable retrieval due to uncorrected liquid, rain or
+                  melting attenuation.""",
+            3: """Retrieval performed with radar corrected for liquid, rain and
+                  melting attenuation.""",
             4: """Ice detected only by the lidar.""",
-            5: """Ice detected by radar but rain below so no retrieval performed
-                  due to very uncertain attenuation.""",
+            5: """Uncorrected rain attenuation (deprecated).""",
             6: """Clear sky above rain and wet-bulb temperature less than 0degC:
                   if rain attenuation is strong, ice could be present but
                   undetected.""",
@@ -200,11 +190,6 @@ IWC_ATTRIBUTES = {
         units="kg m-3",
         ancillary_variables="iwc_error iwc_sensitivity iwc_bias",
     ),
-    "iwc_inc_rain": MetaData(
-        long_name="Ice water content including rain",
-        units="kg m-3",
-        comment=COMMENTS["iwc_inc_rain"],
-    ),
     "iwc_error": MetaData(
         long_name="Random error in ice water content",
         units="dB",

cloudnetpy/products/lwc.py

@@ -6,7 +6,7 @@ import numpy as np
 from numpy import ma
 
 from cloudnetpy import output, utils
-from cloudnetpy.categorize import atmos
+from cloudnetpy.categorize import atmos_utils
 from cloudnetpy.datasource import DataSource
 from cloudnetpy.exceptions import InvalidSourceFileError
 from cloudnetpy.metadata import MetaData
@@ -80,7 +80,7 @@ class LwcSource(DataSource):
         lwp (ndarray): 1D liquid water path.
         lwp_error (ndarray): 1D error of liquid water path.
         is_rain (ndarray): 1D array denoting presence of rain.
-        dheight (float): Median difference in height vector.
+        path_lengths (ndarray): 1D array of path lengths.
         atmosphere (dict): Dictionary containing interpolated fields `temperature`
             and `pressure`.
         categorize_bits (CategorizeBits): The :class:`CategorizeBits` instance.
@@ -96,7 +96,7 @@ class LwcSource(DataSource):
         self.lwp[self.lwp < 0] = 0
         self.lwp_error = self.getvar("lwp_error")
         self.is_rain = get_is_rain(categorize_file)
-        self.dheight = utils.mdiff(self.getvar("height"))
+        self.path_lengths = utils.path_lengths_from_ground(self.getvar("height"))
         self.atmosphere = self._get_atmosphere(categorize_file)
         self.categorize_bits = CategorizeBits(categorize_file)
 
@@ -125,7 +125,6 @@ class Lwc:
 
     Attributes:
         lwc_source (LwcSource): The :class:`LwcSource` instance.
-        dheight (float): Median difference in height vector.
         is_liquid (ndarray): 2D array denoting liquid.
         lwc_adiabatic (ndarray): 2D array storing adiabatic lwc.
         lwc (ndarray): 2D array of liquid water content (scaled with lwp).
@@ -134,7 +133,7 @@ class Lwc:
 
     def __init__(self, lwc_source: LwcSource):
         self.lwc_source = lwc_source
-        self.dheight = self.lwc_source.dheight
+        self.height = lwc_source.getvar("height")
         self.is_liquid = self._get_liquid()
         self.lwc_adiabatic = self._init_lwc_adiabatic()
         self.lwc = self._adiabatic_lwc_to_lwc()
@@ -142,26 +141,26 @@ class Lwc:
 
     def _get_liquid(self) -> np.ndarray:
         category_bits = self.lwc_source.categorize_bits.category_bits
-        return category_bits["droplet"]
+        return category_bits.droplet
 
     def _init_lwc_adiabatic(self) -> np.ndarray:
         """Returns theoretical adiabatic lwc in liquid clouds (kg/m3)."""
-        lwc_dz = atmos.fill_clouds_with_lwc_dz(
-            self.lwc_source.atmosphere,
+        lwc_dz = atmos_utils.fill_clouds_with_lwc_dz(
+            *self.lwc_source.atmosphere,
             self.is_liquid,
         )
-        return atmos.calc_adiabatic_lwc(lwc_dz, self.dheight)
+        return atmos_utils.calc_adiabatic_lwc(lwc_dz, self.height)
 
     def _adiabatic_lwc_to_lwc(self) -> np.ndarray:
         """Initialises liquid water content (kg/m3).
 
         Calculates LWC for ALL profiles (rain, lwp > theoretical, etc.),
         """
-        lwc_scaled = atmos.distribute_lwp_to_liquid_clouds(
+        return atmos_utils.normalize_lwc_by_lwp(
             self.lwc_adiabatic,
             self.lwc_source.lwp,
+            self.height,
         )
-        return lwc_scaled / self.dheight
 
     def _mask_rain(self) -> None:
         is_rain = self.lwc_source.is_rain.astype(bool)
@@ -198,7 +197,7 @@ class CloudAdjustor:
 
     def _get_echo(self) -> dict:
         quality_bits = self.lwc_source.categorize_bits.quality_bits
-        return {key: quality_bits[key] for key in ("radar", "lidar")}
+        return {"radar": quality_bits.radar, "lidar": quality_bits.lidar}
 
     def _init_status(self) -> ma.MaskedArray:
         status = ma.zeros(self.is_liquid.shape, dtype=int)
@@ -232,8 +231,8 @@ class CloudAdjustor:
             distance_from_base += 1
 
     def _has_converged(self, ind: int) -> bool:
-        lwc_sum = ma.sum(self.lwc_adiabatic[ind, :])
-        return lwc_sum * self.lwc_source.dheight > self.lwc_source.lwp[ind]
+        lwc_sum = ma.sum(self.lwc_adiabatic[ind, :] * self.lwc_source.path_lengths)
+        return lwc_sum > self.lwc_source.lwp[ind]
 
     def _out_of_bound(self, ind: int) -> bool:
         return ind >= self.lwc.shape[1] - 1
@@ -288,7 +287,7 @@ class CloudAdjustor:
         In theory, this difference should be always positive. Negative values
         indicate missing (or too narrow) liquid clouds.
         """
-        lwc_sum = ma.sum(self.lwc_adiabatic, axis=1) * self.lwc_source.dheight
+        lwc_sum = ma.sum(self.lwc_adiabatic * self.lwc_source.path_lengths, axis=1)
         return lwc_sum - self.lwc_source.lwp
 
     def _mask_rain(self) -> None:

cloudnetpy/products/mwr_tools.py

@@ -1,8 +1,10 @@
+import os
 import tempfile
 from typing import Literal
 
 import netCDF4
 import numpy as np
+import requests
 from mwrpy.level2.lev2_collocated import generate_lev2_multi as gen_multi
 from mwrpy.level2.lev2_collocated import generate_lev2_single as gen_single
 from mwrpy.level2.write_lev2_nc import MissingInputData
@@ -10,7 +12,6 @@ from mwrpy.version import __version__ as mwrpy_version
 
 from cloudnetpy import output, utils
 from cloudnetpy.exceptions import ValidTimeStampError
-from cloudnetpy.products import product_tools
 
 
 def generate_mwr_single(
@@ -59,7 +60,7 @@ def _generate_product(
 ) -> str:
     fun = gen_multi if product == "multi" else gen_single
     with tempfile.TemporaryDirectory() as temp_dir:
-        coeffs = product_tools.get_read_mwrpy_coeffs(mwr_l1c_file, temp_dir)
+        coeffs = _read_mwrpy_coeffs(mwr_l1c_file, temp_dir)
         try:
             fun(None, mwr_l1c_file, output_file, coeff_files=coeffs)
         except MissingInputData as err:
@@ -136,3 +137,15 @@ class Mwr:
         self.nc_l2.source_file_uuids = self.nc_l1c.file_uuid
         self.nc_l2.mwrpy_version = mwrpy_version
         self.nc_l2.instrument_pid = self.nc_l1c.instrument_pid
+
+
+def _read_mwrpy_coeffs(mwr_l1c_file, folder: str) -> list:
+    with netCDF4.Dataset(mwr_l1c_file) as nc:
+        links = nc.mwrpy_coefficients.split(", ")
+    coeffs = []
+    for link in links:
+        full_path = os.path.join(folder, link.split("/")[-1])
+        with open(full_path, "wb") as f:
+            f.write(requests.get(link, timeout=10).content)
+        coeffs.append(full_path)
+    return coeffs

cloudnetpy/products/product_tools.py

@@ -1,12 +1,12 @@
 """General helper classes and functions for all products."""
 
-import os
+from dataclasses import dataclass
 from typing import NamedTuple
 
 import netCDF4
 import numpy as np
-import requests
 from numpy import ma
+from numpy.typing import NDArray
 
 from cloudnetpy import constants, utils
 from cloudnetpy.categorize import atmos_utils
@@ -23,53 +23,63 @@ class IceCoefficients(NamedTuple):
     c: float
 
 
-class CategorizeBits:
-    """Class holding information about category and quality bits.
+@dataclass
+class CategoryBits:
+    droplet: NDArray[np.bool_]
+    falling: NDArray[np.bool_]
+    freezing: NDArray[np.bool_]
+    melting: NDArray[np.bool_]
+    aerosol: NDArray[np.bool_]
+    insect: NDArray[np.bool_]
 
-    Args:
-        categorize_file (str): Categorize file name.
 
-    Attributes:
-        category_bits (dict): Dictionary containing boolean fields for `droplet`,
-            `falling`, `cold`, `melting`, `aerosol`, `insect`.
-
-        quality_bits (dict): Dictionary containing boolean fields for `radar`,
-            `lidar`, `clutter`, `molecular`, `attenuated` and `corrected`.
-
-    """
+@dataclass
+class QualityBits:
+    radar: NDArray[np.bool_]
+    lidar: NDArray[np.bool_]
+    clutter: NDArray[np.bool_]
+    molecular: NDArray[np.bool_]
+    attenuated_liquid: NDArray[np.bool_]
+    corrected_liquid: NDArray[np.bool_]
+    attenuated_rain: NDArray[np.bool_]
+    corrected_rain: NDArray[np.bool_]
+    attenuated_melting: NDArray[np.bool_]
+    corrected_melting: NDArray[np.bool_]
 
-    category_keys = (
-        "droplet",
-        "falling",
-        "cold",
-        "melting",
-        "aerosol",
-        "insect",
-    )
-
-    quality_keys = (
-        "radar",
-        "lidar",
-        "clutter",
-        "molecular",
-        "attenuated",
-        "corrected",
-    )
 
+class CategorizeBits:
     def __init__(self, categorize_file: str):
         self._categorize_file = categorize_file
-        self.category_bits = self._read_bits("category")
-        self.quality_bits = self._read_bits("quality")
+        self.category_bits = self._read_category_bits()
+        self.quality_bits = self._read_quality_bits()
+
+    def _read_category_bits(self) -> CategoryBits:
+        with netCDF4.Dataset(self._categorize_file) as nc:
+            bits = nc.variables["category_bits"][:]
+            return CategoryBits(
+                droplet=utils.isbit(bits, 0),
+                falling=utils.isbit(bits, 1),
+                freezing=utils.isbit(bits, 2),
+                melting=utils.isbit(bits, 3),
+                aerosol=utils.isbit(bits, 4),
+                insect=utils.isbit(bits, 5),
+            )
 
-    def _read_bits(self, bit_type: str) -> dict:
-        """Converts bitfield into dictionary."""
+    def _read_quality_bits(self) -> QualityBits:
         with netCDF4.Dataset(self._categorize_file) as nc:
-            try:
-                bitfield = nc.variables[f"{bit_type}_bits"][:]
-            except KeyError as err:
-                raise KeyError from err
-            keys = getattr(CategorizeBits, f"{bit_type}_keys")
-            return {key: utils.isbit(bitfield, i) for i, key in enumerate(keys)}
+            bits = nc.variables["quality_bits"][:]
+            return QualityBits(
+                radar=utils.isbit(bits, 0),
+                lidar=utils.isbit(bits, 1),
+                clutter=utils.isbit(bits, 2),
+                molecular=utils.isbit(bits, 3),
+                attenuated_liquid=utils.isbit(bits, 4),
+                corrected_liquid=utils.isbit(bits, 5),
+                attenuated_rain=utils.isbit(bits, 6),
+                corrected_rain=utils.isbit(bits, 7),
+                attenuated_melting=utils.isbit(bits, 8),
+                corrected_melting=utils.isbit(bits, 9),
+            )
 
 
 class ProductClassification(CategorizeBits):
@@ -96,53 +106,75 @@ class IceClassification(ProductClassification):
 
     def __init__(self, categorize_file: str):
         super().__init__(categorize_file)
+        self._is_attenuated = self._find_attenuated()
+        self._is_corrected = self._find_corrected()
         self.is_ice = self._find_ice()
         self.would_be_ice = self._find_would_be_ice()
         self.corrected_ice = self._find_corrected_ice()
         self.uncorrected_ice = self._find_uncorrected_ice()
         self.ice_above_rain = self._find_ice_above_rain()
-        self.cold_above_rain = self._find_cold_above_rain()
+        self.clear_above_rain = self._find_clear_above_rain()
+
+    def _find_clear_above_rain(self) -> np.ndarray:
+        return (
+            utils.transpose(self.is_rain) * ~self.is_ice
+            & self.category_bits.freezing
+            & ~self.category_bits.melting
+        )
+
+    def _find_attenuated(self) -> np.ndarray:
+        return (
+            self.quality_bits.attenuated_liquid
+            | self.quality_bits.attenuated_rain
+            | self.quality_bits.attenuated_melting
+        )
+
+    def _find_corrected(self) -> np.ndarray:
+        return (
+            self.quality_bits.corrected_liquid
+            | self.quality_bits.corrected_rain
+            | self.quality_bits.corrected_melting
+        )
 
     def _find_ice(self) -> np.ndarray:
         return (
-            self.category_bits["falling"]
-            & self.category_bits["cold"]
-            & ~self.category_bits["melting"]
-            & ~self.category_bits["insect"]
+            self.category_bits.falling
+            & self.category_bits.freezing
+            & ~self.category_bits.melting
+            & ~self.category_bits.insect
         )
 
     def _find_would_be_ice(self) -> np.ndarray:
         warm_falling = (
-            self.category_bits["falling"]
-            & ~self.category_bits["cold"]
-            & ~self.category_bits["insect"]
+            self.category_bits.falling
+            & ~self.category_bits.freezing
+            & ~self.category_bits.insect
         )
-        return warm_falling | self.category_bits["melting"]
+        return warm_falling | self.category_bits.melting
 
     def _find_corrected_ice(self) -> np.ndarray:
-        return (
-            self.is_ice
-            & self.quality_bits["attenuated"]
-            & self.quality_bits["corrected"]
-        )
+        return self.is_ice & self._is_attenuated & self._is_corrected
 
     def _find_uncorrected_ice(self) -> np.ndarray:
+        uncorrected_melting = (
+            self.quality_bits.attenuated_melting & ~self.quality_bits.corrected_melting
+        )
+        uncorrected_rain = (
+            self.quality_bits.attenuated_rain & ~self.quality_bits.corrected_rain
+        )
+        uncorrected_liquid = (
+            self.quality_bits.attenuated_liquid & ~self.quality_bits.corrected_liquid
+        )
         return (
             self.is_ice
-            & self.quality_bits["attenuated"]
-            & ~self.quality_bits["corrected"]
+            & self._is_attenuated
+            & (uncorrected_melting | uncorrected_rain | uncorrected_liquid)
         )
 
     def _find_ice_above_rain(self) -> np.ndarray:
         is_rain = utils.transpose(self.is_rain)
         return (self.is_ice * is_rain) == 1
 
-    def _find_cold_above_rain(self) -> np.ndarray:
-        is_cold = self.category_bits["cold"]
-        is_rain = utils.transpose(self.is_rain)
-        is_cold_rain = (is_cold * is_rain) == 1
-        return is_cold_rain & ~self.category_bits["melting"]
-
 
 class IceSource(DataSource):
     """Base class for different ice products."""
@@ -150,24 +182,20 @@ class IceSource(DataSource):
     def __init__(self, categorize_file: str, product: str):
         super().__init__(categorize_file)
         self.wl_band = utils.get_wl_band(float(self.getvar("radar_frequency")))
-        self.temperature = get_temperature(categorize_file)
+        self.temperature = _get_temperature(categorize_file)
         self.product = product
         self.coefficients = self._get_coefficients()
 
-    def append_main_variable_including_rain(
+    def append_icy_data(
         self,
         ice_classification: IceClassification,
     ) -> None:
         """Adds the main variable (including ice above rain)."""
-        data_including_rain = self._convert_z()
-        data_including_rain[~ice_classification.is_ice] = ma.masked
-        self.append_data(data_including_rain, f"{self.product}_inc_rain")
-
-    def append_main_variable(self, ice_classification: IceClassification) -> None:
-        """Adds the main variable (excluding rain)."""
-        data = ma.copy(self.data[f"{self.product}_inc_rain"][:])
-        data[ice_classification.ice_above_rain] = ma.masked
-        self.append_data(data, self.product)
+        data = self._convert_z()
+        data[~ice_classification.is_ice | ice_classification.uncorrected_ice] = (
+            ma.masked
+        )
+        self.append_data(data, f"{self.product}")
 
     def append_status(self, ice_classification: IceClassification) -> None:
         """Adds the status of retrieval."""
@@ -176,10 +204,9 @@ class IceSource(DataSource):
         is_data = ~data.mask
         retrieval_status[is_data] = 1
         retrieval_status[is_data & ice_classification.corrected_ice] = 3
-        retrieval_status[is_data & ice_classification.uncorrected_ice] = 2
         retrieval_status[~is_data & ice_classification.is_ice] = 4
-        retrieval_status[ice_classification.cold_above_rain] = 6
-        retrieval_status[ice_classification.ice_above_rain] = 5
+        retrieval_status[ice_classification.uncorrected_ice] = 2
+        retrieval_status[ice_classification.clear_above_rain] = 6
         retrieval_status[ice_classification.would_be_ice & (retrieval_status == 0)] = 7
         self.append_data(retrieval_status, f"{self.product}_retrieval_status")
 
@@ -232,16 +259,16 @@ class IceSource(DataSource):
 
 
 def get_is_rain(filename: str) -> np.ndarray:
-    try:
-        is_rain = read_nc_field(filename, "rain_detected")
-    except KeyError:
-        try:
-            rainfall_rate = read_nc_field(filename, "rainfall_rate")
-        except KeyError:
-            rainfall_rate = read_nc_field(filename, "rain_rate")
-        is_rain = rainfall_rate != 0
-        is_rain[is_rain.mask] = True
-    return np.array(is_rain)
+    # TODO: Check that this is correct
+    with netCDF4.Dataset(filename) as nc:
+        for name in ["rain_detected", "rainfall_rate", "rain_rate"]:
+            if name in nc.variables:
+                data = nc.variables[name][:]
+                data = data != 0
+                data[data.mask] = True
+                return np.array(data)
+    msg = "No rain data found."
+    raise ValueError(msg)
 
 
 def read_nc_field(nc_file: str, name: str) -> ma.MaskedArray:
@@ -249,21 +276,6 @@ def read_nc_field(nc_file: str, name: str) -> ma.MaskedArray:
         return nc.variables[name][:]
 
 
-def read_nc_fields(nc_file: str, names: list[str]) -> list[ma.MaskedArray]:
-    """Reads selected variables from a netCDF file.
-
-    Args:
-        nc_file: netCDF file name.
-        names: Variables to be read, e.g. ['ldr', 'lwp'].
-
-    Returns:
-        List of numpy arrays.
-
-    """
-    with netCDF4.Dataset(nc_file) as nc:
-        return [nc.variables[name][:] for name in names]
-
-
 def interpolate_model(cat_file: str, names: str | list) -> dict[str, np.ndarray]:
     """Interpolates 2D model field into dense Cloudnet grid.
 
@@ -278,7 +290,7 @@ def interpolate_model(cat_file: str, names: str | list) -> dict[str, np.ndarray]
     """
 
     def _interp_field(var_name: str) -> np.ndarray:
-        values = read_nc_fields(
+        values = _read_nc_fields(
             cat_file,
             ["model_time", "model_height", var_name, "time", "height"],
         )
@@ -288,22 +300,12 @@ def interpolate_model(cat_file: str, names: str | list) -> dict[str, np.ndarray]
     return {name: _interp_field(name) for name in names}
 
 
-def get_temperature(categorize_file: str) -> np.ndarray:
-    """Returns interpolated temperatures in Celsius."""
-    atmosphere = interpolate_model(categorize_file, "temperature")
-    return atmos_utils.k2c(atmosphere["temperature"])
-
-
-def get_mwrpy_coeffs(nc_file: str) -> str:
+def _read_nc_fields(nc_file: str, names: list[str]) -> list[ma.MaskedArray]:
     with netCDF4.Dataset(nc_file) as nc:
-        return nc.mwrpy_coefficients.split(", ")
+        return [nc.variables[name][:] for name in names]
 
 
-def get_read_mwrpy_coeffs(mwr_l1c_file, folder: str) -> list:
-    coeffs = []
-    for link in get_mwrpy_coeffs(mwr_l1c_file):
-        full_path = os.path.join(folder, link.split("/")[-1])
-        with open(full_path, "wb") as f:
-            f.write(requests.get(link, timeout=10).content)
-        coeffs.append(full_path)
-    return coeffs
+def _get_temperature(categorize_file: str) -> np.ndarray:
+    """Returns interpolated temperatures in Celsius."""
+    atmosphere = interpolate_model(categorize_file, "temperature")
+    return atmos_utils.k2c(atmosphere["temperature"])

cloudnetpy/utils.py

@@ -1013,3 +1013,7 @@ def status_field_definition(definitions: dict[T, str]) -> str:
 
 def bit_field_definition(definitions: dict[T, str]) -> str:
     return _format_definition("Bit", definitions)
+
+
+def path_lengths_from_ground(height: np.ndarray) -> np.ndarray:
+    return np.diff(height, prepend=0)

cloudnetpy/version.py

@@ -1,4 +1,4 @@
 MAJOR = 1
-MINOR = 65
-PATCH = 8
+MINOR = 66
+PATCH = 1
 __version__ = f"{MAJOR}.{MINOR}.{PATCH}"

{cloudnetpy-1.65.8.dist-info → cloudnetpy-1.66.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudnetpy
-Version: 1.65.8
+Version: 1.66.1
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License