cloudnetpy 1.57.0__py3-none-any.whl → 1.58.1__py3-none-any.whl

cloudnetpy/categorize/atmos.py

@@ -219,7 +219,7 @@ class LiquidAttenuation(Attenuation):
     def _find_pixels_hard_to_correct(self) -> np.ndarray:
         melting_layer = utils.isbit(self.classification.category_bits, 3)
         hard_to_correct = np.cumsum(melting_layer, axis=1) >= 1
-        hard_to_correct[self.classification.is_rain, :] = True
+        hard_to_correct[self.classification.is_rain == 1, :] = True
         attenuated = self._find_attenuated_part_of_atmosphere()
         hard_to_correct[attenuated & self.atten.mask] = True
         return hard_to_correct

cloudnetpy/categorize/categorize.py

@@ -1,10 +1,12 @@
 """Module that generates Cloudnet categorize file."""
 from cloudnetpy import output, utils
 from cloudnetpy.categorize import atmos, classify
+from cloudnetpy.categorize.disdrometer import Disdrometer
 from cloudnetpy.categorize.lidar import Lidar
 from cloudnetpy.categorize.model import Model
 from cloudnetpy.categorize.mwr import Mwr
 from cloudnetpy.categorize.radar import Radar
+from cloudnetpy.datasource import DataSource
 from cloudnetpy.exceptions import ValidTimeStampError
 from cloudnetpy.metadata import MetaData
 
@@ -59,6 +61,8 @@ def generate_categorize(
     def _interpolate_to_cloudnet_grid() -> list:
         wl_band = utils.get_wl_band(data["radar"].radar_frequency)
         data["mwr"].rebin_to_grid(time)
+        if is_disdrometer:
+            data["disdrometer"].interpolate_to_grid(time)
         data["model"].interpolate_to_common_height(wl_band)
         model_gap_ind = data["model"].interpolate_to_grid(time, height)
         radar_gap_ind = data["radar"].rebin_to_grid(time)
@@ -69,8 +73,10 @@ def generate_categorize(
     def _screen_bad_time_indices(valid_indices: list) -> None:
         n_time_full = len(time)
         data["radar"].time = time[valid_indices]
-        for var in ("radar", "lidar", "mwr", "model"):
-            for key, item in data[var].data.items():
+        for data_key, obj in data.items():
+            if obj is None or data_key == "lv0_files":
+                continue
+            for key, item in obj.data.items():
                 if utils.isscalar(item.data):
                     continue
                 array = item[:]
@@ -81,26 +87,31 @@ def generate_categorize(
                         array = array[valid_indices, :]
                     else:
                         continue
-                    data[var].data[key].data = array
+                    obj.data[key].data = array
         for key, item in data["model"].data_dense.items():
             data["model"].data_dense[key] = item[valid_indices, :]
 
     def _prepare_output() -> dict:
         data["radar"].add_meta()
         data["model"].screen_sparse_fields()
-        for key in ("category_bits", "rainfall_rate", "insect_prob"):
+        if is_disdrometer:
+            data["radar"].data.pop("rainfall_rate", None)
+            data["disdrometer"].data.pop("n_particles", None)
+        for key in ("category_bits", "insect_prob"):
             data["radar"].append_data(getattr(classification, key), key)
         if classification.liquid_prob is not None:
             data["radar"].append_data(classification.liquid_prob, "liquid_prob")
         for key in ("radar_liquid_atten", "radar_gas_atten"):
             data["radar"].append_data(attenuations[key], key)
         data["radar"].append_data(quality["quality_bits"], "quality_bits")
+        data["radar"].append_data(classification.is_rain, "rain_detected")
         return {
             **data["radar"].data,
             **data["lidar"].data,
             **data["model"].data,
             **data["model"].data_sparse,
             **data["mwr"].data,
+            **(data["disdrometer"].data if is_disdrometer else {}),
         }
 
     def _define_dense_grid() -> tuple:
@@ -108,19 +119,20 @@ def generate_categorize(
 
     def _close_all() -> None:
         for obj in data.values():
-            if isinstance(obj, Radar | Lidar | Mwr | Model):
+            if isinstance(obj, DataSource):
                 obj.close()
 
     try:
+        is_disdrometer = "disdrometer" in input_files
         data = {
             "radar": Radar(input_files["radar"]),
             "lidar": Lidar(input_files["lidar"]),
             "mwr": Mwr(input_files["mwr"]),
             "lv0_files": input_files.get("lv0_files", None),
+            "disdrometer": Disdrometer(input_files["disdrometer"])
+            if is_disdrometer
+            else None,
         }
-        if data["radar"].altitude is None:
-            msg = "Radar altitude not defined"
-            raise RuntimeError(msg)
         data["model"] = Model(input_files["model"], data["radar"].altitude)
         time, height = _define_dense_grid()
         valid_ind = _interpolate_to_cloudnet_grid()
@@ -141,7 +153,7 @@ def generate_categorize(
         classification = classify.classify_measurements(data)
         attenuations = atmos.get_attenuations(data, classification)
         data["radar"].correct_atten(attenuations)
-        data["radar"].calc_errors(attenuations, classification)
+        data["radar"].calc_errors(attenuations, classification.is_clutter)
         quality = classify.fetch_quality(data, classification, attenuations)
         cloudnet_arrays = _prepare_output()
         date = data["radar"].get_date()
@@ -382,12 +394,6 @@ CATEGORIZE_ATTRIBUTES = {
         definition=DEFINITIONS["quality_bits"],
         units="1",
     ),
-    "rainfall_rate": MetaData(
-        long_name="Rainfall rate",
-        standard_name="rainfall_rate",
-        units="m s-1",
-        comment="Fill values denote rain with undefined intensity.",
-    ),
     "radar_liquid_atten": MetaData(
         long_name="Two-way radar attenuation due to liquid water",
         units="dB",
@@ -409,4 +415,9 @@ CATEGORIZE_ATTRIBUTES = {
         units="1",
         comment=COMMENTS["liquid_prob"],
     ),
+    "rain_detected": MetaData(
+        long_name="Rain detected",
+        units="1",
+        comment="1 = rain detected, 0 = no rain detected",
+    ),
 }

cloudnetpy/categorize/classify.py

@@ -66,12 +66,11 @@ def classify_measurements(data: dict) -> ClassificationResult:
     bits[4] = _find_aerosols(obs, bits[1], bits[0])
     bits[4][filtered_ice] = False
     return ClassificationResult(
-        _bits_to_integer(bits),
-        obs.is_rain,
-        obs.is_clutter,
-        obs.rainfall_rate,
-        insect_prob,
-        liquid_prob,
+        category_bits=_bits_to_integer(bits),
+        is_rain=obs.is_rain,
+        is_clutter=obs.is_clutter,
+        insect_prob=insect_prob,
+        liquid_prob=liquid_prob,
     )
 
 

cloudnetpy/categorize/containers.py

@@ -1,11 +1,10 @@
-import logging
 from dataclasses import dataclass
 
 import numpy as np
-import skimage
 from numpy import ma
 
 from cloudnetpy import utils
+from cloudnetpy.constants import MM_H_TO_M_S
 
 
 @dataclass
@@ -15,7 +14,6 @@ class ClassificationResult:
     category_bits: np.ndarray
     is_rain: np.ndarray
     is_clutter: np.ndarray
-    rainfall_rate: np.ndarray
     insect_prob: np.ndarray
     liquid_prob: np.ndarray | None
 
@@ -42,12 +40,12 @@ class ClassData:
         radar_type (str): Radar identifier.
         is_rain (ndarray): 2D boolean array denoting rain.
         is_clutter (ndarray): 2D boolean array denoting clutter.
-        rainfall_rate: 1D rain rate.
         altitude: site altitude.
 
     """
 
     def __init__(self, data: dict):
+        self.data = data
         self.z = data["radar"].data["Z"][:]
         self.v = data["radar"].data["v"][:]
         self.v_sigma = data["radar"].data["v_sigma"][:]
@@ -61,61 +59,39 @@ class ClassData:
         self.model_type = data["model"].source_type
         self.beta = data["lidar"].data["beta"][:]
         self.lwp = data["mwr"].data["lwp"][:]
-        self.is_rain = _find_rain_from_radar_echo(self.z, self.time)
-        self.rainfall_rate = _find_rainfall_rate(self.is_rain, data["radar"])
+        self.is_rain = self._find_profiles_with_rain()
         self.is_clutter = _find_clutter(self.v, self.is_rain)
         self.altitude = data["radar"].altitude
         self.lv0_files = data["lv0_files"]
         self.date = data["radar"].get_date()
 
-
-def _find_rain_from_radar_echo(
-    z: np.ndarray,
-    time: np.ndarray,
-    time_buffer: int = 5,
-) -> np.ndarray:
-    """Find profiles affected by rain.
-
-    Rain is present in such profiles where the radar echo in
-    the third range gate is > 0 dB. To make sure we do not include any
-    rainy profiles, we also flag a few profiles before and after
-    detections as raining.
-
-    Args:
-        z: Radar echo.
-        time: Time vector.
-        time_buffer: Time in minutes.
-
-    Returns:
-        1D Boolean array denoting profiles with rain.
-
-    """
-    filled = False
-    is_rain = ma.array(z[:, 3] > 0, dtype=bool).filled(filled)
-    is_rain = skimage.morphology.remove_small_objects(
-        is_rain,
-        2,
-        connectivity=1,
-    )  # Filter hot pixels
-    n_profiles = len(time)
-    n_steps = utils.n_elements(time, time_buffer, "time")
-    for ind in np.where(is_rain)[0]:
-        ind1 = max(0, ind - n_steps)
-        ind2 = min(ind + n_steps, n_profiles)
-        is_rain[ind1 : ind2 + 1] = True
-    return is_rain
-
-
-def _find_rainfall_rate(is_rain: np.ndarray, radar) -> np.ndarray:
-    rainfall_rate = ma.zeros(len(is_rain))
-    rainfall_rate[is_rain] = ma.masked
-    if "rainfall_rate" in radar.data:
-        radar_rainfall_rate = radar.data["rainfall_rate"].data
-        ind = np.where(~radar_rainfall_rate.mask)
-        rainfall_rate[ind] = radar_rainfall_rate[ind]
-    else:
-        logging.info("No measured rain rate available")
-    return rainfall_rate
+    def _find_profiles_with_rain(self) -> np.ndarray:
+        is_rain = self._find_rain_from_radar_echo()
+        rain_from_disdrometer = self._find_rain_from_disdrometer()
+        ind = ~rain_from_disdrometer.mask
+        is_rain[ind] = rain_from_disdrometer[ind]
+        return is_rain
+
+    def _find_rain_from_radar_echo(self) -> np.ndarray:
+        gate_number = 3
+        threshold = 0
+        z = self.z[:, gate_number]
+        return np.where((~ma.getmaskarray(z)) & (z > threshold), 1, 0)
+
+    def _find_rain_from_disdrometer(self) -> ma.MaskedArray:
+        threshold_mm_h = 0.25  # Standard threshold for drizzle -> rain
+        threshold_particles = 10  # This is arbitrary and should be better tested
+        threshold_rate = threshold_mm_h * MM_H_TO_M_S
+        try:
+            rainfall_rate = self.data["disdrometer"].data["rainfall_rate"].data
+            n_particles = self.data["disdrometer"].data["n_particles"].data
+            is_rain = ma.array(
+                (rainfall_rate > threshold_rate) & (n_particles > threshold_particles),
+                dtype=int,
+            )
+        except (AttributeError, KeyError):
+            is_rain = ma.masked_all(self.time.shape, dtype=int)
+        return is_rain
 
 
 def _find_clutter(

cloudnetpy/categorize/disdrometer.py

@@ -0,0 +1,53 @@
+"""Mwr module, containing the :class:`Mwr` class."""
+import logging
+
+import numpy as np
+from numpy import ma
+from scipy.interpolate import interp1d
+
+from cloudnetpy.categorize.lidar import get_gap_ind
+from cloudnetpy.datasource import DataSource
+
+
+class Disdrometer(DataSource):
+    """Disdrometer class, child of DataSource.
+
+    Args:
+    ----
+         full_path: Cloudnet Level 1b disdrometer file.
+
+    """
+
+    def __init__(self, full_path: str):
+        super().__init__(full_path)
+        self._init_rainfall_rate()
+
+    def interpolate_to_grid(self, time_grid: np.ndarray) -> None:
+        for key, array in self.data.items():
+            self.data[key].data = self._interpolate(array.data, time_grid)
+
+    def _init_rainfall_rate(self) -> None:
+        keys = ("rainfall_rate", "n_particles")
+        for key in keys:
+            self.append_data(self.dataset.variables[key][:], key)
+
+    def _interpolate(self, y: ma.MaskedArray, x_new: np.ndarray) -> np.ndarray:
+        if ma.getmask(y) is ma.nomask:
+            non_masked_indices = np.arange(len(y))
+        elif y.mask.all():
+            return ma.masked_all(x_new.shape)
+        else:
+            non_masked_indices = np.where(~y.mask)[0]
+        non_masked_values = y[non_masked_indices]
+        non_masked_time = self.time[non_masked_indices]
+        fun = interp1d(non_masked_time, non_masked_values, fill_value="extrapolate")
+        interpolated_array = ma.array(fun(x_new))
+        max_time = 1 / 60  # min -> fraction hour
+        mask_ind = get_gap_ind(non_masked_time, x_new, max_time)
+
+        if len(mask_ind) > 0:
+            msg = f"Unable to interpolate disdrometer for {len(mask_ind)} time steps"
+            logging.warning(msg)
+            interpolated_array[mask_ind] = ma.masked
+
+        return interpolated_array

cloudnetpy/categorize/insects.py

@@ -147,7 +147,8 @@ def _screen_insects(
         prob[(above_liquid == 1) & (insect_prob_no_ldr > 0)] = 0
 
     def _screen_rainy_profiles() -> None:
-        prob[obs.is_rain == 1, :] = 0
+        rain_smoothed = _smooth_rain(obs.time, obs.is_rain)
+        prob[rain_smoothed == 1, :] = 0
 
     prob = np.copy(insect_prob)
     _screen_liquid_layers()
@@ -155,3 +156,15 @@ def _screen_insects(
     _screen_above_liquid()
     _screen_rainy_profiles()
     return prob
+
+
+def _smooth_rain(time: np.ndarray, is_rain: np.ndarray) -> np.ndarray:
+    is_rain_smoothed = np.copy(is_rain)
+    time_buffer = 5  # minutes
+    n_profiles = len(is_rain)
+    n_steps = utils.n_elements(time, time_buffer, "time")
+    for rain_idx in np.where(is_rain)[0]:
+        idx_start = max(0, rain_idx - n_steps)
+        idx_end = min(rain_idx + n_steps, n_profiles)
+        is_rain_smoothed[idx_start : idx_end + 1] = True
+    return is_rain_smoothed

cloudnetpy/categorize/lidar.py

@@ -44,8 +44,8 @@ class Lidar(DataSource):
             height_new,
         )
         # Mask data points that are too far from the original grid
-        time_gap_ind = _get_gap_ind(self.time[indices], time_new, max_time)
-        height_gap_ind = _get_gap_ind(self.height, height_new, max_height)
+        time_gap_ind = get_gap_ind(self.time[indices], time_new, max_time)
+        height_gap_ind = get_gap_ind(self.height, height_new, max_height)
         self._mask_profiles(beta_interp, time_gap_ind, "time")
         self._mask_profiles(beta_interp, height_gap_ind, "height")
         self.data["beta"].data = beta_interp
@@ -69,7 +69,7 @@ class Lidar(DataSource):
         self.append_data(3.0, "beta_bias")
 
 
-def _get_gap_ind(grid: np.ndarray, new_grid: np.ndarray, threshold: float) -> list[int]:
+def get_gap_ind(grid: np.ndarray, new_grid: np.ndarray, threshold: float) -> list[int]:
     return [
         ind
         for ind, value in enumerate(new_grid)

cloudnetpy/categorize/radar.py

@@ -7,8 +7,7 @@ from numpy import ma
 from scipy import constants
 
 from cloudnetpy import utils
-from cloudnetpy.categorize.classify import ClassificationResult
-from cloudnetpy.constants import SEC_IN_HOUR
+from cloudnetpy.constants import GHZ_TO_HZ, SEC_IN_HOUR, SPEED_OF_LIGHT
 from cloudnetpy.datasource import DataSource
 
 
@@ -69,8 +68,10 @@ class Radar(DataSource):
                     )
                 case "v_sigma":
                     array.calc_linear_std(self.time, time_new)
-                case "width" | "rainfall_rate":
+                case "width":
                     array.rebin_data(self.time, time_new)
+                case "rainfall_rate":
+                    array.rebin_data(self.time, time_new, mask_zeros=False)
                 case _:
                     continue
         return bad_time_indices
@@ -214,7 +215,7 @@ class Radar(DataSource):
     def calc_errors(
         self,
         attenuations: dict,
-        classification: ClassificationResult,
+        is_clutter: np.ndarray,
     ) -> None:
         """Calculates uncertainties of radar echo.
 
@@ -223,7 +224,7 @@ class Radar(DataSource):
 
         Args:
             attenuations: 2-D attenuations due to atmospheric gases.
-            classification: The :class:`ClassificationResult` instance.
+            is_clutter: 2-D boolean array denoting pixels contaminated by clutter.
 
         References:
             The method is based on Hogan R. and O'Connor E., 2004,
@@ -235,17 +236,24 @@ class Radar(DataSource):
             """Returns sensitivity of radar as function of altitude."""
             mean_gas_atten = ma.mean(attenuations["radar_gas_atten"], axis=0)
             z_sensitivity = z_power_min + log_range + mean_gas_atten
-            zc = ma.median(ma.array(z, mask=~classification.is_clutter), axis=0)
+            zc = ma.median(ma.array(z, mask=~is_clutter), axis=0)
             valid_values = np.logical_not(zc.mask)
             z_sensitivity[valid_values] = zc[valid_values]
             return z_sensitivity
 
         def _calc_error() -> np.ndarray | float:
-            if "width" not in self.data:
-                return 0.3
-            z_precision = 4.343 * (
-                1 / np.sqrt(_number_of_independent_pulses())
-                + utils.db2lin(z_power_min - z_power) / 3
+            """Returns error of radar as function of altitude.
+
+            References:
+                Hogan, R. J., 1998: Dual-wavelength radar studies of clouds.
+                PhD Thesis, University of Reading, UK.
+
+            """
+            noise_threshold = 3
+            n_pulses = _number_of_independent_pulses()
+            ln_to_log10 = 10 / np.log(10)
+            z_precision = (ln_to_log10 / np.sqrt(n_pulses)) * (
+                1 + (utils.db2lin(z_power_min - z_power) / noise_threshold)
             )
             gas_error = attenuations["radar_gas_atten"] * 0.1
             liq_error = attenuations["liquid_atten_err"].filled(0)
@@ -254,16 +262,22 @@ class Radar(DataSource):
             return z_error
 
         def _number_of_independent_pulses() -> float:
+            """Returns number of independent pulses.
+
+            References:
+                Atlas, D., 1964: Advances in radar meteorology.
+                Advances in Geophys., 10, 318-478.
+
+            """
+            if "width" not in self.data:
+                default_width = 0.3
+                width = np.zeros_like(self.data["Z"][:])
+                width[~width.mask] = default_width
+            else:
+                width = self.data["width"][:]
             dwell_time = utils.mdiff(self.time) * SEC_IN_HOUR
-            return (
-                dwell_time
-                * self.radar_frequency
-                * 1e9
-                * 4
-                * np.sqrt(math.pi)
-                * self.data["width"][:]
-                / 3e8
-            )
+            wl = SPEED_OF_LIGHT / (self.radar_frequency * GHZ_TO_HZ)
+            return 4 * np.sqrt(math.pi) * dwell_time * width / wl
 
         def _calc_z_power_min() -> float:
             if ma.all(z_power.mask):

cloudnetpy/cloudnetarray.py

@@ -54,7 +54,9 @@ class CloudnetArray:
         """Masks data from given indices."""
         self.data[ind] = ma.masked
 
-    def rebin_data(self, time: np.ndarray, time_new: np.ndarray) -> list:
+    def rebin_data(
+        self, time: np.ndarray, time_new: np.ndarray, *, mask_zeros: bool = True
+    ) -> list:
         """Rebins `data` in time.
 
         Args:
@@ -66,12 +68,14 @@ class CloudnetArray:
 
         """
         if self.data.ndim == 1:
-            self.data = utils.rebin_1d(time, self.data, time_new)
+            self.data = utils.rebin_1d(time, self.data, time_new, mask_zeros=mask_zeros)
             bad_indices = list(np.where(self.data == ma.masked)[0])
         else:
             if not isinstance(self.data, ma.MaskedArray):
                 self.data = ma.masked_array(self.data)
-            self.data, bad_indices = utils.rebin_2d(time, self.data, time_new)
+            self.data, bad_indices = utils.rebin_2d(
+                time, self.data, time_new, mask_zeros=mask_zeros
+            )
         return bad_indices
 
     def fetch_attributes(self) -> list:

cloudnetpy/constants.py

@@ -20,8 +20,12 @@ RS: Final = 287.058
 RHO_ICE: Final = 917
 
 # other
+SPEED_OF_LIGHT: Final = 3.0e8
 SEC_IN_MINUTE: Final = 60
 SEC_IN_HOUR: Final = 3600
 SEC_IN_DAY: Final = 86400
 MM_TO_M: Final = 1e-3
 G_TO_KG: Final = 1e-3
+M_S_TO_MM_H: Final = SEC_IN_HOUR / MM_TO_M
+MM_H_TO_M_S: Final = 1 / M_S_TO_MM_H
+GHZ_TO_HZ: Final = 1e9

cloudnetpy/instruments/disdrometer/parsivel.py

@@ -372,7 +372,10 @@ def _parse_spectrum(tokens: Iterator[str]) -> np.ndarray:
     return np.array(values, dtype="i2").reshape((32, 32))
 
 
-PARSERS: dict[str, Callable[[Iterator[str]], Any]] = {
+ParserType = Callable[[Iterator[str]], Any]
+
+
+PARSERS: dict[str, ParserType] = {
     "I_heating": _parse_float,
     "T_sensor": _parse_int,
     "_T_pcb": _parse_int,
@@ -401,6 +404,16 @@ PARSERS: dict[str, Callable[[Iterator[str]], Any]] = {
     "visibility": _parse_int,
 }
 
+EMPTY_VALUES: dict[ParserType, Any] = {
+    _parse_int: 0,
+    _parse_float: 0.0,
+    _parse_date: datetime.date(2000, 1, 1),
+    _parse_time: datetime.time(12, 0, 0),
+    _parse_datetime: datetime.datetime(2000, 1, 1),
+    _parse_vector: np.zeros(32, dtype=float),
+    _parse_spectrum: np.zeros((32, 32), dtype="i2"),
+}
+
 
 def _parse_headers(line: str) -> list[str]:
     return [CSV_HEADERS[header.strip()] for header in line.split(";")]
@@ -508,7 +521,7 @@ def _read_toa5(filename: str | PathLike) -> dict[str, list]:
         return data
 
 
-def _read_typ_op4a(lines: list[str]) -> dict[str, list]:
+def _read_typ_op4a(lines: list[str]) -> dict[str, Any]:
     """Read output of "CS/PA" command. The output starts with line "TYP OP4A"
     followed by one line per measured variable in format: <number>:<value>.
     Output ends with characters: <ETX><CR><LF><NUL>. Lines are separated by
@@ -527,7 +540,7 @@ def _read_typ_op4a(lines: list[str]) -> dict[str, list]:
             continue
         parser = PARSERS.get(varname, next)
         tokens = value.split(";")
-        data[varname] = [parser(iter(tokens))]
+        data[varname] = parser(iter(tokens))
     return data
 
 
@@ -538,15 +551,26 @@ def _read_fmi(content: str):
     - output of "CS/PA" command without non-printable characters at the end
     - "]\n"
     """
-    output: dict[str, Any] = defaultdict(list)
+    output: dict[str, list] = {"_datetime": []}
     for m in re.finditer(
         r"\[(?P<year>\d+)-(?P<month>\d+)-(?P<day>\d+) "
         r"(?P<hour>\d+):(?P<minute>\d+):(?P<second>\d+)"
         r"(?P<output>[^\]]*)\]",
         content,
     ):
-        for key, value in _read_typ_op4a(m["output"].splitlines()).items():
+        try:
+            record = _read_typ_op4a(m["output"].splitlines())
+        except ValueError:
+            continue
+
+        for key, value in record.items():
+            if key not in output:
+                output[key] = [None] * len(output["_datetime"])
             output[key].append(value)
+        for key in output:
+            if key not in record and key != "_datetime":
+                output[key].append(None)
+
         output["_datetime"].append(
             datetime.datetime(
                 int(m["year"]),
@@ -577,6 +601,7 @@ def _read_parsivel(
             data = _read_toa5(filename)
         elif "TYP OP4A" in lines[0]:
             data = _read_typ_op4a(lines)
+            data = {key: [value] for key, value in data.items()}
         elif "Date" in lines[0]:
             headers = _parse_headers(lines[0])
             data = _read_rows(headers, lines[1:])
@@ -597,6 +622,17 @@ def _read_parsivel(
             combined_data[key].extend(values)
     if timestamps is not None:
         combined_data["_datetime"] = list(timestamps)
-    result = {key: np.array(value) for key, value in combined_data.items()}
+    result = {}
+    for key, value in combined_data.items():
+        array = np.array(
+            [
+                x
+                if x is not None
+                else (EMPTY_VALUES[PARSERS[key]] if key in PARSERS else "")
+                for x in value
+            ]
+        )
+        mask = [np.full(array.shape[1:], x is None) for x in value]
+        result[key] = ma.array(array, mask=mask)
     result["time"] = result["_datetime"].astype("datetime64[s]")
     return result

cloudnetpy/output.py

@@ -317,7 +317,7 @@ def add_time_attribute(
 
 
 def add_source_attribute(attributes: dict, data: dict) -> dict:
-    """Adds source attribute."""
+    """Adds source attribute to variables."""
     variables = {
         "radar": (
             "v",
@@ -334,8 +334,11 @@ def add_source_attribute(attributes: dict, data: dict) -> dict:
         "lidar": ("beta", "lidar_wavelength"),
         "mwr": ("lwp",),
         "model": ("uwind", "vwind", "Tw", "q", "pressure", "temperature"),
+        "disdrometer": ("rainfall_rate",),
     }
     for instrument, keys in variables.items():
+        if data[instrument] is None:
+            continue
         source = data[instrument].dataset.source
         for key in keys:
             if key in attributes:

cloudnetpy/plotting/plotting.py

@@ -514,6 +514,9 @@ class Plot2D(Plot):
         if figure_data.height is None:
             msg = "No height information in the file."
             raise ValueError(msg)
+        if self._data.ndim < 2:
+            msg = "Data has to be 2D."
+            raise PlottingError(msg)
         alt = figure_data.height
         if figure_data.options.max_y is None:
             return alt

cloudnetpy/products/der.py

@@ -214,7 +214,7 @@ class DerSource(DataSource):
         is_retrieved = ~self.data["der"][:].mask
         is_mixed = droplet_classification.is_mixed
         is_ice = droplet_classification.is_ice
-        is_rain = np.tile(self.is_rain, (is_retrieved.shape[1], 1)).T
+        is_rain = np.tile(self.is_rain == 1, (is_retrieved.shape[1], 1)).T
 
         retrieval_status = np.zeros(is_retrieved.shape, dtype=int)
         retrieval_status[is_ice] = 4

cloudnetpy/products/drizzle_tools.py

@@ -4,6 +4,7 @@ from bisect import bisect_left
 
 import netCDF4
 import numpy as np
+from numpy import ma
 from scipy.special import gamma
 
 from cloudnetpy import utils
@@ -101,7 +102,7 @@ class DrizzleClassification(ProductClassification):
 
     @staticmethod
     def _find_v_sigma(cat_file: str) -> np.ndarray:
-        v_sigma = product_tools.read_nc_fields(cat_file, "v_sigma")
+        v_sigma = product_tools.read_nc_field(cat_file, "v_sigma")
         return np.isfinite(v_sigma)
 
     def _find_warm_liquid(self) -> np.ndarray:
@@ -160,11 +161,11 @@ class SpectralWidth:
         self.width_ht = self._calculate_spectral_width()
 
     def _calculate_spectral_width(self) -> np.ndarray:
-        v_sigma = product_tools.read_nc_fields(self.cat_file, "v_sigma")
+        v_sigma = product_tools.read_nc_field(self.cat_file, "v_sigma")
         try:
-            width = product_tools.read_nc_fields(self.cat_file, "width")
+            width = product_tools.read_nc_field(self.cat_file, "width")
         except KeyError:
-            width = [0]
+            width = ma.array([0])
             logging.warning("No spectral width, assuming width = %s", width[0])
         sigma_factor = self._calc_v_sigma_factor()
         return width - sigma_factor * v_sigma
@@ -178,7 +179,7 @@ class SpectralWidth:
 
     def _calc_beam_divergence(self) -> np.ndarray:
         beam_width = 0.5
-        height = product_tools.read_nc_fields(self.cat_file, "height")
+        height = product_tools.read_nc_field(self.cat_file, "height")
         return height * np.deg2rad(beam_width)
 
     def _calc_horizontal_wind(self) -> np.ndarray:

cloudnetpy/products/product_tools.py

@@ -232,32 +232,35 @@ class IceSource(DataSource):
 
 def get_is_rain(filename: str) -> np.ndarray:
     try:
-        rainfall_rate = read_nc_fields(filename, "rainfall_rate")
+        is_rain = read_nc_field(filename, "rain_detected")
     except KeyError:
-        rainfall_rate = read_nc_fields(filename, "rain_rate")
-    is_rain = rainfall_rate != 0
-    if not isinstance(is_rain, ma.MaskedArray):
-        is_rain = ma.array(is_rain)
-    is_rain[is_rain.mask] = True
+        try:
+            rainfall_rate = read_nc_field(filename, "rainfall_rate")
+        except KeyError:
+            rainfall_rate = read_nc_field(filename, "rain_rate")
+        is_rain = rainfall_rate != 0
+        is_rain[is_rain.mask] = True
     return np.array(is_rain)
 
 
-def read_nc_fields(nc_file: str, names: str | list) -> ma.MaskedArray | list:
+def read_nc_field(nc_file: str, name: str) -> ma.MaskedArray:
+    with netCDF4.Dataset(nc_file) as nc:
+        return nc.variables[name][:]
+
+
+def read_nc_fields(nc_file: str, names: list[str]) -> list[ma.MaskedArray]:
     """Reads selected variables from a netCDF file.
 
     Args:
         nc_file: netCDF file name.
-        names: Variables to be read, e.g. 'temperature' or ['ldr', 'lwp'].
+        names: Variables to be read, e.g. ['ldr', 'lwp'].
 
     Returns:
-        ndarray/list: Array in case of one variable passed as a string.
-        List of arrays otherwise.
+        List of numpy arrays.
 
     """
-    names = [names] if isinstance(names, str) else names
     with netCDF4.Dataset(nc_file) as nc:
-        data = [nc.variables[name][:] for name in names]
-    return data[0] if len(data) == 1 else data
+        return [nc.variables[name][:] for name in names]
 
 
 def interpolate_model(cat_file: str, names: str | list) -> dict[str, np.ndarray]:

cloudnetpy/utils.py

@@ -140,6 +140,8 @@ def rebin_2d(
     x_new: np.ndarray,
     statistic: str = "mean",
     n_min: int = 1,
+    *,
+    mask_zeros: bool = True,
 ) -> tuple[ma.MaskedArray, list]:
     """Rebins 2-D data in one dimension.
 
@@ -171,7 +173,10 @@ def rebin_2d(
                 bins=edges,
             )
     result[~np.isfinite(result)] = 0
-    masked_result = ma.masked_equal(result, 0)
+    if mask_zeros is True:
+        masked_result = ma.masked_equal(result, 0)
+    else:
+        masked_result = ma.array(result)
 
     # Fill bins with not enough profiles
     empty_indices = []
@@ -191,6 +196,8 @@ def rebin_1d(
     array: np.ndarray | ma.MaskedArray,
     x_new: np.ndarray,
     statistic: str = "mean",
+    *,
+    mask_zeros: bool = True,
 ) -> ma.MaskedArray:
     """Rebins 1D array.
 
@@ -217,7 +224,9 @@ def rebin_1d(
             bins=edges,
         )
     result[~np.isfinite(result)] = 0
-    return ma.masked_equal(result, 0)
+    if mask_zeros:
+        return ma.masked_equal(result, 0)
+    return ma.array(result)
 
 
 def filter_isolated_pixels(array: np.ndarray) -> np.ndarray:

cloudnetpy/version.py

@@ -1,4 +1,4 @@
 MAJOR = 1
-MINOR = 57
-PATCH = 0
+MINOR = 58
+PATCH = 1
 __version__ = f"{MAJOR}.{MINOR}.{PATCH}"

{cloudnetpy-1.57.0.dist-info → cloudnetpy-1.58.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudnetpy
-Version: 1.57.0
+Version: 1.58.1
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

{cloudnetpy-1.57.0.dist-info → cloudnetpy-1.58.1.dist-info}/RECORD

@@ -1,29 +1,30 @@
 cloudnetpy/__init__.py,sha256=X_FqY-4yg5GUj5Edo14SToLEos6JIsC3fN-v1FUgQoA,43
-cloudnetpy/cloudnetarray.py,sha256=vSI6hqozhS1ntNIgY2kqTt7N6BF_xJtUii_vsuyxTno,6869
+cloudnetpy/cloudnetarray.py,sha256=HT6bLtjnimOVbGrdjQBqD0F8GW0KWkn2qhaIGFMKLAY,6987
 cloudnetpy/concat_lib.py,sha256=YK5ho5msqwNxpPtPT8f2OewIJ8hTrbhCkaxHaBKLTI0,9809
-cloudnetpy/constants.py,sha256=OMp3pKHCZmdKyRvfO35E7vE3FrS_DHIs_GJuexJLCQk,600
+cloudnetpy/constants.py,sha256=l7_ohQgLEQ6XEG9AMBarTPKp9OM8B1ElJ6fSN0ScdmM,733
 cloudnetpy/datasource.py,sha256=-6oLC5bsn9sIoaN0glV88owFyTeGRsW1ZVJSV8rM5rE,7813
 cloudnetpy/exceptions.py,sha256=nA6YieylwKSp5KQOh3DhhcTmUBsd0tBZnedgmlWck-w,1415
 cloudnetpy/metadata.py,sha256=Bcu1a9UyUq61jomuZ0_6hYIOzf61e5qCXeiwLm46ikw,5040
-cloudnetpy/output.py,sha256=jD1pfBb4OQhVOrlhPEk-8FAi4bUW7zjAL468r6BPkJg,14586
+cloudnetpy/output.py,sha256=WoVTNuxni0DUr163vZ-_mDr1brXhY15XSlGMrq9Aoqw,14700
 cloudnetpy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cloudnetpy/utils.py,sha256=yY5a5HLuAks2uzA4XbbqsGFEmXoyqECn_TjD3sMa0lI,27193
-cloudnetpy/version.py,sha256=qC35wCh1KxhBnDTG1irZ-STVg6O7ebRlvaGi3fZqxjA,72
+cloudnetpy/utils.py,sha256=0TlHm71YtSrKXBsRKctitnhQrvZPE-ulEVeAQW-oK58,27398
+cloudnetpy/version.py,sha256=ACE_5AEhiiFIDukxdKYlkIvBmsdfhExmwRoHxF3tRPs,72
 cloudnetpy/categorize/__init__.py,sha256=gP5q3Vis1y9u9OWgA_idlbjfWXYN_S0IBSWdwBhL_uU,69
-cloudnetpy/categorize/atmos.py,sha256=cax3iRmvr7S-VkUZqz0JCfAN3WEsUVbGfH4zSHy1APo,12384
+cloudnetpy/categorize/atmos.py,sha256=fWW8ye_8HZASRAiYwURFKWzcGOYIA2RKeVxCq0lVOuM,12389
 cloudnetpy/categorize/atmos_utils.py,sha256=wndpwJxc2-QnNTkV8tc8I11Vs_WkNz9sVMX1fuGgUC4,3777
-cloudnetpy/categorize/categorize.py,sha256=88gHVRHDB0eyapWUrKwo_T6ep-vtAT1_NS1yy3f59lo,16970
-cloudnetpy/categorize/classify.py,sha256=NwJXt44bynlNOuYAlt0i3rOgbuU4zVMQDnYxG92jOx0,8875
-cloudnetpy/categorize/containers.py,sha256=bt6q7vUGn9V8mpNcB44mWNZj92rYmaIh-SDlRIrQY_g,4608
+cloudnetpy/categorize/categorize.py,sha256=_chLFT0l9ll78y3oaxFwOTBbv2raxxjghCz-_KGdImQ,17476
+cloudnetpy/categorize/classify.py,sha256=vIR7Ztu41kQbRaA2xIy_mm08KxVgQczYXpNSo0vz2Yg,8905
+cloudnetpy/categorize/containers.py,sha256=j6oSKPeZcq9vFthYaocAw1m6yReRNNPYUQF5UTDq4YM,4232
+cloudnetpy/categorize/disdrometer.py,sha256=0Z0nvUtoZKDxiUfBZzoYZxUFOVjq-thmYfaCkskeECs,1799
 cloudnetpy/categorize/droplet.py,sha256=pUmB-gN0t9sVgsGLof6X9N0nuEb4EBtEUswwpoQapTY,8687
 cloudnetpy/categorize/falling.py,sha256=xES5ZdYs34tbX1p4a9kzt9r3G5s25Mpvs5WeFs1KNzo,4385
 cloudnetpy/categorize/freezing.py,sha256=684q83TPQ5hHrbbHX-E36VoTlWLSOlGfOW1FC8b3wfI,3754
-cloudnetpy/categorize/insects.py,sha256=ubNp7-lZWXsQ3qYgyEqqjSF2LWnHdlYTInUGKYl_p_s,5242
-cloudnetpy/categorize/lidar.py,sha256=4dRbD3E1r36d1cEIaHBVh8BiXXlEFBdqhNIyH682K4k,2607
+cloudnetpy/categorize/insects.py,sha256=7m31aQSO9nekf_d3TgXMnPpgHIP7J_xhHLShfQ9JS9E,5764
+cloudnetpy/categorize/lidar.py,sha256=LYqXw30sLOYxhKRcO3k5r0uVLGRYmJ5k0KuVOMduY5A,2604
 cloudnetpy/categorize/melting.py,sha256=AOq36yLntDXYbeMw5QhZ7kMLwt0INyUbhzv-rSILLyo,6261
 cloudnetpy/categorize/model.py,sha256=xWB6XOSz9p0h4b4m6ImMmzcTImOmz54d093WmsLogdQ,5535
 cloudnetpy/categorize/mwr.py,sha256=-KMoYlch_C79bqgcEiRDCTRCcQf1ZsYxU90GQ8hzMgs,1435
-cloudnetpy/categorize/radar.py,sha256=gephjRnEEMMCfTGX-HVT5kklAZblvkkGwf3SPCngqe4,12981
+cloudnetpy/categorize/radar.py,sha256=S3561FVK6J4NUjqPi2S6fKnjnOm0oLwbIC54J64AN4Y,13659
 cloudnetpy/instruments/__init__.py,sha256=_jejVwi_viSZehmAOkEqTNI-0-exGgAJ_bHW1IRRwTI,398
 cloudnetpy/instruments/basta.py,sha256=OeWBP_5b2xjOzPaFWEFC0irgLhSqX1YdcPkVBRbxd4c,3742
 cloudnetpy/instruments/campbell_scientific.py,sha256=2WHfBKQjtRSl0AqvtPeX7G8Hdi3Dn0WbvoAppFOMbA8,5270
@@ -48,7 +49,7 @@ cloudnetpy/instruments/vaisala.py,sha256=E6PaK26lHprqOJUCEDZPZQu83Qan9n_THudTFQM
 cloudnetpy/instruments/weather_station.py,sha256=IMJHGXfMhb4jJw_i66oGDCkeeRn3_eko8zVehu6Fte0,5970
 cloudnetpy/instruments/disdrometer/__init__.py,sha256=lyjwttWvFvuwYxEkusoAvgRcbBmglmOp5HJOpXUqLWo,93
 cloudnetpy/instruments/disdrometer/common.py,sha256=TTsvWzhHg5tansTs47WB-7uuBRCZdjbFQMAyAQtFkSU,15636
-cloudnetpy/instruments/disdrometer/parsivel.py,sha256=myBHJw4VDPJErPlzAgwLoTjUQOGcvXdbvvy0MEjyGBc,21293
+cloudnetpy/instruments/disdrometer/parsivel.py,sha256=TWq8VgG8U75AJQfCX-V2y8qy-nO6dKMOGd2eG8-u7to,22342
 cloudnetpy/instruments/disdrometer/thies.py,sha256=h7EwZ9tn47UUMiYqDQ68vkXv4q0rEqX1ZeFXd7XJYNg,5050
 cloudnetpy/model_evaluation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloudnetpy/model_evaluation/file_handler.py,sha256=oUGIblcEWLLv16YKUch-M5KA-dGRAcuHa-9anP3xtX4,6447
@@ -92,22 +93,22 @@ cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py,sha256=Ra3r4V
 cloudnetpy/model_evaluation/tests/unit/test_tools.py,sha256=Ia_VrLdV2NstX5gbx_3AZTOAlrgLAy_xFZ8fHYVX0xI,3817
 cloudnetpy/plotting/__init__.py,sha256=lg9Smn4BI0dVBgnDLC3JVJ4GmwoSnO-qoSd4ApvwV6Y,107
 cloudnetpy/plotting/plot_meta.py,sha256=MK_-fByZgihXkTXVJhs942qvq9SLfyzLa911y6x8cV0,14791
-cloudnetpy/plotting/plotting.py,sha256=LzO0UIMSGLsF0Q5Tkr5G5Xa8Ah7pi1ZzibksBIOLoEI,30755
+cloudnetpy/plotting/plotting.py,sha256=TUC6kZqpBDE-EgehkFFebw4y9tZOIVP3kH6UHh41H3o,30863
 cloudnetpy/products/__init__.py,sha256=2hRb5HG9hNrxH1if5laJkLeFeaZCd5W1q3hh4ewsX0E,273
 cloudnetpy/products/classification.py,sha256=0E9OUGR3uLCsS1nORwQu0SqW0_8uX7n6LlRcVhtzKw4,7845
-cloudnetpy/products/der.py,sha256=HAdPvbJySEqkIwDrdZDPnli_wnN2qwm72_D1a82ZWIs,12398
+cloudnetpy/products/der.py,sha256=mam6jWV7A2h8V5WC3DIeFp6ou7UD1JOw9r7h2B0su-s,12403
 cloudnetpy/products/drizzle.py,sha256=BY2HvJeWt_ps6KKCGXwUUNRTy78q0cQM8bOCCoj8TWA,10803
 cloudnetpy/products/drizzle_error.py,sha256=4GwlHRtNbk9ks7bGtXCco-wXbcDOKeAQwKmbhzut6Qk,6132
-cloudnetpy/products/drizzle_tools.py,sha256=wGnYVPCzQpvIRV0OJoL-5NavstEA1JuHsj3Is2uxp48,10820
+cloudnetpy/products/drizzle_tools.py,sha256=UhcJbPa4tXHbuVlegIRfOl5nZ_E6ddKv20aghfP0hdg,10847
 cloudnetpy/products/ier.py,sha256=IcGPlQahbwJjp3vOOrxWSYW2FPzbSV0KQL5eYECc4kU,7777
 cloudnetpy/products/iwc.py,sha256=MUPuVKWgqOuuLRCGk3QY74uBZB_7P1qlinlP8nEvz9o,10124
 cloudnetpy/products/lwc.py,sha256=TbIR6kMwjbm63ed5orB1pkqx9ZBm8C5TF2JmT8WKdKI,18794
 cloudnetpy/products/mie_lu_tables.nc,sha256=It4fYpqJXlqOgL8jeZ-PxGzP08PMrELIDVe55y9ob58,16637951
 cloudnetpy/products/mwr_tools.py,sha256=PRm5aCULccUehU-Byk55wYhhEHseMjoAjGBu5TSyHao,4621
-cloudnetpy/products/product_tools.py,sha256=E8CSijBY8cr70BH2JFa0lGQ-RzI9EcHQ0Fzt8CQ8rY4,10442
+cloudnetpy/products/product_tools.py,sha256=rhx_Ru9FLlQqCNM-awoiHx18-Aq1eBwL9LiUaQoJs6k,10412
 docs/source/conf.py,sha256=IKiFWw6xhUd8NrCg0q7l596Ck1d61XWeVjIFHVSG9Og,1490
-cloudnetpy-1.57.0.dist-info/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
-cloudnetpy-1.57.0.dist-info/METADATA,sha256=YfJf13JTJBey4rImv0KbljSElInh9z4EygDLWZPDF08,5733
-cloudnetpy-1.57.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-cloudnetpy-1.57.0.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
-cloudnetpy-1.57.0.dist-info/RECORD,,
+cloudnetpy-1.58.1.dist-info/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
+cloudnetpy-1.58.1.dist-info/METADATA,sha256=CqdJACjYG1_pszHVdzrrr6RZGJNO-YUvhSmrNEuBhz8,5733
+cloudnetpy-1.58.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+cloudnetpy-1.58.1.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
+cloudnetpy-1.58.1.dist-info/RECORD,,