cloudnetpy 1.57.0__tar.gz → 1.58.1__tar.gz

{cloudnetpy-1.57.0/cloudnetpy.egg-info → cloudnetpy-1.58.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudnetpy
-Version: 1.57.0
+Version: 1.58.1
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

{cloudnetpy-1.57.0 → cloudnetpy-1.58.1}/cloudnetpy/categorize/atmos.py

@@ -219,7 +219,7 @@ class LiquidAttenuation(Attenuation):
     def _find_pixels_hard_to_correct(self) -> np.ndarray:
         melting_layer = utils.isbit(self.classification.category_bits, 3)
         hard_to_correct = np.cumsum(melting_layer, axis=1) >= 1
-        hard_to_correct[self.classification.is_rain, :] = True
+        hard_to_correct[self.classification.is_rain == 1, :] = True
         attenuated = self._find_attenuated_part_of_atmosphere()
         hard_to_correct[attenuated & self.atten.mask] = True
         return hard_to_correct

{cloudnetpy-1.57.0 → cloudnetpy-1.58.1}/cloudnetpy/categorize/categorize.py

@@ -1,10 +1,12 @@
 """Module that generates Cloudnet categorize file."""
 from cloudnetpy import output, utils
 from cloudnetpy.categorize import atmos, classify
+from cloudnetpy.categorize.disdrometer import Disdrometer
 from cloudnetpy.categorize.lidar import Lidar
 from cloudnetpy.categorize.model import Model
 from cloudnetpy.categorize.mwr import Mwr
 from cloudnetpy.categorize.radar import Radar
+from cloudnetpy.datasource import DataSource
 from cloudnetpy.exceptions import ValidTimeStampError
 from cloudnetpy.metadata import MetaData
 
@@ -59,6 +61,8 @@ def generate_categorize(
     def _interpolate_to_cloudnet_grid() -> list:
         wl_band = utils.get_wl_band(data["radar"].radar_frequency)
         data["mwr"].rebin_to_grid(time)
+        if is_disdrometer:
+            data["disdrometer"].interpolate_to_grid(time)
         data["model"].interpolate_to_common_height(wl_band)
         model_gap_ind = data["model"].interpolate_to_grid(time, height)
         radar_gap_ind = data["radar"].rebin_to_grid(time)
@@ -69,8 +73,10 @@ def generate_categorize(
     def _screen_bad_time_indices(valid_indices: list) -> None:
         n_time_full = len(time)
         data["radar"].time = time[valid_indices]
-        for var in ("radar", "lidar", "mwr", "model"):
-            for key, item in data[var].data.items():
+        for data_key, obj in data.items():
+            if obj is None or data_key == "lv0_files":
+                continue
+            for key, item in obj.data.items():
                 if utils.isscalar(item.data):
                     continue
                 array = item[:]
@@ -81,26 +87,31 @@ def generate_categorize(
                         array = array[valid_indices, :]
                     else:
                         continue
-                    data[var].data[key].data = array
+                    obj.data[key].data = array
         for key, item in data["model"].data_dense.items():
             data["model"].data_dense[key] = item[valid_indices, :]
 
     def _prepare_output() -> dict:
         data["radar"].add_meta()
         data["model"].screen_sparse_fields()
-        for key in ("category_bits", "rainfall_rate", "insect_prob"):
+        if is_disdrometer:
+            data["radar"].data.pop("rainfall_rate", None)
+            data["disdrometer"].data.pop("n_particles", None)
+        for key in ("category_bits", "insect_prob"):
             data["radar"].append_data(getattr(classification, key), key)
         if classification.liquid_prob is not None:
             data["radar"].append_data(classification.liquid_prob, "liquid_prob")
         for key in ("radar_liquid_atten", "radar_gas_atten"):
             data["radar"].append_data(attenuations[key], key)
         data["radar"].append_data(quality["quality_bits"], "quality_bits")
+        data["radar"].append_data(classification.is_rain, "rain_detected")
         return {
             **data["radar"].data,
             **data["lidar"].data,
             **data["model"].data,
             **data["model"].data_sparse,
             **data["mwr"].data,
+            **(data["disdrometer"].data if is_disdrometer else {}),
         }
 
     def _define_dense_grid() -> tuple:
@@ -108,19 +119,20 @@ def generate_categorize(
 
     def _close_all() -> None:
         for obj in data.values():
-            if isinstance(obj, Radar | Lidar | Mwr | Model):
+            if isinstance(obj, DataSource):
                 obj.close()
 
     try:
+        is_disdrometer = "disdrometer" in input_files
         data = {
             "radar": Radar(input_files["radar"]),
             "lidar": Lidar(input_files["lidar"]),
             "mwr": Mwr(input_files["mwr"]),
             "lv0_files": input_files.get("lv0_files", None),
+            "disdrometer": Disdrometer(input_files["disdrometer"])
+            if is_disdrometer
+            else None,
         }
-        if data["radar"].altitude is None:
-            msg = "Radar altitude not defined"
-            raise RuntimeError(msg)
         data["model"] = Model(input_files["model"], data["radar"].altitude)
         time, height = _define_dense_grid()
         valid_ind = _interpolate_to_cloudnet_grid()
@@ -141,7 +153,7 @@ def generate_categorize(
         classification = classify.classify_measurements(data)
         attenuations = atmos.get_attenuations(data, classification)
         data["radar"].correct_atten(attenuations)
-        data["radar"].calc_errors(attenuations, classification)
+        data["radar"].calc_errors(attenuations, classification.is_clutter)
         quality = classify.fetch_quality(data, classification, attenuations)
         cloudnet_arrays = _prepare_output()
         date = data["radar"].get_date()
@@ -382,12 +394,6 @@ CATEGORIZE_ATTRIBUTES = {
         definition=DEFINITIONS["quality_bits"],
         units="1",
     ),
-    "rainfall_rate": MetaData(
-        long_name="Rainfall rate",
-        standard_name="rainfall_rate",
-        units="m s-1",
-        comment="Fill values denote rain with undefined intensity.",
-    ),
     "radar_liquid_atten": MetaData(
         long_name="Two-way radar attenuation due to liquid water",
         units="dB",
@@ -409,4 +415,9 @@ CATEGORIZE_ATTRIBUTES = {
         units="1",
         comment=COMMENTS["liquid_prob"],
     ),
+    "rain_detected": MetaData(
+        long_name="Rain detected",
+        units="1",
+        comment="1 = rain detected, 0 = no rain detected",
+    ),
 }

{cloudnetpy-1.57.0 → cloudnetpy-1.58.1}/cloudnetpy/categorize/classify.py

@@ -66,12 +66,11 @@ def classify_measurements(data: dict) -> ClassificationResult:
     bits[4] = _find_aerosols(obs, bits[1], bits[0])
     bits[4][filtered_ice] = False
     return ClassificationResult(
-        _bits_to_integer(bits),
-        obs.is_rain,
-        obs.is_clutter,
-        obs.rainfall_rate,
-        insect_prob,
-        liquid_prob,
+        category_bits=_bits_to_integer(bits),
+        is_rain=obs.is_rain,
+        is_clutter=obs.is_clutter,
+        insect_prob=insect_prob,
+        liquid_prob=liquid_prob,
     )
 
 

{cloudnetpy-1.57.0 → cloudnetpy-1.58.1}/cloudnetpy/categorize/containers.py

@@ -1,11 +1,10 @@
-import logging
 from dataclasses import dataclass
 
 import numpy as np
-import skimage
 from numpy import ma
 
 from cloudnetpy import utils
+from cloudnetpy.constants import MM_H_TO_M_S
 
 
 @dataclass
@@ -15,7 +14,6 @@ class ClassificationResult:
     category_bits: np.ndarray
     is_rain: np.ndarray
     is_clutter: np.ndarray
-    rainfall_rate: np.ndarray
     insect_prob: np.ndarray
     liquid_prob: np.ndarray | None
 
@@ -42,12 +40,12 @@ class ClassData:
         radar_type (str): Radar identifier.
         is_rain (ndarray): 2D boolean array denoting rain.
         is_clutter (ndarray): 2D boolean array denoting clutter.
-        rainfall_rate: 1D rain rate.
         altitude: site altitude.
 
     """
 
     def __init__(self, data: dict):
+        self.data = data
         self.z = data["radar"].data["Z"][:]
         self.v = data["radar"].data["v"][:]
         self.v_sigma = data["radar"].data["v_sigma"][:]
@@ -61,61 +59,39 @@ class ClassData:
         self.model_type = data["model"].source_type
         self.beta = data["lidar"].data["beta"][:]
         self.lwp = data["mwr"].data["lwp"][:]
-        self.is_rain = _find_rain_from_radar_echo(self.z, self.time)
-        self.rainfall_rate = _find_rainfall_rate(self.is_rain, data["radar"])
+        self.is_rain = self._find_profiles_with_rain()
         self.is_clutter = _find_clutter(self.v, self.is_rain)
         self.altitude = data["radar"].altitude
         self.lv0_files = data["lv0_files"]
         self.date = data["radar"].get_date()
 
-
-def _find_rain_from_radar_echo(
-    z: np.ndarray,
-    time: np.ndarray,
-    time_buffer: int = 5,
-) -> np.ndarray:
-    """Find profiles affected by rain.
-
-    Rain is present in such profiles where the radar echo in
-    the third range gate is > 0 dB. To make sure we do not include any
-    rainy profiles, we also flag a few profiles before and after
-    detections as raining.
-
-    Args:
-        z: Radar echo.
-        time: Time vector.
-        time_buffer: Time in minutes.
-
-    Returns:
-        1D Boolean array denoting profiles with rain.
-
-    """
-    filled = False
-    is_rain = ma.array(z[:, 3] > 0, dtype=bool).filled(filled)
-    is_rain = skimage.morphology.remove_small_objects(
-        is_rain,
-        2,
-        connectivity=1,
-    )  # Filter hot pixels
-    n_profiles = len(time)
-    n_steps = utils.n_elements(time, time_buffer, "time")
-    for ind in np.where(is_rain)[0]:
-        ind1 = max(0, ind - n_steps)
-        ind2 = min(ind + n_steps, n_profiles)
-        is_rain[ind1 : ind2 + 1] = True
-    return is_rain
-
-
-def _find_rainfall_rate(is_rain: np.ndarray, radar) -> np.ndarray:
-    rainfall_rate = ma.zeros(len(is_rain))
-    rainfall_rate[is_rain] = ma.masked
-    if "rainfall_rate" in radar.data:
-        radar_rainfall_rate = radar.data["rainfall_rate"].data
-        ind = np.where(~radar_rainfall_rate.mask)
-        rainfall_rate[ind] = radar_rainfall_rate[ind]
-    else:
-        logging.info("No measured rain rate available")
-    return rainfall_rate
+    def _find_profiles_with_rain(self) -> np.ndarray:
+        is_rain = self._find_rain_from_radar_echo()
+        rain_from_disdrometer = self._find_rain_from_disdrometer()
+        ind = ~rain_from_disdrometer.mask
+        is_rain[ind] = rain_from_disdrometer[ind]
+        return is_rain
+
+    def _find_rain_from_radar_echo(self) -> np.ndarray:
+        gate_number = 3
+        threshold = 0
+        z = self.z[:, gate_number]
+        return np.where((~ma.getmaskarray(z)) & (z > threshold), 1, 0)
+
+    def _find_rain_from_disdrometer(self) -> ma.MaskedArray:
+        threshold_mm_h = 0.25  # Standard threshold for drizzle -> rain
+        threshold_particles = 10  # This is arbitrary and should be better tested
+        threshold_rate = threshold_mm_h * MM_H_TO_M_S
+        try:
+            rainfall_rate = self.data["disdrometer"].data["rainfall_rate"].data
+            n_particles = self.data["disdrometer"].data["n_particles"].data
+            is_rain = ma.array(
+                (rainfall_rate > threshold_rate) & (n_particles > threshold_particles),
+                dtype=int,
+            )
+        except (AttributeError, KeyError):
+            is_rain = ma.masked_all(self.time.shape, dtype=int)
+        return is_rain
 
 
 def _find_clutter(

cloudnetpy-1.58.1/cloudnetpy/categorize/disdrometer.py

@@ -0,0 +1,53 @@
+"""Mwr module, containing the :class:`Mwr` class."""
+import logging
+
+import numpy as np
+from numpy import ma
+from scipy.interpolate import interp1d
+
+from cloudnetpy.categorize.lidar import get_gap_ind
+from cloudnetpy.datasource import DataSource
+
+
+class Disdrometer(DataSource):
+    """Disdrometer class, child of DataSource.
+
+    Args:
+    ----
+         full_path: Cloudnet Level 1b disdrometer file.
+
+    """
+
+    def __init__(self, full_path: str):
+        super().__init__(full_path)
+        self._init_rainfall_rate()
+
+    def interpolate_to_grid(self, time_grid: np.ndarray) -> None:
+        for key, array in self.data.items():
+            self.data[key].data = self._interpolate(array.data, time_grid)
+
+    def _init_rainfall_rate(self) -> None:
+        keys = ("rainfall_rate", "n_particles")
+        for key in keys:
+            self.append_data(self.dataset.variables[key][:], key)
+
+    def _interpolate(self, y: ma.MaskedArray, x_new: np.ndarray) -> np.ndarray:
+        if ma.getmask(y) is ma.nomask:
+            non_masked_indices = np.arange(len(y))
+        elif y.mask.all():
+            return ma.masked_all(x_new.shape)
+        else:
+            non_masked_indices = np.where(~y.mask)[0]
+        non_masked_values = y[non_masked_indices]
+        non_masked_time = self.time[non_masked_indices]
+        fun = interp1d(non_masked_time, non_masked_values, fill_value="extrapolate")
+        interpolated_array = ma.array(fun(x_new))
+        max_time = 1 / 60  # min -> fraction hour
+        mask_ind = get_gap_ind(non_masked_time, x_new, max_time)
+
+        if len(mask_ind) > 0:
+            msg = f"Unable to interpolate disdrometer for {len(mask_ind)} time steps"
+            logging.warning(msg)
+            interpolated_array[mask_ind] = ma.masked
+
+        return interpolated_array

{cloudnetpy-1.57.0 → cloudnetpy-1.58.1}/cloudnetpy/categorize/insects.py

@@ -147,7 +147,8 @@ def _screen_insects(
         prob[(above_liquid == 1) & (insect_prob_no_ldr > 0)] = 0
 
     def _screen_rainy_profiles() -> None:
-        prob[obs.is_rain == 1, :] = 0
+        rain_smoothed = _smooth_rain(obs.time, obs.is_rain)
+        prob[rain_smoothed == 1, :] = 0
 
     prob = np.copy(insect_prob)
     _screen_liquid_layers()
@@ -155,3 +156,15 @@ def _screen_insects(
     _screen_above_liquid()
     _screen_rainy_profiles()
     return prob
+
+
+def _smooth_rain(time: np.ndarray, is_rain: np.ndarray) -> np.ndarray:
+    is_rain_smoothed = np.copy(is_rain)
+    time_buffer = 5  # minutes
+    n_profiles = len(is_rain)
+    n_steps = utils.n_elements(time, time_buffer, "time")
+    for rain_idx in np.where(is_rain)[0]:
+        idx_start = max(0, rain_idx - n_steps)
+        idx_end = min(rain_idx + n_steps, n_profiles)
+        is_rain_smoothed[idx_start : idx_end + 1] = True
+    return is_rain_smoothed

{cloudnetpy-1.57.0 → cloudnetpy-1.58.1}/cloudnetpy/categorize/lidar.py

@@ -44,8 +44,8 @@ class Lidar(DataSource):
             height_new,
         )
         # Mask data points that are too far from the original grid
-        time_gap_ind = _get_gap_ind(self.time[indices], time_new, max_time)
-        height_gap_ind = _get_gap_ind(self.height, height_new, max_height)
+        time_gap_ind = get_gap_ind(self.time[indices], time_new, max_time)
+        height_gap_ind = get_gap_ind(self.height, height_new, max_height)
         self._mask_profiles(beta_interp, time_gap_ind, "time")
         self._mask_profiles(beta_interp, height_gap_ind, "height")
         self.data["beta"].data = beta_interp
@@ -69,7 +69,7 @@ class Lidar(DataSource):
         self.append_data(3.0, "beta_bias")
 
 
-def _get_gap_ind(grid: np.ndarray, new_grid: np.ndarray, threshold: float) -> list[int]:
+def get_gap_ind(grid: np.ndarray, new_grid: np.ndarray, threshold: float) -> list[int]:
     return [
         ind
         for ind, value in enumerate(new_grid)

{cloudnetpy-1.57.0 → cloudnetpy-1.58.1}/cloudnetpy/categorize/radar.py

@@ -7,8 +7,7 @@ from numpy import ma
 from scipy import constants
 
 from cloudnetpy import utils
-from cloudnetpy.categorize.classify import ClassificationResult
-from cloudnetpy.constants import SEC_IN_HOUR
+from cloudnetpy.constants import GHZ_TO_HZ, SEC_IN_HOUR, SPEED_OF_LIGHT
 from cloudnetpy.datasource import DataSource
 
 
@@ -69,8 +68,10 @@ class Radar(DataSource):
                     )
                 case "v_sigma":
                     array.calc_linear_std(self.time, time_new)
-                case "width" | "rainfall_rate":
+                case "width":
                     array.rebin_data(self.time, time_new)
+                case "rainfall_rate":
+                    array.rebin_data(self.time, time_new, mask_zeros=False)
                 case _:
                     continue
         return bad_time_indices
@@ -214,7 +215,7 @@ class Radar(DataSource):
     def calc_errors(
         self,
         attenuations: dict,
-        classification: ClassificationResult,
+        is_clutter: np.ndarray,
     ) -> None:
         """Calculates uncertainties of radar echo.
 
@@ -223,7 +224,7 @@ class Radar(DataSource):
 
         Args:
             attenuations: 2-D attenuations due to atmospheric gases.
-            classification: The :class:`ClassificationResult` instance.
+            is_clutter: 2-D boolean array denoting pixels contaminated by clutter.
 
         References:
             The method is based on Hogan R. and O'Connor E., 2004,
@@ -235,17 +236,24 @@ class Radar(DataSource):
             """Returns sensitivity of radar as function of altitude."""
             mean_gas_atten = ma.mean(attenuations["radar_gas_atten"], axis=0)
             z_sensitivity = z_power_min + log_range + mean_gas_atten
-            zc = ma.median(ma.array(z, mask=~classification.is_clutter), axis=0)
+            zc = ma.median(ma.array(z, mask=~is_clutter), axis=0)
             valid_values = np.logical_not(zc.mask)
             z_sensitivity[valid_values] = zc[valid_values]
             return z_sensitivity
 
         def _calc_error() -> np.ndarray | float:
-            if "width" not in self.data:
-                return 0.3
-            z_precision = 4.343 * (
-                1 / np.sqrt(_number_of_independent_pulses())
-                + utils.db2lin(z_power_min - z_power) / 3
+            """Returns error of radar as function of altitude.
+
+            References:
+                Hogan, R. J., 1998: Dual-wavelength radar studies of clouds.
+                PhD Thesis, University of Reading, UK.
+
+            """
+            noise_threshold = 3
+            n_pulses = _number_of_independent_pulses()
+            ln_to_log10 = 10 / np.log(10)
+            z_precision = (ln_to_log10 / np.sqrt(n_pulses)) * (
+                1 + (utils.db2lin(z_power_min - z_power) / noise_threshold)
             )
             gas_error = attenuations["radar_gas_atten"] * 0.1
             liq_error = attenuations["liquid_atten_err"].filled(0)
@@ -254,16 +262,22 @@ class Radar(DataSource):
             return z_error
 
         def _number_of_independent_pulses() -> float:
+            """Returns number of independent pulses.
+
+            References:
+                Atlas, D., 1964: Advances in radar meteorology.
+                Advances in Geophys., 10, 318-478.
+
+            """
+            if "width" not in self.data:
+                default_width = 0.3
+                width = np.zeros_like(self.data["Z"][:])
+                width[~width.mask] = default_width
+            else:
+                width = self.data["width"][:]
             dwell_time = utils.mdiff(self.time) * SEC_IN_HOUR
-            return (
-                dwell_time
-                * self.radar_frequency
-                * 1e9
-                * 4
-                * np.sqrt(math.pi)
-                * self.data["width"][:]
-                / 3e8
-            )
+            wl = SPEED_OF_LIGHT / (self.radar_frequency * GHZ_TO_HZ)
+            return 4 * np.sqrt(math.pi) * dwell_time * width / wl
 
         def _calc_z_power_min() -> float:
             if ma.all(z_power.mask):

{cloudnetpy-1.57.0 → cloudnetpy-1.58.1}/cloudnetpy/cloudnetarray.py

@@ -54,7 +54,9 @@ class CloudnetArray:
         """Masks data from given indices."""
         self.data[ind] = ma.masked
 
-    def rebin_data(self, time: np.ndarray, time_new: np.ndarray) -> list:
+    def rebin_data(
+        self, time: np.ndarray, time_new: np.ndarray, *, mask_zeros: bool = True
+    ) -> list:
         """Rebins `data` in time.
 
         Args:
@@ -66,12 +68,14 @@ class CloudnetArray:
 
         """
         if self.data.ndim == 1:
-            self.data = utils.rebin_1d(time, self.data, time_new)
+            self.data = utils.rebin_1d(time, self.data, time_new, mask_zeros=mask_zeros)
             bad_indices = list(np.where(self.data == ma.masked)[0])
         else:
             if not isinstance(self.data, ma.MaskedArray):
                 self.data = ma.masked_array(self.data)
-            self.data, bad_indices = utils.rebin_2d(time, self.data, time_new)
+            self.data, bad_indices = utils.rebin_2d(
+                time, self.data, time_new, mask_zeros=mask_zeros
+            )
         return bad_indices
 
     def fetch_attributes(self) -> list:

{cloudnetpy-1.57.0 → cloudnetpy-1.58.1}/cloudnetpy/constants.py

@@ -20,8 +20,12 @@ RS: Final = 287.058
 RHO_ICE: Final = 917
 
 # other
+SPEED_OF_LIGHT: Final = 3.0e8
 SEC_IN_MINUTE: Final = 60
 SEC_IN_HOUR: Final = 3600
 SEC_IN_DAY: Final = 86400
 MM_TO_M: Final = 1e-3
 G_TO_KG: Final = 1e-3
+M_S_TO_MM_H: Final = SEC_IN_HOUR / MM_TO_M
+MM_H_TO_M_S: Final = 1 / M_S_TO_MM_H
+GHZ_TO_HZ: Final = 1e9

{cloudnetpy-1.57.0 → cloudnetpy-1.58.1}/cloudnetpy/instruments/disdrometer/parsivel.py

@@ -372,7 +372,10 @@ def _parse_spectrum(tokens: Iterator[str]) -> np.ndarray:
     return np.array(values, dtype="i2").reshape((32, 32))
 
 
-PARSERS: dict[str, Callable[[Iterator[str]], Any]] = {
+ParserType = Callable[[Iterator[str]], Any]
+
+
+PARSERS: dict[str, ParserType] = {
     "I_heating": _parse_float,
     "T_sensor": _parse_int,
     "_T_pcb": _parse_int,
@@ -401,6 +404,16 @@ PARSERS: dict[str, Callable[[Iterator[str]], Any]] = {
     "visibility": _parse_int,
 }
 
+EMPTY_VALUES: dict[ParserType, Any] = {
+    _parse_int: 0,
+    _parse_float: 0.0,
+    _parse_date: datetime.date(2000, 1, 1),
+    _parse_time: datetime.time(12, 0, 0),
+    _parse_datetime: datetime.datetime(2000, 1, 1),
+    _parse_vector: np.zeros(32, dtype=float),
+    _parse_spectrum: np.zeros((32, 32), dtype="i2"),
+}
+
 
 def _parse_headers(line: str) -> list[str]:
     return [CSV_HEADERS[header.strip()] for header in line.split(";")]
@@ -508,7 +521,7 @@ def _read_toa5(filename: str | PathLike) -> dict[str, list]:
         return data
 
 
-def _read_typ_op4a(lines: list[str]) -> dict[str, list]:
+def _read_typ_op4a(lines: list[str]) -> dict[str, Any]:
     """Read output of "CS/PA" command. The output starts with line "TYP OP4A"
     followed by one line per measured variable in format: <number>:<value>.
     Output ends with characters: <ETX><CR><LF><NUL>. Lines are separated by
@@ -527,7 +540,7 @@ def _read_typ_op4a(lines: list[str]) -> dict[str, list]:
             continue
         parser = PARSERS.get(varname, next)
         tokens = value.split(";")
-        data[varname] = [parser(iter(tokens))]
+        data[varname] = parser(iter(tokens))
     return data
 
 
@@ -538,15 +551,26 @@ def _read_fmi(content: str):
     - output of "CS/PA" command without non-printable characters at the end
     - "]\n"
     """
-    output: dict[str, Any] = defaultdict(list)
+    output: dict[str, list] = {"_datetime": []}
     for m in re.finditer(
         r"\[(?P<year>\d+)-(?P<month>\d+)-(?P<day>\d+) "
         r"(?P<hour>\d+):(?P<minute>\d+):(?P<second>\d+)"
         r"(?P<output>[^\]]*)\]",
         content,
     ):
-        for key, value in _read_typ_op4a(m["output"].splitlines()).items():
+        try:
+            record = _read_typ_op4a(m["output"].splitlines())
+        except ValueError:
+            continue
+
+        for key, value in record.items():
+            if key not in output:
+                output[key] = [None] * len(output["_datetime"])
             output[key].append(value)
+        for key in output:
+            if key not in record and key != "_datetime":
+                output[key].append(None)
+
         output["_datetime"].append(
             datetime.datetime(
                 int(m["year"]),
@@ -577,6 +601,7 @@ def _read_parsivel(
             data = _read_toa5(filename)
         elif "TYP OP4A" in lines[0]:
             data = _read_typ_op4a(lines)
+            data = {key: [value] for key, value in data.items()}
         elif "Date" in lines[0]:
             headers = _parse_headers(lines[0])
             data = _read_rows(headers, lines[1:])
@@ -597,6 +622,17 @@ def _read_parsivel(
             combined_data[key].extend(values)
     if timestamps is not None:
         combined_data["_datetime"] = list(timestamps)
-    result = {key: np.array(value) for key, value in combined_data.items()}
+    result = {}
+    for key, value in combined_data.items():
+        array = np.array(
+            [
+                x
+                if x is not None
+                else (EMPTY_VALUES[PARSERS[key]] if key in PARSERS else "")
+                for x in value
+            ]
+        )
+        mask = [np.full(array.shape[1:], x is None) for x in value]
+        result[key] = ma.array(array, mask=mask)
     result["time"] = result["_datetime"].astype("datetime64[s]")
     return result

{cloudnetpy-1.57.0 → cloudnetpy-1.58.1}/cloudnetpy/output.py

@@ -317,7 +317,7 @@ def add_time_attribute(
 
 
 def add_source_attribute(attributes: dict, data: dict) -> dict:
-    """Adds source attribute."""
+    """Adds source attribute to variables."""
     variables = {
         "radar": (
             "v",
@@ -334,8 +334,11 @@ def add_source_attribute(attributes: dict, data: dict) -> dict:
         "lidar": ("beta", "lidar_wavelength"),
         "mwr": ("lwp",),
         "model": ("uwind", "vwind", "Tw", "q", "pressure", "temperature"),
+        "disdrometer": ("rainfall_rate",),
     }
     for instrument, keys in variables.items():
+        if data[instrument] is None:
+            continue
         source = data[instrument].dataset.source
         for key in keys:
             if key in attributes:

{cloudnetpy-1.57.0 → cloudnetpy-1.58.1}/cloudnetpy/plotting/plotting.py

@@ -514,6 +514,9 @@ class Plot2D(Plot):
         if figure_data.height is None:
             msg = "No height information in the file."
             raise ValueError(msg)
+        if self._data.ndim < 2:
+            msg = "Data has to be 2D."
+            raise PlottingError(msg)
         alt = figure_data.height
         if figure_data.options.max_y is None:
             return alt