cloudnetpy 1.60.3__py3-none-any.whl → 1.61.0__py3-none-any.whl

cloudnetpy/concat_lib.py

@@ -5,8 +5,10 @@ import numpy as np
 from cloudnetpy.exceptions import InconsistentDataError
 
 
-def truncate_netcdf_file(filename: str, output_file: str, n_profiles: int) -> None:
-    """Truncates netcdf file in 'time' dimension taking only n_profiles.
+def truncate_netcdf_file(
+    filename: str, output_file: str, n_profiles: int, dim_name: str = "time"
+) -> None:
+    """Truncates netcdf file in dim_name dimension taking only n_profiles.
     Useful for creating small files for tests.
     """
     with (
@@ -14,7 +16,7 @@ def truncate_netcdf_file(filename: str, output_file: str, n_profiles: int) -> No
         netCDF4.Dataset(output_file, "w", format=nc.data_model) as nc_new,
     ):
         for dim in nc.dimensions:
-            dim_len = None if dim == "time" else nc.dimensions[dim].size
+            dim_len = None if dim == dim_name else nc.dimensions[dim].size
             nc_new.createDimension(dim, dim_len)
         for attr in nc.ncattrs():
             value = getattr(nc, attr)
@@ -30,7 +32,7 @@ def truncate_netcdf_file(filename: str, output_file: str, n_profiles: int) -> No
                 zlib=True,
                 fill_value=fill_value,
             )
-            if dimensions and "time" in dimensions[0]:
+            if dimensions and dim_name in dimensions[0]:
                 if array.ndim == 1:
                     var[:] = array[:n_profiles]
                 if array.ndim == 2:

cloudnetpy/instruments/disdrometer/common.py

@@ -1,207 +1,46 @@
 """Module for reading / converting disdrometer data."""
-import logging
 
 import numpy as np
-from numpy import ma
 
-from cloudnetpy import utils
 from cloudnetpy.cloudnetarray import CloudnetArray
-from cloudnetpy.constants import MM_TO_M, SEC_IN_HOUR, SEC_IN_MINUTE
-from cloudnetpy.exceptions import DisdrometerDataError, ValidTimeStampError
 from cloudnetpy.instruments.cloudnet_instrument import CloudnetInstrument
-from cloudnetpy.instruments.vaisala import values_to_dict
 from cloudnetpy.metadata import MetaData
 
-PARSIVEL = "OTT Parsivel-2"
-THIES = "Thies-LNM"
-
 
 class Disdrometer(CloudnetInstrument):
-    def __init__(self, filename: str, site_meta: dict, source: str):
-        super().__init__()
-        self.filename = filename
-        self.site_meta = site_meta
-        self.source = source
-        self.date: list[str] = []
-        self.sensor_id = None
-        self.n_diameter: int = 0
-        self.n_velocity: int = 0
-        self._file_data = self._read_file()
-
-    def convert_units(self) -> None:
-        mmh_to_ms = SEC_IN_HOUR / MM_TO_M
-        c_to_k = 273.15
-        self._convert_data(("rainfall_rate_1min_total",), mmh_to_ms)
-        self._convert_data(("rainfall_rate",), mmh_to_ms)
-        self._convert_data(("rainfall_rate_1min_solid",), mmh_to_ms)
-        self._convert_data(("diameter", "diameter_spread", "diameter_bnds"), 1e3)
-        self._convert_data(("V_sensor_supply",), 10)
-        self._convert_data(("I_mean_laser",), 100)
-        self._convert_data(("T_sensor",), c_to_k, method="add")
-        self._convert_data(("T_interior",), c_to_k, method="add")
-        self._convert_data(("T_ambient",), c_to_k, method="add")
-        self._convert_data(("T_laser_driver",), c_to_k, method="add")
-
     def add_meta(self) -> None:
-        valid_names = ("latitude", "longitude", "altitude")
+        valid_keys = ("latitude", "longitude", "altitude")
         for key, value in self.site_meta.items():
             name = key.lower()
-            if name in valid_names:
+            if name in valid_keys:
                 self.data[name] = CloudnetArray(float(value), name)
 
-    def validate_date(self, expected_date: str) -> None:
-        valid_ind = []
-        for ind, row in enumerate(self._file_data["scalars"]):
-            if self.source == PARSIVEL:
-                raise NotImplementedError
-            date = _format_thies_date(row[3])
-            if date == expected_date:
-                valid_ind.append(ind)
-        if not valid_ind:
-            raise ValidTimeStampError
-        for key, value in self._file_data.items():
-            if value:
-                self._file_data[key] = [self._file_data[key][ind] for ind in valid_ind]
-        self.date = expected_date.split("-")
-
-    def sort_time(self) -> None:
-        time = self.data["time"][:]
-        ind = time.argsort()
-        for _, data in self.data.items():
-            if data.data.shape[0] == len(time):
-                data.data[:] = data.data[ind]
-
-    def _read_file(self) -> dict:
-        data: dict = {"scalars": [], "vectors": [], "spectra": []}
-        with open(self.filename, encoding="utf8", errors="ignore") as file:
-            for row in file:
-                if row == "\n":
-                    continue
-                if self.source == PARSIVEL:
-                    values = row.split(";")
-                    if "\n" in values:
-                        values.remove("\n")
-                    if len(values) != 1106:
-                        continue
-                    data["scalars"].append(values[:18])
-                    data["vectors"].append(values[18 : 18 + 64])
-                    data["spectra"].append(values[18 + 64 :])
-                else:
-                    values = row.split(";")
-                    data["scalars"].append(values[:79])
-                    data["spectra"].append(values[79:-2])
-        if len(data["scalars"]) == 0:
-            raise ValueError
-        return data
-
-    def _append_data(self, column_and_key: list) -> None:
-        indices, keys = zip(*column_and_key, strict=True)
-        data = self._parse_useful_data(indices)
-        data_dict = values_to_dict(keys, data)
-        for key in keys:
-            if key.startswith("_"):
-                continue
-            invalid_value = -9999.0
-            float_array = ma.array([])
-            for value_str in data_dict[key]:
-                try:
-                    float_array = ma.append(float_array, float(value_str))
-                except ValueError:
-                    logging.warning(
-                        "Invalid character: %s, masking a data point",
-                        value_str,
-                    )
-                    float_array = ma.append(float_array, invalid_value)
-            float_array[float_array == invalid_value] = ma.masked
-            if key in (
-                "rainfall_rate",
-                "radar_reflectivity",
-                "T_sensor",
-                "I_heating",
-                "V_power_supply",
-                "T_interior",
-                "T_ambient",
-                "T_laser_driver",
-            ):
-                data_type = "f4"
-            else:
-                data_type = "i4"
-            self.data[key] = CloudnetArray(float_array, key, data_type=data_type)
-        self.data["time"] = self._convert_time(data_dict)
-        if "_serial_number" in data_dict:
-            first_id = data_dict["_serial_number"][0]
-            for sensor_id in data_dict["_serial_number"]:
-                if sensor_id != first_id:
-                    msg = "Multiple serial numbers are not supported"
-                    raise DisdrometerDataError(msg)
-
-            self.serial_number = first_id
-
-    def _parse_useful_data(self, indices: tuple) -> list:
-        data = []
-        for row in self._file_data["scalars"]:
-            useful_data = [row[ind] for ind in indices]
-            data.append(useful_data)
-        return data
-
-    def _convert_time(self, data: dict) -> CloudnetArray:
-        seconds = []
-        for timestamp in data["_time"]:
-            if self.source == PARSIVEL:
-                raise NotImplementedError
-            hour, minute, sec = timestamp.split(":")
-            seconds.append(
-                int(hour) * SEC_IN_HOUR + int(minute) * SEC_IN_MINUTE + int(sec)
-            )
-        return CloudnetArray(utils.seconds2hours(np.array(seconds)), "time")
-
     def _convert_data(self, keys: tuple, value: float, method: str = "divide") -> None:
         for key in keys:
             if key in self.data:
                 if method == "divide":
-                    self.data[key].data /= value
+                    self.data[key].data = self.data[key].data / value
                 elif method == "add":
-                    self.data[key].data += value
+                    self.data[key].data = self.data[key].data + value
                 else:
                     raise ValueError
 
-    def _append_spectra(self) -> None:
-        array = ma.masked_all(
-            (len(self._file_data["scalars"]), self.n_diameter, self.n_velocity),
-        )
-        for time_ind, row in enumerate(self._file_data["spectra"]):
-            values = _parse_int(row)
-            if len(values) != self.n_diameter * self.n_velocity:
-                continue
-            array[time_ind, :, :] = np.reshape(
-                values,
-                (self.n_diameter, self.n_velocity),
-            )
-        self.data["data_raw"] = CloudnetArray(
-            array,
-            "data_raw",
-            dimensions=("time", "diameter", "velocity"),
-            data_type="i2",
-        )
-
-    @classmethod
     def store_vectors(
-        cls,
-        data,
+        self,
         n_values: list,
         spreads: list,
         name: str,
         start: float = 0.0,
     ):
-        mid, bounds, spread = cls._create_vectors(n_values, spreads, start)
-        data[name] = CloudnetArray(mid, name, dimensions=(name,))
+        mid, bounds, spread = self._create_vectors(n_values, spreads, start)
+        self.data[name] = CloudnetArray(mid, name, dimensions=(name,))
         key = f"{name}_spread"
-        data[key] = CloudnetArray(spread, key, dimensions=(name,))
+        self.data[key] = CloudnetArray(spread, key, dimensions=(name,))
         key = f"{name}_bnds"
-        data[key] = CloudnetArray(bounds, key, dimensions=(name, "nv"))
+        self.data[key] = CloudnetArray(bounds, key, dimensions=(name, "nv"))
 
-    @staticmethod
     def _create_vectors(
+        self,
         n_values: list[int],
         spreads: list[float],
         start: float,
@@ -221,24 +60,6 @@ class Disdrometer(CloudnetInstrument):
         return mid_value, bounds, spread
 
 
-def _format_thies_date(date: str) -> str:
-    day, month, year = date.split(".")
-    year = f"20{year}"
-    return f"{year}-{month.zfill(2)}-{day.zfill(2)}"
-
-
-def _parse_int(row: np.ndarray) -> np.ndarray:
-    values = ma.masked_all((len(row),))
-    for ind, value in enumerate(row):
-        try:
-            value_int = int(value)
-            if value_int != 0:
-                values[ind] = value_int
-        except ValueError:
-            pass
-    return values
-
-
 ATTRIBUTES = {
     "velocity": MetaData(
         long_name="Center fall velocity of precipitation particles",

cloudnetpy/instruments/disdrometer/parsivel.py

@@ -6,7 +6,7 @@ from collections import defaultdict
 from collections.abc import Callable, Iterable, Iterator, Sequence
 from itertools import islice
 from os import PathLike
-from typing import Any, Literal
+from typing import Any
 
 import numpy as np
 from numpy import ma
@@ -16,7 +16,6 @@ from cloudnetpy.cloudnetarray import CloudnetArray
 from cloudnetpy.constants import MM_TO_M, SEC_IN_HOUR
 from cloudnetpy.exceptions import DisdrometerDataError
 from cloudnetpy.instruments import instruments
-from cloudnetpy.instruments.cloudnet_instrument import CloudnetInstrument
 
 from .common import ATTRIBUTES, Disdrometer
 
@@ -78,7 +77,7 @@ def parsivel2nc(
     return output.save_level1b(disdrometer, output_file, uuid)
 
 
-class Parsivel(CloudnetInstrument):
+class Parsivel(Disdrometer):
     def __init__(
         self,
         filenames: Iterable[str | PathLike],
@@ -142,12 +141,12 @@ class Parsivel(CloudnetInstrument):
     def _create_velocity_vectors(self) -> None:
         n_values = [10, 5, 5, 5, 5, 2]
         spreads = [0.1, 0.2, 0.4, 0.8, 1.6, 3.2]
-        Disdrometer.store_vectors(self.data, n_values, spreads, "velocity")
+        self.store_vectors(n_values, spreads, "velocity")
 
     def _create_diameter_vectors(self) -> None:
         n_values = [10, 5, 5, 5, 5, 2]
         spreads = [0.125, 0.25, 0.5, 1, 2, 3]
-        Disdrometer.store_vectors(self.data, n_values, spreads, "diameter")
+        self.store_vectors(n_values, spreads, "diameter")
 
     def mask_invalid_values(self) -> None:
         if variable := self.data.get("number_concentration"):
@@ -166,32 +165,6 @@ class Parsivel(CloudnetInstrument):
         if variable := self.data.get("number_concentration"):
             variable.data = np.power(10, variable.data).round().astype(np.uint32)
 
-    def add_meta(self) -> None:
-        valid_keys = ("latitude", "longitude", "altitude")
-        for key, value in self.site_meta.items():
-            name = key.lower()
-            if name in valid_keys:
-                self.data[name] = CloudnetArray(float(value), name)
-
-    def _convert_data(
-        self,
-        keys: tuple[str, ...],
-        value: float,
-        method: Literal["divide", "add"] = "divide",
-    ) -> None:
-        for key in keys:
-            if key not in self.data:
-                continue
-            variable = self.data[key]
-            if method == "divide":
-                variable.data = variable.data.astype("f4") / value
-                variable.data_type = "f4"
-            elif method == "add":
-                variable.data = variable.data.astype("f4") + value
-                variable.data_type = "f4"
-            else:
-                raise ValueError
-
 
 CSV_HEADERS = {
     "Date": "_date",

cloudnetpy/instruments/disdrometer/thies.py

@@ -1,8 +1,70 @@
+import datetime
+from collections import defaultdict
+from os import PathLike
+from typing import Any
+
+import numpy as np
+
 from cloudnetpy import output
+from cloudnetpy.cloudnetarray import CloudnetArray
+from cloudnetpy.constants import MM_TO_M, SEC_IN_HOUR
 from cloudnetpy.exceptions import DisdrometerDataError
 from cloudnetpy.instruments import instruments
+from cloudnetpy.instruments.toa5 import read_toa5
 
-from .common import ATTRIBUTES, THIES, Disdrometer, _format_thies_date
+from .common import ATTRIBUTES, Disdrometer
+
+TELEGRAM4 = [
+    (1, "_serial_number"),
+    (2, "_software_version"),
+    (3, "_date"),
+    (4, "_time"),
+    (5, "_synop_5min_ww"),
+    (6, "_synop_5min_WaWa"),
+    (7, "_metar_5min_4678"),
+    (8, "_rainfall_rate_5min"),
+    (9, "synop_WW"),  # 1min
+    (10, "synop_WaWa"),  # 1min
+    (11, "_metar_1_min_4678"),
+    (12, "rainfall_rate_1min_total"),
+    (13, "rainfall_rate"),  # liquid, mm h-1
+    (14, "rainfall_rate_1min_solid"),
+    (15, "_precipition_amount"),  # mm
+    (16, "visibility"),
+    (17, "radar_reflectivity"),
+    (18, "measurement_quality"),
+    (19, "maximum_hail_diameter"),
+    (20, "status_laser"),
+    (21, "static_signal"),
+    (22, "status_T_laser_analogue"),
+    (23, "status_T_laser_digital"),
+    (24, "status_I_laser_analogue"),
+    (25, "status_I_laser_digital"),
+    (26, "status_sensor_supply"),
+    (27, "status_laser_heating"),
+    (28, "status_receiver_heating"),
+    (29, "status_temperature_sensor"),
+    (30, "status_heating_supply"),
+    (31, "status_heating_housing"),
+    (32, "status_heating_heads"),
+    (33, "status_heating_carriers"),
+    (34, "status_laser_power"),
+    (35, "_status_reserve"),
+    (36, "T_interior"),
+    (37, "T_laser_driver"),  # 0-80 C
+    (38, "I_mean_laser"),
+    (39, "V_control"),  # mV 4005-4015
+    (40, "V_optical_output"),  # mV 2300-6500
+    (41, "V_sensor_supply"),  # 1/10V
+    (42, "I_heating_laser_head"),  # mA
+    (43, "I_heating_receiver_head"),  # mA
+    (44, "T_ambient"),  # C
+    (45, "_V_heating_supply"),
+    (46, "_I_housing"),
+    (47, "_I_heating_heads"),
+    (48, "_I_heating_carriers"),
+    (49, "n_particles"),
+]
 
 
 def thies2nc(
@@ -10,7 +72,7 @@ def thies2nc(
     output_file: str,
     site_meta: dict,
     uuid: str | None = None,
-    date: str | None = None,
+    date: str | datetime.date | None = None,
 ) -> str:
     """Converts Thies-LNM disdrometer data into Cloudnet Level 1b netCDF file.
 
@@ -36,17 +98,15 @@ def thies2nc(
         >>> uuid = thies2nc('thies-lnm.log', 'thies-lnm.nc', site_meta)
 
     """
+    if isinstance(date, str):
+        date = datetime.date.fromisoformat(date)
     try:
-        disdrometer = Thies(disdrometer_file, site_meta)
+        disdrometer = Thies(disdrometer_file, site_meta, date)
     except (ValueError, IndexError) as err:
         msg = "Unable to read disdrometer file"
         raise DisdrometerDataError(msg) from err
-    if date is not None:
-        disdrometer.validate_date(date)
-    disdrometer.init_data()
-    if date is not None:
-        disdrometer.sort_timestamps()
-        disdrometer.remove_duplicate_timestamps()
+    disdrometer.sort_timestamps()
+    disdrometer.remove_duplicate_timestamps()
     disdrometer.add_meta()
     disdrometer.convert_units()
     attributes = output.add_time_attribute(ATTRIBUTES, disdrometer.date)
@@ -55,84 +115,158 @@ def thies2nc(
 
 
 class Thies(Disdrometer):
-    def __init__(self, filename: str, site_meta: dict):
-        super().__init__(filename, site_meta, THIES)
+    def __init__(
+        self,
+        filename: str | PathLike,
+        site_meta: dict,
+        expected_date: datetime.date | None = None,
+    ):
+        super().__init__()
+        self.instrument = instruments.THIES
         self.n_velocity = 20
         self.n_diameter = 22
-        self.date = self._init_date()
+        self.site_meta = site_meta
+        self.raw_data: dict[str, Any] = defaultdict(list)
+        self._read_data(filename)
+        self._screen_time(expected_date)
+        self.data = {}
+        self._append_data()
         self._create_velocity_vectors()
         self._create_diameter_vectors()
-        self.instrument = instruments.THIES
 
-    def init_data(self) -> None:
-        """According to
-        https://www.biral.com/wp-content/uploads/2015/01/5.4110.xx_.xxx_.pdf
-        """
-        column_and_key = [
-            (1, "_serial_number"),
-            (2, "_software_version"),
-            (3, "_date"),
-            (4, "_time"),
-            (5, "_synop_5min_ww"),
-            (6, "_synop_5min_WaWa"),
-            (7, "_metar_5min_4678"),
-            (8, "_rainfall_rate_5min"),
-            (9, "synop_WW"),  # 1min
-            (10, "synop_WaWa"),  # 1min
-            (11, "_metar_1_min_4678"),
-            (12, "rainfall_rate_1min_total"),
-            (13, "rainfall_rate"),  # liquid, mm h-1
-            (14, "rainfall_rate_1min_solid"),
-            (15, "_precipition_amount"),  # mm
-            (16, "visibility"),
-            (17, "radar_reflectivity"),
-            (18, "measurement_quality"),
-            (19, "maximum_hail_diameter"),
-            (20, "status_laser"),
-            (21, "static_signal"),
-            (22, "status_T_laser_analogue"),
-            (23, "status_T_laser_digital"),
-            (24, "status_I_laser_analogue"),
-            (25, "status_I_laser_digital"),
-            (26, "status_sensor_supply"),
-            (27, "status_laser_heating"),
-            (28, "status_receiver_heating"),
-            (29, "status_temperature_sensor"),
-            (30, "status_heating_supply"),
-            (31, "status_heating_housing"),
-            (32, "status_heating_heads"),
-            (33, "status_heating_carriers"),
-            (34, "status_laser_power"),
-            (35, "_status_reserve"),
-            (36, "T_interior"),
-            (37, "T_laser_driver"),  # 0-80 C
-            (38, "I_mean_laser"),
-            (39, "V_control"),  # mV 4005-4015
-            (40, "V_optical_output"),  # mV 2300-6500
-            (41, "V_sensor_supply"),  # 1/10V
-            (42, "I_heating_laser_head"),  # mA
-            (43, "I_heating_receiver_head"),  # mA
-            (44, "T_ambient"),  # C
-            (45, "_V_heating_supply"),
-            (46, "_I_housing"),
-            (47, "_I_heating_heads"),
-            (48, "_I_heating_carriers"),
-            (49, "n_particles"),
-        ]
-        self._append_data(column_and_key)
-        self._append_spectra()
-
-    def _init_date(self) -> list:
-        first_date = self._file_data["scalars"][0][3]
-        first_date = _format_thies_date(first_date)
-        return first_date.split("-")
+    def convert_units(self) -> None:
+        mmh_to_ms = SEC_IN_HOUR / MM_TO_M
+        c_to_k = 273.15
+        self._convert_data(("rainfall_rate_1min_total",), mmh_to_ms)
+        self._convert_data(("rainfall_rate",), mmh_to_ms)
+        self._convert_data(("rainfall_rate_1min_solid",), mmh_to_ms)
+        self._convert_data(("diameter", "diameter_spread", "diameter_bnds"), 1e3)
+        self._convert_data(("V_sensor_supply",), 10)
+        self._convert_data(("I_mean_laser",), 100)
+        self._convert_data(("T_interior",), c_to_k, method="add")
+        self._convert_data(("T_ambient",), c_to_k, method="add")
+        self._convert_data(("T_laser_driver",), c_to_k, method="add")
+
+    def _read_data(self, filename: str | PathLike) -> None:
+        with open(filename) as file:
+            first_line = file.readline()
+        if "TOA5" in first_line:
+            for row in read_toa5(filename):
+                self._read_line(row["RawString"], row["TIMESTAMP"])
+        else:
+            with open(filename) as file:
+                for line in file:
+                    self._read_line(line)
+        for key, value in self.raw_data.items():
+            array = np.array(value)
+            if key == "time":
+                array = array.astype("datetime64[s]")
+            self.raw_data[key] = array
+
+    def _append_data(self) -> None:
+        for key, values in self.raw_data.items():
+            if key.startswith("_"):
+                continue
+            name_out = key
+            values_out = values
+            match key:
+                case "spectrum":
+                    name_out = "data_raw"
+                    dimensions = ["time", "diameter", "velocity"]
+                case "time":
+                    dimensions = []
+                    base = values[0].astype("datetime64[D]")
+                    values_out = (values - base) / np.timedelta64(1, "h")
+                case _:
+                    dimensions = ["time"]
+            self.data[name_out] = CloudnetArray(
+                values_out, name_out, dimensions=dimensions
+            )
+
+        first_id = self.raw_data["_serial_number"][0]
+        for sensor_id in self.raw_data["_serial_number"]:
+            if sensor_id != first_id:
+                msg = "Multiple serial numbers are not supported"
+                raise DisdrometerDataError(msg)
+        self.serial_number = first_id
+
+    def _read_line(self, line: str, timestamp: datetime.datetime | None = None):
+        raw_values = line.split(";")
+        if len(raw_values) != 521:
+            return
+        for i, key in TELEGRAM4:
+            value: Any
+            if key == "_date":
+                value = _parse_date(raw_values[i])
+            elif key == "_time":
+                value = _parse_time(raw_values[i])
+            elif key in (
+                "I_heating",
+                "T_ambient",
+                "T_interior",
+                "T_laser_driver",
+                "V_power_supply",
+                "_precipition_amount",
+                "_rainfall_rate_5min",
+                "maximum_hail_diameter",
+                "radar_reflectivity",
+                "rainfall_rate",
+                "rainfall_rate_1min_solid",
+                "rainfall_rate_1min_total",
+            ):
+                value = float(raw_values[i])
+            elif key in (
+                "_serial_number",
+                "_software_version",
+                "_metar_5min_4678",
+                "_metar_1_min_4678",
+            ):
+                value = raw_values[i]
+            else:
+                value = int(raw_values[i])
+            self.raw_data[key].append(value)
+        self.raw_data["spectrum"].append(
+            np.array(list(map(int, raw_values[79:-2])), dtype="i2").reshape(
+                self.n_diameter, self.n_velocity
+            )
+        )
+        if timestamp is not None:
+            self.raw_data["time"].append(timestamp)
+        else:
+            self.raw_data["time"].append(
+                datetime.datetime.combine(
+                    self.raw_data["_date"][-1], self.raw_data["_time"][-1]
+                )
+            )
+
+    def _screen_time(self, expected_date: datetime.date | None = None) -> None:
+        if expected_date is None:
+            self.date = self.raw_data["time"][0].astype(object).date()
+            return
+        self.date = expected_date
+        valid_mask = self.raw_data["time"].astype("datetime64[D]") == self.date
+        if np.count_nonzero(valid_mask) == 0:
+            msg = f"No data found on {expected_date}"
+            raise DisdrometerDataError(msg)
+        for key in self.raw_data:
+            self.raw_data[key] = self.raw_data[key][valid_mask]
 
     def _create_velocity_vectors(self) -> None:
         n_values = [5, 6, 7, 1, 1]
         spreads = [0.2, 0.4, 0.8, 1, 10]
-        self.store_vectors(self.data, n_values, spreads, "velocity")
+        self.store_vectors(n_values, spreads, "velocity")
 
     def _create_diameter_vectors(self) -> None:
         n_values = [3, 6, 13]
         spreads = [0.125, 0.25, 0.5]
-        self.store_vectors(self.data, n_values, spreads, "diameter", start=0.125)
+        self.store_vectors(n_values, spreads, "diameter", start=0.125)
+
+
+def _parse_date(date: str) -> datetime.date:
+    day, month, year = map(int, date.split("."))
+    return datetime.date(2000 + year, month, day)
+
+
+def _parse_time(time: str) -> datetime.time:
+    hour, minute, second = map(int, time.split(":"))
+    return datetime.time(hour, minute, second)

cloudnetpy/instruments/toa5.py

@@ -0,0 +1,45 @@
+import csv
+import datetime
+from os import PathLike
+from typing import Any
+
+
+def read_toa5(filename: str | PathLike) -> list[dict[str, Any]]:
+    """Read ASCII data from Campbell Scientific datalogger such as CR1000.
+
+    References
+        CR1000 Measurement and Control System.
+        https://s.campbellsci.com/documents/us/manuals/cr1000.pdf
+    """
+    with open(filename) as file:
+        reader = csv.reader(file)
+        origin_line = next(reader)
+        if len(origin_line) == 0 or origin_line[0] != "TOA5":
+            msg = "Invalid TOA5 file"
+            raise ValueError(msg)
+        header_line = next(reader)
+        _units_line = next(reader)
+        _process_line = next(reader)
+        output = []
+
+        row_template: dict[str, Any] = {}
+        for header in header_line:
+            if "(" in header:
+                row_template[header[: header.index("(")]] = []
+
+        for data_line in reader:
+            row = row_template.copy()
+            for key, value in zip(header_line, data_line, strict=False):
+                parsed_value: Any = value
+                if key == "TIMESTAMP":
+                    parsed_value = datetime.datetime.strptime(
+                        parsed_value, "%Y-%m-%d %H:%M:%S"
+                    )
+                elif key == "RECORD":
+                    parsed_value = int(parsed_value)
+                if "(" in key:
+                    row[key[: key.index("(")]].append(parsed_value)
+                else:
+                    row[key] = parsed_value
+            output.append(row)
+        return output

cloudnetpy/plotting/plotting.py

@@ -281,10 +281,11 @@ class SubPlot:
 
     def _read_plot_meta(self, file_type: str | None) -> PlotMeta:
         if self.options.plot_meta is not None:
-            return self.options.plot_meta
-        fallback = ATTRIBUTES["fallback"].get(self.variable.name, PlotMeta())
-        file_attributes = ATTRIBUTES.get(file_type or "", {})
-        plot_meta = file_attributes.get(self.variable.name, fallback)
+            plot_meta = self.options.plot_meta
+        else:
+            fallback = ATTRIBUTES["fallback"].get(self.variable.name, PlotMeta())
+            file_attributes = ATTRIBUTES.get(file_type or "", {})
+            plot_meta = file_attributes.get(self.variable.name, fallback)
         if plot_meta.clabel is None:
             plot_meta = plot_meta._replace(clabel=_reformat_units(self.variable.units))
         return plot_meta

cloudnetpy/version.py

@@ -1,4 +1,4 @@
 MAJOR = 1
-MINOR = 60
-PATCH = 3
+MINOR = 61
+PATCH = 0
 __version__ = f"{MAJOR}.{MINOR}.{PATCH}"

{cloudnetpy-1.60.3.dist-info → cloudnetpy-1.61.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudnetpy
-Version: 1.60.3
+Version: 1.61.0
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

{cloudnetpy-1.60.3.dist-info → cloudnetpy-1.61.0.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cloudnetpy/__init__.py,sha256=X_FqY-4yg5GUj5Edo14SToLEos6JIsC3fN-v1FUgQoA,43
 cloudnetpy/cloudnetarray.py,sha256=HT6bLtjnimOVbGrdjQBqD0F8GW0KWkn2qhaIGFMKLAY,6987
-cloudnetpy/concat_lib.py,sha256=YK5ho5msqwNxpPtPT8f2OewIJ8hTrbhCkaxHaBKLTI0,9809
+cloudnetpy/concat_lib.py,sha256=-pXH7xjU7nm7tWdgwnrV6wC-g4PZOzYVPMYm1oOud-M,9845
 cloudnetpy/constants.py,sha256=l7_ohQgLEQ6XEG9AMBarTPKp9OM8B1ElJ6fSN0ScdmM,733
 cloudnetpy/datasource.py,sha256=CSiKQGVEX459tagRjLrww6hZMZcc3r1sR2WcaTKTTWo,7864
 cloudnetpy/exceptions.py,sha256=wrI0bZTwmS5C_cqOmvlJ8XJSEFyzuD1eD4voGJc_Gjg,1584
@@ -8,7 +8,7 @@ cloudnetpy/metadata.py,sha256=v_VDo2vbdTxB0zIsfP69IcrwSKiRlLpsGdq6JPI4CoA,5306
 cloudnetpy/output.py,sha256=WoVTNuxni0DUr163vZ-_mDr1brXhY15XSlGMrq9Aoqw,14700
 cloudnetpy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloudnetpy/utils.py,sha256=0TlHm71YtSrKXBsRKctitnhQrvZPE-ulEVeAQW-oK58,27398
-cloudnetpy/version.py,sha256=fW-PBXVNQlnZz6np_OiXBmF_xTaIE-Lpj9n9IljdZOw,72
+cloudnetpy/version.py,sha256=yXaNRiv7_J3_1S8AdHMUGGwtfSHvLCTJ991S3zY0sbw,72
 cloudnetpy/categorize/__init__.py,sha256=gP5q3Vis1y9u9OWgA_idlbjfWXYN_S0IBSWdwBhL_uU,69
 cloudnetpy/categorize/atmos.py,sha256=fWW8ye_8HZASRAiYwURFKWzcGOYIA2RKeVxCq0lVOuM,12389
 cloudnetpy/categorize/atmos_utils.py,sha256=wndpwJxc2-QnNTkV8tc8I11Vs_WkNz9sVMX1fuGgUC4,3777
@@ -45,12 +45,13 @@ cloudnetpy/instruments/pollyxt.py,sha256=SccV9htZ5MWrK7JEleOr4hbmeTr-lKktUzAt7H9
 cloudnetpy/instruments/radiometrics.py,sha256=2ofeZ6KJ_JOWTd3UA-wSzJpM5cjN7R4jZeBLJCQKEYc,7624
 cloudnetpy/instruments/rpg.py,sha256=U8nEOlOI74f2lk2w4C4xKZCrW6AkDZpQZYE3yv7SNHE,17130
 cloudnetpy/instruments/rpg_reader.py,sha256=LAdXL3TmD5QzQbqtPOcemZji_qkXwmw6a6F8NmF6Zg8,11355
+cloudnetpy/instruments/toa5.py,sha256=xYJYEVNykCWqIsESno0eBIWqkYb-LHXjFjUp3EoqGDU,1565
 cloudnetpy/instruments/vaisala.py,sha256=GzESZvboOoXzWmmr9dC-y6oM6ogc-M-zT3KmBTaD0LI,14512
 cloudnetpy/instruments/weather_station.py,sha256=gTY3Y5UATqJo9Gld4hm7WdsKBwcF8WgNTIK2nOfl3Nc,5739
 cloudnetpy/instruments/disdrometer/__init__.py,sha256=lyjwttWvFvuwYxEkusoAvgRcbBmglmOp5HJOpXUqLWo,93
-cloudnetpy/instruments/disdrometer/common.py,sha256=3z2pVgDuUMxM5sIJc-wYhdGYSOUsAG_Mp6MI8mPlJ5w,15843
-cloudnetpy/instruments/disdrometer/parsivel.py,sha256=_W0uW3DhD2myzuOWi6Izv6W6dcg9EDQrolSnMRVnrSI,26545
-cloudnetpy/instruments/disdrometer/thies.py,sha256=h7EwZ9tn47UUMiYqDQ68vkXv4q0rEqX1ZeFXd7XJYNg,5050
+cloudnetpy/instruments/disdrometer/common.py,sha256=A9k4z4SwZaqORUca3fYVCG0aS1Emy84B2755kShGeY8,9040
+cloudnetpy/instruments/disdrometer/parsivel.py,sha256=WiL-vCjw9Gmb5irvW3AXddsyprp8MGOfqcVAlfy0zpc,25521
+cloudnetpy/instruments/disdrometer/thies.py,sha256=dNSpRlyZPjQDVA6cm-Xkh-ub1fwLxok-9-xZBxcYIJA,9645
 cloudnetpy/model_evaluation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloudnetpy/model_evaluation/file_handler.py,sha256=oUGIblcEWLLv16YKUch-M5KA-dGRAcuHa-9anP3xtX4,6447
 cloudnetpy/model_evaluation/metadata.py,sha256=7ZL87iDbaQJIMu8wfnMvb01cGVPkl8RtvEm_tt9uIHE,8413
@@ -93,7 +94,7 @@ cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py,sha256=Ra3r4V
 cloudnetpy/model_evaluation/tests/unit/test_tools.py,sha256=Ia_VrLdV2NstX5gbx_3AZTOAlrgLAy_xFZ8fHYVX0xI,3817
 cloudnetpy/plotting/__init__.py,sha256=lg9Smn4BI0dVBgnDLC3JVJ4GmwoSnO-qoSd4ApvwV6Y,107
 cloudnetpy/plotting/plot_meta.py,sha256=cLdCZrhbP-gaobS_zjcf8d2xVALzl7zh2qpttxCHyrg,15983
-cloudnetpy/plotting/plotting.py,sha256=Qm1eQK1eTnXtHzmhxerYrKMpZTCxoOvwS8f1IxsZGaM,32428
+cloudnetpy/plotting/plotting.py,sha256=bve91iM9RcWmKaZOFWxVh2y3DPmupI1944MMYDdv17I,32459
 cloudnetpy/products/__init__.py,sha256=2hRb5HG9hNrxH1if5laJkLeFeaZCd5W1q3hh4ewsX0E,273
 cloudnetpy/products/classification.py,sha256=0E9OUGR3uLCsS1nORwQu0SqW0_8uX7n6LlRcVhtzKw4,7845
 cloudnetpy/products/der.py,sha256=mam6jWV7A2h8V5WC3DIeFp6ou7UD1JOw9r7h2B0su-s,12403
@@ -107,8 +108,8 @@ cloudnetpy/products/mie_lu_tables.nc,sha256=It4fYpqJXlqOgL8jeZ-PxGzP08PMrELIDVe5
 cloudnetpy/products/mwr_tools.py,sha256=PRm5aCULccUehU-Byk55wYhhEHseMjoAjGBu5TSyHao,4621
 cloudnetpy/products/product_tools.py,sha256=rhx_Ru9FLlQqCNM-awoiHx18-Aq1eBwL9LiUaQoJs6k,10412
 docs/source/conf.py,sha256=IKiFWw6xhUd8NrCg0q7l596Ck1d61XWeVjIFHVSG9Og,1490
-cloudnetpy-1.60.3.dist-info/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
-cloudnetpy-1.60.3.dist-info/METADATA,sha256=iMOBIyUZ82_uBThmxlSrdizjFks8-tsGx14amFGcahI,5784
-cloudnetpy-1.60.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-cloudnetpy-1.60.3.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
-cloudnetpy-1.60.3.dist-info/RECORD,,
+cloudnetpy-1.61.0.dist-info/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
+cloudnetpy-1.61.0.dist-info/METADATA,sha256=ZxuqGJhzfPtzacU04IO88UaXFvUlchwavhzhduLpPtA,5784
+cloudnetpy-1.61.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+cloudnetpy-1.61.0.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
+cloudnetpy-1.61.0.dist-info/RECORD,,