cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/instruments/disdrometer/thies.py

@@ -1,17 +1,82 @@
+import datetime
+from collections import defaultdict
+from os import PathLike
+from typing import Any
+from uuid import UUID
+
+import numpy as np
+from numpy import ma
+
 from cloudnetpy import output
-from cloudnetpy.exceptions import DisdrometerDataError
+from cloudnetpy.cloudnetarray import CloudnetArray
+from cloudnetpy.constants import MM_TO_M, SEC_IN_HOUR
+from cloudnetpy.exceptions import DisdrometerDataError, ValidTimeStampError
 from cloudnetpy.instruments import instruments
+from cloudnetpy.instruments.toa5 import read_toa5
+from cloudnetpy.utils import get_uuid
 
-from .common import ATTRIBUTES, THIES, Disdrometer, _format_thies_date
+from .common import ATTRIBUTES, Disdrometer
+
+TELEGRAM4 = [
+    (1, "_serial_number"),
+    (2, "_software_version"),
+    (3, "_date"),
+    (4, "_time"),
+    (5, "_synop_5min_ww"),
+    (6, "_synop_5min_WaWa"),
+    (7, "_metar_5min_4678"),
+    (8, "_rainfall_rate_5min"),
+    (9, "synop_WW"),  # 1min
+    (10, "synop_WaWa"),  # 1min
+    (11, "_metar_1_min_4678"),
+    (12, "rainfall_rate_1min_total"),
+    (13, "rainfall_rate"),  # liquid, mm h-1
+    (14, "rainfall_rate_1min_solid"),
+    (15, "_precipition_amount"),  # mm
+    (16, "visibility"),
+    (17, "radar_reflectivity"),
+    (18, "measurement_quality"),
+    (19, "maximum_hail_diameter"),
+    (20, "status_laser"),
+    (21, "static_signal"),
+    (22, "status_T_laser_analogue"),
+    (23, "status_T_laser_digital"),
+    (24, "status_I_laser_analogue"),
+    (25, "status_I_laser_digital"),
+    (26, "status_sensor_supply"),
+    (27, "status_laser_heating"),
+    (28, "status_receiver_heating"),
+    (29, "status_temperature_sensor"),
+    (30, "status_heating_supply"),
+    (31, "status_heating_housing"),
+    (32, "status_heating_heads"),
+    (33, "status_heating_carriers"),
+    (34, "status_laser_power"),
+    (35, "_status_reserve"),
+    (36, "T_interior"),
+    (37, "T_laser_driver"),  # 0-80 C
+    (38, "I_mean_laser"),
+    (39, "V_control"),  # mV 4005-4015
+    (40, "V_optical_output"),  # mV 2300-6500
+    (41, "V_sensor_supply"),  # 1/10V
+    (42, "I_heating_laser_head"),  # mA
+    (43, "I_heating_receiver_head"),  # mA
+    (44, "T_ambient"),  # C
+    (45, "_V_heating_supply"),
+    (46, "_I_housing"),
+    (47, "_I_heating_heads"),
+    (48, "_I_heating_carriers"),
+    (49, "n_particles"),
+]
 
 
 def thies2nc(
-    disdrometer_file: str,
-    output_file: str,
+    disdrometer_file: str | PathLike,
+    output_file: str | PathLike,
     site_meta: dict,
-    uuid: str | None = None,
-    date: str | None = None,
-) -> str:
+    uuid: str | UUID | None = None,
+    date: str | datetime.date | None = None,
+) -> UUID:
     """Converts Thies-LNM disdrometer data into Cloudnet Level 1b netCDF file.
 
     Args:
@@ -36,103 +101,205 @@ def thies2nc(
         >>> uuid = thies2nc('thies-lnm.log', 'thies-lnm.nc', site_meta)
 
     """
+    if isinstance(date, str):
+        date = datetime.date.fromisoformat(date)
+    uuid = get_uuid(uuid)
     try:
-        disdrometer = Thies(disdrometer_file, site_meta)
+        disdrometer = Thies(disdrometer_file, site_meta, date)
     except (ValueError, IndexError) as err:
-        raise DisdrometerDataError("Can not read disdrometer file") from err
-    if date is not None:
-        disdrometer.validate_date(date)
-    disdrometer.init_data()
-    if date is not None:
-        disdrometer.sort_timestamps()
-        disdrometer.remove_duplicate_timestamps()
+        msg = "Unable to read disdrometer file"
+        raise DisdrometerDataError(msg) from err
+    disdrometer.sort_timestamps()
+    disdrometer.remove_duplicate_timestamps()
+    disdrometer.mask_invalid_values()
     disdrometer.add_meta()
     disdrometer.convert_units()
     attributes = output.add_time_attribute(ATTRIBUTES, disdrometer.date)
     output.update_attributes(disdrometer.data, attributes)
-    uuid = output.save_level1b(disdrometer, output_file, uuid)
+    output.save_level1b(disdrometer, output_file, uuid)
     return uuid
 
 
 class Thies(Disdrometer):
-    def __init__(self, filename: str, site_meta: dict):
-        super().__init__(filename, site_meta, THIES)
+    def __init__(
+        self,
+        filename: str | PathLike,
+        site_meta: dict,
+        expected_date: datetime.date | None = None,
+    ) -> None:
+        super().__init__()
+        self.instrument = instruments.THIES
         self.n_velocity = 20
         self.n_diameter = 22
-        self.date = self._init_date()
+        self.site_meta = site_meta
+        self.raw_data: dict[str, Any] = defaultdict(list)
+        self._read_data(filename)
+        self._screen_time(expected_date)
+        self.data = {}
+        self._append_data()
         self._create_velocity_vectors()
         self._create_diameter_vectors()
-        self.instrument = instruments.THIES
 
-    def init_data(self):
-        """According to
-        https://www.biral.com/wp-content/uploads/2015/01/5.4110.xx_.xxx_.pdf
-        """
-        column_and_key = [
-            (1, "_serial_number"),
-            (2, "_software_version"),
-            (3, "_date"),
-            (4, "_time"),
-            (5, "_synop_5min_ww"),
-            (6, "_synop_5min_WaWa"),
-            (7, "_metar_5min_4678"),
-            (8, "_rainfall_rate_5min"),
-            (9, "synop_WW"),  # 1min
-            (10, "synop_WaWa"),  # 1min
-            (11, "_metar_1_min_4678"),
-            (12, "rainfall_rate_1min_total"),
-            (13, "rainfall_rate"),  # liquid, mm h-1
-            (14, "rainfall_rate_1min_solid"),
-            (15, "_precipition_amount"),  # mm
-            (16, "visibility"),
-            (17, "radar_reflectivity"),
-            (18, "measurement_quality"),
-            (19, "maximum_hail_diameter"),
-            (20, "status_laser"),
-            (21, "static_signal"),
-            (22, "status_T_laser_analogue"),
-            (23, "status_T_laser_digital"),
-            (24, "status_I_laser_analogue"),
-            (25, "status_I_laser_digital"),
-            (26, "status_sensor_supply"),
-            (27, "status_laser_heating"),
-            (28, "status_receiver_heating"),
-            (29, "status_temperature_sensor"),
-            (30, "status_heating_supply"),
-            (31, "status_heating_housing"),
-            (32, "status_heating_heads"),
-            (33, "status_heating_carriers"),
-            (34, "status_laser_power"),
-            (35, "_status_reserve"),
-            (36, "T_interior"),
-            (37, "T_laser_driver"),  # 0-80 C
-            (38, "I_mean_laser"),
-            (39, "V_control"),  # mV 4005-4015
-            (40, "V_optical_output"),  # mV 2300-6500
-            (41, "V_sensor_supply"),  # 1/10V
-            (42, "I_heating_laser_head"),  # mA
-            (43, "I_heating_receiver_head"),  # mA
-            (44, "T_ambient"),  # C
-            (45, "_V_heating_supply"),
-            (46, "_I_housing"),
-            (47, "_I_heating_heads"),
-            (48, "_I_heating_carriers"),
-            (49, "n_particles"),
-        ]
-        self._append_data(column_and_key)
-        self._append_spectra()
-
-    def _init_date(self) -> list:
-        first_date = self._file_data["scalars"][0][3]
-        first_date = _format_thies_date(first_date)
-        return first_date.split("-")
-
-    def _create_velocity_vectors(self):
+    def convert_units(self) -> None:
+        mmh_to_ms = SEC_IN_HOUR / MM_TO_M
+        c_to_k = 273.15
+        self._convert_data(("rainfall_rate_1min_total",), mmh_to_ms)
+        self._convert_data(("rainfall_rate",), mmh_to_ms)
+        self._convert_data(("rainfall_rate_1min_solid",), mmh_to_ms)
+        self._convert_data(("diameter", "diameter_spread", "diameter_bnds"), 1e3)
+        self._convert_data(("V_sensor_supply",), 10)
+        self._convert_data(("I_mean_laser",), 100)
+        self._convert_data(("T_interior",), c_to_k, method="add")
+        self._convert_data(("T_ambient",), c_to_k, method="add")
+        self._convert_data(("T_laser_driver",), c_to_k, method="add")
+
+    def _read_data(self, filename: str | PathLike) -> None:
+        with open(filename, errors="ignore") as file:
+            first_line = file.readline()
+        if "TOA5" in first_line:
+            _units, _process, rows = read_toa5(filename)
+            for row in rows:
+                self._read_line(row["RawString"], row["TIMESTAMP"])
+        elif first_line.lower().startswith("datetime [utc];"):
+            with open(filename, errors="ignore") as file:
+                first_line = file.readline()
+                for line in file:
+                    timestamp, telegram = line.split(";", maxsplit=1)
+                    fixed_telegram = telegram.strip().rstrip(";") + ";"
+                    parsed_timestamp = datetime.datetime.strptime(
+                        timestamp, "%Y-%m-%d %H:%M:%S"
+                    )
+                    self._read_line(fixed_telegram, parsed_timestamp)
+        else:
+            with open(filename, errors="ignore") as file:
+                for line in file:
+                    self._read_line(line)
+        if len(self.raw_data["time"]) == 0:
+            raise ValidTimeStampError
+        for key, value in self.raw_data.items():
+            array = np.array(value)
+            if key == "time":
+                array = array.astype("datetime64[s]")
+            self.raw_data[key] = array
+
+    def _append_data(self) -> None:
+        for key, values in self.raw_data.items():
+            if key.startswith("_"):
+                continue
+            name_out = key
+            values_out = values
+            match key:
+                case "spectrum":
+                    name_out = "data_raw"
+                    dimensions = ["time", "diameter", "velocity"]
+                case "time":
+                    dimensions = []
+                    base = values[0].astype("datetime64[D]")
+                    values_out = (values - base) / np.timedelta64(1, "h")
+                case _:
+                    dimensions = ["time"]
+            self.data[name_out] = CloudnetArray(
+                values_out, name_out, dimensions=dimensions
+            )
+
+        first_id = self.raw_data["_serial_number"][0]
+        for sensor_id in self.raw_data["_serial_number"]:
+            if sensor_id != first_id:
+                msg = "Multiple serial numbers are not supported"
+                raise DisdrometerDataError(msg)
+        self.serial_number = first_id
+
+    def _read_line(self, line: str, timestamp: datetime.datetime | None = None) -> None:
+        raw_values = line.strip().strip(";").split(";")
+        # Support custom truncated format used in Leipzig LIM.
+        expected_columns = self.site_meta.get("truncate_columns", 521) - 1
+        # Length matches telegram 4 or 5 (has 4 additional columns).
+        if len(raw_values) not in (expected_columns, expected_columns + 4):
+            return
+        for i, key in TELEGRAM4:
+            if i >= expected_columns - 1:
+                break
+            value: Any
+            if key == "_date":
+                value = _parse_date(raw_values[i])
+            elif key == "_time":
+                value = _parse_time(raw_values[i])
+            elif key in (
+                "I_heating",
+                "T_ambient",
+                "T_interior",
+                "T_laser_driver",
+                "V_power_supply",
+                "_precipition_amount",
+                "_rainfall_rate_5min",
+                "maximum_hail_diameter",
+                "radar_reflectivity",
+                "rainfall_rate",
+                "rainfall_rate_1min_solid",
+                "rainfall_rate_1min_total",
+            ):
+                value = float(raw_values[i])
+            elif key in (
+                "_serial_number",
+                "_software_version",
+                "_metar_5min_4678",
+                "_metar_1_min_4678",
+            ):
+                value = raw_values[i]
+            else:
+                value = int(raw_values[i])
+            self.raw_data[key].append(value)
+        if expected_columns > 79:
+            self.raw_data["spectrum"].append(
+                np.array(list(map(int, raw_values[79:519])), dtype="i2").reshape(
+                    self.n_diameter, self.n_velocity
+                )
+            )
+        if timestamp is not None:
+            self.raw_data["time"].append(timestamp)
+        else:
+            self.raw_data["time"].append(
+                datetime.datetime.combine(
+                    self.raw_data["_date"][-1], self.raw_data["_time"][-1]
+                )
+            )
+
+    def _screen_time(self, expected_date: datetime.date | None = None) -> None:
+        if expected_date is None:
+            self.date = self.raw_data["time"][0].astype(object).date()
+            return
+        self.date = expected_date
+        valid_mask = self.raw_data["time"].astype("datetime64[D]") == self.date
+        if np.count_nonzero(valid_mask) == 0:
+            msg = f"No data found on {expected_date}"
+            raise DisdrometerDataError(msg)
+        for key in self.raw_data:
+            self.raw_data[key] = self.raw_data[key][valid_mask]
+
+    def mask_invalid_values(self) -> None:
+        rainfall_rate = self.data["rainfall_rate"]
+        rainfall_rate.data = ma.masked_where(
+            rainfall_rate.data > 999, rainfall_rate.data
+        )
+
+    def _create_velocity_vectors(self) -> None:
         n_values = [5, 6, 7, 1, 1]
         spreads = [0.2, 0.4, 0.8, 1, 10]
-        self.store_vectors(self.data, n_values, spreads, "velocity")
+        self.store_vectors(n_values, spreads, "velocity")
 
-    def _create_diameter_vectors(self):
+    def _create_diameter_vectors(self) -> None:
         n_values = [3, 6, 13]
         spreads = [0.125, 0.25, 0.5]
-        self.store_vectors(self.data, n_values, spreads, "diameter", start=0.125)
+        self.store_vectors(n_values, spreads, "diameter", start=0.125)
+
+
+def _parse_date(date: str) -> datetime.date:
+    day, month, year = map(int, date.split("."))
+    if year < 100:
+        year += 2000
+    return datetime.date(year, month, day)
+
+
+def _parse_time(time: str) -> datetime.time:
+    hour, minute, second = map(int, time.split(":"))
+    return datetime.time(hour, minute, second)

cloudnetpy/instruments/fd12p.py

@@ -0,0 +1,201 @@
+import datetime
+import logging
+import math
+import re
+from os import PathLike
+from uuid import UUID
+
+import numpy as np
+from numpy import ma
+
+from cloudnetpy import output
+from cloudnetpy.exceptions import ValidTimeStampError
+from cloudnetpy.instruments import instruments
+from cloudnetpy.instruments.cloudnet_instrument import CSVFile
+from cloudnetpy.metadata import MetaData
+from cloudnetpy.utils import get_uuid
+
+
+def fd12p2nc(
+    input_file: str | PathLike,
+    output_file: str | PathLike,
+    site_meta: dict,
+    uuid: str | UUID | None = None,
+    date: str | datetime.date | None = None,
+) -> UUID:
+    """Converts Vaisala FD12P into Cloudnet Level 1b netCDF file.
+
+    Args:
+        input_file: Filename of input file.
+        output_file: Output filename.
+        site_meta: Dictionary containing information about the site. Required key
+            is `name`.
+        uuid: Set specific UUID for the file.
+        date: Expected date of the measurements as YYYY-MM-DD or datetime.date object.
+
+    Returns:
+        UUID of the generated file.
+
+    Raises:
+        ValidTimeStampError: No valid timestamps found.
+    """
+    if isinstance(date, str):
+        date = datetime.date.fromisoformat(date)
+    uuid = get_uuid(uuid)
+    fd12p = FD12P(site_meta)
+    fd12p.parse_input_file(input_file, date)
+    fd12p.add_data()
+    fd12p.add_date()
+    fd12p.screen_all_masked()
+    fd12p.sort_timestamps()
+    fd12p.remove_duplicate_timestamps()
+    fd12p.convert_units()
+    fd12p.normalize_cumulative_amount("precipitation_amount")
+    fd12p.normalize_cumulative_amount("snowfall_amount")
+    fd12p.add_site_geolocation()
+    attributes = output.add_time_attribute(ATTRIBUTES, fd12p.date)
+    output.update_attributes(fd12p.data, attributes)
+    output.save_level1b(fd12p, output_file, uuid)
+    return uuid
+
+
+class FD12P(CSVFile):
+    def __init__(self, site_meta: dict) -> None:
+        super().__init__(site_meta)
+        self.instrument = instruments.FD12P
+        self._data = {
+            key: []
+            for key in (
+                "time",
+                "visibility",
+                "synop_WaWa",
+                "precipitation_rate",
+                "precipitation_amount",
+                "snowfall_amount",
+            )
+        }
+
+    def parse_input_file(
+        self, filename: str | PathLike, expected_date: datetime.date | None = None
+    ) -> None:
+        # In Lindenberg, format is date and time followed by Message 2 without
+        # non-printable characters.
+        with open(filename) as file:
+            invalid_lines = 0
+            for line in file:
+                try:
+                    columns = line.split()
+                    if len(columns) != 13:
+                        msg = "Invalid column count"
+                        raise ValueError(msg)  # noqa: TRY301
+                    date = _parse_date(columns[0])
+                    time = _parse_time(columns[1])
+                    visibility = _parse_int(columns[4])
+                    synop = _parse_int(columns[7])
+                    p_rate = _parse_float(columns[10])  # mm/h
+                    p_amount = _parse_float(columns[11])  # mm
+                    s_amount = _parse_int(columns[12])  # mm
+                    self._data["time"].append(datetime.datetime.combine(date, time))
+                    self._data["visibility"].append(visibility)
+                    self._data["synop_WaWa"].append(synop)
+                    self._data["precipitation_rate"].append(p_rate)
+                    self._data["precipitation_amount"].append(p_amount)
+                    self._data["snowfall_amount"].append(s_amount)
+                except ValueError:
+                    invalid_lines += 1
+                    continue
+        if invalid_lines:
+            logging.info("Skipped %d lines", invalid_lines)
+        for key in ("visibility", "synop_WaWa", "snowfall_amount"):
+            values = np.array(
+                [0 if math.isnan(x) else x for x in self._data[key]], dtype=np.int32
+            )
+            mask = np.array([math.isnan(x) for x in self._data[key]])
+            self._data[key] = ma.array(values, mask=mask)
+        self._data["snowfall_amount"] = self._data["snowfall_amount"].astype(np.float32)
+        if expected_date:
+            self._data["time"] = [
+                d for d in self._data["time"] if d.date() == expected_date
+            ]
+        if not self._data["time"]:
+            raise ValidTimeStampError
+
+    def convert_units(self) -> None:
+        precipitation_rate = self.data["precipitation_rate"][:]
+        self.data["precipitation_rate"].data = (
+            precipitation_rate / 3600 / 1000
+        )  # mm/h -> m/s
+        for key in ("precipitation_amount", "snowfall_amount"):
+            self.data[key].data = self.data[key][:] / 1000  # mm -> m
+
+    def screen_all_masked(self) -> None:
+        is_valid = np.ones_like(self.data["time"][:], dtype=np.bool_)
+        for key in self.data:
+            if key == "time":
+                continue
+            is_valid &= ma.getmaskarray(self.data[key][:])
+        self.screen_time_indices(~is_valid)
+
+
+def _parse_date(date: str) -> datetime.date:
+    match = re.fullmatch(r"(?P<day>\d{2})\.(?P<month>\d{2})\.(?P<year>\d{4})", date)
+    if match is None:
+        msg = f"Invalid date: {date}"
+        raise ValueError(msg)
+    return datetime.date(int(match["year"]), int(match["month"]), int(match["day"]))
+
+
+def _parse_time(time: str) -> datetime.time:
+    match = re.fullmatch(
+        r"(?P<hour>\d{2}):(?P<minute>\d{2})(:(?P<second>\d{2}))?", time
+    )
+    if match is None:
+        msg = f"Invalid time: {time}"
+        raise ValueError(msg)
+    return datetime.time(
+        int(match["hour"]),
+        int(match["minute"]),
+        int(match["second"]) if match["second"] is not None else 0,
+    )
+
+
+def _parse_int(value: str) -> float:
+    if "/" in value:
+        return math.nan
+    return int(value)
+
+
+def _parse_float(value: str) -> float:
+    if "/" in value:
+        return math.nan
+    return float(value)
+
+
+ATTRIBUTES = {
+    "visibility": MetaData(
+        long_name="Meteorological optical range (MOR) visibility",
+        units="m",
+        standard_name="visibility_in_air",
+        dimensions=("time",),
+    ),
+    "precipitation_rate": MetaData(
+        long_name="Precipitation rate",
+        standard_name="lwe_precipitation_rate",
+        units="m s-1",
+        dimensions=("time",),
+    ),
+    "precipitation_amount": MetaData(
+        long_name="Precipitation amount",
+        standard_name="lwe_thickness_of_precipitation_amount",
+        units="m",
+        comment="Cumulated precipitation since 00:00 UTC",
+        dimensions=("time",),
+    ),
+    "snowfall_amount": MetaData(
+        long_name="Snowfall amount",
+        units="m",
+        standard_name="thickness_of_snowfall_amount",
+        comment="Cumulated snow since 00:00 UTC",
+        dimensions=("time",),
+    ),
+}