cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/instruments/radiometrics.py

@@ -1,26 +1,35 @@
 """Module for reading Radiometrics MP3014 microwave radiometer data."""
+
 import csv
 import datetime
 import logging
+import math
 import os
+import re
 from operator import attrgetter
+from os import PathLike
+from pathlib import Path
 from typing import Any, NamedTuple
+from uuid import UUID
 
 import numpy as np
+import numpy.typing as npt
+from numpy import ma
 
 from cloudnetpy import output, utils
 from cloudnetpy.cloudnetarray import CloudnetArray
-from cloudnetpy.exceptions import ValidTimeStampError
+from cloudnetpy.exceptions import InconsistentDataError, ValidTimeStampError
 from cloudnetpy.instruments import instruments
+from cloudnetpy.metadata import MetaData
 
 
 def radiometrics2nc(
-    full_path: str,
-    output_file: str,
+    full_path: str | PathLike,
+    output_file: str | PathLike,
     site_meta: dict,
-    uuid: str | None = None,
+    uuid: str | UUID | None = None,
     date: str | datetime.date | None = None,
-) -> str:
+) -> UUID:
     """Converts Radiometrics .csv file into Cloudnet Level 1b netCDF file.
 
     Args:
@@ -43,28 +52,24 @@ def radiometrics2nc(
     """
     if isinstance(date, str):
         date = datetime.date.fromisoformat(date)
-
+    uuid = utils.get_uuid(uuid)
     if os.path.isdir(full_path):
-        valid_filenames = utils.get_sorted_filenames(full_path, ".csv")
+        valid_filenames = list(Path(full_path).iterdir())
     else:
-        valid_filenames = [full_path]
-
-    objs = []
-    for filename in valid_filenames:
-        obj = Radiometrics(filename)
-        obj.read_raw_data()
-        obj.read_data()
-        objs.append(obj)
-
+        valid_filenames = [Path(full_path)]
+    objs = [_read_file(filename) for filename in valid_filenames]
     radiometrics = RadiometricsCombined(objs, site_meta)
     radiometrics.screen_time(date)
+    radiometrics.sort_timestamps()
     radiometrics.time_to_fractional_hours()
     radiometrics.data_to_cloudnet_arrays()
     radiometrics.add_meta()
-    assert radiometrics.date is not None
-    attributes = output.add_time_attribute({}, radiometrics.date)
+    if radiometrics.date is None:
+        msg = "Failed to find valid timestamps from Radiometrics file(s)."
+        raise ValidTimeStampError(msg)
+    attributes = output.add_time_attribute(ATTRIBUTES, radiometrics.date)
     output.update_attributes(radiometrics.data, attributes)
-    uuid = output.save_level1b(radiometrics, output_file, uuid)
+    output.save_level1b(radiometrics, output_file, uuid)
     return uuid
 
 
@@ -76,32 +81,43 @@ class Record(NamedTuple):
     values: dict[str, Any]
 
 
-class Radiometrics:
-    """Reader for level 2 files of Radiometrics microwave radiometers.
+class RadiometricsMP:
+    """Reader for level 2 files (*.csv) from Radiometrics MP-3000A and similar
+    microwave radiometers.
 
     References:
         Radiometrics (2008). Profiler Operator's Manual: MP-3000A, MP-2500A,
         MP-1500A, MP-183A.
     """
 
-    def __init__(self, filename: str):
+    def __init__(self, filename: Path) -> None:
         self.filename = filename
         self.raw_data: list[Record] = []
         self.data: dict = {}
         self.instrument = instruments.RADIOMETRICS
+        self.ranges: list[str] = []
 
-    def read_raw_data(self):
+    def read_raw_data(self) -> None:
         """Reads Radiometrics raw data."""
         record_columns = {}
         unknown_record_types = set()
         rows = []
-        with open(self.filename, mode="r", encoding="utf8") as infile:
-            reader = csv.reader(infile)
+        with open(self.filename, encoding="utf8") as infile:
+            reader = csv.reader(infile, skipinitialspace=True)
             for row in reader:
                 if row[0] == "Record":
-                    assert row[1] == "Date/Time"
+                    if row[1] != "Date/Time":
+                        msg = "Unexpected header in Radiometrics file"
+                        raise RuntimeError(msg)
                     record_type = int(row[2])
-                    record_columns[record_type] = row[3:]
+                    columns = row[3:]
+                    record_columns[record_type] = columns
+                    if record_type in (10, 400):
+                        self.ranges = [
+                            column
+                            for column in columns
+                            if re.fullmatch(r"\d+\.\d+", column)
+                        ]
                 else:
                     record_type = int(row[2])
                     block_type = record_type // 10 * 10
@@ -109,45 +125,201 @@ class Radiometrics:
                     column_names = record_columns.get(block_type)
                     if column_names is None:
                         if record_type not in unknown_record_types:
-                            logging.info(f"Skipping unknown record type {record_type}")
+                            logging.info("Skipping unknown record type %d", record_type)
                             unknown_record_types.add(record_type)
                         continue
+
+                    row_trimmed = [value for value in row if value != ""]
+
                     record = Record(
                         row_number=int(row[0]),
                         timestamp=_parse_datetime(row[1]),
                         block_type=block_type,
                         block_index=block_index,
-                        values=dict(zip(column_names, row[3:])),
+                        values=dict(zip(column_names, row_trimmed[3:], strict=True)),
                     )
                     rows.append(record)
 
-        # The rows should be in sequence but sort them just to be sure.
-        rows.sort(key=attrgetter("row_number"))
+        self.raw_data = sorted(rows, key=attrgetter("row_number"))
 
-        for data_row in rows:
-            # Use the first row of a block and skip the rest which should
-            # contain the same values in the first columns.
-            if data_row.block_index == 0:
-                self.raw_data.append(data_row)
-
-    def read_data(self):
+    def read_data(self) -> None:
         """Reads values."""
         times = []
         lwps = []
         iwvs = []
+        irts = []
+        irt_times = []
+        temps = []
+        temp_times = []
+        rhs = []
+        rh_times = []
+        ahs = []
+        ah_times = []
+        block_titles = {}
+        superblock: list[Record] = []
+
+        def _parse_floats(record: Record) -> list[float]:
+            return [
+                float(record.values[column])
+                if record.values[column].replace(".", "", 1).isdigit()
+                else math.nan
+                for column in self.ranges
+            ]
+
+        def _process_superblock() -> None:
+            # There can be multiple 301 records but they don't have "LV2
+            # Processor" column. We can deduce the "LV2 Processor" values from
+            # 401 records in the same "superblock" before or after the 301
+            # records.
+            procs = []
+            for record in superblock:
+                if record.block_type == 400 and record.block_index == 1:
+                    procs.append(record.values["LV2 Processor"])
+            good_procs = ["0.00:19.80", "Zenith", "Zenith16", "Zenith18"]
+            curr_proc = 0
+            for record in superblock:
+                if record.block_type == 100:
+                    block_type = int(record.values["Record Type"]) - 1
+                    title = record.values["Title"]
+                    block_titles[block_type] = title
+                if title := block_titles.get(record.block_type + record.block_index):
+                    if record.values["LV2 Processor"] not in good_procs:
+                        continue
+                    if title == "Temperature (K)":
+                        temp_times.append(record.timestamp)
+                        temps.append(_parse_floats(record))
+                    elif title == "Relative Humidity (%)":
+                        rh_times.append(record.timestamp)
+                        rhs.append(_parse_floats(record))
+                    elif title == "Vapor Density (g/m^3)":
+                        ah_times.append(record.timestamp)
+                        ahs.append(_parse_floats(record))
+                elif record.block_type == 10:
+                    if record.block_index == 0:
+                        lwp = record.values["Lqint(mm)"]
+                        iwv = record.values["Vint(cm)"]
+                        irt = record.values["Tir(K)"]
+                        times.append(record.timestamp)
+                        lwps.append(float(lwp))
+                        iwvs.append(float(iwv))
+                        irt_times.append(record.timestamp)
+                        irts.append([float(irt)])
+                        temp_times.append(record.timestamp)
+                        temps.append(_parse_floats(record))
+                    elif record.block_index == 1:
+                        ah_times.append(record.timestamp)
+                        ahs.append(_parse_floats(record))
+                    elif record.block_index == 2:
+                        rh_times.append(record.timestamp)
+                        rhs.append(_parse_floats(record))
+                elif record.block_type == 200:
+                    irt_times.append(record.timestamp)
+                    irt = record.values["Tir(K)"]
+                    irts.append([float(irt)])
+                elif record.block_type == 300:
+                    if procs:
+                        curr_proc += 1
+                        if procs[curr_proc - 1] not in good_procs:
+                            continue
+                    lwp = record.values["Int. Liquid(mm)"]
+                    iwv = record.values["Int. Vapor(cm)"]
+                    times.append(record.timestamp)
+                    lwps.append(float(lwp))
+                    iwvs.append(float(iwv))
+
         for record in self.raw_data:
-            lwp = record.values.get("Lqint(mm)", record.values.get("Int. Liquid(mm)"))
-            iwv = record.values.get("Vint(cm)", record.values.get("Int. Vapor(cm)"))
-            if lwp is None and iwv is None:
-                continue
-            times.append(record.timestamp)
-            if lwp is not None:
-                lwps.append(float(lwp) * 1000)  # g / m2
-            if iwv is not None:
-                iwvs.append(float(iwv) * 100)  # g / m2
+            if record.block_type == 200 and superblock:
+                _process_superblock()
+                superblock.clear()
+            superblock.append(record)
+        if superblock:
+            _process_superblock()
+
         self.data["time"] = np.array(times, dtype="datetime64[s]")
-        self.data["lwp"] = np.array(lwps, dtype=float)
-        self.data["iwv"] = np.array(iwvs, dtype=float)
+        self.data["lwp"] = np.array(lwps)  # mm => kg m-2
+        self.data["iwv"] = np.array(iwvs) * 10  # cm => kg m-2
+        if irt_times:
+            self.data["irt"] = _find_closest(
+                np.array(irt_times, dtype="datetime64[s]"),
+                np.array(irts),
+                self.data["time"],
+            )
+        if temp_times:
+            self.data["temperature"] = _find_closest(
+                np.array(temp_times, dtype="datetime64[s]"),
+                np.array(temps),
+                self.data["time"],
+            )
+        if rh_times:
+            self.data["relative_humidity"] = _find_closest(
+                np.array(rh_times, dtype="datetime64[s]"),
+                np.array(rhs) / 100,  # % => 1
+                self.data["time"],
+            )
+        if ah_times:
+            self.data["absolute_humidity"] = _find_closest(
+                np.array(ah_times, dtype="datetime64[s]"),
+                np.array(ahs) / 1000,  # g m-3 => kg m-3
+                self.data["time"],
+            )
+
+
+class RadiometricsWVR:
+    """Reader for *.los files from Radiometrics WVR-1100 microwave
+    radiometer.
+    """
+
+    def __init__(self, filename: Path) -> None:
+        self.filename = filename
+        self.raw_data: dict = {}
+        self.data: dict = {}
+        self.instrument = instruments.RADIOMETRICS
+        self.ranges: list[str] = []
+
+    def read_raw_data(self) -> None:
+        with open(self.filename, encoding="utf8") as file:
+            for line in file:
+                columns = line.split()
+                if columns[:2] == ["date", "time"]:
+                    headers = columns
+                    break
+            else:
+                msg = "No headers found"
+                raise RuntimeError(msg)
+            for header in headers:
+                self.raw_data[header] = []
+            for line in file:
+                columns = line.split()
+                if len(columns) != len(headers):
+                    continue
+                for header, column in zip(headers, columns, strict=True):
+                    value: datetime.date | datetime.time | float
+                    if header == "date":
+                        value = _parse_date(column)
+                    elif header == "time":
+                        value = _parse_time(column)
+                    else:
+                        value = _parse_value(column)
+                    self.raw_data[header].append(value)
+
+    def read_data(self) -> None:
+        self.data["time"] = np.array(
+            [
+                datetime.datetime.combine(date, time)
+                for date, time in zip(
+                    self.raw_data["date"], self.raw_data["time"], strict=True
+                )
+            ],
+            dtype="datetime64[s]",
+        )
+        self.data["lwp"] = np.array(self.raw_data["LiqCM"]) * 10  # cm => kg m-2
+        self.data["iwv"] = np.array(self.raw_data["VapCM"]) * 10  # cm => kg m-2
+        is_zenith = np.abs(np.array(self.raw_data["ELact"]) - 90.0) < 1.0
+        tb23_valid = np.array(self.raw_data["TbSky23"]) > 0
+        tb31_valid = np.array(self.raw_data["TbSky31"]) > 0
+        is_valid = is_zenith & tb23_valid & tb31_valid
+        for key in self.data:
+            self.data[key] = self.data[key][is_valid]
 
 
 class RadiometricsCombined:
@@ -156,16 +328,25 @@ class RadiometricsCombined:
     date: datetime.date | None
     instrument: instruments.Instrument
 
-    def __init__(self, objs: list[Radiometrics], site_meta: dict):
+    def __init__(
+        self, objs: list[RadiometricsMP | RadiometricsWVR], site_meta: dict
+    ) -> None:
         self.site_meta = site_meta
         self.data = {}
         self.date = None
         for obj in objs:
+            if obj.ranges != objs[0].ranges:
+                msg = "Inconsistent range between files"
+                raise InconsistentDataError(msg)
             for key in obj.data:
                 self.data = utils.append_data(self.data, key, obj.data[key])
+        if objs[0].ranges:
+            ranges = [float(x) for x in objs[0].ranges]
+            self.data["range"] = np.array(ranges) * 1000  # m => km
+            self.data["height"] = self.data["range"] + self.site_meta["altitude"]
         self.instrument = instruments.RADIOMETRICS
 
-    def screen_time(self, expected_date: datetime.date | None):
+    def screen_time(self, expected_date: datetime.date | None) -> None:
         """Screens timestamps."""
         if expected_date is None:
             self.date = self.data["time"][0].astype(object).date()
@@ -175,24 +356,68 @@ class RadiometricsCombined:
         if np.count_nonzero(valid_mask) == 0:
             raise ValidTimeStampError
         for key in self.data:
+            if key in ("range", "height"):
+                continue
             self.data[key] = self.data[key][valid_mask]
 
-    def time_to_fractional_hours(self):
+    def sort_timestamps(self) -> None:
+        ind = np.argsort(self.data["time"])
+        for key in self.data:
+            if key in ("range", "height"):
+                continue
+            self.data[key] = self.data[key][ind]
+
+    def time_to_fractional_hours(self) -> None:
         base = self.data["time"][0].astype("datetime64[D]")
         self.data["time"] = (self.data["time"] - base) / np.timedelta64(1, "h")
 
-    def data_to_cloudnet_arrays(self):
+    def data_to_cloudnet_arrays(self) -> None:
         """Converts arrays to CloudnetArrays."""
         for key, array in self.data.items():
-            self.data[key] = CloudnetArray(array, key)
+            dimensions = (
+                ("time", "range")
+                if key in ("temperature", "relative_humidity", "absolute_humidity")
+                else None
+            )
+            array_masked = ma.masked_invalid(array) if np.isnan(array).any() else array
+            self.data[key] = CloudnetArray(array_masked, key, dimensions=dimensions)
 
-    def add_meta(self):
+    def add_meta(self) -> None:
         """Adds some metadata."""
         valid_keys = ("latitude", "longitude", "altitude")
         for key, value in self.site_meta.items():
-            key = key.lower()
-            if key in valid_keys:
-                self.data[key] = CloudnetArray(float(value), key)
+            name = key.lower()
+            if name in valid_keys:
+                self.data[name] = CloudnetArray(float(value), key)
+
+
+def _read_file(filename: Path) -> RadiometricsMP | RadiometricsWVR:
+    with open(filename) as f:
+        first_line = f.readline()
+    obj = (
+        RadiometricsWVR(filename)
+        if "RETRIEVAL COEFFICIENTS" in first_line
+        else RadiometricsMP(filename)
+    )
+    obj.read_raw_data()
+    obj.read_data()
+    return obj
+
+
+def _parse_value(text: str) -> float:
+    return math.nan if "*" in text else float(text)
+
+
+def _parse_date(text: str) -> datetime.date:
+    month, day, year = map(int, text.split("/"))
+    if year < 100:
+        year += 2000
+    return datetime.date(year, month, day)
+
+
+def _parse_time(text: str) -> datetime.time:
+    hour, minute, second = map(int, text.split(":"))
+    return datetime.time(hour, minute, second)
 
 
 def _parse_datetime(text: str) -> datetime.datetime:
@@ -201,4 +426,33 @@ def _parse_datetime(text: str) -> datetime.datetime:
     hour, minute, second = map(int, time.split(":"))
     if year < 100:
         year += 2000
-    return datetime.datetime(year, month, day, hour, minute, second)
+    return datetime.datetime(
+        year,
+        month,
+        day,
+        hour,
+        minute,
+        second,
+    )
+
+
+def _find_closest(x: npt.NDArray, y: npt.NDArray, x_new: npt.NDArray) -> npt.NDArray:
+    return y[np.argmin(np.abs(x_new - x[:, np.newaxis]), axis=0)]
+
+
+ATTRIBUTES = {
+    "irt": MetaData(
+        long_name="Infrared brightness temperatures",
+        units="K",
+        dimensions=("time", "ir_channel"),
+    ),
+    "temperature": MetaData(
+        long_name="Temperature", units="K", dimensions=("time", "range")
+    ),
+    "relative_humidity": MetaData(
+        long_name="Relative humidity",
+        standard_name="relative_humidity",
+        units="1",
+        dimensions=("time", "range"),
+    ),
+}

cloudnetpy/instruments/rain_e_h3.py

@@ -0,0 +1,171 @@
+import csv
+import datetime
+from os import PathLike
+from uuid import UUID
+
+import numpy as np
+
+from cloudnetpy import output
+from cloudnetpy.exceptions import ValidTimeStampError
+from cloudnetpy.instruments import instruments
+from cloudnetpy.instruments.cloudnet_instrument import CSVFile
+from cloudnetpy.utils import get_uuid
+
+
+def rain_e_h32nc(
+    input_file: str | PathLike,
+    output_file: str | PathLike,
+    site_meta: dict,
+    uuid: str | UUID | None = None,
+    date: str | datetime.date | None = None,
+) -> UUID:
+    """Converts rain_e_h3 rain-gauge into Cloudnet Level 1b netCDF file.
+
+    Args:
+        input_file: Filename of rain_e_h3 CSV file.
+        output_file: Output filename.
+        site_meta: Dictionary containing information about the site. Required key
+            is `name`.
+        uuid: Set specific UUID for the file.
+        date: Expected date of the measurements as YYYY-MM-DD or datetime.date object.
+
+    Returns:
+        UUID of the generated file.
+
+    Raises:
+        ValidTimeStampError: No valid timestamps found.
+    """
+    rain = RainEH3(site_meta)
+    if isinstance(date, str):
+        date = datetime.date.fromisoformat(date)
+    uuid = get_uuid(uuid)
+    rain.parse_input_file(input_file, date)
+    rain.add_data()
+    rain.add_date()
+    rain.convert_units()
+    rain.normalize_cumulative_amount("rainfall_amount")
+    rain.add_site_geolocation()
+    rain.sort_timestamps()
+    rain.remove_duplicate_timestamps()
+    attributes = output.add_time_attribute({}, rain.date)
+    output.update_attributes(rain.data, attributes)
+    output.save_level1b(rain, output_file, uuid)
+    return uuid
+
+
+class RainEH3(CSVFile):
+    def __init__(self, site_meta: dict) -> None:
+        super().__init__(site_meta)
+        self.instrument = instruments.RAIN_E_H3
+        self._data = {
+            "time": [],
+            "rainfall_rate": [],
+            "rainfall_amount": [],
+        }
+
+    def parse_input_file(
+        self, filepath: str | PathLike, date: datetime.date | None = None
+    ) -> None:
+        with open(filepath, encoding="latin1") as f:
+            data = list(csv.reader(f, delimiter=";"))
+        n_values = np.median([len(row) for row in data]).astype(int)
+
+        if n_values == 22:
+            self._read_talker_protocol_22_columns(data, date)
+        elif n_values == 16:
+            self._read_talker_protocol_16_columns(data, date)
+        else:
+            msg = "Only talker protocol with 16 or 22 columns is supported."
+            raise NotImplementedError(msg)
+
+    def _read_talker_protocol_16_columns(
+        self, data: list, date: datetime.date | None = None
+    ) -> None:
+        """Old Lindenberg data format.
+
+        0  date  DD.MM.YYYY
+        1  time
+        2  precipitation intensity in mm/h
+        3  precipitation accumulation in mm
+        4  housing contact
+        5  top temperature
+        6  bottom temperature
+        7  heater status
+        8  error code
+        9  system status
+        10 talker interval in seconds
+        11 operating hours
+        12 device type
+        13 user data storage 1
+        14 user data storage 2
+        15 user data storage 3
+
+        """
+        for row in data:
+            if len(row) != 16:
+                continue
+            try:
+                dt = datetime.datetime.strptime(
+                    f"{row[0]} {row[1]}", "%d.%m.%Y %H:%M:%S"
+                )
+            except ValueError:
+                continue
+            if date and date != dt.date():
+                continue
+            self._data["time"].append(dt)
+            self._data["rainfall_rate"].append(float(row[2]))
+            self._data["rainfall_amount"].append(float(row[3]))
+        if not self._data["time"]:
+            raise ValidTimeStampError
+
+    def _read_talker_protocol_22_columns(
+        self, data: list, date: datetime.date | None = None
+    ) -> None:
+        """Columns according to header in Lindenberg data.
+
+        0  datetime utc
+        1  date
+        2  time
+        3  precipitation intensity in mm/h
+        4  precipitation accumulation in mm
+        5  housing contact
+        6  top temperature
+        7  bottom temperature
+        8  heater status
+        9  error code
+        10 system status
+        11 talker interval in seconds
+        12 operating hours
+        13 device type
+        14 user data storage 1
+        15 user data storage 2
+        16 user data storage 3
+        17 user data storage 4
+        18 serial number
+        19 hardware version
+        20 firmware version
+        21 external temperature * checksum
+
+        """
+        for row in data:
+            if len(row) != 22:
+                continue
+            try:
+                dt = datetime.datetime.strptime(f"{row[0]}", "%Y-%m-%d %H:%M:%S")
+            except ValueError:
+                continue
+            if date and date != dt.date():
+                continue
+            self._data["time"].append(dt)
+            self._data["rainfall_rate"].append(float(row[3]))
+            self._data["rainfall_amount"].append(float(row[4]))
+            self.serial_number = row[18]
+        if not self._data["time"]:
+            raise ValidTimeStampError
+
+    def convert_units(self) -> None:
+        rainfall_rate = self.data["rainfall_rate"][:]
+        self.data["rainfall_rate"].data = rainfall_rate / 3600 / 1000  # mm/h -> m/s
+        self.data["rainfall_amount"].data = (
+            self.data["rainfall_amount"][:] / 1000
+        )  # mm -> m