cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/instruments/rpg_reader.py

@@ -1,15 +1,22 @@
+import logging
+from os import PathLike
+from pathlib import Path
 from typing import BinaryIO, Literal
 
 import numpy as np
+import numpy.typing as npt
 from numpy import ma
 from numpy.lib import recfunctions as rfn
 from rpgpy import read_rpg
 
+from cloudnetpy.constants import G_TO_KG
+from cloudnetpy.exceptions import ValidTimeStampError
+
 
 class Fmcw94Bin:
     """RPG Cloud Radar Level 1 data reader."""
 
-    def __init__(self, filename):
+    def __init__(self, filename: str | PathLike) -> None:
         self.filename = filename
         self.header, self.data = read_rpg(filename)
 
@@ -31,7 +38,7 @@ class Fmcw94Bin:
             "CGProg": "program_number",
             "ModelNo": "model_number",
             "ProgName": "_program_name",
-            "CustName": "_customer_name",
+            "CustName": "customer_name",
             "Freq": "radar_frequency",
             "AntSep": "antenna_separation",
             "AntDia": "antenna_diameter",
@@ -67,8 +74,8 @@ class Fmcw94Bin:
             "QF": "quality_flag",
             "RR": "rainfall_rate",
             "RelHum": "relative_humidity",
-            "EnvTemp": "temperature",
-            "BaroP": "pressure",
+            "EnvTemp": "air_temperature",
+            "BaroP": "air_pressure",
             "WS": "wind_speed",
             "WD": "wind_direction",
             "DDVolt": "voltage",
@@ -97,7 +104,7 @@ class Fmcw94Bin:
         self.replace_keys(self.data, data_keymap)
 
     @staticmethod
-    def replace_keys(d: dict, keymap: dict):
+    def replace_keys(d: dict, keymap: dict) -> None:
         for key in d.copy():
             if key in keymap:
                 new_key = keymap[key]
@@ -105,20 +112,22 @@ class Fmcw94Bin:
 
 
 def _read_from_file(
-    file: BinaryIO, fields: list[tuple[str, str]], count: int = 1
+    file: BinaryIO,
+    fields: list[tuple[str, str]],
+    count: int | None = None,
 ) -> ma.MaskedArray:
-    arr = np.fromfile(file, np.dtype(fields), count)
+    arr = np.fromfile(file, np.dtype(fields), 1 if count is None else count)
     masked_arr = ma.array(arr)
-    if count == 1:
+    if count is None:
         return masked_arr[0]
     return masked_arr
 
 
 def _decode_angles(
-    x: np.ndarray, version: Literal[1, 2]
-) -> tuple[np.ndarray, np.ndarray]:
-    """
-    Decode elevation and azimuth angles.
+    x: npt.NDArray,
+    version: Literal[1, 2],
+) -> tuple[npt.NDArray, npt.NDArray]:
+    """Decode elevation and azimuth angles.
 
     >>> _decode_angles(np.array([1267438.5]), version=1)
     (array([138.5]), array([267.4]))
@@ -128,7 +137,6 @@ def _decode_angles(
     Based on `interpret_angle` from mwr_raw2l1 licensed under BSD 3-Clause:
     https://github.com/MeteoSwiss/mwr_raw2l1/blob/0738490d22f77138cdf9329bf102f319c78be584/mwr_raw2l1/readers/reader_rpg_helpers.py#L30
     """
-
     if version == 1:
         # Description in the manual is quite unclear so here's an improved one:
         # Ang=sign(El)*(|El|+1000*Az), -90°<=El<100°, 0°<=Az<360°. If El>=100°
@@ -150,9 +158,8 @@ def _decode_angles(
         ele = np.sign(x) * (np.abs(x) // 1e5) / 100
         azi = (np.abs(x) - np.abs(ele) * 1e7) / 100
     else:
-        raise NotImplementedError(
-            f"Known versions for angle encoding are 1 and 2, but received {version}"
-        )
+        msg = f"Known versions for angle encoding are 1 and 2, but received {version}"
+        raise NotImplementedError(msg)
 
     return ele, azi
 
@@ -177,7 +184,7 @@ class HatproBin:
     QUALITY_MEDIUM = 2
     QUALITY_LOW = 3
 
-    def __init__(self, filename):
+    def __init__(self, filename: Path) -> None:
         self.filename = filename
         with open(self.filename, "rb") as file:
             self._read_header(file)
@@ -185,21 +192,39 @@ class HatproBin:
         self._remove_duplicate_timestamps()
         self._add_zenith_angle()
 
-    def screen_bad_profiles(self):
+    def screen_bad_profiles(self) -> None:
+        # In Limassol 2025-06-06 for instance, LWP is all zero but IWV has
+        # values.
+        if ma.all(self.data[self.variable] == 0):
+            self.data[self.variable][:] = ma.masked
+            return
+        # Screen by quality flag.
         is_bad = self.data["_quality_flag"] & 0b110 == self.QUALITY_LOW << 1
+        n_bad = np.count_nonzero(is_bad)
+        if n_bad == len(is_bad):
+            msg = "All data are low quality"
+            raise ValidTimeStampError(msg)
+        if n_bad:
+            percentage = round(100 * n_bad / len(is_bad))
+            logging.info(
+                "Screening %s %% (%s/%s) data points with low quality",
+                percentage,
+                n_bad,
+                len(is_bad),
+            )
         self.data[self.variable][is_bad] = ma.masked
 
-    def _remove_duplicate_timestamps(self):
+    def _remove_duplicate_timestamps(self) -> None:
         _, ind = np.unique(self.data["time"], return_index=True)
         self.data = self.data[ind]
 
-    def _read_header(self, file: BinaryIO):
-        raise NotImplementedError()
+    def _read_header(self, file: BinaryIO) -> None:
+        raise NotImplementedError
 
-    def _read_data(self, file: BinaryIO):
-        raise NotImplementedError()
+    def _read_data(self, file: BinaryIO) -> None:
+        raise NotImplementedError
 
-    def _add_zenith_angle(self):
+    def _add_zenith_angle(self) -> None:
         ele, _azi = _decode_angles(self.data["_instrument_angles"], self.version)
         self.data = rfn.append_fields(self.data, "zenith_angle", 90 - ele)
 
@@ -209,7 +234,7 @@ class HatproBinLwp(HatproBin):
 
     variable = "lwp"
 
-    def _read_header(self, file):
+    def _read_header(self, file: BinaryIO) -> None:
         self.header = _read_from_file(
             file,
             [
@@ -226,9 +251,10 @@ class HatproBinLwp(HatproBin):
         elif self.header["file_code"] == 934501000:
             self.version = 2
         else:
-            raise ValueError(f'Unknown HATPRO version. {self.header["file_code"]}')
+            msg = f"Unknown HATPRO version. {self.header['file_code']}"
+            raise ValueError(msg)
 
-    def _read_data(self, file):
+    def _read_data(self, file: BinaryIO) -> None:
         self.data = _read_from_file(
             file,
             [
@@ -237,9 +263,9 @@ class HatproBinLwp(HatproBin):
                 ("lwp", "<f"),
                 ("_instrument_angles", "<f" if self.version == 1 else "<i4"),
             ],
-            self.header["_n_samples"],
+            int(self.header["_n_samples"]),
         )
-        self.data["lwp"] *= 1e-3  # Convert from g/m^2 to kg/m^2
+        self.data["lwp"] *= G_TO_KG
 
 
 class HatproBinIwv(HatproBin):
@@ -247,7 +273,7 @@ class HatproBinIwv(HatproBin):
 
     variable = "iwv"
 
-    def _read_header(self, file):
+    def _read_header(self, file: BinaryIO) -> None:
         self.header = _read_from_file(
             file,
             [
@@ -264,9 +290,10 @@ class HatproBinIwv(HatproBin):
         elif self.header["file_code"] == 594811000:
             self.version = 2
         else:
-            raise ValueError(f'Unknown HATPRO version. {self.header["file_code"]}')
+            msg = f"Unknown HATPRO version. {self.header['file_code']}"
+            raise ValueError(msg)
 
-    def _read_data(self, file):
+    def _read_data(self, file: BinaryIO) -> None:
         self.data = _read_from_file(
             file,
             [
@@ -275,17 +302,17 @@ class HatproBinIwv(HatproBin):
                 ("iwv", "<f"),
                 ("_instrument_angles", "<f" if self.version == 1 else "<i4"),
             ],
-            self.header["_n_samples"],
+            int(self.header["_n_samples"]),
         )
 
 
 class HatproBinCombined:
     """Combine HATPRO objects that share values of the given dimensions."""
 
-    header: dict[str, np.ndarray]
-    data: dict[str, np.ndarray]
+    header: dict[str, npt.NDArray]
+    data: dict[str, npt.NDArray]
 
-    def __init__(self, files: list[HatproBin]):
+    def __init__(self, files: list[HatproBin]) -> None:
         self.header = {}
         if len(files) == 1:
             arr = files[0].data
@@ -298,16 +325,23 @@ class HatproBinCombined:
                 _combine_values(arr["zenith_angle1"], arr["zenith_angle2"]),
             )
             # Workaround because rfn.drop_fields seems to incorrectly drop mask...
-            # arr = rfn.drop_fields(arr, ["zenith_angle1", "zenith_angle2"])
             arr = rfn.rename_fields(
-                arr, {"zenith_angle1": "_tmp1", "zenith_angle2": "_tmp2"}
+                arr,
+                {"zenith_angle1": "_tmp1", "zenith_angle2": "_tmp2"},
             )
         else:
-            raise NotImplementedError("Only implemented up to 2 files")
+            msg = "Only implemented up to 2 files"
+            raise NotImplementedError(msg)
+
+        if arr.dtype.fields is None:
+            msg = "Data has no fields"
+            raise ValueError(msg)
+
         self.data = {field: arr[field] for field in arr.dtype.fields}
 
 
 def _combine_values(arr1: ma.MaskedArray, arr2: ma.MaskedArray) -> ma.MaskedArray:
     if not ma.allequal(arr1, arr2):
-        raise ValueError("Inconsistent values")
+        msg = "Inconsistent values"
+        raise ValueError(msg)
     return ma.where(~arr1.mask, arr1, arr2)

cloudnetpy/instruments/toa5.py

@@ -0,0 +1,49 @@
+import csv
+import datetime
+from os import PathLike
+from typing import Any
+
+
+def read_toa5(
+    filename: str | PathLike,
+) -> tuple[dict[str, str], dict[str, str], list[dict[str, Any]]]:
+    """Read ASCII data from Campbell Scientific datalogger such as CR1000.
+
+    References:
+        CR1000 Measurement and Control System.
+        https://s.campbellsci.com/documents/us/manuals/cr1000.pdf
+    """
+    with open(filename) as file:
+        reader = csv.reader(file)
+        origin_line = next(reader)
+        if len(origin_line) == 0 or origin_line[0] != "TOA5":
+            msg = "Invalid TOA5 file"
+            raise ValueError(msg)
+        header_line = next(reader)
+        units_line = next(reader)
+        process_line = next(reader)
+        output = []
+        units = dict(zip(header_line, units_line, strict=False))
+        process = dict(zip(header_line, process_line, strict=False))
+
+        row_template: dict[str, Any] = {}
+        for header in header_line:
+            if "(" in header:
+                row_template[header[: header.index("(")]] = []
+
+        for data_line in reader:
+            row = row_template.copy()
+            for key, value in zip(header_line, data_line, strict=False):
+                parsed_value: Any = value
+                if key == "TIMESTAMP":
+                    parsed_value = datetime.datetime.strptime(
+                        parsed_value, "%Y-%m-%d %H:%M:%S"
+                    )
+                elif key == "RECORD":
+                    parsed_value = int(parsed_value)
+                if "(" in key:
+                    row[key[: key.index("(")]].append(parsed_value)
+                else:
+                    row[key] = parsed_value
+            output.append(row)
+        return units, process, output