cloudnetpy 1.55.20__py3-none-any.whl → 1.55.22__py3-none-any.whl

cloudnetpy/instruments/disdrometer/parsivel.py

@@ -3,15 +3,16 @@ import datetime
 import logging
 import re
 from collections import defaultdict
-from collections.abc import Callable, Iterator, Sequence
+from collections.abc import Callable, Iterable, Iterator, Sequence
 from itertools import islice
 from os import PathLike
-from typing import Any, Iterable, Literal
+from typing import Any, Literal
 
 import numpy as np
 
 from cloudnetpy import output
 from cloudnetpy.cloudnetarray import CloudnetArray
+from cloudnetpy.constants import MM_TO_M, SEC_IN_HOUR
 from cloudnetpy.exceptions import DisdrometerDataError
 from cloudnetpy.instruments import instruments
 from cloudnetpy.instruments.cloudnet_instrument import CloudnetInstrument
@@ -32,6 +33,7 @@ def parsivel2nc(
     file.
 
     Args:
+    ----
         disdrometer_file: Filename of disdrometer file or list of filenames.
         output_file: Output filename.
         site_meta: Dictionary containing information about the site. Required key
@@ -45,13 +47,16 @@ def parsivel2nc(
         timestamps:
 
     Returns:
+    -------
         UUID of the generated file.
 
     Raises:
+    ------
         DisdrometerDataError: Timestamps do not match the expected date, or unable
             to read the disdrometer file.
 
     Examples:
+    --------
         >>> from cloudnetpy.instruments import parsivel2nc
         >>> site_meta = {'name': 'Lindenberg', 'altitude': 104, 'latitude': 52.2,
         'longitude': 14.1}
@@ -69,8 +74,7 @@ def parsivel2nc(
     disdrometer.add_meta()
     attributes = output.add_time_attribute(ATTRIBUTES, disdrometer.date)
     output.update_attributes(disdrometer.data, attributes)
-    uuid = output.save_level1b(disdrometer, output_file, uuid)
-    return uuid
+    return output.save_level1b(disdrometer, output_file, uuid)
 
 
 class Parsivel(CloudnetInstrument):
@@ -95,74 +99,77 @@ class Parsivel(CloudnetInstrument):
         self._create_velocity_vectors()
         self._create_diameter_vectors()
 
-    def _screen_time(self, expected_date: datetime.date | None = None):
+    def _screen_time(self, expected_date: datetime.date | None = None) -> None:
         if expected_date is None:
             self.date = self.raw_data["time"][0].astype(object).date()
             return
         self.date = expected_date
         valid_mask = self.raw_data["time"].astype("datetime64[D]") == self.date
         if np.count_nonzero(valid_mask) == 0:
-            raise DisdrometerDataError(f"No data found on {expected_date}")
+            msg = f"No data found on {expected_date}"
+            raise DisdrometerDataError(msg)
         for key in self.raw_data:
             self.raw_data[key] = self.raw_data[key][valid_mask]
 
-    def _append_data(self):
+    def _append_data(self) -> None:
         for key, values in self.raw_data.items():
             if key.startswith("_"):
                 continue
+            name = key
+            values_out = values
             match key:
                 case "spectrum":
-                    key = "data_raw"
+                    name = "data_raw"
                     dimensions = ["time", "diameter", "velocity"]
                 case "number_concentration" | "fall_velocity":
                     dimensions = ["time", "diameter"]
                 case "time":
                     dimensions = []
                     base = values[0].astype("datetime64[D]")
-                    values = (values - base) / np.timedelta64(1, "h")
+                    values_out = (values - base) / np.timedelta64(1, "h")
                 case _:
                     dimensions = ["time"]
-            self.data[key] = CloudnetArray(values, key, dimensions=dimensions)
+            self.data[name] = CloudnetArray(values_out, name, dimensions=dimensions)
         if "_sensor_id" in self.raw_data:
             first_id = self.raw_data["_sensor_id"][0]
             for sensor_id in self.raw_data["_sensor_id"]:
                 if sensor_id != first_id:
-                    raise DisdrometerDataError("Multiple sensor IDs are not supported")
+                    msg = "Multiple sensor IDs are not supported"
+                    raise DisdrometerDataError(msg)
             self.serial_number = first_id
 
-    def _create_velocity_vectors(self):
+    def _create_velocity_vectors(self) -> None:
         n_values = [10, 5, 5, 5, 5, 2]
         spreads = [0.1, 0.2, 0.4, 0.8, 1.6, 3.2]
         Disdrometer.store_vectors(self.data, n_values, spreads, "velocity")
 
-    def _create_diameter_vectors(self):
+    def _create_diameter_vectors(self) -> None:
         n_values = [10, 5, 5, 5, 5, 2]
         spreads = [0.125, 0.25, 0.5, 1, 2, 3]
         Disdrometer.store_vectors(self.data, n_values, spreads, "diameter")
 
-    def convert_units(self):
-        mm_to_m = 1e3
-        mmh_to_ms = 3600 * mm_to_m
+    def convert_units(self) -> None:
+        mmh_to_ms = SEC_IN_HOUR / MM_TO_M
         c_to_k = 273.15
         self._convert_data(("rainfall_rate",), mmh_to_ms)
         self._convert_data(("snowfall_rate",), mmh_to_ms)
-        self._convert_data(("diameter", "diameter_spread", "diameter_bnds"), mm_to_m)
+        self._convert_data(("diameter", "diameter_spread", "diameter_bnds"), 1e3)
         self._convert_data(("V_sensor_supply",), 10)
         self._convert_data(("T_sensor",), c_to_k, method="add")
 
-    def add_meta(self):
+    def add_meta(self) -> None:
         valid_keys = ("latitude", "longitude", "altitude")
         for key, value in self.site_meta.items():
-            key = key.lower()
-            if key in valid_keys:
-                self.data[key] = CloudnetArray(float(value), key)
+            name = key.lower()
+            if name in valid_keys:
+                self.data[name] = CloudnetArray(float(value), name)
 
     def _convert_data(
         self,
         keys: tuple[str, ...],
         value: float,
         method: Literal["divide", "add"] = "divide",
-    ):
+    ) -> None:
         for key in keys:
             if key not in self.data:
                 continue
@@ -298,9 +305,11 @@ def _parse_date(tokens: Iterator[str]) -> datetime.date:
     elif "." in token:
         day, month, year = token.split(".")
     else:
-        raise ValueError(f"Unsupported date: '{input}'")
+        msg = f"Unsupported date: '{input}'"
+        raise ValueError(msg)
     if len(year) != 4:
-        raise ValueError(f"Unsupported date: '{input}'")
+        msg = f"Unsupported date: '{input}'"
+        raise ValueError(msg)
     return datetime.date(int(year), int(month), int(day))
 
 
@@ -318,7 +327,14 @@ def _parse_datetime(tokens: Iterator[str]) -> datetime.datetime:
     hour = int(token[8:10])
     minute = int(token[10:12])
     second = int(token[12:14])
-    return datetime.datetime(year, month, day, hour, minute, second)
+    return datetime.datetime(
+        year,
+        month,
+        day,
+        hour,
+        minute,
+        second,
+    )
 
 
 def _parse_vector(tokens: Iterator[str]) -> np.ndarray:
@@ -333,13 +349,15 @@ def _parse_spectrum(tokens: Iterator[str]) -> np.ndarray:
         raw = [first.removeprefix("<SPECTRUM>")]
         raw.extend(islice(tokens, 1023))
         if next(tokens) != "</SPECTRUM>":
-            raise ValueError("Invalid spectrum format")
+            msg = "Invalid spectrum format"
+            raise ValueError(msg)
         values = [int(x) if x != "" else 0 for x in raw]
     else:
         values = [int(first)]
         values.extend(int(x) for x in islice(tokens, 1023))
     if len(values) != 1024:
-        raise ValueError("Invalid length")
+        msg = f"Invalid spectrum length: {len(values)}"
+        raise ValueError(msg)
     return np.array(values, dtype="i2").reshape((32, 32))
 
 
@@ -391,32 +409,37 @@ def _read_rows(headers: list[str], rows: list[str]) -> dict[str, list]:
         if row == "":
             continue
         try:
-            tokens = iter(row.removesuffix(";").split(";"))
-            parsed = [PARSERS.get(header, next)(tokens) for header in headers]
-            unread_tokens = list(tokens)
-            if unread_tokens:
-                raise ValueError("More values than expected")
-            for header, value in zip(headers, parsed):
+            parsed = _parse_row(row, headers)
+            for header, value in zip(headers, parsed, strict=True):
                 result[header].append(value)
         except (ValueError, StopIteration):
             invalid_rows += 1
             continue
     if invalid_rows == len(rows):
-        raise DisdrometerDataError("No valid data in file")
+        msg = "No valid data in file"
+        raise DisdrometerDataError(msg)
     if invalid_rows > 0:
-        logging.info(f"Skipped {invalid_rows} invalid rows")
+        logging.info("Skipped %s invalid rows", invalid_rows)
     return result
 
 
+def _parse_row(row_in: str, headers: list[str]) -> list:
+    tokens = iter(row_in.removesuffix(";").split(";"))
+    parsed = [PARSERS.get(header, next)(tokens) for header in headers]
+    if unread_tokens := list(tokens):
+        msg = f"Unused tokens: {unread_tokens}"
+        raise ValueError(msg)
+    return parsed
+
+
 def _read_toa5(filename: str | PathLike) -> dict[str, list]:
-    """
-    Read ASCII data from Campbell Scientific datalogger such as CR1000.
+    """Read ASCII data from Campbell Scientific datalogger such as CR1000.
 
-    References:
+    References
+    ----------
         CR1000 Measurement and Control System.
         https://s.campbellsci.com/documents/us/manuals/cr1000.pdf
     """
-    # pylint: disable=too-many-branches,comparison-with-callable
     with open(filename, encoding="latin1", errors="ignore") as file:
         reader = csv.reader(file)
         _origin_line = next(reader)
@@ -438,12 +461,13 @@ def _read_toa5(filename: str | PathLike) -> dict[str, list]:
                 "spectrum": [],
             }
             try:
-                for header, value in zip(headers, data_line):
+                for header, value in zip(headers, data_line, strict=True):
                     if header is None:
                         continue
                     if header == "_datetime":
                         scalars[header] = datetime.datetime.strptime(
-                            value, "%Y-%m-%d %H:%M:%S"
+                            value,
+                            "%Y-%m-%d %H:%M:%S",
                         )
                     elif header in ("number_concentration", "fall_velocity"):
                         arrays[header].append(float(value))
@@ -460,21 +484,26 @@ def _read_toa5(filename: str | PathLike) -> dict[str, list]:
                 data[header].append(scalar)
             if "spectrum" in headers:
                 data["spectrum"].append(
-                    np.array(arrays["spectrum"], dtype="i2").reshape((32, 32))
+                    np.array(arrays["spectrum"], dtype="i2").reshape((32, 32)),
                 )
             if "number_concentration" in headers:
                 data["number_concentration"].append(arrays["number_concentration"])
             if "fall_velocity" in headers:
                 data["fall_velocity"].append(arrays["fall_velocity"])
         if n_invalid_rows == n_rows:
-            raise DisdrometerDataError("No valid data in file")
+            msg = "No valid data in file"
+            raise DisdrometerDataError(msg)
         if n_invalid_rows > 0:
-            logging.info(f"Skipped {n_invalid_rows} invalid rows")
+            logging.info("Skipped %s invalid rows", n_invalid_rows)
         return data
 
 
 def _read_typ_op4a(filename: str | PathLike) -> dict[str, list]:
-    """Read file starting with "TYP OP4A" that contains one measurement."""
+    """Read output of "CS/PA" command. The output starts with line "TYP OP4A"
+    followed by one line per measured variable in format: <number>:<value>.
+    Output ends with characters: <ETX><CR><LF><NUL>. Lines are separated by
+    <CR><LF>.
+    """
     data = {}
     with open(filename, encoding="latin1", errors="ignore") as file:
         for line in file:
@@ -503,7 +532,8 @@ def _read_parsivel(
         with open(filename, encoding="latin1", errors="ignore") as file:
             lines = file.read().splitlines()
         if not lines:
-            raise DisdrometerDataError("File is empty")
+            msg = f"File '{filename}' is empty"
+            raise DisdrometerDataError(msg)
         if "TOA5" in lines[0]:
             data = _read_toa5(filename)
         elif "TYP OP4A" in lines[0]:
@@ -515,11 +545,12 @@ def _read_parsivel(
             headers = _parse_telegram(telegram)
             data = _read_rows(headers, lines)
         else:
-            raise ValueError("telegram must be specified for files without header")
+            msg = "telegram must be specified for files without header"
+            raise ValueError(msg)
         if "_datetime" not in data and timestamps is None:
             data["_datetime"] = [
                 datetime.datetime.combine(date, time)
-                for date, time in zip(data["_date"], data["_time"])
+                for date, time in zip(data["_date"], data["_time"], strict=True)
             ]
         for key, values in data.items():
             combined_data[key].extend(values)

cloudnetpy/instruments/disdrometer/thies.py

@@ -15,6 +15,7 @@ def thies2nc(
     """Converts Thies-LNM disdrometer data into Cloudnet Level 1b netCDF file.
 
     Args:
+    ----
         disdrometer_file: Filename of disdrometer .log file.
         output_file: Output filename.
         site_meta: Dictionary containing information about the site. Required key
@@ -23,13 +24,16 @@ def thies2nc(
         date: Expected date of the measurements as YYYY-MM-DD.
 
     Returns:
+    -------
         UUID of the generated file.
 
     Raises:
+    ------
         DisdrometerDataError: Timestamps do not match the expected date, or unable
             to read the disdrometer file.
 
     Examples:
+    --------
         >>> from cloudnetpy.instruments import thies2nc
         >>> site_meta = {'name': 'Lindenberg', 'altitude': 104, 'latitude': 52.2,
         'longitude': 14.1}
@@ -39,7 +43,8 @@ def thies2nc(
     try:
         disdrometer = Thies(disdrometer_file, site_meta)
     except (ValueError, IndexError) as err:
-        raise DisdrometerDataError("Can not read disdrometer file") from err
+        msg = "Unable to read disdrometer file"
+        raise DisdrometerDataError(msg) from err
     if date is not None:
         disdrometer.validate_date(date)
     disdrometer.init_data()
@@ -50,8 +55,7 @@ def thies2nc(
     disdrometer.convert_units()
     attributes = output.add_time_attribute(ATTRIBUTES, disdrometer.date)
     output.update_attributes(disdrometer.data, attributes)
-    uuid = output.save_level1b(disdrometer, output_file, uuid)
-    return uuid
+    return output.save_level1b(disdrometer, output_file, uuid)
 
 
 class Thies(Disdrometer):
@@ -64,7 +68,7 @@ class Thies(Disdrometer):
         self._create_diameter_vectors()
         self.instrument = instruments.THIES
 
-    def init_data(self):
+    def init_data(self) -> None:
         """According to
         https://www.biral.com/wp-content/uploads/2015/01/5.4110.xx_.xxx_.pdf
         """
@@ -127,12 +131,12 @@ class Thies(Disdrometer):
         first_date = _format_thies_date(first_date)
         return first_date.split("-")
 
-    def _create_velocity_vectors(self):
+    def _create_velocity_vectors(self) -> None:
         n_values = [5, 6, 7, 1, 1]
         spreads = [0.2, 0.4, 0.8, 1, 10]
         self.store_vectors(self.data, n_values, spreads, "velocity")
 
-    def _create_diameter_vectors(self):
+    def _create_diameter_vectors(self) -> None:
         n_values = [3, 6, 13]
         spreads = [0.125, 0.25, 0.5]
         self.store_vectors(self.data, n_values, spreads, "diameter", start=0.125)

cloudnetpy/instruments/galileo.py

@@ -1,6 +1,6 @@
 """Module for reading raw Galileo cloud radar data."""
 import os
-from tempfile import TemporaryDirectory
+from tempfile import NamedTemporaryFile, TemporaryDirectory
 
 import numpy as np
 
@@ -20,6 +20,7 @@ def galileo2nc(
     """Converts 'Galileo' cloud radar data into Cloudnet Level 1b netCDF file.
 
     Args:
+    ----
         raw_files: Input file name or folder containing multiple input files.
         output_file: Output filename.
         site_meta: Dictionary containing information about the site. Required key
@@ -29,12 +30,15 @@ def galileo2nc(
         date: Expected date as YYYY-MM-DD of all profiles in the file.
 
     Returns:
+    -------
         UUID of the generated file.
 
     Raises:
+    ------
         ValidTimeStampError: No valid timestamps found.
 
     Examples:
+    --------
           >>> from cloudnetpy.instruments import galileo2nc
           >>> site_meta = {'name': 'Chilbolton'}
           >>> galileo2nc('raw_radar.nc', 'radar.nc', site_meta)
@@ -57,13 +61,20 @@ def galileo2nc(
 
     with TemporaryDirectory() as temp_dir:
         if os.path.isdir(raw_files):
-            nc_filename = f"{temp_dir}/tmp.nc"
-            valid_filenames = utils.get_sorted_filenames(raw_files, ".nc")
-            valid_filenames = utils.get_files_with_common_range(valid_filenames)
-            variables = list(keymap.keys())
-            concat_lib.concatenate_files(
-                valid_filenames, nc_filename, variables=variables
-            )
+            with NamedTemporaryFile(
+                dir=temp_dir,
+                suffix=".nc",
+                delete=False,
+            ) as temp_file:
+                nc_filename = temp_file.name
+                valid_filenames = utils.get_sorted_filenames(raw_files, ".nc")
+                valid_filenames = utils.get_files_with_common_range(valid_filenames)
+                variables = list(keymap.keys())
+                concat_lib.concatenate_files(
+                    valid_filenames,
+                    nc_filename,
+                    variables=variables,
+                )
         else:
             nc_filename = raw_files
 
@@ -87,14 +98,14 @@ def galileo2nc(
             galileo.add_height()
         attributes = output.add_time_attribute(ATTRIBUTES, galileo.date)
         output.update_attributes(galileo.data, attributes)
-        uuid = output.save_level1b(galileo, output_file, uuid)
-        return uuid
+        return output.save_level1b(galileo, output_file, uuid)
 
 
 class Galileo(ChilboltonRadar):
     """Class for Galileo raw radar data. Child of ChilboltonRadar().
 
     Args:
+    ----
         full_path: Filename of a daily Galileo .nc NetCDF file.
         site_meta: Site properties in a dictionary. Required keys are: `name`.
 
@@ -105,12 +116,12 @@ class Galileo(ChilboltonRadar):
         self.date = self._init_date()
         self.instrument = GALILEO
 
-    def mask_clutter(self):
+    def mask_clutter(self) -> None:
         """Masks clutter."""
         # Only strong Z values are valid
         n_low_gates = 15
         ind = np.where(self.data["Zh"][:, :n_low_gates] < -15) and np.where(
-            self.data["ldr"][:, :n_low_gates] > -5
+            self.data["ldr"][:, :n_low_gates] > -5,
         )
         self.data["v"].mask_indices(ind)
 

cloudnetpy/instruments/hatpro.py

@@ -34,6 +34,7 @@ def hatpro2l1c(
     """Converts RPG HATPRO microwave radiometer data into Cloudnet Level 1c netCDF file.
 
     Args:
+    ----
         mwr_dir: Folder containing one day of HATPRO files.
         output_file: Output file name.
         site_meta: Dictionary containing information about the site and instrument
@@ -41,9 +42,9 @@ def hatpro2l1c(
         date: Expected date in the input files.
 
     Returns:
+    -------
         UUID of the generated file.
     """
-
     coeff_files = site_meta.get("coefficientFiles", None)
 
     hatpro_raw = mwrpy.lev1_to_nc(
@@ -59,9 +60,16 @@ def hatpro2l1c(
     timestamps = hatpro.data["time"][:]
     if date is not None:
         # Screen timestamps if these assertions start to fail
-        assert np.all(np.diff(timestamps) > 0)
-        dates = [str(datetime.datetime.utcfromtimestamp(t).date()) for t in timestamps]
-        assert len(set(dates)) == 1
+        if not np.all(np.diff(timestamps) > 0):
+            msg = "Timestamps are not increasing"
+            raise RuntimeError(msg)
+        dates = [
+            str(datetime.datetime.fromtimestamp(t, tz=datetime.timezone.utc).date())
+            for t in timestamps
+        ]
+        if len(set(dates)) != 1:
+            msg = f"Several dates, something is wrong: {set(dates)}"
+            raise RuntimeError(msg)
 
     decimal_hours = utils.seconds2hours(timestamps)
     hatpro.data["time"] = CloudnetArray(decimal_hours, "time", data_type="f8")
@@ -115,6 +123,7 @@ def hatpro2nc(
     concatenates the data and writes it into netCDF file.
 
     Args:
+    ----
         path_to_files: Folder containing one day of RPG HATPRO files.
         output_file: Output file name.
         site_meta: Dictionary containing information about the site with keys:
@@ -131,15 +140,18 @@ def hatpro2nc(
             only files that match the date will be used.
 
     Returns:
+    -------
         2-element tuple containing
 
         - UUID of the generated file.
         - Files used in the processing.
 
     Raises:
+    ------
         ValidTimeStampError: No valid timestamps found.
 
     Examples:
+    --------
         >>> from cloudnetpy.instruments import hatpro2nc
         >>> site_meta = {'name': 'Hyytiala', 'altitude': 174}
         >>> hatpro2nc('/path/to/files/', 'hatpro.nc', site_meta)
@@ -165,7 +177,8 @@ def hatpro2nc(
 
 
 def _get_hatpro_objects(
-    directory: Path, expected_date: str | None
+    directory: Path,
+    expected_date: str | None,
 ) -> tuple[list[HatproBinCombined], list[str]]:
     objects = defaultdict(list)
     for filename in directory.iterdir():
@@ -183,7 +196,7 @@ def _get_hatpro_objects(
                 obj = _validate_date(obj, expected_date)
             objects[filename.stem].append(obj)
         except (TypeError, ValueError, ValidTimeStampError) as err:
-            logging.warning(f"Ignoring file '{filename}': {err}")
+            logging.warning("Ignoring file '%s': %s", filename, err)
             continue
 
     valid_files: list[str] = []
@@ -194,28 +207,31 @@ def _get_hatpro_objects(
             valid_files.extend(str(obj.filename) for obj in objs)
         except (TypeError, ValueError) as err:
             files = "'" + "', '".join(str(obj.filename) for obj in objs) + "'"
-            logging.warning(f"Ignoring files {files}: {err}")
+            logging.warning("Ignoring files %s: %s", files, err)
             continue
 
     return combined_objs, valid_files
 
 
-def _validate_date(obj: HatproBin, expected_date: str):
+def _validate_date(obj: HatproBin, expected_date: str) -> HatproBin:
     if obj.header["_time_reference"] != 1:
-        raise ValueError("Can not validate non-UTC dates")
+        msg = "Can not validate non-UTC dates"
+        raise ValueError(msg)
     inds = []
     for ind, timestamp in enumerate(obj.data["time"][:]):
         date = "-".join(utils.seconds2date(timestamp)[:3])
         if date == expected_date:
             inds.append(ind)
     if not inds:
-        raise ValueError("Timestamps not what expected")
+        msg = f"No valid timestamps found for date {expected_date}"
+        raise ValueError(msg)
     obj.data = obj.data[:][inds]
     return obj
 
 
 def _add_missing_variables(
-    hatpro_objects: list[HatproBinCombined], keys: tuple
+    hatpro_objects: list[HatproBinCombined],
+    keys: tuple,
 ) -> list[HatproBinCombined]:
     for obj in hatpro_objects:
         for key in keys:

cloudnetpy/instruments/instruments.py

@@ -116,7 +116,10 @@ FMCW35 = Instrument(
 )
 
 BASTA = Instrument(
-    domain="radar", category="cloud radar", model="BASTA", frequency=95.0
+    domain="radar",
+    category="cloud radar",
+    model="BASTA",
+    frequency=95.0,
 )
 
 HATPRO = Instrument(