cloudnetpy 1.55.20__py3-none-any.whl → 1.55.22__py3-none-any.whl

cloudnetpy/instruments/pollyxt.py

@@ -22,10 +22,10 @@ def pollyxt2nc(
     uuid: str | None = None,
     date: str | None = None,
 ) -> str:
-    """
-    Converts PollyXT Raman lidar data into Cloudnet Level 1b netCDF file.
+    """Converts PollyXT Raman lidar data into Cloudnet Level 1b netCDF file.
 
     Args:
+    ----
         input_folder: Path to pollyxt netCDF files.
         output_file: Output filename.
         site_meta: Dictionary containing information about the site with keys:
@@ -40,9 +40,11 @@ def pollyxt2nc(
         date: Expected date of the measurements as YYYY-MM-DD.
 
     Returns:
+    -------
         UUID of the generated file.
 
     Examples:
+    --------
         >>> from cloudnetpy.instruments import pollyxt2nc
         >>> site_meta = {'name': 'Mindelo', 'altitude': 13, 'zenith_angle': 6,
         'snr_limit': 3}
@@ -61,8 +63,7 @@ def pollyxt2nc(
     attributes = output.add_time_attribute(ATTRIBUTES, polly.date)
     output.update_attributes(polly.data, attributes)
     polly.add_snr_info("beta", snr_limit)
-    uuid = output.save_level1b(polly, output_file, uuid)
-    return uuid
+    return output.save_level1b(polly, output_file, uuid)
 
 
 class PollyXt(Ceilometer):
@@ -72,16 +73,17 @@ class PollyXt(Ceilometer):
         self.expected_date = expected_date
         self.instrument = instruments.POLLYXT
 
-    def mask_nan_values(self):
+    def mask_nan_values(self) -> None:
         for array in self.data.values():
             if getattr(array, "ndim", 0) > 0:
                 array[np.isnan(array)] = ma.masked
 
-    def calc_screened_products(self, snr_limit: float = 5.0):
+    def calc_screened_products(self, snr_limit: float = 5.0) -> None:
         keys = ("beta", "depolarisation")
         for key in keys:
             self.data[key] = ma.masked_where(
-                self.data["snr"] < snr_limit, self.data[f"{key}_raw"]
+                self.data["snr"] < snr_limit,
+                self.data[f"{key}_raw"],
             )
         self.data["depolarisation"][self.data["depolarisation"] > 1] = ma.masked
         self.data["depolarisation"][self.data["depolarisation"] < 0] = ma.masked
@@ -99,19 +101,21 @@ class PollyXt(Ceilometer):
         bsc_files.sort()
         depol_files.sort()
         if not bsc_files:
-            raise RuntimeError("No pollyxt files found")
+            msg = "No pollyxt bsc files found"
+            raise RuntimeError(msg)
         if len(bsc_files) != len(depol_files):
-            raise InconsistentDataError(
-                "Inconsistent number of pollyxt bsc / depol files"
-            )
+            msg = "Inconsistent number of pollyxt bsc / depol files"
+            raise InconsistentDataError(msg)
         self._fetch_attributes(bsc_files[0])
         self.data["range"] = _read_array_from_multiple_files(
-            bsc_files, depol_files, "height"
+            bsc_files,
+            depol_files,
+            "height",
         )
         calibration_factors: np.ndarray = np.array([])
         beta_channel = self._get_valid_beta_channel(bsc_files)
         bsc_key = f"attenuated_backscatter_{beta_channel}nm"
-        for bsc_file, depol_file in zip(bsc_files, depol_files):
+        for bsc_file, depol_file in zip(bsc_files, depol_files, strict=True):
             with (
                 netCDF4.Dataset(bsc_file, "r") as nc_bsc,
                 netCDF4.Dataset(depol_file, "r") as nc_depol,
@@ -119,11 +123,14 @@ class PollyXt(Ceilometer):
                 epoch = utils.get_epoch(nc_bsc["time"].unit)
                 try:
                     time = np.array(
-                        _read_array_from_file_pair(nc_bsc, nc_depol, "time")
+                        _read_array_from_file_pair(nc_bsc, nc_depol, "time"),
                     )
                 except AssertionError as err:
                     logging.warning(
-                        f"Ignoring files '{nc_bsc}' and '{nc_depol}': {err}"
+                        "Ignoring files '%s' and '%s': %s",
+                        nc_bsc,
+                        nc_depol,
+                        err,
                     )
                     continue
                 beta_raw = nc_bsc.variables[bsc_key][:]
@@ -132,6 +139,7 @@ class PollyXt(Ceilometer):
                 for array, key in zip(
                     [beta_raw, depol_raw, time, snr],
                     ["beta_raw", "depolarisation_raw", "time", "snr"],
+                    strict=True,
                 ):
                     self.data = utils.append_data(self.data, key, array)
                 calibration_factor = nc_bsc.variables[
@@ -139,7 +147,7 @@ class PollyXt(Ceilometer):
                 ].Lidar_calibration_constant_used
                 calibration_factor = np.repeat(calibration_factor, len(time))
                 calibration_factors = np.concatenate(
-                    [calibration_factors, calibration_factor]
+                    [calibration_factors, calibration_factor],
                 )
         self.data["calibration_factor"] = calibration_factors
         return epoch
@@ -153,11 +161,16 @@ class PollyXt(Ceilometer):
                     if not _only_zeros_or_masked(beta):
                         if channel != polly_channels[0]:
                             logging.warning(
-                                f"Using {channel}nm pollyXT channel for backscatter"
+                                "Using %s nm pollyXT channel for backscatter",
+                                channel,
                             )
-                            self.instrument.wavelength = float(channel)  # type: ignore
+                            if self.instrument is None:
+                                msg = "No instrument defined"
+                                raise RuntimeError(msg)
+                            self.instrument.wavelength = float(channel)
                         return channel
-        raise ValidTimeStampError("No functional pollyXT backscatter channels found")
+        msg = "No functional pollyXT backscatter channels found"
+        raise ValidTimeStampError(msg)
 
     def _fetch_attributes(self, file: str) -> None:
         with netCDF4.Dataset(file, "r") as nc:
@@ -167,7 +180,7 @@ class PollyXt(Ceilometer):
 
 def _read_array_from_multiple_files(files1: list, files2: list, key) -> np.ndarray:
     array: np.ndarray = np.array([])
-    for ind, (file1, file2) in enumerate(zip(files1, files2)):
+    for ind, (file1, file2) in enumerate(zip(files1, files2, strict=True)):
         with netCDF4.Dataset(file1, "r") as nc1, netCDF4.Dataset(file2, "r") as nc2:
             array1 = _read_array_from_file_pair(nc1, nc2, key)
             if ind == 0:
@@ -177,7 +190,9 @@ def _read_array_from_multiple_files(files1: list, files2: list, key) -> np.ndarr
 
 
 def _read_array_from_file_pair(
-    nc_file1: netCDF4.Dataset, nc_file2: netCDF4.Dataset, key: str
+    nc_file1: netCDF4.Dataset,
+    nc_file2: netCDF4.Dataset,
+    key: str,
 ) -> np.ndarray:
     array1 = nc_file1.variables[key][:]
     array2 = nc_file2.variables[key][:]

cloudnetpy/instruments/radiometrics.py

@@ -24,6 +24,7 @@ def radiometrics2nc(
     """Converts Radiometrics .csv file into Cloudnet Level 1b netCDF file.
 
     Args:
+    ----
         full_path: Input file name or folder containing multiple input files.
         output_file: Output file name, e.g. 'radiometrics.nc'.
         site_meta: Dictionary containing information about the site and instrument.
@@ -33,9 +34,11 @@ def radiometrics2nc(
         date: Expected date as YYYY-MM-DD of all profiles in the file.
 
     Returns:
+    -------
         UUID of the generated file.
 
     Examples:
+    --------
         >>> from cloudnetpy.instruments import radiometrics2nc
         >>> site_meta = {'name': 'Soverato', 'altitude': 21}
         >>> radiometrics2nc('radiometrics.csv', 'radiometrics.nc', site_meta)
@@ -61,11 +64,12 @@ def radiometrics2nc(
     radiometrics.time_to_fractional_hours()
     radiometrics.data_to_cloudnet_arrays()
     radiometrics.add_meta()
-    assert radiometrics.date is not None
+    if radiometrics.date is None:
+        msg = "Failed to find valid timestamps from Radiometrics file(s)."
+        raise ValidTimeStampError(msg)
     attributes = output.add_time_attribute({}, radiometrics.date)
     output.update_attributes(radiometrics.data, attributes)
-    uuid = output.save_level1b(radiometrics, output_file, uuid)
-    return uuid
+    return output.save_level1b(radiometrics, output_file, uuid)
 
 
 class Record(NamedTuple):
@@ -79,7 +83,8 @@ class Record(NamedTuple):
 class Radiometrics:
     """Reader for level 2 files of Radiometrics microwave radiometers.
 
-    References:
+    References
+    ----------
         Radiometrics (2008). Profiler Operator's Manual: MP-3000A, MP-2500A,
         MP-1500A, MP-183A.
     """
@@ -90,16 +95,18 @@ class Radiometrics:
         self.data: dict = {}
         self.instrument = instruments.RADIOMETRICS
 
-    def read_raw_data(self):
+    def read_raw_data(self) -> None:
         """Reads Radiometrics raw data."""
         record_columns = {}
         unknown_record_types = set()
         rows = []
-        with open(self.filename, mode="r", encoding="utf8") as infile:
+        with open(self.filename, encoding="utf8") as infile:
             reader = csv.reader(infile)
             for row in reader:
                 if row[0] == "Record":
-                    assert row[1] == "Date/Time"
+                    if row[1] != "Date/Time":
+                        msg = "Unexpected header in Radiometrics file"
+                        raise RuntimeError(msg)
                     record_type = int(row[2])
                     record_columns[record_type] = row[3:]
                 else:
@@ -109,7 +116,7 @@ class Radiometrics:
                     column_names = record_columns.get(block_type)
                     if column_names is None:
                         if record_type not in unknown_record_types:
-                            logging.info(f"Skipping unknown record type {record_type}")
+                            logging.info("Skipping unknown record type %d", record_type)
                             unknown_record_types.add(record_type)
                         continue
                     record = Record(
@@ -117,7 +124,7 @@ class Radiometrics:
                         timestamp=_parse_datetime(row[1]),
                         block_type=block_type,
                         block_index=block_index,
-                        values=dict(zip(column_names, row[3:])),
+                        values=dict(zip(column_names, row[3:], strict=True)),
                     )
                     rows.append(record)
 
@@ -130,7 +137,7 @@ class Radiometrics:
             if data_row.block_index == 0:
                 self.raw_data.append(data_row)
 
-    def read_data(self):
+    def read_data(self) -> None:
         """Reads values."""
         times = []
         lwps = []
@@ -165,7 +172,7 @@ class RadiometricsCombined:
                 self.data = utils.append_data(self.data, key, obj.data[key])
         self.instrument = instruments.RADIOMETRICS
 
-    def screen_time(self, expected_date: datetime.date | None):
+    def screen_time(self, expected_date: datetime.date | None) -> None:
         """Screens timestamps."""
         if expected_date is None:
             self.date = self.data["time"][0].astype(object).date()
@@ -177,22 +184,22 @@ class RadiometricsCombined:
         for key in self.data:
             self.data[key] = self.data[key][valid_mask]
 
-    def time_to_fractional_hours(self):
+    def time_to_fractional_hours(self) -> None:
         base = self.data["time"][0].astype("datetime64[D]")
         self.data["time"] = (self.data["time"] - base) / np.timedelta64(1, "h")
 
-    def data_to_cloudnet_arrays(self):
+    def data_to_cloudnet_arrays(self) -> None:
         """Converts arrays to CloudnetArrays."""
         for key, array in self.data.items():
             self.data[key] = CloudnetArray(array, key)
 
-    def add_meta(self):
+    def add_meta(self) -> None:
         """Adds some metadata."""
         valid_keys = ("latitude", "longitude", "altitude")
         for key, value in self.site_meta.items():
-            key = key.lower()
-            if key in valid_keys:
-                self.data[key] = CloudnetArray(float(value), key)
+            name = key.lower()
+            if name in valid_keys:
+                self.data[name] = CloudnetArray(float(value), key)
 
 
 def _parse_datetime(text: str) -> datetime.datetime:
@@ -201,4 +208,11 @@ def _parse_datetime(text: str) -> datetime.datetime:
     hour, minute, second = map(int, time.split(":"))
     if year < 100:
         year += 2000
-    return datetime.datetime(year, month, day, hour, minute, second)
+    return datetime.datetime(
+        year,
+        month,
+        day,
+        hour,
+        minute,
+        second,
+    )

cloudnetpy/instruments/rpg.py

@@ -1,7 +1,8 @@
 """This module contains RPG Cloud Radar related functions."""
 import logging
 import math
-from typing import Sequence
+from collections.abc import Sequence
+from typing import TYPE_CHECKING
 
 import numpy as np
 from numpy import ma
@@ -10,13 +11,16 @@ from rpgpy import RPGFileError
 from cloudnetpy import output, utils
 from cloudnetpy.categorize.atmos_utils import mmh2ms
 from cloudnetpy.cloudnetarray import CloudnetArray
+from cloudnetpy.constants import G_TO_KG
 from cloudnetpy.exceptions import InconsistentDataError, ValidTimeStampError
 from cloudnetpy.instruments import instruments
 from cloudnetpy.instruments.cloudnet_instrument import CloudnetInstrument
-from cloudnetpy.instruments.instruments import Instrument
 from cloudnetpy.instruments.rpg_reader import Fmcw94Bin, HatproBinCombined
 from cloudnetpy.metadata import MetaData
 
+if TYPE_CHECKING:
+    from cloudnetpy.instruments.instruments import Instrument
+
 
 def rpg2nc(
     path_to_l1_files: str,
@@ -31,6 +35,7 @@ def rpg2nc(
     concatenates the data and writes a netCDF file.
 
     Args:
+    ----
         path_to_l1_files: Folder containing one day of RPG LV1 files.
         output_file: Output file name.
         site_meta: Dictionary containing information about the
@@ -43,15 +48,18 @@ def rpg2nc(
             only files that match the date will be used.
 
     Returns:
+    -------
         2-element tuple containing
 
         - UUID of the generated file.
         - Files used in the processing.
 
     Raises:
+    ------
         ValidTimeStampError: No valid timestamps found.
 
     Examples:
+    --------
         >>> from cloudnetpy.instruments import rpg2nc
         >>> site_meta = {'name': 'Hyytiala', 'altitude': 174}
         >>> rpg2nc('/path/to/files/', 'test.nc', site_meta)
@@ -89,7 +97,7 @@ def print_info(data: dict) -> None:
         mode = "LDR"
     else:
         mode = "STSR"
-    logging.info(f"RPG cloud radar in {mode} mode")
+    logging.info("RPG cloud radar in %s mode", mode)
 
 
 RpgObjects = Sequence[Fmcw94Bin] | Sequence[HatproBinCombined]
@@ -107,12 +115,13 @@ def create_one_day_data_record(rpg_objects: RpgObjects) -> dict:
 def _stack_rpg_data(rpg_objects: RpgObjects) -> tuple[dict, dict]:
     """Combines data from hourly RPG objects.
 
-    Notes:
+    Notes
+    -----
         Ignores variable names starting with an underscore.
 
     """
 
-    def _stack(source, target, fun):
+    def _stack(source, target, fun) -> None:
         for name, value in source.items():
             if not name.startswith("_"):
                 target[name] = fun((target[name], value)) if name in target else value
@@ -131,7 +140,7 @@ def _reduce_header(header: dict) -> dict:
     for key, data in header.items():
         first_profile_value = data[0]
         is_identical_value = bool(
-            np.isclose(data, first_profile_value, rtol=1e-2).all()
+            np.isclose(data, first_profile_value, rtol=1e-2).all(),
         )
         if is_identical_value is False:
             msg = f"Inconsistent header: {key}: {data}"
@@ -154,6 +163,8 @@ def _mask_invalid_data(data_in: dict) -> dict:
     data = data_in.copy()
     fill_values = (-999, 1e-10)
     for name in data:
+        if np.issubdtype(data[name].dtype, np.integer) or name == "rainfall_rate":
+            continue
         data[name] = ma.masked_equal(data[name], 0)
         for value in fill_values:
             data[name][data[name] == value] = ma.masked
@@ -188,14 +199,19 @@ def _remove_files_with_bad_height(objects: list, files: list) -> tuple[list, lis
     most_common = np.bincount(lengths).argmax()
     files = [
         file
-        for file, obj, length in zip(files, objects, lengths)
+        for file, obj, length in zip(files, objects, lengths, strict=True)
+        if length == most_common
+    ]
+    objects = [
+        obj
+        for obj, length in zip(objects, lengths, strict=True)
         if length == most_common
     ]
-    objects = [obj for obj, length in zip(objects, lengths) if length == most_common]
     n_removed = len(lengths) - len(files)
     if n_removed > 0:
         logging.warning(
-            f"Removed {n_removed} RPG-FMCW-94 files due to inconsistent height vector"
+            "Removed %s RPG-FMCW-94 files due to inconsistent height vector",
+            n_removed,
         )
     return objects, files
 
@@ -204,7 +220,8 @@ def _validate_date(obj, expected_date: str) -> None:
     for t in obj.data["time"][:]:
         date_str = "-".join(utils.seconds2date(t)[:3])
         if date_str != expected_date:
-            raise ValueError("Ignoring a file (time stamps not what expected)")
+            msg = "Ignoring a file (time stamps not what expected)"
+            raise ValueError(msg)
 
 
 class Rpg(CloudnetInstrument):
@@ -220,10 +237,14 @@ class Rpg(CloudnetInstrument):
 
     def convert_time_to_fraction_hour(self, data_type: str | None = None) -> None:
         """Converts time to fraction hour."""
-        key = "time"
-        fraction_hour = utils.seconds2hours(self.raw_data[key])
-        self.data[key] = CloudnetArray(
-            np.array(fraction_hour), key, data_type=data_type
+        ms2s = 1e-3
+        total_time_sec = self.raw_data["time"] + self.raw_data.get("time_ms", 0) * ms2s
+        fraction_hour = utils.seconds2hours(total_time_sec)
+
+        self.data["time"] = CloudnetArray(
+            np.array(fraction_hour),
+            "time",
+            data_type=data_type,
         )
 
     def _get_date(self) -> list:
@@ -254,7 +275,8 @@ class Fmcw(Rpg):
         threshold = -35
         if "ldr" in self.data:
             self.data["ldr"].data = ma.masked_less_equal(
-                self.data["ldr"].data, threshold
+                self.data["ldr"].data,
+                threshold,
             )
 
     def mask_invalid_width(self) -> None:
@@ -276,19 +298,21 @@ class Fmcw(Rpg):
         is_valid_zenith = _filter_zenith_angle(zenith)
         n_removed = len(is_valid_zenith) - np.count_nonzero(is_valid_zenith)
         if n_removed == len(zenith):
-            raise ValidTimeStampError("No profiles with valid zenith angle")
+            msg = "No profiles with valid zenith angle"
+            raise ValidTimeStampError(msg)
         if n_removed > 0:
             logging.warning(
-                f"Filtering {n_removed} profiles due to invalid zenith angle"
+                "Filtering %s profiles due to invalid zenith angle",
+                n_removed,
             )
         self.data["zenith_angle"] = CloudnetArray(zenith, "zenith_angle")
         del self.data["elevation"]
         return list(is_valid_zenith)
 
-    def convert_units(self):
+    def convert_units(self) -> None:
         """Converts units."""
         self.data["rainfall_rate"].data = mmh2ms(self.data["rainfall_rate"].data)
-        self.data["lwp"].data *= 1e-3  # g -> kg
+        self.data["lwp"].data *= G_TO_KG
 
     @staticmethod
     def _get_instrument(data: dict):
@@ -297,7 +321,8 @@ class Fmcw(Rpg):
             return instruments.FMCW35
         if math.isclose(frequency, 94, abs_tol=0.1):
             return instruments.FMCW94
-        raise RuntimeError(f"Unknown RPG cloud radar frequency: {frequency}")
+        msg = f"Unknown RPG cloud radar frequency: {frequency}"
+        raise RuntimeError(msg)
 
 
 class Hatpro(Rpg):
@@ -315,7 +340,7 @@ def _filter_zenith_angle(zenith: ma.MaskedArray) -> np.ndarray:
         logging.warning("Can not determine zenith angle, assuming 0 degrees")
     limits = [-5, 15]
     ind_close_to_zenith = np.where(
-        np.logical_and(zenith > limits[0], zenith < limits[1])
+        np.logical_and(zenith > limits[0], zenith < limits[1]),
     )
     if not ind_close_to_zenith[0].size:
         return np.zeros_like(zenith, dtype=bool)
@@ -365,7 +390,8 @@ RPG_ATTRIBUTES = {
     "srho_hv": MetaData(long_name="Slanted correlation coefficient", units="1"),
     "kdp": MetaData(long_name="Specific differential phase shift", units="rad km-1"),
     "differential_attenuation": MetaData(
-        long_name="Differential attenuation", units="dB km-1"
+        long_name="Differential attenuation",
+        units="dB km-1",
     ),
     # All radars
     "file_code": MetaData(