cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/instruments/mrr.py

@@ -0,0 +1,187 @@
+import datetime
+import logging
+import re
+from collections.abc import Iterable
+from os import PathLike
+from pathlib import Path
+from tempfile import NamedTemporaryFile, TemporaryDirectory
+from uuid import UUID
+
+import netCDF4
+
+from cloudnetpy import concat_lib, output, utils
+from cloudnetpy.instruments import instruments
+from cloudnetpy.instruments.nc_radar import NcRadar
+from cloudnetpy.metadata import MetaData
+
+
+def mrr2nc(
+    input_file: PathLike | str | Iterable[PathLike | str],
+    output_file: PathLike | str,
+    site_meta: dict,
+    uuid: str | UUID | None = None,
+    date: str | datetime.date | None = None,
+) -> UUID:
+    """Converts METEK MRR-PRO data into Cloudnet Level 1b netCDF file.
+
+    This function converts raw MRR file(s) into a much smaller file that
+    contains only the relevant data.
+
+    Args:
+        input_file: Filename of a daily MMR-PRO .nc file, path to directory
+            containing several non-concatenated .nc files from one day, or list
+            of filenames.
+        output_file: Output filename.
+        site_meta: Dictionary containing information about the site. Required key
+            value pairs are `name`, `latitude`, `longitude` and `altitude`.
+        uuid: Set specific UUID for the file.
+        date: Expected date as YYYY-MM-DD of all profiles in the file.
+
+    Returns:
+        UUID of the generated file.
+
+    Raises:
+        ValidTimeStampError: No valid timestamps found.
+
+    Examples:
+          >>> from cloudnetpy.instruments import mira2nc
+          >>> site_meta = {'name': 'LIM', 'latitude': 51.333, 'longitude': 12.389}
+          >>> mrr2nc('input.nc', 'output.nc', site_meta)
+    """
+    if isinstance(date, str):
+        date = datetime.date.fromisoformat(date)
+    uuid = utils.get_uuid(uuid)
+
+    keymap = {
+        "RR": "rainfall_rate",
+        "WIDTH": "width",
+        "VEL": "v",
+        "LWC": "lwc",
+        "Ze": "Zh",
+        "PIA": "pia",
+    }
+
+    def valid_nc_files(files: Iterable[PathLike | str]) -> Iterable[PathLike | str]:
+        for file in files:
+            try:
+                with netCDF4.Dataset(file):
+                    yield file
+            except OSError:
+                logging.warning("Skipping invalid file: %s", file)
+
+    def concat_files(dir_name: str, files: Iterable[PathLike | str]) -> str:
+        with NamedTemporaryFile(
+            dir=dir_name,
+            suffix=".nc",
+            delete=False,
+        ) as temp_file:
+            tmp_filename = temp_file.name
+            variables = [*keymap.keys(), "elevation"]
+            valid_files = list(valid_nc_files(files))
+            concat_lib.concatenate_files(
+                valid_files,
+                tmp_filename,
+                variables=variables,
+                ignore=["time_coverage_start", "time_coverage_end"],
+            )
+            return tmp_filename
+
+    with TemporaryDirectory() as temp_dir:
+        if isinstance(input_file, PathLike | str):
+            path = Path(input_file)
+            if path.is_dir():
+                input_file = concat_files(
+                    temp_dir,
+                    (p for p in path.iterdir() if p.suffix.lower() == ".nc"),
+                )
+        else:
+            input_file = concat_files(temp_dir, input_file)
+
+        with MrrPro(input_file, site_meta) as mrr:
+            mrr.init_data(keymap)
+            mrr.fix_units()
+            mrr.date = mrr.init_date()
+            if date:
+                mrr.screen_by_date(date)
+            mrr.add_time_and_range()
+            mrr.fix_range()
+            mrr.add_site_geolocation()
+            mrr.add_zenith_angle()
+            mrr.add_radar_specific_variables()
+            mrr.add_height()
+            mrr.sort_timestamps()
+        attributes = output.add_time_attribute(ATTRIBUTES, mrr.date)
+        output.update_attributes(mrr.data, attributes)
+        output.save_level1b(mrr, output_file, uuid)
+        return uuid
+
+
+class MrrPro(NcRadar):
+    """Class for MRR-PRO raw data. Child of NcRadar().
+
+    Args:
+        full_path: MRR-PRO netCDF filename.
+        site_meta: Site properties in a dictionary. Required keys are `name`,
+            `latitude`, `longitude` and `altitude`.
+
+    """
+
+    epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
+
+    def __init__(self, full_path: PathLike | str, site_meta: dict) -> None:
+        super().__init__(full_path, site_meta)
+        self.instrument = instruments.MRR_PRO
+        if m := re.search(
+            r"serial number:\s*(\w+)",
+            self.dataset.instrument_name,
+            re.IGNORECASE,
+        ):
+            self.serial_number = m[1]
+
+    def init_date(self) -> datetime.date:
+        time_stamps = self.getvar("time")
+        return utils.seconds2date(time_stamps[0], self.epoch).date()
+
+    def fix_units(self) -> None:
+        self.data["v"].data *= -1  # towards -> away from instrument
+        self.data["rainfall_rate"].data /= 3600000  # mm h-1 -> m s-1
+        self.data["lwc"].data *= 0.001  # g m-3 -> kg m-3
+
+    def add_zenith_angle(self) -> None:
+        elevation = self.getvar("elevation")
+        zenith = 90 - elevation
+        self.append_data(zenith, "zenith_angle")
+
+    def screen_by_date(self, expected_date: datetime.date) -> None:
+        """Screens incorrect time stamps."""
+        time_stamps = self.getvar("time")
+        valid_indices = []
+        for ind, timestamp in enumerate(time_stamps):
+            date = utils.seconds2date(timestamp, self.epoch).date()
+            if date == expected_date:
+                valid_indices.append(ind)
+        self.screen_time_indices(valid_indices)
+
+    def fix_range(self) -> None:
+        # It seems like the "range" variable in MRR-PRO raw files is actually
+        # defined above mean sea level -> convert to range above ground level
+        range_data = self.data["range"].data
+        range_data -= range_data[0]
+
+
+ATTRIBUTES = {
+    "lwc": MetaData(
+        long_name="Liquid water content", units="kg m-3", dimensions=("time", "range")
+    ),
+    "pia": MetaData(
+        long_name="Path integrated rain attenuation",
+        units="dB",
+        dimensions=("time", "range"),
+    ),
+    "rainfall_rate": MetaData(
+        long_name="Rainfall rate",
+        standard_name="rainfall_rate",
+        units="m s-1",
+        dimensions=("time", "range"),
+    ),
+}

cloudnetpy/instruments/nc_lidar.py

@@ -1,39 +1,50 @@
 """Module with a class for Lufft chm15k ceilometer."""
+
 import logging
+from typing import TYPE_CHECKING, Literal
 
-import netCDF4
 import numpy as np
+from numpy import ma
 
 from cloudnetpy import utils
 from cloudnetpy.instruments.ceilometer import Ceilometer
 
+if TYPE_CHECKING:
+    import netCDF4
+
 
 class NcLidar(Ceilometer):
     """Class for all lidars using netCDF files."""
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__()
         self.dataset: netCDF4.Dataset | None = None
 
-    def _fetch_range(self, reference: str) -> None:
-        assert self.dataset is not None
+    def _fetch_range(self, reference: Literal["upper", "lower"]) -> None:
+        if self.dataset is None:
+            msg = "No dataset found"
+            raise RuntimeError(msg)
         range_instrument = self.dataset.variables["range"][:]
         self.data["range"] = utils.edges2mid(range_instrument, reference)
 
     def _fetch_time_and_date(self) -> None:
-        assert self.dataset is not None
+        if self.dataset is None:
+            msg = "No dataset found"
+            raise RuntimeError(msg)
         time = self.dataset.variables["time"]
         self.data["time"] = time[:]
         epoch = utils.get_epoch(time.units)
         self.get_date_and_time(epoch)
 
     def _fetch_zenith_angle(self, key: str, default: float = 3.0) -> None:
-        assert self.dataset is not None
+        if self.dataset is None:
+            msg = "No dataset found"
+            raise RuntimeError(msg)
         if key in self.dataset.variables:
-            zenith_angle = self.dataset.variables[key][:]
+            zenith_angle = ma.median(self.dataset.variables[key][:])
         else:
             zenith_angle = float(default)
-            logging.warning(f"No zenith angle found, assuming {zenith_angle} degrees")
+            logging.warning("No zenith angle found, assuming %s degrees", zenith_angle)
         if zenith_angle == 0:
             logging.warning("Zenith angle 0 degrees - risk of specular reflection")
         self.data["zenith_angle"] = np.array(zenith_angle)

cloudnetpy/instruments/nc_radar.py

@@ -1,5 +1,9 @@
 """Module for reading raw cloud radar data."""
+
+import datetime
 import logging
+from os import PathLike
+from typing import TYPE_CHECKING
 
 import numpy as np
 from numpy import ma
@@ -7,9 +11,11 @@ from numpy import ma
 from cloudnetpy import utils
 from cloudnetpy.cloudnetarray import CloudnetArray
 from cloudnetpy.datasource import DataSource
-from cloudnetpy.exceptions import ValidTimeStampError
+from cloudnetpy.exceptions import RadarDataError, ValidTimeStampError
 from cloudnetpy.instruments.cloudnet_instrument import CloudnetInstrument
-from cloudnetpy.instruments.instruments import Instrument
+
+if TYPE_CHECKING:
+    from cloudnetpy.instruments.instruments import Instrument
 
 
 class NcRadar(DataSource, CloudnetInstrument):
@@ -23,10 +29,10 @@ class NcRadar(DataSource, CloudnetInstrument):
         Used with BASTA, MIRA and Copernicus radars.
     """
 
-    def __init__(self, full_path: str, site_meta: dict):
+    def __init__(self, full_path: PathLike | str, site_meta: dict) -> None:
         super().__init__(full_path)
         self.site_meta = site_meta
-        self.date: list[str]
+        self.date: datetime.date
         self.instrument: Instrument | None = None
 
     def init_data(self, keymap: dict) -> None:
@@ -35,28 +41,39 @@ class NcRadar(DataSource, CloudnetInstrument):
             name = keymap[key]
             try:
                 array = self.getvar(key)
-            except RuntimeError:
-                logging.warning(f"Can not find variable {key} from the input file")
+            except KeyError:
+                logging.warning("Can not find variable %s from the input file", key)
                 continue
             array = np.array(array) if utils.isscalar(array) else array
             array[~np.isfinite(array)] = ma.masked
             self.append_data(array, name)
 
-    def add_time_and_range(self) -> None:
+    def add_time_and_range(self, time_dtype: str | None = None) -> None:
         """Adds time and range."""
         range_instru = np.array(
-            self.getvar("range", "height")
+            self.getvar("range", "height"),
         )  # "height" in old BASTA files
         time = np.array(self.time)
         self.append_data(range_instru, "range")
-        self.append_data(time, "time")
+        self.append_data(time, "time", dtype=time_dtype)
 
-    def screen_by_snr(self, snr_limit: float = -17) -> None:
+    def screen_by_snr(self, snr_limit: float) -> None:
         """Mask values where SNR smaller than threshold."""
         ind = np.where(self.data["SNR"][:] < snr_limit)
         for cloudnet_array in self.data.values():
             if cloudnet_array.data.ndim == 2:
                 cloudnet_array.mask_indices(ind)
+        self.append_data(float(snr_limit), "snr_limit")
+
+    def screen_using_top_gates_snr(self, snr_limit: float = 2) -> None:
+        """Masks values where SNR is smaller than mean SNR of top gates."""
+        n_gates = 50
+        snr = self.data["SNR"][:]
+        mean_snr = np.mean(snr[:, -n_gates:], axis=1)
+        for time_ind, snr_profile in enumerate(snr):
+            alt_ind = np.where(snr_profile < mean_snr[time_ind] + snr_limit)[0]
+            if len(alt_ind) > 0:
+                self.data["Zh"][:][time_ind, alt_ind] = ma.masked
 
     def mask_invalid_data(self) -> None:
         """Makes sure Z and v masks are also in other 2d variables."""
@@ -67,76 +84,113 @@ class NcRadar(DataSource, CloudnetInstrument):
                 cloudnet_array.mask_indices(z_mask)
                 cloudnet_array.mask_indices(v_mask)
 
-    def add_zenith_and_azimuth_angles(self) -> list:
+    def mask_first_range_gates(self, range_limit: float = 150) -> None:
+        """Masks first range gates."""
+        if "v" not in self.data or "range" not in self.data:
+            return
+        ind = np.where(self.data["range"][:] < range_limit)[0]
+        if len(ind) > 0:
+            self.data["v"].data[:, ind] = ma.masked
+
+    def add_zenith_and_azimuth_angles(
+        self,
+        elevation_threshold: float,
+        elevation_diff_threshold: float,
+        azimuth_diff_threshold: float,
+        zenith_offset: float | None = None,
+        azimuth_offset: float | None = None,
+    ) -> list:
         """Adds non-varying instrument zenith and azimuth angles and returns valid
-        time indices."""
-        if "azimuth_velocity" in self.data:
-            azimuth = self.data["azimuth_velocity"].data
-            if np.all(azimuth == azimuth[0]):
-                azimuth_reference = azimuth[0]
-            else:
-                azimuth_reference = 0
-            azimuth_tolerance = 1e-6
-        else:
-            azimuth = self.data["azimuth_angle"].data
-            azimuth_reference = ma.median(azimuth)
-            azimuth_tolerance = 0.1
-
+        time indices.
+        """
         elevation = self.data["elevation"].data
-        zenith = 90 - elevation
-        is_stable_zenith = np.isclose(zenith, ma.median(zenith), atol=0.1)
-        is_stable_azimuth = np.isclose(
-            azimuth, azimuth_reference, atol=azimuth_tolerance
-        )
-        is_stable_profile = is_stable_zenith & is_stable_azimuth
-        if ma.isMaskedArray(is_stable_profile):
-            is_stable_profile[is_stable_profile.mask] = False
-        n_removed = np.count_nonzero(~is_stable_profile)
-        if n_removed >= len(zenith) - 1:
-            raise ValidTimeStampError(
-                "Less than two profiles with valid zenith / azimuth angles"
-            )
+        if zenith_offset is not None:
+            zenith_offset = float(zenith_offset)
+            self.append_data(zenith_offset, "zenith_offset")
+            elevation -= zenith_offset
+
+        azimuth = self.data["azimuth_angle"].data
+        if azimuth_offset is not None:
+            azimuth_offset = float(azimuth_offset)
+            self.append_data(azimuth_offset, "azimuth_offset")
+            azimuth += azimuth_offset
+
+        if len(elevation) < 2 or len(azimuth) < 2:
+            msg = "Less than two profiles with valid zenith / azimuth angles"
+            raise ValidTimeStampError(msg)
+
+        elevation_diff = ma.diff(elevation, prepend=elevation[1])
+        azimuth_diff = ma.diff(azimuth, prepend=azimuth[1])
+
+        is_stable = np.abs(elevation - 90) < elevation_threshold
+        is_stable &= np.abs(elevation_diff) < elevation_diff_threshold
+        is_stable &= np.abs(azimuth_diff) < azimuth_diff_threshold
+
+        # If scanning unit is broken, data are missing
+        # (assume it's vertically pointing)
+        missing_info = elevation.mask & azimuth.mask
+        is_stable[missing_info] = True
+
+        if ma.isMaskedArray(is_stable):
+            is_stable[is_stable.mask] = False
+        n_removed = np.count_nonzero(~is_stable)
+
+        if n_removed >= len(elevation) - 1:
+            msg = "Less than two profiles with valid zenith / azimuth angles"
+            raise ValidTimeStampError(msg)
+
         if n_removed > 0:
             logging.warning(
-                f"Filtering {n_removed} profiles due to varying zenith / azimuth angle"
+                "Filtering %s profiles due to varying zenith / azimuth angle",
+                n_removed,
             )
+        zenith = 90 - elevation
         self.append_data(zenith, "zenith_angle")
-        for key in ("elevation", "azimuth_velocity"):
-            if key in self.data:
-                del self.data[key]
-        return list(is_stable_profile)
+        del self.data["elevation"]
+        return list(is_stable)
 
-    def add_radar_specific_variables(self):
+    def add_radar_specific_variables(self) -> None:
         """Adds radar specific variables."""
-        assert self.instrument is not None
+        if self.instrument is None:
+            msg = "Instrument not defined"
+            raise RuntimeError(msg)
         key = "radar_frequency"
-        self.data[key] = CloudnetArray(self.instrument.frequency, key)
+        if self.instrument.frequency is not None:
+            self.data[key] = CloudnetArray(self.instrument.frequency, key)
         try:
             possible_nyquist_names = ("ambiguous_velocity", "NyquistVelocity")
             data = self.getvar(*possible_nyquist_names)
             key = "nyquist_velocity"
-            self.data[key] = CloudnetArray(np.array(data), key)
-        except RuntimeError:
+            self.data[key] = CloudnetArray(np.median(np.array(data)), key)
+            if "NyquistVelocity" in self.data:
+                del self.data["NyquistVelocity"]
+        except KeyError:
             logging.warning("Unable to find nyquist_velocity")
 
+    def test_if_all_masked(self) -> None:
+        """Tests if all data are masked."""
+        v = self.data["v"][:]
+        if ma.isMaskedArray(v) and np.all(v.mask):
+            msg = "All radar data are masked"
+            raise RadarDataError(msg)
+
 
 class ChilboltonRadar(NcRadar):
     """Class for Chilbolton cloud radars Galileo and Copernicus."""
 
-    def __init__(self, full_path: str, site_meta: dict):
+    def __init__(self, full_path: str | PathLike, site_meta: dict) -> None:
         super().__init__(full_path, site_meta)
         self.date = self._init_date()
 
-    def add_nyquist_velocity(self, keymap: dict):
+    def add_nyquist_velocity(self, keymap: dict) -> None:
         """Adds nyquist velocity."""
-        key = [key for key, value in keymap.items() if value == "v"][0]
+        key = [key for key, value in keymap.items() if value == "v"][0]  # noqa: RUF015
         folding_velocity = self.dataset.variables[key].folding_velocity
         self.append_data(np.array(folding_velocity), "nyquist_velocity")
 
-    def check_date(self, date: str):
-        if self.date != date.split("-"):
+    def check_date(self, date: datetime.date) -> None:
+        if self.date != date:
             raise ValidTimeStampError
 
-    def _init_date(self) -> list[str]:
-        epoch = utils.get_epoch(self.dataset["time"].units)
-        return [str(x).zfill(2) for x in epoch]
+    def _init_date(self) -> datetime.date:
+        return utils.get_epoch(self.dataset["time"].units).date()