cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/instruments/lufft.py

@@ -1,7 +1,11 @@
 """Module with a class for Lufft chm15k ceilometer."""
+
+import datetime
 import logging
+from os import PathLike
 
 import netCDF4
+import numpy as np
 from numpy import ma
 
 from cloudnetpy import utils
@@ -12,16 +16,17 @@ from cloudnetpy.instruments.nc_lidar import NcLidar
 class LufftCeilo(NcLidar):
     """Class for Lufft chm15k ceilometer."""
 
-    serial_number: str | None
-
     def __init__(
-        self, file_name: str, site_meta: dict, expected_date: str | None = None
-    ):
+        self,
+        file_name: str | PathLike,
+        site_meta: dict,
+        expected_date: datetime.date | None = None,
+    ) -> None:
         super().__init__()
         self.file_name = file_name
         self.site_meta = site_meta
         self.expected_date = expected_date
-        self.serial_number = None
+        self.is_old_version = False
 
     def read_ceilometer_file(self, calibration_factor: float | None = None) -> None:
         """Reads data and metadata from Jenoptik netCDF file."""
@@ -34,32 +39,46 @@ class LufftCeilo(NcLidar):
             self._fetch_zenith_angle("zenith")
 
     def _fetch_beta_raw(self, calibration_factor: float | None = None) -> None:
-        assert self.dataset is not None
         if calibration_factor is None:
             logging.warning("Using default calibration factor")
             calibration_factor = 3e-12
         beta_raw = self._getvar("beta_raw", "beta_att")
+        beta_raw = ma.masked_array(beta_raw)
         old_version = self._get_old_software_version()
         if old_version is not None:
+            self.is_old_version = True
             logging.warning(
-                f"Software version {old_version}. Assuming data not range corrected."
+                "Software version %s. Assuming data not range corrected.",
+                old_version,
             )
             data_std = self._getvar("stddev")
             normalised_apd = self._get_nn()
-            beta_raw *= utils.transpose(data_std / normalised_apd)
+            beta_raw *= utils.transpose(ma.masked_array(data_std / normalised_apd))
             beta_raw *= self.data["range"] ** 2
         beta_raw *= calibration_factor
         self.data["calibration_factor"] = float(calibration_factor)
         self.data["beta_raw"] = beta_raw
 
     def _get_old_software_version(self) -> str | None:
-        assert self.dataset is not None
+        if self.dataset is None:
+            msg = "No dataset found"
+            raise RuntimeError(msg)
         version = self.dataset.software_version
-        if len(str(version)) > 4:
+        # In old files, the version is a single integer.
+        if isinstance(version, np.integer):
+            return str(version)
+        # In newer files, the version is a space-separated list: Operating
+        # system, FPGA, firmware, CloudDetectionMode (added in firmware 0.747).
+        if isinstance(version, str):
+            parts = version.split()
+            firmware = parts[2]
+            if firmware < "0.702":
+                return firmware
             return None
-        return version
+        msg = f"Cannot determine version: {version}"
+        raise RuntimeError(msg)
 
-    def _get_nn(self):
+    def _get_nn(self) -> float | ma.MaskedArray:
         nn1 = self._getvar("nn1", "NN1")
         median_nn1 = ma.median(nn1)
         # Parameters taken from the matlab code and should be verified
@@ -72,18 +91,21 @@ class LufftCeilo(NcLidar):
             return 1
         return step_factor ** (-(nn1 - reference) / scale)
 
-    def _getvar(self, *args):
-        assert self.dataset is not None
+    def _getvar(self, *args: str) -> float | ma.MaskedArray:
+        if self.dataset is None:
+            msg = "No dataset found"
+            raise RuntimeError(msg)
         for arg in args:
             if arg in self.dataset.variables:
                 var = self.dataset.variables[arg]
                 return var[0] if utils.isscalar(var) else var[:]
-        raise ValueError("Unknown variable")
+        msg = f"Unable to find variable {args[0]}"
+        raise ValueError(msg)
 
-    def _fetch_attributes(self):
+    def _fetch_attributes(self) -> None:
         self.serial_number = getattr(self.dataset, "device_name", None)
         if self.serial_number is None:
-            self.serial_number = getattr(self.dataset, "source")
+            self.serial_number = getattr(self.dataset, "source", "")
         self.instrument = (
             instruments.CHM15KX
             if self.serial_number.startswith("CHX")

cloudnetpy/instruments/mira.py

@@ -1,21 +1,31 @@
 """Module for reading raw cloud radar data."""
+
+import datetime
+import logging
 import os
-from tempfile import TemporaryDirectory
+import re
+from collections import OrderedDict
+from collections.abc import Sequence
+from os import PathLike
+from tempfile import NamedTemporaryFile, TemporaryDirectory
+from uuid import UUID
+
+import numpy as np
+from numpy import ma
 
 from cloudnetpy import concat_lib, output, utils
-from cloudnetpy.exceptions import ValidTimeStampError
-from cloudnetpy.instruments.instruments import MIRA35
+from cloudnetpy.instruments.instruments import MIRA10, MIRA35
 from cloudnetpy.instruments.nc_radar import NcRadar
 from cloudnetpy.metadata import MetaData
 
 
 def mira2nc(
-    raw_mira: str | list[str],
-    output_file: str,
+    raw_mira: str | PathLike | Sequence[str | PathLike],
+    output_file: str | PathLike,
     site_meta: dict,
-    uuid: str | None = None,
-    date: str | None = None,
-) -> str:
+    uuid: str | UUID | None = None,
+    date: str | datetime.date | None = None,
+) -> UUID:
     """Converts METEK MIRA-35 cloud radar data into Cloudnet Level 1b netCDF file.
 
     This function converts raw MIRA file(s) into a much smaller file that
@@ -23,8 +33,10 @@ def mira2nc(
     steps.
 
     Args:
-        raw_mira: Filename of a daily MIRA .mmclx file. Can be also a folder containing
-            several non-concatenated .mmclx files from one day or list of files.
+        raw_mira: Filename of a daily MIRA .mmclx or .zncfile. Can be also a folder
+            containing several non-concatenated .mmclx or .znc files from one day
+            or list of files. znc files take precedence because they are the newer
+            filetype
         output_file: Output filename.
         site_meta: Dictionary containing information about the site. Required key
             value pair is `name`.
@@ -36,66 +48,68 @@ def mira2nc(
 
     Raises:
         ValidTimeStampError: No valid timestamps found.
+        FileNotFoundError: No suitable input files found.
+        ValueError: Wrong suffix in input file(s).
+        TypeError: Mixed mmclx and znc files.
 
     Examples:
           >>> from cloudnetpy.instruments import mira2nc
           >>> site_meta = {'name': 'Vehmasmaki'}
           >>> mira2nc('raw_radar.mmclx', 'radar.nc', site_meta)
+          >>> mira2nc('raw_radar.znc', 'radar.nc', site_meta)
           >>> mira2nc('/one/day/of/mira/mmclx/files/', 'radar.nc', site_meta)
+          >>> mira2nc('/one/day/of/mira/znc/files/', 'radar.nc', site_meta)
 
     """
-    keymap = {
-        "Zg": "Zh",
-        "VELg": "v",
-        "RMSg": "width",
-        "LDRg": "ldr",
-        "SNRg": "SNR",
-        "elv": "elevation",
-        "azi": "azimuth_angle",
-        "aziv": "azimuth_velocity",
-        "nfft": "nfft",
-        "nave": "nave",
-        "prf": "prf",
-        "rg0": "rg0",
-    }
+    if isinstance(date, str):
+        date = datetime.date.fromisoformat(date)
+    uuid = utils.get_uuid(uuid)
 
     with TemporaryDirectory() as temp_dir:
-        if isinstance(raw_mira, list) or os.path.isdir(raw_mira):
-            mmclx_filename = f"{temp_dir}/tmp.mmclx"
-            if isinstance(raw_mira, list):
-                valid_filenames = sorted(raw_mira)
-            else:
-                valid_filenames = utils.get_sorted_filenames(raw_mira, ".mmclx")
-            valid_filenames = utils.get_files_with_common_range(valid_filenames)
-            variables = list(keymap.keys())
-            concat_lib.concatenate_files(
-                valid_filenames,
-                mmclx_filename,
-                variables=variables,
-                allow_difference=["nave", "ovl"],
-            )
-        else:
-            mmclx_filename = raw_mira
+        input_filename, keymap = _parse_input_files(raw_mira, temp_dir)
 
-        with Mira(mmclx_filename, site_meta) as mira:
+        with Mira(input_filename, site_meta) as mira:
             mira.init_data(keymap)
             if date is not None:
                 mira.screen_by_date(date)
-                mira.date = date.split("-")
+                mira.date = date
             mira.sort_timestamps()
             mira.remove_duplicate_timestamps()
             mira.linear_to_db(("Zh", "ldr", "SNR"))
-            mira.screen_by_snr()
+            mira.screen_low_power()
+
+            if "snr_limit" in site_meta and site_meta["snr_limit"] is not None:
+                snr_limit = site_meta["snr_limit"]
+            else:
+                # Empirical values, should be checked
+                snr_limit = -30 if mira.instrument == MIRA10 else -17
+
+            # Old MIRA files don't have angle variables.
+            if "elevation" not in mira.data:
+                mira.append_data(ma.masked_all_like(mira.time.data), "elevation")
+            if "azimuth_angle" not in mira.data:
+                mira.append_data(ma.masked_all_like(mira.time.data), "azimuth_angle")
+
+            mira.screen_by_snr(snr_limit)
+            mira.screen_invalid_ldr()
             mira.mask_invalid_data()
+            mira.mask_bad_angles()
             mira.add_time_and_range()
             mira.add_site_geolocation()
             mira.add_radar_specific_variables()
-            valid_indices = mira.add_zenith_and_azimuth_angles()
+            valid_indices = mira.add_zenith_and_azimuth_angles(
+                elevation_threshold=1.1,
+                elevation_diff_threshold=1e-6,
+                azimuth_diff_threshold=1e-3,
+                zenith_offset=site_meta.get("zenith_offset"),
+                azimuth_offset=site_meta.get("azimuth_offset"),
+            )
             mira.screen_time_indices(valid_indices)
             mira.add_height()
+            mira.test_if_all_masked()
         attributes = output.add_time_attribute(ATTRIBUTES, mira.date)
         output.update_attributes(mira.data, attributes)
-        uuid = output.save_level1b(mira, output_file, uuid)
+        output.save_level1b(mira, output_file, uuid)
         return uuid
 
 
@@ -108,42 +122,235 @@ class Mira(NcRadar):
 
     """
 
-    epoch = (1970, 1, 1)
+    epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
 
-    def __init__(self, full_path: str, site_meta: dict):
+    def __init__(self, full_path: str | PathLike, site_meta: dict) -> None:
         super().__init__(full_path, site_meta)
         self.date = self._init_mira_date()
-        self.instrument = MIRA35
+        if "model" not in site_meta or site_meta["model"] == "mira-35":
+            self.instrument = MIRA35
+        elif site_meta["model"] == "mira-10":
+            self.instrument = MIRA10
+        else:
+            msg = f"Invalid model: {site_meta['model']}"
+            raise ValueError(msg)
 
-    def screen_by_date(self, expected_date: str) -> None:
+    def screen_by_date(self, expected_date: datetime.date) -> None:
         """Screens incorrect time stamps."""
         time_stamps = self.getvar("time")
         valid_indices = []
         for ind, timestamp in enumerate(time_stamps):
-            date = "-".join(utils.seconds2date(timestamp, self.epoch)[:3])
+            if not timestamp:
+                continue
+            date = utils.seconds2date(timestamp, self.epoch).date()
             if date == expected_date:
                 valid_indices.append(ind)
-        if not valid_indices:
-            raise ValidTimeStampError
         self.screen_time_indices(valid_indices)
 
-    def _init_mira_date(self) -> list[str]:
+    def _init_mira_date(self) -> datetime.date:
         time_stamps = self.getvar("time")
-        return utils.seconds2date(time_stamps[0], self.epoch)[:3]
+        return utils.seconds2date(float(time_stamps[0]), self.epoch).date()
+
+    def screen_low_power(self) -> None:
+        """Screen times with average transmit power close to zero."""
+        if "tpow" not in self.data:
+            logging.warning("Variable tpow is missing")
+            return
+        tpow = self.data["tpow"][:]
+        # Threshold for abnormally low power e.g. Limassol 2024-10-20. Average
+        # power should 30 to 60 W according to MIRA-35 data sheet. Based on a
+        # random sample, typical range is 15 to 25 W. In Lampedusa, the power is
+        # constantly as low as 1.9 W.
+        is_low = tpow < 1
+        n_removed = np.count_nonzero(is_low)
+        if n_removed > 0:
+            logging.warning(
+                "Filtering %s profiles due to low average transmit power", n_removed
+            )
+            self.screen_time_indices(~is_low)
+
+    def screen_invalid_ldr(self) -> None:
+        """Masks LDR in MIRA STSR mode data.
+        Is there a better way to identify this mode?
+        """
+        if "ldr" not in self.data:
+            return
+        ldr = self.data["ldr"][:]
+        if ma.mean(ldr) > 0:
+            logging.warning(
+                "LDR values suspiciously high. Mira in STSR mode? "
+                "Screening all LDR for now.",
+            )
+            self.data["ldr"].data[:] = ma.masked
+
+    def mask_bad_angles(self) -> None:
+        """Masks clearly bad elevation and azimuth angles."""
+        limits = {
+            "elevation": (0, 180),
+            "azimuth_angle": (-360, 360),
+        }
+        for key, (lower, upper) in limits.items():
+            if (array := self.data[key].data) is not None:
+                margin = (upper - lower) * 0.05
+                array[array < (lower - margin)] = ma.masked
+                array[array > (upper + margin)] = ma.masked
+
+
+def _parse_input_files(
+    input_files: str | PathLike | Sequence[str | PathLike], temp_dir: str
+) -> tuple[str | PathLike, dict[str, str]]:
+    input_filename: str | PathLike
+    if (
+        not isinstance(input_files, str) and isinstance(input_files, Sequence)
+    ) or os.path.isdir(input_files):
+        with NamedTemporaryFile(
+            dir=temp_dir,
+            suffix=".nc",
+            delete=False,
+        ) as temp_file:
+            input_filename = temp_file.name
+            if not isinstance(input_files, str) and isinstance(input_files, Sequence):
+                valid_files = sorted(map(str, input_files))
+            else:
+                valid_files = utils.get_sorted_filenames(input_files, ".znc")
+                if not valid_files:
+                    valid_files = utils.get_sorted_filenames(input_files, ".mmclx")
+
+            if not valid_files:
+                msg = (
+                    (
+                        f"Neither znc nor mmclx files found {input_files}. "
+                        f"Please check your input."
+                    ),
+                )
+                raise FileNotFoundError(msg)
+
+            filetypes = list({_get_suffix(f) for f in valid_files})
+
+            if len(filetypes) > 1:
+                err_msg = "Mixed mmclx and znc files. Please use only one filetype."
+                raise TypeError(err_msg)
+
+            keymap = _get_keymap(filetypes[0])
+
+            variables = list(keymap.keys())
+            concat_lib.concatenate_files(
+                valid_files,
+                input_filename,
+                variables=variables,
+                ignore=_get_ignored_variables(filetypes[0]),
+            )
+    else:
+        input_filename = input_files
+        keymap = _get_keymap(_get_suffix(input_filename))
+
+    return input_filename, keymap
+
+
+def _get_ignored_variables(filetype: str) -> list | None:
+    """Returns variables to ignore for METEK MIRA-35 cloud radar concat."""
+    _check_file_type(filetype)
+    # Ignore spectral variables for now
+    keymaps = {
+        "znc": ["DropSize", "SPCco", "SPCcx", "SPCcocxRe", "SPCcocxIm", "doppler"],
+        "mmclx": None,
+    }
+
+    return keymaps.get(filetype.lower(), keymaps.get("mmclx"))
+
+
+def _get_suffix(filename: str | PathLike) -> str:
+    m = re.search(r"\.(\w+)(\.\d+)?$", str(filename))
+    if m is None:
+        return ""
+    return m[1].lower()
+
+
+def _get_keymap(filetype: str) -> dict[str, str]:
+    """Returns a dictionary mapping the variables in the raw data to the processed
+    Cloudnet file.
+    """
+    _check_file_type(filetype)
+
+    # Order is relevant with the new znc files from STSR radar
+    keymaps = {
+        "znc": OrderedDict(
+            [
+                ("Zg", "Zh"),  # fallback
+                ("Zh2l", "Zh"),
+                ("VELg", "v"),  # fallback
+                ("VELh2l", "v"),
+                ("RMSg", "width"),  # fallback
+                ("RMSh2l", "width"),
+                ("LDRg", "ldr"),  # fallback
+                ("LDRh2l", "ldr"),
+                ("SNRg", "SNR"),  # fallback
+                ("SNRh2l", "SNR"),
+                ("elv", "elevation"),
+                ("azi", "azimuth_angle"),
+                ("nfft", "nfft"),
+                ("nave", "nave"),
+                ("prf", "prf"),
+                ("rg0", "rg0"),
+                ("tpow", "tpow"),
+            ],
+        ),
+        "mmclx": OrderedDict(
+            [
+                ("Ze", "Zh"),  # fallback for old mmclx files
+                ("Zg", "Zh"),
+                ("VELg", "v"),
+                ("RMSg", "width"),
+                ("LDRg", "ldr"),
+                ("SNRg", "SNR"),
+                ("elv", "elevation"),
+                ("azi", "azimuth_angle"),
+                ("nfft", "nfft"),
+                ("nave", "nave"),
+                ("prf", "prf"),
+                ("rg0", "rg0"),
+                ("NyquistVelocity", "NyquistVelocity"),  # variable in some mmclx files
+                ("tpow", "tpow"),
+            ]
+        ),
+    }
+
+    return keymaps.get(filetype.lower(), keymaps["mmclx"])
+
+
+def _check_file_type(filetype: str) -> None:
+    known_filetypes = ["znc", "mmclx"]
+    if filetype.lower() not in known_filetypes:
+        msg = f"Filetype must be one of {known_filetypes}"
+        raise ValueError(msg)
 
 
 ATTRIBUTES = {
-    "nfft": MetaData(
-        long_name="Number of FFT points",
-        units="1",
-    ),
+    "nfft": MetaData(long_name="Number of FFT points", units="1", dimensions=("time",)),
     "nave": MetaData(
         long_name="Number of spectral averages (not accounting for overlapping FFTs)",
         units="1",
+        dimensions=("time",),
+    ),
+    "rg0": MetaData(
+        long_name="Number of lowest range gates", units="1", dimensions=("time",)
     ),
-    "rg0": MetaData(long_name="Number of lowest range gates", units="1"),
     "prf": MetaData(
-        long_name="Pulse Repetition Frequency",
-        units="Hz",
+        long_name="Pulse Repetition Frequency", units="Hz", dimensions=("time",)
+    ),
+    "tpow": MetaData(
+        long_name="Average Transmit Power", units="W", dimensions=("time",)
+    ),
+    "zenith_offset": MetaData(
+        long_name="Zenith offset of the instrument",
+        units="degrees",
+        comment="Zenith offset applied.",
+        dimensions=None,
+    ),
+    "azimuth_offset": MetaData(
+        long_name="Azimuth offset of the instrument (positive clockwise from north)",
+        units="degrees",
+        comment="Azimuth offset applied.",
+        dimensions=None,
     ),
 }