cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/instruments/galileo.py

@@ -1,6 +1,10 @@
 """Module for reading raw Galileo cloud radar data."""
+
+import datetime
 import os
-from tempfile import TemporaryDirectory
+from os import PathLike
+from tempfile import NamedTemporaryFile, TemporaryDirectory
+from uuid import UUID
 
 import numpy as np
 
@@ -11,12 +15,12 @@ from cloudnetpy.metadata import MetaData
 
 
 def galileo2nc(
-    raw_files: str,
-    output_file: str,
+    raw_files: str | PathLike,
+    output_file: str | PathLike,
     site_meta: dict,
-    uuid: str | None = None,
-    date: str | None = None,
-) -> str:
+    uuid: str | UUID | None = None,
+    date: str | datetime.date | None = None,
+) -> UUID:
     """Converts 'Galileo' cloud radar data into Cloudnet Level 1b netCDF file.
 
     Args:
@@ -41,6 +45,10 @@ def galileo2nc(
           >>> galileo2nc('/one/day/of/galileo/files/', 'radar.nc', site_meta)
 
     """
+    if isinstance(date, str):
+        date = datetime.date.fromisoformat(date)
+    uuid = utils.get_uuid(uuid)
+
     keymap = {
         "ZED_HC": "Zh",
         "VEL_HC": "v",
@@ -55,15 +63,25 @@ def galileo2nc(
         "beamwidthH": "beamwidthH",
     }
 
+    nc_filename: str | PathLike
     with TemporaryDirectory() as temp_dir:
         if os.path.isdir(raw_files):
-            nc_filename = f"{temp_dir}/tmp.nc"
-            valid_filenames = utils.get_sorted_filenames(raw_files, ".nc")
-            valid_filenames = utils.get_files_with_common_range(valid_filenames)
-            variables = list(keymap.keys())
-            concat_lib.concatenate_files(
-                valid_filenames, nc_filename, variables=variables
-            )
+            with NamedTemporaryFile(
+                dir=temp_dir,
+                suffix=".nc",
+                delete=False,
+            ) as temp_file:
+                nc_filename = temp_file.name
+                valid_filenames = utils.get_sorted_filenames(raw_files, ".nc")
+                valid_filenames = utils.get_files_with_variables(
+                    valid_filenames, ["time", "ZED_HC"]
+                )
+                variables = list(keymap.keys())
+                concat_lib.concatenate_files(
+                    valid_filenames,
+                    nc_filename,
+                    variables=variables,
+                )
         else:
             nc_filename = raw_files
 
@@ -74,20 +92,25 @@ def galileo2nc(
                 galileo.check_date(date)
             galileo.sort_timestamps()
             galileo.remove_duplicate_timestamps()
+            galileo.screen_negative_altitudes()
             snr_limit = site_meta.get("snr_limit", 3)
             galileo.screen_by_snr(snr_limit=snr_limit)
             galileo.mask_clutter()
             galileo.mask_invalid_data()
-            galileo.add_time_and_range()
             galileo.add_radar_specific_variables()
             galileo.add_nyquist_velocity(keymap)
             galileo.add_site_geolocation()
-            valid_indices = galileo.add_zenith_and_azimuth_angles()
+            valid_indices = galileo.add_zenith_and_azimuth_angles(
+                elevation_threshold=1.1,
+                elevation_diff_threshold=0.1,
+                azimuth_diff_threshold=0.1,
+            )
             galileo.screen_time_indices(valid_indices)
             galileo.add_height()
+            galileo.test_if_all_masked()
         attributes = output.add_time_attribute(ATTRIBUTES, galileo.date)
         output.update_attributes(galileo.data, attributes)
-        uuid = output.save_level1b(galileo, output_file, uuid)
+        output.save_level1b(galileo, output_file, uuid)
         return uuid
 
 
@@ -100,23 +123,42 @@ class Galileo(ChilboltonRadar):
 
     """
 
-    def __init__(self, full_path: str, site_meta: dict):
+    def __init__(self, full_path: str | PathLike, site_meta: dict) -> None:
         super().__init__(full_path, site_meta)
         self.date = self._init_date()
         self.instrument = GALILEO
 
-    def mask_clutter(self):
-        """Masks clutter."""
+    def mask_clutter(self) -> None:
         # Only strong Z values are valid
         n_low_gates = 15
         ind = np.where(self.data["Zh"][:, :n_low_gates] < -15) and np.where(
-            self.data["ldr"][:, :n_low_gates] > -5
+            self.data["ldr"][:, :n_low_gates] > -5,
         )
         self.data["v"].mask_indices(ind)
 
+    def screen_negative_altitudes(self) -> None:
+        range_var = self.data["range"][:]
+        valid_idx = np.where(range_var > 0)[0]
+        if valid_idx.size == 0:
+            msg = "No valid altitudes found."
+            raise ValueError(msg)
+        if valid_idx.size == range_var.shape[0]:
+            return
+        for arr in self.data.values():
+            if utils.isscalar(arr.data):
+                continue
+            if arr[:].shape[-1] == range_var.shape[0]:
+                arr.data = arr[..., valid_idx]
+
 
 ATTRIBUTES = {
-    "antenna_diameter": MetaData(long_name="Antenna diameter", units="m"),
-    "beamwidthV": MetaData(long_name="Vertical angular beamwidth", units="degree"),
-    "beamwidthH": MetaData(long_name="Horizontal angular beamwidth", units="degree"),
+    "antenna_diameter": MetaData(
+        long_name="Antenna diameter", units="m", dimensions=("time",)
+    ),
+    "beamwidthV": MetaData(
+        long_name="Vertical angular beamwidth", units="degree", dimensions=("time",)
+    ),
+    "beamwidthH": MetaData(
+        long_name="Horizontal angular beamwidth", units="degree", dimensions=("time",)
+    ),
 }

cloudnetpy/instruments/hatpro.py

@@ -1,21 +1,27 @@
 """This module contains RPG Cloud Radar related functions."""
+
 import datetime
 import logging
 from collections import defaultdict
+from os import PathLike
 from pathlib import Path
+from typing import Literal
+from uuid import UUID
 
-import mwrpy
+import mwrpy.rpg_mwr
 import netCDF4
 import numpy as np
+from mwrpy.exceptions import MissingInputData
 from mwrpy.level1.lev1_meta_nc import ATTRIBUTES_1B01
+from mwrpy.level1.write_lev1_nc import lev1_to_nc
 from mwrpy.version import __version__ as mwrpy_version
 from numpy import ma
 
 from cloudnetpy import output, utils
 from cloudnetpy.cloudnetarray import CloudnetArray
-from cloudnetpy.exceptions import ValidTimeStampError
+from cloudnetpy.exceptions import HatproDataError, ValidTimeStampError
 from cloudnetpy.instruments import rpg
-from cloudnetpy.instruments.instruments import HATPRO
+from cloudnetpy.instruments.instruments import HATPRO, LHATPRO, LHUMPRO_U90, Instrument
 from cloudnetpy.instruments.rpg_reader import (
     HatproBin,
     HatproBinCombined,
@@ -23,36 +29,83 @@ from cloudnetpy.instruments.rpg_reader import (
     HatproBinLwp,
 )
 
+IType = Literal["hatpro", "lhatpro", "lhumpro_u90"]
+ITYPE_MAP: dict[IType, Instrument] = {
+    "hatpro": HATPRO,
+    "lhatpro": LHATPRO,
+    "lhumpro_u90": LHUMPRO_U90,
+}
+
 
 def hatpro2l1c(
-    mwr_dir: str,
-    output_file: str,
+    mwr_dir: str | PathLike,
+    output_file: str | PathLike,
     site_meta: dict,
-    uuid: str | None = None,
-    date: str | None = None,
-) -> str:
+    instrument_type: IType = "hatpro",
+    lidar_file: str | PathLike | None = None,
+    uuid: str | UUID | None = None,
+    date: str | datetime.date | None = None,
+) -> UUID:
     """Converts RPG HATPRO microwave radiometer data into Cloudnet Level 1c netCDF file.
 
     Args:
         mwr_dir: Folder containing one day of HATPRO files.
         output_file: Output file name.
-        site_meta: Dictionary containing information about the site
+        site_meta: Dictionary containing information about the site and instrument
+        instrument_type: Specific type of the RPG microwave radiometer.
+        lidar_file: Path to a lidar file.
         uuid: Set specific UUID for the file.
         date: Expected date in the input files.
 
     Returns:
         UUID of the generated file.
     """
-
-    hatpro_raw = mwrpy.lev1_to_nc(site_meta["coeffs_dir"], "1C01", mwr_dir)
-    hatpro = HatproL1c(hatpro_raw, site_meta)
+    if isinstance(date, str):
+        date = datetime.date.fromisoformat(date)
+    uuid = utils.get_uuid(uuid)
+
+    coeff_files = site_meta.get("coefficientFiles")
+    time_offset = site_meta.get("time_offset")
+
+    try:
+        hatpro_raw = lev1_to_nc(
+            "1C01",
+            str(mwr_dir),
+            instrument_type=instrument_type,
+            output_file=str(output_file),
+            lidar_path=lidar_file,
+            coeff_files=coeff_files,
+            instrument_config=site_meta,
+            date=date,
+            time_offset=time_offset,
+        )
+    except MissingInputData as err:
+        raise HatproDataError(str(err)) from err
+
+    hatpro = HatproL1c(hatpro_raw, site_meta, ITYPE_MAP[instrument_type])
+
+    flags = hatpro.data["quality_flag"][:]
+    bad_percentage = ma.sum(flags != 0) / flags.size * 100
+    if bad_percentage > 90:
+        msg = "More than 90% of brightness temperatures are flagged"
+        raise HatproDataError(msg)
 
     timestamps = hatpro.data["time"][:]
     if date is not None:
         # Screen timestamps if these assertions start to fail
-        assert np.all(np.diff(timestamps) > 0)
-        dates = [str(datetime.datetime.utcfromtimestamp(t).date()) for t in timestamps]
-        assert len(set(dates)) == 1
+        if not np.all(np.diff(timestamps) > 0):
+            msg = "Timestamps are not increasing"
+            raise RuntimeError(msg)
+        dates = [
+            datetime.datetime.fromtimestamp(t, tz=datetime.timezone.utc).date()
+            for t in timestamps
+        ]
+        if len(set(dates)) != 1:
+            msg = f"Several dates, something is wrong: {set(dates)}"
+            raise RuntimeError(msg)
+        if date != dates[0]:
+            msg = f"Expected date {date}, got {dates[0]}"
+            raise RuntimeError(msg)
 
     decimal_hours = utils.seconds2hours(timestamps)
     hatpro.data["time"] = CloudnetArray(decimal_hours, "time", data_type="f8")
@@ -60,42 +113,55 @@ def hatpro2l1c(
     hatpro.data["t_amb"].dimensions = ("time", "t_amb_nb")
 
     for key in ("elevation_angle", "ir_elevation_angle"):
-        zenith_angle = 90 - hatpro.data.pop(key)[:]
+        if key not in hatpro.data:
+            continue
+        zenith_angle = 90 - hatpro.data[key][:]
         new_key = key.replace("elevation", "zenith")
         hatpro.data[new_key] = CloudnetArray(zenith_angle, new_key)
 
-    for key in ("latitude", "longitude", "altitude"):
-        if key in site_meta:
-            hatpro.data[key] = CloudnetArray(site_meta[key], key, data_type="f4")
+    if "ir_wavelength" in hatpro.data:
+        hatpro.data["ir_wavelength"].dimensions = ("ir_channel",)
+    if "irt" in hatpro.data:
+        hatpro.data["irt"].dimensions = ("time", "ir_channel")
+
+    utils.add_site_geolocation(hatpro.data, gps=True, site_meta=site_meta)
 
-    attributes = output.add_time_attribute(ATTRIBUTES_1B01, hatpro.date)
+    attrs_copy = ATTRIBUTES_1B01.copy()
+    attributes = output.add_time_attribute(attrs_copy, hatpro.date)
     output.update_attributes(hatpro.data, attributes)
-    uuid = output.save_level1b(hatpro, output_file, uuid)
+    output.save_level1b(hatpro, output_file, uuid)
     with netCDF4.Dataset(output_file, "a") as nc:
         nc.cloudnet_file_type = "mwr-l1c"
         nc.title = nc.title.replace("radiometer", "radiometer Level 1c")
         nc.mwrpy_version = mwrpy_version
-        nc.mwrpy_coefficients = site_meta["coeffs_dir"]
-
+        nc.mwrpy_coefficients = ", ".join(site_meta["coefficientLinks"])
+        nc.history = nc.history.replace("mwr", "mwr-l1c")
+        if lidar_file is not None:
+            with netCDF4.Dataset(lidar_file) as lidar_nc:
+                nc.source = f"{nc.source}\n{lidar_nc.source}"
+                nc.history = f"{nc.history}\n{lidar_nc.history}"
     return uuid
 
 
 class HatproL1c:
-    def __init__(self, hatpro: mwrpy.Rpg, site_meta: dict):
+    def __init__(
+        self, hatpro: mwrpy.rpg_mwr.Rpg, site_meta: dict, instrument: Instrument
+    ) -> None:
         self.raw_data = hatpro.raw_data
         self.data = hatpro.data
-        self.date = hatpro.date.split("-")
+        self.date = hatpro.date
         self.site_meta = site_meta
-        self.instrument = HATPRO
+        self.instrument = instrument
 
 
 def hatpro2nc(
-    path_to_files: str,
-    output_file: str,
+    path_to_files: str | PathLike,
+    output_file: str | PathLike,
     site_meta: dict,
-    uuid: str | None = None,
-    date: str | None = None,
-) -> tuple[str, list]:
+    instrument_type: IType = "hatpro",
+    uuid: str | UUID | None = None,
+    date: str | datetime.date | None = None,
+) -> tuple[UUID, list[Path]]:
     """Converts RPG HATPRO microwave radiometer data into Cloudnet Level 1b
     netCDF file.
 
@@ -112,6 +178,7 @@ def hatpro2nc(
             - `latitude` (optional).
             - `longitude` (optional).
 
+        instrument_type: Specific type of the RPG microwave radiometer.
         uuid: Set specific UUID for the file.
         date: Expected date in the input files. If not set,
             all files will be used. This might cause unexpected behavior if
@@ -133,28 +200,32 @@ def hatpro2nc(
         >>> hatpro2nc('/path/to/files/', 'hatpro.nc', site_meta)
 
     """
+    if isinstance(date, str):
+        date = datetime.date.fromisoformat(date)
+    uuid = utils.get_uuid(uuid)
     hatpro_objects, valid_files = _get_hatpro_objects(Path(path_to_files), date)
-    is_lwp_files = any(f.endswith(".LWP") for f in valid_files)
-    is_iwv_files = any(f.endswith(".IWV") for f in valid_files)
+    is_lwp_files = any(f.suffix == ".LWP" for f in valid_files)
+    is_iwv_files = any(f.suffix == ".IWV" for f in valid_files)
     if not is_lwp_files:
         raise ValidTimeStampError
     if is_iwv_files:
         _add_missing_variables(hatpro_objects, ("lwp", "iwv"))
     one_day_of_data = rpg.create_one_day_data_record(hatpro_objects)
-    hatpro = rpg.Hatpro(one_day_of_data, site_meta)
-    hatpro.sort_timestamps()
-    hatpro.convert_time_to_fraction_hour("float64")
+    hatpro = rpg.Hatpro(one_day_of_data, site_meta, ITYPE_MAP[instrument_type])
     hatpro.add_site_geolocation()
+    hatpro.convert_time_to_fraction_hour("float64")
+    hatpro.sort_timestamps()
     hatpro.remove_duplicate_timestamps()
     attributes = output.add_time_attribute({}, hatpro.date)
     output.update_attributes(hatpro.data, attributes)
-    uuid = output.save_level1b(hatpro, output_file, uuid)
+    output.save_level1b(hatpro, output_file, uuid)
     return uuid, valid_files
 
 
 def _get_hatpro_objects(
-    directory: Path, expected_date: str | None
-) -> tuple[list[HatproBinCombined], list[str]]:
+    directory: Path,
+    expected_date: datetime.date | None,
+) -> tuple[list[HatproBinCombined], list[Path]]:
     objects = defaultdict(list)
     for filename in directory.iterdir():
         try:
@@ -170,40 +241,43 @@ def _get_hatpro_objects(
             if expected_date is not None:
                 obj = _validate_date(obj, expected_date)
             objects[filename.stem].append(obj)
-        except (TypeError, ValueError) as err:
-            logging.warning(f"Ignoring file '{filename}': {err}")
+        except (TypeError, ValueError, ValidTimeStampError) as err:
+            logging.warning("Ignoring file '%s': %s", filename, err)
             continue
 
-    valid_files: list[str] = []
+    valid_files: list[Path] = []
     combined_objs = []
     for _stem, objs in sorted(objects.items()):
         try:
             combined_objs.append(HatproBinCombined(objs))
-            valid_files.extend(str(obj.filename) for obj in objs)
+            valid_files.extend(obj.filename for obj in objs)
         except (TypeError, ValueError) as err:
             files = "'" + "', '".join(str(obj.filename) for obj in objs) + "'"
-            logging.warning(f"Ignoring files {files}: {err}")
+            logging.warning("Ignoring files %s: %s", files, err)
             continue
 
     return combined_objs, valid_files
 
 
-def _validate_date(obj: HatproBin, expected_date: str):
+def _validate_date(obj: HatproBin, expected_date: datetime.date) -> HatproBin:
     if obj.header["_time_reference"] != 1:
-        raise ValueError("Can not validate non-UTC dates")
+        msg = "Can not validate non-UTC dates"
+        raise ValueError(msg)
     inds = []
     for ind, timestamp in enumerate(obj.data["time"][:]):
-        date = "-".join(utils.seconds2date(timestamp)[:3])
+        date = utils.seconds2date(timestamp).date()
         if date == expected_date:
             inds.append(ind)
     if not inds:
-        raise ValueError("Timestamps not what expected")
+        msg = f"No valid timestamps found for date {expected_date}"
+        raise ValueError(msg)
     obj.data = obj.data[:][inds]
     return obj
 
 
 def _add_missing_variables(
-    hatpro_objects: list[HatproBinCombined], keys: tuple
+    hatpro_objects: list[HatproBinCombined],
+    keys: tuple,
 ) -> list[HatproBinCombined]:
     for obj in hatpro_objects:
         for key in keys:

cloudnetpy/instruments/instruments.py

@@ -35,6 +35,14 @@ CL31 = Instrument(
     wavelength=910.0,
 )
 
+CS135 = Instrument(
+    manufacturer="Campbell Scientific",
+    domain="lidar",
+    category="ceilometer",
+    model="CS135",
+    wavelength=905.0,
+)
+
 CT25K = Instrument(
     manufacturer="Vaisala",
     domain="lidar",
@@ -67,6 +75,14 @@ CHM15KX = Instrument(
     wavelength=1064.0,
 )
 
+MIRA10 = Instrument(
+    manufacturer="METEK",
+    domain="radar",
+    category="cloud radar",
+    model="MIRA-10",
+    frequency=9.4,  # 9.2 - 9.6 GHz
+)
+
 MIRA35 = Instrument(
     manufacturer="METEK",
     domain="radar",
@@ -108,7 +124,10 @@ FMCW35 = Instrument(
 )
 
 BASTA = Instrument(
-    domain="radar", category="cloud radar", model="BASTA", frequency=95.0
+    domain="radar",
+    category="cloud radar",
+    model="BASTA",
+    frequency=95.0,
 )
 
 HATPRO = Instrument(
@@ -118,6 +137,20 @@ HATPRO = Instrument(
     model="HATPRO",
 )
 
+LHATPRO = Instrument(
+    manufacturer="RPG-Radiometer Physics",
+    domain="mwr",
+    category="microwave radiometer",
+    model="LHATPRO",
+)
+
+LHUMPRO_U90 = Instrument(
+    manufacturer="RPG-Radiometer Physics",
+    domain="mwr",
+    category="microwave radiometer",
+    model="LHUMPRO-U90",
+)
+
 RADIOMETRICS = Instrument(
     manufacturer="Radiometrics",
     domain="mwr",
@@ -131,6 +164,34 @@ HALO = Instrument(
     model="StreamLine",
 )
 
+WINDCUBE_WLS100S = Instrument(
+    manufacturer="Vaisala",
+    domain="lidar",
+    category="Doppler lidar",
+    model="WindCube WLS100S",
+)
+
+WINDCUBE_WLS200S = Instrument(
+    manufacturer="Vaisala",
+    domain="lidar",
+    category="Doppler lidar",
+    model="WindCube WLS200S",
+)
+
+WINDCUBE_WLS400S = Instrument(
+    manufacturer="Vaisala",
+    domain="lidar",
+    category="Doppler lidar",
+    model="WindCube WLS400S",
+)
+
+WINDCUBE_WLS70 = Instrument(
+    manufacturer="Vaisala",
+    domain="lidar",
+    category="Doppler lidar",
+    model="WindCube WLS70",
+)
+
 PARSIVEL2 = Instrument(
     manufacturer="OTT HydroMet",
     domain="disdrometer",
@@ -145,7 +206,58 @@ THIES = Instrument(
     model="LNM",
 )
 
+PLUVIO2 = Instrument(
+    manufacturer="OTT HydroMet",
+    domain="rain-gauge",
+    category="rain-gauge",
+    model="Pluvio2",
+)
+
+PLUVIO2S = Instrument(
+    manufacturer="OTT HydroMet",
+    domain="rain-gauge",
+    category="rain-gauge",
+    model="Pluvio2S",
+)
+
+THIES_PT = Instrument(
+    manufacturer="Thies Clima",
+    domain="rain-gauge",
+    category="rain-gauge",
+    model="Precipitation Transmitter",
+)
+
+RAIN_E_H3 = Instrument(
+    manufacturer="LAMBRECHT meteo GmbH",
+    domain="rain-gauge",
+    category="rain-gauge",
+    model="rain[e]H3",
+)
+
 GENERIC_WEATHER_STATION = Instrument(
     domain="weather-station",
     category="weather station",
 )
+
+MRR_PRO = Instrument(
+    manufacturer="METEK",
+    domain="rain-radar",
+    category="rain radar",
+    model="MRR-PRO",
+    frequency=24.23,
+)
+
+FD12P = Instrument(
+    manufacturer="Vaisala",
+    domain="weather-station",
+    category="present weather sensor",
+    model="FD12P",
+)
+
+DA10 = Instrument(
+    manufacturer="Vaisala",
+    domain="lidar",
+    category="differential absorption lidar",
+    model="DA10",
+    wavelength=911.0,
+)