cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/datasource.py

@@ -1,11 +1,16 @@
-"""Datasource module, containing the :class:`DataSource class.`"""
+"""Datasource module, containing the :class:`DataSource` class."""
+
+import datetime
 import logging
 import os
-from datetime import datetime
-from typing import Callable
+from collections.abc import Callable
+from os import PathLike
+from types import TracebackType
 
 import netCDF4
 import numpy as np
+import numpy.typing as npt
+from typing_extensions import Self
 
 from cloudnetpy import utils
 from cloudnetpy.cloudnetarray import CloudnetArray
@@ -44,19 +49,25 @@ class DataSource:
     radar_frequency: float
     data_dense: dict
     data_sparse: dict
-    type: str
+    source_type: str
 
-    def __init__(self, full_path: str, radar: bool = False):
+    def __init__(self, full_path: PathLike | str, *, radar: bool = False) -> None:
         self.filename = os.path.basename(full_path)
         self.dataset = netCDF4.Dataset(full_path)
         self.source = getattr(self.dataset, "source", "")
-        self.time: np.ndarray = self._init_time()
+        self.instrument_pid = getattr(self.dataset, "instrument_pid", "")
+        self.time: npt.NDArray = self._init_time()
         self.altitude = self._init_altitude()
         self.height = self._init_height()
+        self.height_agl = (
+            self.height - self.altitude
+            if self.height is not None and self.altitude is not None
+            else None
+        )
         self.data: dict = {}
         self._is_radar = radar
 
-    def getvar(self, *args) -> np.ndarray:
+    def getvar(self, *args: str) -> npt.NDArray:
         """Returns data array from the source file variables.
 
         Returns just the data (and no attributes) from the original
@@ -69,21 +80,23 @@ class DataSource:
             ndarray: The actual data.
 
         Raises:
-             RuntimeError: The variable is not found.
+             KeyError: The variable is not found.
 
         """
         for arg in args:
             if arg in self.dataset.variables:
                 return self.dataset.variables[arg][:]
-        raise RuntimeError("Missing variable in the input file.")
+        msg = f"Missing variable {args[0]} in the input file."
+        raise KeyError(msg)
 
     def append_data(
         self,
-        variable: netCDF4.Variable | np.ndarray | float | int,
+        variable: netCDF4.Variable | npt.NDArray | float,
         key: str,
         name: str | None = None,
         units: str | None = None,
-    ):
+        dtype: str | None = None,
+    ) -> None:
         """Adds new CloudnetVariable or RadarVariable into `data` attribute.
 
         Args:
@@ -92,123 +105,85 @@ class DataSource:
                 attribute (dictionary).
             name: CloudnetArray.name attribute. Default value is *key*.
             units: CloudnetArray.units attribute.
+            dtype: CloudnetArray.data_type attribute.
 
         """
-        self.data[key] = CloudnetArray(variable, name or key, units)
+        self.data[key] = CloudnetArray(variable, name or key, units, data_type=dtype)
 
-    def get_date(self) -> list:
+    def get_date(self) -> datetime.date:
         """Returns date components.
 
         Returns:
-            list: Date components [YYYY, MM, DD].
+            date object
 
         Raises:
              RuntimeError: Not found or invalid date.
 
         """
         try:
-            year = str(self.dataset.year)
-            month = str(self.dataset.month).zfill(2)
-            day = str(self.dataset.day).zfill(2)
-            datetime.strptime(f"{year}{month}{day}", "%Y%m%d")
+            year = int(self.dataset.year)
+            month = int(self.dataset.month)
+            day = int(self.dataset.day)
+            return datetime.date(year, month, day)
         except (AttributeError, ValueError) as read_error:
-            raise RuntimeError(
-                "Missing or invalid date in global attributes."
-            ) from read_error
-        return [year, month, day]
+            msg = "Missing or invalid date in global attributes."
+            raise RuntimeError(msg) from read_error
 
     def close(self) -> None:
         """Closes the open file."""
         self.dataset.close()
 
     @staticmethod
-    def km2m(var: netCDF4.Variable) -> np.ndarray:
+    def to_m(var: netCDF4.Variable) -> npt.NDArray:
         """Converts km to m."""
         alt = var[:]
         if var.units == "km":
             alt *= 1000
+        elif var.units not in ("m", "meters"):
+            msg = f"Unexpected unit: {var.units}"
+            raise ValueError(msg)
         return alt
 
-    @staticmethod
-    def m2km(var: netCDF4.Variable) -> np.ndarray:
-        """Converts m to km."""
-        alt = var[:]
-        if var.units == "m":
-            alt /= 1000
-        return alt
-
-    def _init_time(self) -> np.ndarray:
+    def _init_time(self) -> npt.NDArray:
         time = self.getvar("time")
         if len(time) == 0:
-            raise ValidTimeStampError("Empty time vector")
+            msg = "Empty time vector"
+            raise ValidTimeStampError(msg)
         if max(time) > 25:
-            logging.warning("Assuming time as seconds, converting to fraction hour")
+            logging.debug("Assuming time as seconds, converting to fraction hour")
             time = utils.seconds2hours(time)
         return time
 
     def _init_altitude(self) -> float | None:
         """Returns altitude of the instrument (m)."""
         if "altitude" in self.dataset.variables:
-            altitude_above_sea = self.km2m(self.dataset.variables["altitude"])
-            return float(np.mean(altitude_above_sea))
+            var = self.dataset.variables["altitude"]
+            if utils.is_all_masked(var[:]):
+                return None
+            altitude_above_sea = self.to_m(var)
+            return float(
+                altitude_above_sea
+                if utils.isscalar(altitude_above_sea)
+                else np.mean(altitude_above_sea),
+            )
         return None
 
-    def _init_height(self) -> np.ndarray | None:
+    def _init_height(self) -> npt.NDArray | None:
         """Returns height array above mean sea level (m)."""
         if "height" in self.dataset.variables:
-            return self.km2m(self.dataset.variables["height"])
+            return self.to_m(self.dataset.variables["height"])
         if "range" in self.dataset.variables and self.altitude is not None:
-            range_instrument = self.km2m(self.dataset.variables["range"])
+            range_instrument = self.to_m(self.dataset.variables["range"])
             return np.array(range_instrument + self.altitude)
         return None
 
-    def _variables_to_cloudnet_arrays(self, keys: tuple) -> None:
-        """Transforms netCDF4-variables into CloudnetArrays.
-
-        Args:
-            keys: netCDF4-variables to be converted. The results
-                are saved in *self.data* dictionary with *fields*
-                strings as keys.
-
-        Notes:
-            The attributes of the variables are not copied. Just the data.
-
-        """
-        for key in keys:
-            self.append_data(self.dataset.variables[key], key)
-
-    def _unknown_variable_to_cloudnet_array(
-        self,
-        possible_names: tuple,
-        key: str,
-        units: str | None = None,
-        ignore_mask: bool = False,
-    ):
-        """Transforms single netCDF4 variable into CloudnetArray.
-
-        Args:
-            possible_names: Tuple of strings containing the possible
-                names of the variable in the input NetCDF file.
-            key: Key for self.data dictionary and name-attribute
-                for the saved CloudnetArray object.
-            units: Units attribute for the CloudnetArray object.
-            ignore_mask: If true, always writes an ordinary numpy array.
-
-        Raises:
-            RuntimeError: No variable found.
-
-        """
-        for name in possible_names:
-            if name in self.dataset.variables:
-                array = self.dataset.variables[name]
-                if ignore_mask is True:
-                    array = np.array(array)
-                self.append_data(array, key, units=units)
-                return
-        raise RuntimeError("Missing variable in the input file.")
-
-    def __enter__(self):
+    def __enter__(self) -> Self:
         return self
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
         self.close()

cloudnetpy/exceptions.py

@@ -1,38 +1,67 @@
-class InconsistentDataError(Exception):
+class CloudnetException(Exception):
+    """Base class for exceptions in this module."""
+
+
+class InconsistentDataError(CloudnetException):
+    """Internal exception class."""
+
+    def __init__(self, msg: str) -> None:
+        super().__init__(msg)
+
+
+class DisdrometerDataError(CloudnetException):
+    """Internal exception class."""
+
+    def __init__(self, msg: str) -> None:
+        super().__init__(msg)
+
+
+class RadarDataError(CloudnetException):
+    """Internal exception class."""
+
+    def __init__(self, msg: str) -> None:
+        super().__init__(msg)
+
+
+class LidarDataError(CloudnetException):
+    """Internal exception class."""
+
+    def __init__(self, msg: str) -> None:
+        super().__init__(msg)
+
+
+class PlottingError(CloudnetException):
     """Internal exception class."""
 
-    def __init__(self, msg: str = ""):
-        self.message = msg
-        super().__init__(self.message)
+    def __init__(self, msg: str) -> None:
+        super().__init__(msg)
 
 
-class DisdrometerDataError(Exception):
+class ModelDataError(CloudnetException):
     """Internal exception class."""
 
-    def __init__(self, msg: str = ""):
-        self.message = msg
-        super().__init__(self.message)
+    def __init__(
+        self, msg: str = "Invalid model file: not enough proper profiles"
+    ) -> None:
+        super().__init__(msg)
 
 
-class WeatherStationDataError(Exception):
+class ValidTimeStampError(CloudnetException):
     """Internal exception class."""
 
-    def __init__(self, msg: str = ""):
-        self.message = msg
-        super().__init__(self.message)
+    def __init__(self, msg: str = "No valid timestamps found") -> None:
+        super().__init__(msg)
 
 
-class ModelDataError(Exception):
+class HatproDataError(CloudnetException):
     """Internal exception class."""
 
-    def __init__(self, msg: str = "Invalid model file: not enough proper profiles"):
-        self.message = msg
-        super().__init__(self.message)
+    def __init__(self, msg: str = "Invalid HATPRO file") -> None:
+        super().__init__(msg)
 
 
-class ValidTimeStampError(Exception):
+class InvalidSourceFileError(CloudnetException):
     """Internal exception class."""
 
-    def __init__(self, msg: str = "No valid timestamps found"):
-        self.message = msg
-        super().__init__(self.message)
+    def __init__(self, msg: str = "Invalid source file") -> None:
+        super().__init__(msg)

cloudnetpy/instruments/__init__.py

@@ -1,11 +1,16 @@
 from .basta import basta2nc
+from .bowtie import bowtie2nc
 from .ceilo import ceilo2nc
 from .copernicus import copernicus2nc
 from .disdrometer import parsivel2nc, thies2nc
+from .fd12p import fd12p2nc
 from .galileo import galileo2nc
 from .hatpro import hatpro2l1c, hatpro2nc
+from .instruments import Instrument
 from .mira import mira2nc
+from .mrr import mrr2nc
 from .pollyxt import pollyxt2nc
 from .radiometrics import radiometrics2nc
+from .rain_e_h3 import rain_e_h32nc
 from .rpg import rpg2nc
 from .weather_station import ws2nc

cloudnetpy/instruments/basta.py

@@ -1,4 +1,9 @@
 """Module for reading / converting BASTA radar data."""
+
+import datetime
+from os import PathLike
+from uuid import UUID
+
 import numpy as np
 
 from cloudnetpy import output
@@ -6,15 +11,16 @@ from cloudnetpy.exceptions import ValidTimeStampError
 from cloudnetpy.instruments import instruments
 from cloudnetpy.instruments.nc_radar import NcRadar
 from cloudnetpy.metadata import MetaData
+from cloudnetpy.utils import get_uuid
 
 
 def basta2nc(
-    basta_file: str,
-    output_file: str,
+    basta_file: str | PathLike,
+    output_file: str | PathLike,
     site_meta: dict,
-    uuid: str | None = None,
-    date: str | None = None,
-) -> str:
+    uuid: str | UUID | None = None,
+    date: str | datetime.date | None = None,
+) -> UUID:
     """Converts BASTA cloud radar data into Cloudnet Level 1b netCDF file.
 
     This function converts daily BASTA file into a much smaller file that
@@ -41,6 +47,10 @@ def basta2nc(
           >>> basta2nc('basta_file.nc', 'radar.nc', site_meta)
 
     """
+    if isinstance(date, str):
+        date = datetime.date.fromisoformat(date)
+    uuid = get_uuid(uuid)
+
     keymap = {
         "reflectivity": "Zh",
         "velocity": "v",
@@ -54,15 +64,17 @@ def basta2nc(
         if date is not None:
             basta.validate_date(date)
         basta.screen_data(keymap)
-        basta.add_time_and_range()
+        basta.add_time_and_range(time_dtype="f8")
         basta.add_site_geolocation()
         basta.add_zenith_angle()
         basta.add_radar_specific_variables()
         basta.add_height()
         basta.sort_timestamps()
+        basta.remove_duplicate_timestamps()
+        basta.test_if_all_masked()
     attributes = output.add_time_attribute(ATTRIBUTES, basta.date)
     output.update_attributes(basta.data, attributes)
-    uuid = output.save_level1b(basta, output_file, uuid)
+    output.save_level1b(basta, output_file, uuid)
     return uuid
 
 
@@ -75,9 +87,9 @@ class Basta(NcRadar):
 
     """
 
-    def __init__(self, full_path: str, site_meta: dict):
+    def __init__(self, full_path: str | PathLike, site_meta: dict) -> None:
         super().__init__(full_path, site_meta)
-        self.date: list[str] = self.get_date()
+        self.date = self.get_date()
         self.instrument = instruments.BASTA
 
     def screen_data(self, keymap: dict) -> None:
@@ -87,12 +99,14 @@ class Basta(NcRadar):
             if key in self.data and self.data[key].data.ndim == mask.ndim:
                 self.data[key].mask_indices(np.where(mask != 1))
 
-    def validate_date(self, expected_date: str) -> None:
+    def validate_date(self, expected_date: datetime.date) -> None:
         """Validates expected data."""
         date_units = self.dataset.variables["time"].units
-        date = date_units.split()[2]
+        date_str = date_units.split()[2]
+        date = datetime.date.fromisoformat(date_str)
         if expected_date != date:
-            raise ValidTimeStampError
+            msg = "Time units doesn't match expected date"
+            raise ValidTimeStampError(msg)
 
     def add_zenith_angle(self) -> None:
         elevation = self.getvar("elevation")
@@ -105,15 +119,18 @@ ATTRIBUTES = {
         long_name="Radar pitch angle",
         units="degree",
         standard_name="platform_roll",
+        dimensions=("time",),
     ),
     "radar_yaw": MetaData(
         long_name="Radar yaw angle",
         units="degree",
         standard_name="platform_yaw",
+        dimensions=("time",),
     ),
     "radar_roll": MetaData(
         long_name="Radar roll angle",
         units="degree",
         standard_name="platform_roll",
+        dimensions=("time",),
     ),
 }

cloudnetpy/instruments/bowtie.py

@@ -0,0 +1,135 @@
+import datetime
+from os import PathLike
+from uuid import UUID
+
+import numpy as np
+from numpy import ma
+
+from cloudnetpy import output
+from cloudnetpy.constants import G_TO_KG, MM_H_TO_M_S
+from cloudnetpy.exceptions import ValidTimeStampError
+from cloudnetpy.instruments.instruments import FMCW94
+from cloudnetpy.instruments.nc_radar import NcRadar
+from cloudnetpy.instruments.rpg import RPG_ATTRIBUTES
+from cloudnetpy.metadata import MetaData
+from cloudnetpy.utils import bit_field_definition, get_uuid
+
+
+def bowtie2nc(
+    bowtie_file: str | PathLike,
+    output_file: str | PathLike,
+    site_meta: dict,
+    uuid: str | UUID | None = None,
+    date: str | datetime.date | None = None,
+) -> UUID:
+    """Converts data from 'BOW-TIE' campaign cloud radar on RV-Meteor into
+       Cloudnet Level 1b netCDF file.
+
+    Args:
+        bowtie_file: Input filename.
+        output_file: Output filename.
+        site_meta: Dictionary containing information about the site. Required key
+            value pair is `name`. Optional are `latitude`, `longitude`, `altitude`.
+        uuid: Set specific UUID for the file.
+        date: Expected date as YYYY-MM-DD of all profiles in the file.
+
+    Returns:
+        UUID of the generated file.
+
+    Raises:
+        ValidTimeStampError: No valid timestamps found.
+
+    """
+    keymap = {
+        "Zh": "Zh",
+        "v": "v",
+        "width": "width",
+        "ldr": "ldr",
+        "kurt": "kurtosis",
+        "Skew": "skewness",
+        "SNR": "SNR",
+        "time": "time",
+        "range": "range",
+        "lwp": "lwp",
+        "SurfRelHum": "relative_humidity",
+        "rain": "rainfall_rate",
+        "Nyquist_velocity": "nyquist_velocity",
+        "range_offsets": "chirp_start_indices",
+    }
+
+    if isinstance(date, str):
+        date = datetime.date.fromisoformat(date)
+    uuid = get_uuid(uuid)
+
+    with Bowtie(bowtie_file, site_meta) as bowtie:
+        bowtie.init_data(keymap)
+        bowtie.add_time_and_range()
+        if date is not None:
+            bowtie.check_date(date)
+        bowtie.add_radar_specific_variables()
+        bowtie.add_site_geolocation()
+        bowtie.add_height()
+        bowtie.convert_units()
+        bowtie.fix_chirp_start_indices()
+        bowtie.test_if_all_masked()
+        bowtie.add_correction_bits()
+    attributes = output.add_time_attribute(ATTRIBUTES, bowtie.date)
+    output.update_attributes(bowtie.data, attributes)
+    output.save_level1b(bowtie, output_file, uuid)
+    return uuid
+
+
+class Bowtie(NcRadar):
+    def __init__(self, full_path: str | PathLike, site_meta: dict) -> None:
+        super().__init__(full_path, site_meta)
+        self.instrument = FMCW94
+        self.date = self.get_date()
+
+    def convert_units(self) -> None:
+        self.data["lwp"].data *= G_TO_KG
+        self.data["rainfall_rate"].data *= MM_H_TO_M_S
+        self.data["relative_humidity"].data /= 100
+
+    def fix_chirp_start_indices(self) -> None:
+        array = self.data["chirp_start_indices"].data
+        self.data["chirp_start_indices"].data = np.array(array, dtype=np.int32)
+        self.data["chirp_start_indices"].data_type = "int32"
+
+    def add_correction_bits(self) -> None:
+        bits = ma.ones(self.data["v"].data.shape, dtype=np.uint32)
+        bits.mask = self.data["v"].data.mask
+        self.append_data(bits, "correction_bits")
+
+    def check_date(self, date: datetime.date) -> None:
+        if self.date != date:
+            raise ValidTimeStampError
+
+
+ATTRIBUTES = RPG_ATTRIBUTES | {
+    "v": MetaData(
+        long_name="Doppler velocity",
+        units="m s-1",
+        comment=(
+            "This parameter is the radial component of the velocity, with positive\n"
+            "velocities are away from the radar. It was corrected for the heave\n"
+            "motion of the ship. A rolling average over 3 time steps has been\n"
+            "applied to it."
+        ),
+        dimensions=("time", "range"),
+    ),
+    "correction_bits": MetaData(
+        long_name="Correction bits",
+        units="1",
+        definition=bit_field_definition({0: """Doppler velocity is dealiased."""}),
+        comment=(
+            "This parameter is a bit field that indicates which corrections have\n"
+            "been applied to radar measurements."
+        ),
+        dimensions=("time", "range"),
+    ),
+    "nyquist_velocity": MetaData(
+        long_name="Nyquist velocity",
+        units="m s-1",
+        dimensions=("time", "chirp_sequence"),
+    ),
+}