cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/cloudnetarray.py

@@ -1,9 +1,10 @@
 """CloudnetArray class."""
-import math
-from collections.abc import Sequence
+
+from collections.abc import Callable, Sequence
 
 import netCDF4
 import numpy as np
+import numpy.typing as npt
 from numpy import ma
 
 from cloudnetpy import utils
@@ -20,23 +21,26 @@ class CloudnetArray:
         units_from_user: Explicit units, optional.
         dimensions: Explicit dimension names, optional.
         data_type: Explicit data type, optional.
+        source: Source attribute, optional.
 
     """
 
     def __init__(
         self,
-        variable: netCDF4.Variable | np.ndarray | float | int,
+        variable: netCDF4.Variable | npt.NDArray | float,
         name: str,
         units_from_user: str | None = None,
         dimensions: Sequence[str] | None = None,
         data_type: str | None = None,
-    ):
+        source: str | None = None,
+    ) -> None:
         self.variable = variable
         self.name = name
         self.data = self._init_data()
         self.units = units_from_user or self._init_units()
         self.data_type = data_type or self._init_data_type()
         self.dimensions = dimensions
+        self.source = source
 
     def lin2db(self) -> None:
         """Converts linear units to log."""
@@ -54,7 +58,7 @@ class CloudnetArray:
         """Masks data from given indices."""
         self.data[ind] = ma.masked
 
-    def rebin_data(self, time: np.ndarray, time_new: np.ndarray) -> list:
+    def rebin_data(self, time: npt.NDArray, time_new: npt.NDArray) -> npt.NDArray:
         """Rebins `data` in time.
 
         Args:
@@ -67,41 +71,59 @@ class CloudnetArray:
         """
         if self.data.ndim == 1:
             self.data = utils.rebin_1d(time, self.data, time_new)
-            bad_indices = list(np.where(self.data == ma.masked)[0])
+            bad_indices = np.nonzero(self.data.mask)[0]
         else:
-            assert isinstance(self.data, ma.MaskedArray)
             self.data, bad_indices = utils.rebin_2d(time, self.data, time_new)
         return bad_indices
 
     def fetch_attributes(self) -> list:
         """Returns list of user-defined attributes."""
         attributes = []
-        for attr in self.__dict__:
-            if attr not in (
-                "variable",
-                "name",
-                "data",
-                "data_type",
-                "dimensions",
+        for key, value in self.__dict__.items():
+            if (
+                key
+                not in (
+                    "variable",
+                    "name",
+                    "data",
+                    "data_type",
+                    "dimensions",
+                )
+                and value is not None
             ):
-                attributes.append(attr)
+                attributes.append(key)
         return attributes
 
     def set_attributes(self, attributes: MetaData) -> None:
         """Overwrites existing instance attributes."""
         for key in attributes._fields:  # To iterate namedtuple fields.
             data = getattr(attributes, key)
-            if data:
+            if key == "dimensions" or data:
                 setattr(self, key, data)
 
-    def _init_data(self) -> np.ndarray:
+    def filter_isolated_pixels(self) -> None:
+        """Filters hot pixels from radar data."""
+        self._filter(utils.filter_isolated_pixels)
+
+    def filter_vertical_stripes(self) -> None:
+        """Filters vertical artifacts from radar data."""
+        self._filter(utils.filter_x_pixels)
+
+    def _filter(self, fun: Callable[[npt.NDArray], npt.NDArray]) -> None:
+        if not isinstance(self.data, ma.MaskedArray):
+            self.data = ma.masked_array(self.data)
+        is_data = (~self.data.mask).astype(int)
+        is_data_filtered = fun(is_data)
+        self.data[is_data_filtered == 0] = ma.masked
+
+    def _init_data(self) -> npt.NDArray:
         if isinstance(self.variable, netCDF4.Variable):
             return self.variable[:]
         if isinstance(self.variable, np.ndarray):
             return self.variable
         if isinstance(
             self.variable,
-            (int, float, np.float32, np.int8, np.float64, np.int32, np.uint16),
+            int | float | np.float32 | np.int8 | np.float64 | np.int32 | np.uint16,
         ):
             return np.array(self.variable)
         if isinstance(self.variable, str):
@@ -110,7 +132,8 @@ class CloudnetArray:
                 return np.array(numeric_value)
             except ValueError:
                 pass
-        raise ValueError(f"Incorrect CloudnetArray input: {self.variable}")
+        msg = f"Incorrect CloudnetArray input: {self.variable}"
+        raise ValueError(msg)
 
     def _init_units(self) -> str:
         return getattr(self.variable, "units", "")
@@ -122,74 +145,5 @@ class CloudnetArray:
             return "i2"
         return "i4"
 
-    def __getitem__(self, ind: tuple) -> np.ndarray:
+    def __getitem__(self, ind: tuple) -> npt.NDArray:
         return self.data[ind]
-
-    def filter_isolated_pixels(self) -> None:
-        """Filters hot pixels from radar data."""
-        self._filter(utils.filter_isolated_pixels)
-
-    def filter_vertical_stripes(self) -> None:
-        """Filters vertical artifacts from radar data."""
-        self._filter(utils.filter_x_pixels)
-
-    def _filter(self, fun) -> None:
-        assert isinstance(self.data, ma.MaskedArray)
-        is_data = (~self.data.mask).astype(int)
-        is_data_filtered = fun(is_data)
-        self.data[is_data_filtered == 0] = ma.masked
-
-    def calc_linear_std(self, time: np.ndarray, time_new: np.ndarray) -> None:
-        """Calculates std of radar velocity.
-
-        Args:
-            time: 1D time array.
-            time_new: 1D new time array.
-
-        Notes:
-            The result is masked if the bin contains masked values.
-        """
-        data_as_float = self.data.astype(float)
-        assert isinstance(data_as_float, ma.MaskedArray)
-        self.data, _ = utils.rebin_2d(time, data_as_float, time_new, "std")
-
-    def rebin_velocity(
-        self,
-        time: np.ndarray,
-        time_new: np.ndarray,
-        folding_velocity: float | np.ndarray,
-        sequence_indices: list,
-    ) -> None:
-        """Rebins Doppler velocity in polar coordinates.
-
-        Args:
-            time: 1D time array.
-            time_new: 1D new time array.
-            folding_velocity: Folding velocity (m/s). Can be a float when
-                it's the same for all altitudes, or np.ndarray when it
-                matches difference altitude regions (defined in `sequence_indices`).
-            sequence_indices: List containing indices of different folding regions,
-                e.g. [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10]].
-
-        """
-
-        def _get_scaled_vfold() -> np.ndarray:
-            vfold_scaled = math.pi / folding_velocity
-            if isinstance(vfold_scaled, float):
-                vfold_scaled = np.array([float(vfold_scaled)])
-            return vfold_scaled
-
-        def _scale_by_vfold(data_in: np.ndarray, fun) -> np.ndarray:
-            data_out = ma.copy(data_in)
-            for i, ind in enumerate(sequence_indices):
-                data_out[:, ind] = fun(data_in[:, ind], folding_velocity_scaled[i])
-            return data_out
-
-        folding_velocity_scaled = _get_scaled_vfold()
-        data_scaled = _scale_by_vfold(self.data, np.multiply)
-        vel_x = ma.cos(data_scaled)
-        vel_y = ma.sin(data_scaled)
-        vel_x_mean, _ = utils.rebin_2d(time, vel_x, time_new)
-        vel_y_mean, _ = utils.rebin_2d(time, vel_y, time_new)
-        mean_vel_scaled = np.arctan2(vel_y_mean, vel_x_mean)
-        self.data = _scale_by_vfold(mean_vel_scaled, np.divide)

cloudnetpy/concat_lib.py

@@ -1,20 +1,36 @@
 """Module for concatenating netCDF files."""
+
+import datetime
+import logging
+import shutil
+from collections.abc import Iterable, Sequence
+from os import PathLike
+from pathlib import Path
+from types import TracebackType
+from typing import Literal
+
 import netCDF4
 import numpy as np
+import numpy.typing as npt
+from numpy import ma
+from typing_extensions import Self
 
-from cloudnetpy.exceptions import InconsistentDataError
+from cloudnetpy import utils
+from cloudnetpy.exceptions import ValidTimeStampError
 
 
-def truncate_netcdf_file(filename: str, output_file: str, n_profiles: int):
-    """Truncates netcdf file in 'time' dimension taking only n_profiles.
+def truncate_netcdf_file(
+    filename: str, output_file: str, n_profiles: int, dim_name: str = "time"
+) -> None:
+    """Truncates netcdf file in dim_name dimension taking only n_profiles.
     Useful for creating small files for tests.
     """
     with (
         netCDF4.Dataset(filename, "r") as nc,
         netCDF4.Dataset(output_file, "w", format=nc.data_model) as nc_new,
     ):
-        for dim in nc.dimensions.keys():
-            dim_len = None if dim == "time" else nc.dimensions[dim].size
+        for dim in nc.dimensions:
+            dim_len = None if dim == dim_name else nc.dimensions[dim].size
             nc_new.createDimension(dim, dim_len)
         for attr in nc.ncattrs():
             value = getattr(nc, attr)
@@ -24,9 +40,13 @@ def truncate_netcdf_file(filename: str, output_file: str, n_profiles: int):
             dimensions = nc.variables[key].dimensions
             fill_value = getattr(nc.variables[key], "_FillValue", None)
             var = nc_new.createVariable(
-                key, array.dtype, dimensions, zlib=True, fill_value=fill_value
+                key,
+                array.dtype,
+                dimensions,
+                zlib=True,
+                fill_value=fill_value,
             )
-            if dimensions and "time" in dimensions[0]:
+            if dimensions and dim_name in dimensions[0]:
                 if array.ndim == 1:
                     var[:] = array[:n_profiles]
                 if array.ndim == 2:
@@ -68,14 +88,14 @@ def update_nc(old_file: str, new_file: str) -> int:
 
 
 def concatenate_files(
-    filenames: list,
-    output_file: str,
+    filenames: Iterable[PathLike | str],
+    output_file: str | PathLike,
     concat_dimension: str = "time",
-    variables: list | None = None,
-    new_attributes: dict | None = None,
-    ignore: list | None = None,
-    allow_difference: list | None = None,
-) -> None:
+    variables: list[str] | None = None,
+    new_attributes: dict[str, str | int] | None = None,
+    ignore: list[str] | None = None,
+    interp_dimension: str = "range",
+) -> list[Path]:
     """Concatenate netCDF files in one dimension.
 
     Args:
@@ -86,39 +106,53 @@ def concatenate_files(
             Default is None when all variables with 'concat_dimension' will be saved.
         new_attributes: Optional new global attributes as {'attribute_name': value}.
         ignore: List of variables to be ignored.
-        allow_difference: Names of scalar variables that can differ from one file to
-            another (value from the first file is saved).
+        interp_dimension: Dimension name for interpolation if the dimensions
+            are not the same.
+
+    Returns:
+        List of filenames that were successfully concatenated.
 
     Notes:
-        Arrays without 'concat_dimension', scalars, and global attributes will be taken
-        from the first file. Groups, possibly present in a NETCDF4 formatted file,
-        are ignored.
+        Arrays without 'concat_dimension' and scalars are expanded to the
+        concat_dimension. Global attributes are taken from the first file.
+        Groups, possibly present in a NETCDF4 formatted file, are ignored.
 
     """
-    with _Concat(filenames, output_file, concat_dimension) as concat:
-        concat.get_common_variables()
+    with _Concat(filenames, output_file, concat_dimension, interp_dimension) as concat:
         concat.create_global_attributes(new_attributes)
-        concat.concat_data(variables, ignore, allow_difference)
+        return concat.concat_data(variables, ignore)
 
 
 class _Concat:
     common_variables: set[str]
 
     def __init__(
-        self, filenames: list, output_file: str, concat_dimension: str = "time"
-    ):
-        self.filenames = sorted(filenames)
+        self,
+        filenames: Iterable[str | PathLike],
+        output_file: str | PathLike,
+        concat_dimension: str = "time",
+        interp_dim: str = "range",
+    ) -> None:
         self.concat_dimension = concat_dimension
+        self.interp_dim = interp_dim
+        self.filenames = sorted(
+            [Path(filename) for filename in filenames if self._is_valid_file(filename)],
+            key=lambda f: f.name,
+        )
+        if not self.filenames:
+            msg = "No valid files to concatenate."
+            raise ValidTimeStampError(msg)
         self.first_filename = self.filenames[0]
         self.first_file = netCDF4.Dataset(self.first_filename)
         self.concatenated_file = self._init_output_file(output_file)
-        self.common_variables = set()
 
-    def get_common_variables(self):
-        """Finds variables which should have the same values in all files."""
-        for key, value in self.first_file.variables.items():
-            if self.concat_dimension not in value.dimensions:
-                self.common_variables.add(key)
+    def _is_valid_file(self, filename: str | PathLike) -> bool:
+        # Added to handle strange .znc files with no time and huge range
+        # dimension resulting in large memory usage (e.g. Jülich 2019-05-18).
+        with netCDF4.Dataset(filename) as nc:
+            return (
+                nc[self.concat_dimension].size > 0 and nc[self.interp_dim].size < 10_000
+            )
 
     def create_global_attributes(self, new_attributes: dict | None) -> None:
         """Copies global attributes from one of the source files."""
@@ -129,33 +163,47 @@ class _Concat:
 
     def concat_data(
         self,
-        variables: list | None,
-        ignore: list | None,
-        allow_vary: list | None,
-    ):
+        keep: list | None = None,
+        ignore: list | None = None,
+    ) -> list[Path]:
         """Concatenates data arrays."""
-        self._write_initial_data(variables, ignore)
+        self._write_initial_data(keep, ignore)
+        output = [self.first_filename]
         if len(self.filenames) > 1:
             for filename in self.filenames[1:]:
-                self._append_data(filename, allow_vary)
+                try:
+                    self._append_data(filename)
+                except RuntimeError as e:
+                    if "NetCDF: HDF error" in str(e):
+                        msg = f"Caught a NetCDF HDF error. Skipping file '{filename}'."
+                        logging.exception(msg)
+                        continue
+                    raise
+                output.append(filename)
+        return output
 
-    def _write_initial_data(self, variables: list | None, ignore: list | None) -> None:
-        for key in self.first_file.variables.keys():
+    def _write_initial_data(self, keep: list | None, ignore: list | None) -> None:
+        len_concat_dim = self.first_file[self.concat_dimension].size
+        auto_scale = False
+
+        for key, var in self.first_file.variables.items():
             if (
-                variables is not None
-                and key not in variables
-                and key not in self.common_variables
+                # This filtering only affects variables having the concat_dimension
+                keep is not None
+                and key not in keep
                 and key != self.concat_dimension
+                and self.concat_dimension in var.dimensions
             ):
                 continue
             if ignore and key in ignore:
                 continue
 
-            self.first_file[key].set_auto_scale(False)
-            array = self.first_file[key][:]
-            dimensions = self.first_file[key].dimensions
-            fill_value = getattr(self.first_file[key], "_FillValue", None)
-            var = self.concatenated_file.createVariable(
+            var.set_auto_scale(auto_scale)
+            array, dimensions = self._expand_array(var, len_concat_dim)
+
+            fill_value = var.get_fill_value()
+
+            var_new = self.concatenated_file.createVariable(
                 key,
                 array.dtype,
                 dimensions,
@@ -164,39 +212,56 @@ class _Concat:
                 shuffle=False,
                 fill_value=fill_value,
             )
-            var.set_auto_scale(False)
-            var[:] = array
-            _copy_attributes(self.first_file[key], var)
-
-    def _append_data(self, filename: str, allow_vary: list | None) -> None:
+            var_new.set_auto_scale(auto_scale)
+            var_new[:] = array
+            _copy_attributes(var, var_new)
+
+    def _expand_array(
+        self, var: netCDF4.Variable, n_data: int
+    ) -> tuple[ma.MaskedArray, tuple[str, ...]]:
+        dimensions = var.dimensions
+        arr = var[:]
+        if self.concat_dimension not in dimensions and var.name != self.interp_dim:
+            dimensions = (self.concat_dimension, *dimensions)
+            arr = np.repeat(arr[np.newaxis, ...], n_data, axis=0)
+
+        return arr, dimensions
+
+    def _append_data(self, filename: str | PathLike) -> None:
         with netCDF4.Dataset(filename) as file:
-            file.set_auto_scale(False)
+            auto_scale = False
+            file.set_auto_scale(auto_scale)
             ind0 = len(self.concatenated_file.variables[self.concat_dimension])
             ind1 = ind0 + len(file.variables[self.concat_dimension])
-            for key in self.concatenated_file.variables.keys():
-                array = file[key][:]
-                if key in self.common_variables:
-                    if allow_vary is not None and key in allow_vary:
-                        continue
-                    if not np.array_equal(self.first_file[key][:], array):
-                        raise InconsistentDataError(
-                            f"Inconsistent values in variable '{key}' between "
-                            f"files '{self.first_filename}' and '{filename}'"
-                        )
-                    continue
-                if array.ndim == 0:
+            n_points = ind1 - ind0
+
+            for key in self.concatenated_file.variables:
+                if key not in file.variables or key == self.interp_dim:
                     continue
-                if array.ndim == 1:
-                    self.concatenated_file.variables[key][ind0:ind1] = array
-                else:
-                    self.concatenated_file.variables[key][ind0:ind1, :] = array
 
-    def _init_output_file(self, output_file: str) -> netCDF4.Dataset:
-        data_model = (
+                array, dimensions = self._expand_array(file[key], n_points)
+
+                # Nearest neighbour interpolation in the interp_dim dimension
+                # if the dimensions are not the same between the files
+                if self.interp_dim in dimensions and (
+                    self.first_file[self.interp_dim].size != file[self.interp_dim].size
+                ):
+                    x = file.variables[self.interp_dim][:]
+                    x_target = self.first_file.variables[self.interp_dim][:]
+                    idx = np.abs(x[:, None] - x_target[None, :]).argmin(axis=0)
+                    array = array[:, idx]
+                    out_of_bounds = (x_target < x.min()) | (x_target > x.max())
+                    fill_value = self.first_file.variables[key].get_fill_value()
+                    array[:, out_of_bounds] = fill_value
+
+                self.concatenated_file.variables[key][ind0:ind1, ...] = array
+
+    def _init_output_file(self, output_file: str | PathLike) -> netCDF4.Dataset:
+        data_model: Literal["NETCDF4", "NETCDF4_CLASSIC"] = (
             "NETCDF4" if self.first_file.data_model == "NETCDF4" else "NETCDF4_CLASSIC"
         )
         nc = netCDF4.Dataset(output_file, "w", format=data_model)
-        for dim in self.first_file.dimensions.keys():
+        for dim in self.first_file.dimensions:
             dim_len = (
                 None
                 if dim == self.concat_dimension
@@ -205,29 +270,44 @@ class _Concat:
             nc.createDimension(dim, dim_len)
         return nc
 
-    def _close(self):
+    def _close(self) -> None:
         self.first_file.close()
         self.concatenated_file.close()
 
-    def __enter__(self):
+    def __enter__(self) -> Self:
         return self
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
         self._close()
 
 
-def _copy_attributes(source: netCDF4.Dataset, target: netCDF4.Dataset) -> None:
+def _copy_attributes(
+    source: netCDF4.Dataset | netCDF4.Variable,
+    target: netCDF4.Dataset | netCDF4.Variable,
+) -> None:
     for attr in source.ncattrs():
         if attr != "_FillValue":
             value = getattr(source, attr)
             setattr(target, attr, value)
 
 
-def _find_valid_time_indices(nc_old: netCDF4.Dataset, nc_new: netCDF4.Dataset):
+def _find_valid_time_indices(
+    nc_old: netCDF4.Dataset,
+    nc_new: netCDF4.Dataset,
+) -> npt.NDArray:
     return np.where(nc_new.variables["time"][:] > nc_old.variables["time"][-1])[0]
 
 
-def _update_fields(nc_old: netCDF4.Dataset, nc_new: netCDF4.Dataset, valid_ind: list):
+def _update_fields(
+    nc_old: netCDF4.Dataset,
+    nc_new: netCDF4.Dataset,
+    valid_ind: npt.NDArray,
+) -> None:
     ind0 = len(nc_old.variables["time"])
     idx = [ind0 + x for x in valid_ind]
     concat_dimension = nc_old.variables["time"].dimensions[0]
@@ -243,3 +323,63 @@ def _update_fields(nc_old: netCDF4.Dataset, nc_new: netCDF4.Dataset, valid_ind:
                 nc_old.variables[field][idx, :] = nc_new.variables[field][valid_ind, :]
             elif len(dimensions) == 2 and concat_ind == 1:
                 nc_old.variables[field][:, idx] = nc_new.variables[field][:, valid_ind]
+
+
+def concatenate_text_files(filenames: list, output_filename: str | PathLike) -> None:
+    """Concatenates text files."""
+    with open(output_filename, "wb") as target:
+        for filename in filenames:
+            with open(filename, "rb") as source:
+                shutil.copyfileobj(source, target)
+
+
+def bundle_netcdf_files(
+    files: Sequence[str | PathLike],
+    date: datetime.date,
+    output_file: str | PathLike,
+    concat_dimensions: tuple[str, ...] = ("time", "profile"),
+    variables: list | None = None,
+) -> list:
+    """Concatenates several netcdf files into daily file with
+    some extra data manipulation.
+    """
+    with netCDF4.Dataset(files[0]) as nc:
+        concat_dimension = None
+        for key in concat_dimensions:
+            if key in nc.dimensions:
+                concat_dimension = key
+                break
+        if concat_dimension is None:
+            msg = f"Dimension '{concat_dimensions}' not found in the files."
+            raise KeyError(msg)
+    if len(files) == 1:
+        shutil.copy(files[0], output_file)
+        return list(files)
+    valid_files = []
+    for file in files:
+        try:
+            with netCDF4.Dataset(file) as nc:
+                time = nc.variables["time"]
+                time_array = time[:]
+                time_units = time.units
+        except OSError:
+            continue
+        epoch = utils.get_epoch(time_units)
+        for timestamp in time_array:
+            if utils.seconds2date(timestamp, epoch).date() == date:
+                valid_files.append(file)
+                break
+    concatenate_files(
+        valid_files,
+        output_file,
+        concat_dimension=concat_dimension,
+        variables=variables,
+        ignore=[
+            "minimum",
+            "maximum",
+            "number_integrated_samples",
+            "Min_LWP",
+            "Max_LWP",
+        ],
+    )
+    return valid_files

cloudnetpy/constants.py

@@ -1,19 +1,37 @@
 """Constants used in Cloudnet processing."""
 
+from typing import Final
+
 # Triple point of water
-T0 = 273.16
+T0: Final = 273.16
 
 # Ratio of the molecular weight of water vapor to dry air
-MW_RATIO = 0.62198
-
-# Specific heat capacity of air at around 275K (J kg-1 K-1)
-SPECIFIC_HEAT = 1004
-
-# Latent heat of evaporation (J kg-1)
-LATENT_HEAT = 2.26e6
+MW_RATIO: Final = 0.62198
 
 # Specific gas constant for dry  air (J kg-1 K-1)
-RS = 287.058
+RS: Final = 287.058
 
 # ice density kg m-3
-RHO_ICE = 917
+RHO_ICE: Final = 917
+
+# Standard atmospheric pressure at sea level Pa
+P0: Final = 1013_25
+
+# other
+SPEED_OF_LIGHT: Final = 3.0e8
+SEC_IN_MINUTE: Final = 60
+SEC_IN_HOUR: Final = 3600
+SEC_IN_DAY: Final = 86400
+MM_TO_M: Final = 1e-3
+G_TO_KG: Final = 1e-3
+M_TO_KM: Final = 1e-3
+KG_TO_G: Final = 1e3
+M_TO_MM: Final = 1e3
+M_S_TO_MM_H: Final = SEC_IN_HOUR / MM_TO_M
+MM_H_TO_M_S: Final = 1 / M_S_TO_MM_H
+GHZ_TO_HZ: Final = 1e9
+HPA_TO_PA: Final = 100
+PA_TO_HPA: Final = 1 / HPA_TO_PA
+KM_H_TO_M_S: Final = 1000 / SEC_IN_HOUR
+TWO_WAY: Final = 2
+G: Final = 9.80665