cloudnetpy 1.77.2__py3-none-any.whl → 1.78.0__py3-none-any.whl

cloudnetpy/concat_lib.py

@@ -9,9 +9,9 @@ from typing import Literal
 
 import netCDF4
 import numpy as np
+from numpy import ma
 
 from cloudnetpy import utils
-from cloudnetpy.exceptions import InconsistentDataError
 
 
 def truncate_netcdf_file(
@@ -89,7 +89,7 @@ def concatenate_files(
     variables: list | None = None,
     new_attributes: dict | None = None,
     ignore: list | None = None,
-    allow_difference: list | None = None,
+    interp_dimension: str = "range",
 ) -> list:
     """Concatenate netCDF files in one dimension.
 
@@ -101,22 +101,21 @@ def concatenate_files(
             Default is None when all variables with 'concat_dimension' will be saved.
         new_attributes: Optional new global attributes as {'attribute_name': value}.
         ignore: List of variables to be ignored.
-        allow_difference: Names of scalar variables that can differ from one file to
-            another (value from the first file is saved).
+        interp_dimension: Dimension name for interpolation if the dimensions
+            are not the same.
 
     Returns:
         List of filenames that were successfully concatenated.
 
     Notes:
-        Arrays without 'concat_dimension', scalars, and global attributes will be taken
-        from the first file. Groups, possibly present in a NETCDF4 formatted file,
-        are ignored.
+        Arrays without 'concat_dimension' and scalars are expanded to the
+        concat_dimension. Global attributes are taken from the first file.
+        Groups, possibly present in a NETCDF4 formatted file, are ignored.
 
     """
-    with _Concat(filenames, output_file, concat_dimension) as concat:
-        concat.get_common_variables()
+    with _Concat(filenames, output_file, concat_dimension, interp_dimension) as concat:
         concat.create_global_attributes(new_attributes)
-        return concat.concat_data(variables, ignore, allow_difference)
+        return concat.concat_data(variables, ignore)
 
 
 class _Concat:
@@ -127,19 +126,14 @@ class _Concat:
         filenames: Iterable[PathLike | str],
         output_file: str,
         concat_dimension: str = "time",
+        interp_dim: str = "range",
     ):
         self.filenames = sorted(map(Path, filenames), key=lambda f: f.name)
         self.concat_dimension = concat_dimension
+        self.interp_dim = interp_dim
         self.first_filename = self.filenames[0]
         self.first_file = netCDF4.Dataset(self.first_filename)
         self.concatenated_file = self._init_output_file(output_file)
-        self.common_variables = set()
-
-    def get_common_variables(self) -> None:
-        """Finds variables which should have the same values in all files."""
-        for key, value in self.first_file.variables.items():
-            if self.concat_dimension not in value.dimensions:
-                self.common_variables.add(key)
 
     def create_global_attributes(self, new_attributes: dict | None) -> None:
         """Copies global attributes from one of the source files."""
@@ -150,17 +144,16 @@ class _Concat:
 
     def concat_data(
         self,
-        variables: list | None,
-        ignore: list | None,
-        allow_vary: list | None,
+        keep: list | None = None,
+        ignore: list | None = None,
     ) -> list:
         """Concatenates data arrays."""
-        self._write_initial_data(variables, ignore)
+        self._write_initial_data(keep, ignore)
         output = [self.first_filename]
         if len(self.filenames) > 1:
             for filename in self.filenames[1:]:
                 try:
-                    self._append_data(filename, allow_vary)
+                    self._append_data(filename)
                 except RuntimeError as e:
                     if "NetCDF: HDF error" in str(e):
                         msg = f"Caught a NetCDF HDF error. Skipping file '{filename}'."
@@ -170,24 +163,28 @@ class _Concat:
                 output.append(filename)
         return output
 
-    def _write_initial_data(self, variables: list | None, ignore: list | None) -> None:
-        for key in self.first_file.variables:
+    def _write_initial_data(self, keep: list | None, ignore: list | None) -> None:
+        len_concat_dim = self.first_file[self.concat_dimension].size
+        auto_scale = False
+
+        for key, var in self.first_file.variables.items():
             if (
-                variables is not None
-                and key not in variables
-                and key not in self.common_variables
+                # This filtering only affects variables having the concat_dimension
+                keep is not None
+                and key not in keep
                 and key != self.concat_dimension
+                and self.concat_dimension in var.dimensions
             ):
                 continue
             if ignore and key in ignore:
                 continue
 
-            auto_scale = False
-            self.first_file[key].set_auto_scale(auto_scale)
-            array = self.first_file[key][:]
-            dimensions = self.first_file[key].dimensions
-            fill_value = getattr(self.first_file[key], "_FillValue", None)
-            var = self.concatenated_file.createVariable(
+            var.set_auto_scale(auto_scale)
+            array, dimensions = self._expand_array(var, len_concat_dim)
+
+            fill_value = var.get_fill_value()
+
+            var_new = self.concatenated_file.createVariable(
                 key,
                 array.dtype,
                 dimensions,
@@ -196,37 +193,49 @@ class _Concat:
                 shuffle=False,
                 fill_value=fill_value,
             )
-            auto_scale = False
-            var.set_auto_scale(auto_scale)
-            var[:] = array
-            _copy_attributes(self.first_file[key], var)
-
-    def _append_data(self, filename: str | PathLike, allow_vary: list | None) -> None:
+            var_new.set_auto_scale(auto_scale)
+            var_new[:] = array
+            _copy_attributes(var, var_new)
+
+    def _expand_array(
+        self, var: netCDF4.Variable, n_data: int
+    ) -> tuple[ma.MaskedArray, tuple[str, ...]]:
+        dimensions = var.dimensions
+        arr = var[:]
+        if self.concat_dimension not in dimensions and var.name != self.interp_dim:
+            dimensions = (self.concat_dimension, *dimensions)
+            arr = np.repeat(arr[np.newaxis, ...], n_data, axis=0)
+
+        return arr, dimensions
+
+    def _append_data(self, filename: str | PathLike) -> None:
         with netCDF4.Dataset(filename) as file:
             auto_scale = False
             file.set_auto_scale(auto_scale)
             ind0 = len(self.concatenated_file.variables[self.concat_dimension])
             ind1 = ind0 + len(file.variables[self.concat_dimension])
+            n_points = ind1 - ind0
+
             for key in self.concatenated_file.variables:
-                if key not in file.variables:
-                    continue
-                array = file[key][:]
-                if key in self.common_variables:
-                    if allow_vary is not None and key in allow_vary:
-                        continue
-                    if not np.array_equal(self.first_file[key][:], array):
-                        msg = (
-                            f"Inconsistent values in variable '{key}' between "
-                            f"files '{self.first_filename}' and '{filename}'"
-                        )
-                        raise InconsistentDataError(msg)
+                if key not in file.variables or key == self.interp_dim:
                     continue
-                if array.ndim == 0:
-                    continue
-                if array.ndim == 1:
-                    self.concatenated_file.variables[key][ind0:ind1] = array
-                else:
-                    self.concatenated_file.variables[key][ind0:ind1, :] = array
+
+                array, dimensions = self._expand_array(file[key], n_points)
+
+                # Nearest neighbour interpolation in the interp_dim dimension
+                # if the dimensions are not the same between the files
+                if self.interp_dim in dimensions and (
+                    self.first_file[self.interp_dim].size != file[self.interp_dim].size
+                ):
+                    x = file.variables[self.interp_dim][:]
+                    x_target = self.first_file.variables[self.interp_dim][:]
+                    idx = np.abs(x[:, None] - x_target[None, :]).argmin(axis=0)
+                    array = array[:, idx]
+                    out_of_bounds = (x_target < x.min()) | (x_target > x.max())
+                    fill_value = self.first_file.variables[key].get_fill_value()
+                    array[:, out_of_bounds] = fill_value
+
+                self.concatenated_file.variables[key][ind0:ind1, ...] = array
 
     def _init_output_file(self, output_file: str) -> netCDF4.Dataset:
         data_model: Literal["NETCDF4", "NETCDF4_CLASSIC"] = (

cloudnetpy/instruments/bowtie.py

@@ -80,8 +80,8 @@ class Bowtie(NcRadar):
         self.data["relative_humidity"].data /= 100
 
     def fix_chirp_start_indices(self) -> None:
-        ind = self.data["chirp_start_indices"].data
-        self.data["chirp_start_indices"].data = np.array([int(i) for i in ind])
+        array = self.data["chirp_start_indices"].data
+        self.data["chirp_start_indices"].data = np.array(array, dtype=np.int32)
         self.data["chirp_start_indices"].data_type = "int32"
 
     def check_date(self, date: str):

cloudnetpy/instruments/copernicus.py

@@ -69,7 +69,6 @@ def copernicus2nc(
                 valid_filenames = utils.get_files_with_variables(
                     valid_filenames, ["time", "ZED_HC"]
                 )
-                valid_filenames = utils.get_files_with_common_range(valid_filenames)
                 variables = list(keymap.keys())
                 concat_lib.concatenate_files(
                     valid_filenames,

cloudnetpy/instruments/galileo.py

@@ -68,7 +68,6 @@ def galileo2nc(
                 valid_filenames = utils.get_files_with_variables(
                     valid_filenames, ["time", "ZED_HC"]
                 )
-                valid_filenames = utils.get_files_with_common_range(valid_filenames)
                 variables = list(keymap.keys())
                 concat_lib.concatenate_files(
                     valid_filenames,

cloudnetpy/instruments/mira.py

@@ -193,7 +193,6 @@ def _parse_input_files(input_files: str | list[str], temp_dir: str) -> tuple:
                 )
                 raise FileNotFoundError(msg)
 
-            valid_files = utils.get_files_with_common_range(valid_files)
             filetypes = list({f.split(".")[-1].lower() for f in valid_files})
 
             if len(filetypes) > 1:
@@ -208,15 +207,6 @@ def _parse_input_files(input_files: str | list[str], temp_dir: str) -> tuple:
                 input_filename,
                 variables=variables,
                 ignore=_get_ignored_variables(filetypes[0]),
-                # It's somewhat risky to use varying nfft values as the velocity
-                # resolution may differ, but this enables concatenation when switching
-                # between different nfft configurations. Spectral data is ignored
-                # anyway for now.
-                allow_difference=[
-                    "nave",
-                    "ovl",
-                    "nfft",
-                ],
             )
     else:
         input_filename = input_files

cloudnetpy/output.py

@@ -57,7 +57,11 @@ def _get_netcdf_dimensions(obj) -> dict:
     }
     # RPG cloud radar
     if "chirp_start_indices" in obj.data:
-        dimensions["chirp_sequence"] = len(obj.data["chirp_start_indices"][:])
+        if obj.data["chirp_start_indices"][:].ndim == 1:
+            dimensions["chirp_start_indices"] = len(obj.data["chirp_start_indices"][:])
+        else:
+            dimensions["chirp"] = obj.data["chirp_start_indices"][:].shape[1]
+
     # disdrometer
     if hasattr(obj, "n_diameter") and hasattr(obj, "n_velocity"):
         dimensions["diameter"] = obj.n_diameter

cloudnetpy/utils.py

@@ -960,21 +960,6 @@ def get_file_type(filename: str) -> str:
     raise ValueError(msg)
 
 
-def get_files_with_common_range(filenames: list) -> list:
-    """Returns files with the same (most common) number of range gates."""
-    n_range = []
-    for file in filenames:
-        with netCDF4.Dataset(file) as nc:
-            n_range.append(len(nc.variables["range"]))
-    most_common = np.bincount(n_range).argmax()
-    n_removed = len(filenames) - n_range.count(int(most_common))
-    if n_removed > 0:
-        logging.warning(
-            "Removing %s files due to inconsistent height vector", n_removed
-        )
-    return [file for i, file in enumerate(filenames) if n_range[i] == most_common]
-
-
 def get_files_with_variables(filenames: list, variables: list[str]) -> list:
     """Returns files where all variables exist."""
     valid_files = []

cloudnetpy/version.py

@@ -1,4 +1,4 @@
 MAJOR = 1
-MINOR = 77
-PATCH = 2
+MINOR = 78
+PATCH = 0
 __version__ = f"{MAJOR}.{MINOR}.{PATCH}"

{cloudnetpy-1.77.2.dist-info → cloudnetpy-1.78.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnetpy
-Version: 1.77.2
+Version: 1.78.0
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

{cloudnetpy-1.77.2.dist-info → cloudnetpy-1.78.0.dist-info}/RECORD

@@ -1,15 +1,15 @@
 cloudnetpy/__init__.py,sha256=X_FqY-4yg5GUj5Edo14SToLEos6JIsC3fN-v1FUgQoA,43
 cloudnetpy/cli.py,sha256=lHkeAErmAijI-Ugpd4DHRHfbZP4SXOake0LIY5Ovv_Q,20782
 cloudnetpy/cloudnetarray.py,sha256=XFyXZwR4QWPyo7WLmvsu7DEELZQp1vi5FZ8F7tX_tM0,7307
-cloudnetpy/concat_lib.py,sha256=jcLppqAmVHVkykcXBcpwUr8MS_k8v2Xl2xBLmVRE_DI,12624
+cloudnetpy/concat_lib.py,sha256=XQ5Sk8kfXqI0Q5HoomKWWhdZ1-m2thYDKGL7SKapITE,12851
 cloudnetpy/constants.py,sha256=YnoSzZm35NDooJfhlulSJBc7g0eSchT3yGytRaTaJEI,845
 cloudnetpy/datasource.py,sha256=FcWS77jz56gIzwnbafDLdj-HjAyu0P_VtY7gkeVZThU,7952
 cloudnetpy/exceptions.py,sha256=hYbUtBwjCIfxnPe_5mELDEw87AWITBrwuo7WYIEKmJ8,1579
 cloudnetpy/metadata.py,sha256=lO7BCbVAzFoH3Nq-VuezYX0f7MnbG1Zp11g5GSiuQwM,6189
-cloudnetpy/output.py,sha256=l0LoOhcGCBrg2EJ4NT1xZ7-UKWdV7X7yQ0fJmhkwJVc,15829
+cloudnetpy/output.py,sha256=gupxt4f_-eUrFsWMto8tnknoV-p9QauC9L6CJAqBILU,15988
 cloudnetpy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cloudnetpy/utils.py,sha256=FFV4MRoiJfcbbknMD5DM-VesY-7yCb8jzeeadyyijLg,33532
-cloudnetpy/version.py,sha256=MBEzrrjK2C6qfoMHCRMYDIMl98CZIE1aPKuEZsjCPeo,72
+cloudnetpy/utils.py,sha256=O57jqUX61V_Jodrs_04Js9cmTUbK0cymW9NcHzKVrx0,32926
+cloudnetpy/version.py,sha256=Umza9M-fEEsgi9FrAC0XLoVQcbh-gNPHmP_sNLxDOD4,72
 cloudnetpy/categorize/__init__.py,sha256=s-SJaysvVpVVo5kidiruWQO6p3gv2TXwY1wEHYO5D6I,44
 cloudnetpy/categorize/atmos_utils.py,sha256=RcmbKxm2COkE7WEya0mK3yX5rzUbrewRVh3ekm01RtM,10598
 cloudnetpy/categorize/attenuation.py,sha256=Y_-fzmQTltWTqIZTulJhovC7a6ifpMcaAazDJcnMIOc,990
@@ -34,18 +34,18 @@ cloudnetpy/categorize/attenuations/melting_attenuation.py,sha256=9c9xoZHtGUbjFYJ
 cloudnetpy/categorize/attenuations/rain_attenuation.py,sha256=qazJzRyXf9vbjJhh4yiFmABI4L57j5W_6YZ-6qjRiBI,2839
 cloudnetpy/instruments/__init__.py,sha256=PEgrrQNoiOuN_ctYilmt4LV2QCLg1likPjJdWtuGlLs,528
 cloudnetpy/instruments/basta.py,sha256=Lb_EhQTI93S5Bd9osDbCE_tC8gZreRsHz7D2_dFOjmE,3793
-cloudnetpy/instruments/bowtie.py,sha256=jrEkOsZ4ScwmTN35rAwIbVHkHxQatFCHiuzBjVA4pJ0,3279
+cloudnetpy/instruments/bowtie.py,sha256=GlWCxemeXrIvWnnQRvZa1ttl4J0NmLUdc7xngIH5WBI,3281
 cloudnetpy/instruments/ceilo.py,sha256=qM3AkQKHUblhRCD42HsB6lr82giBH-0g_VzoWHZDgeA,9535
 cloudnetpy/instruments/ceilometer.py,sha256=ati9-fUQ54K9tvynIPB-nlBYwtvBVaQtUCjVCLNB67w,12059
 cloudnetpy/instruments/cl61d.py,sha256=g6DNBFju3wYhLFl32DKmC8pUup7y-EupXoUU0fuoGGA,1990
 cloudnetpy/instruments/cloudnet_instrument.py,sha256=SGPsRYYoGPoRoDY7hHJcKUVX0A23X0Telc00Fu01PnY,4495
-cloudnetpy/instruments/copernicus.py,sha256=99idcn6-iKOSvSslNjwFRng3gwlTLFjKPiT1tnVytpQ,6613
+cloudnetpy/instruments/copernicus.py,sha256=hCphEKyFCc3f1uLRdjL2435kuh64M5q-V1bI68bzGbA,6528
 cloudnetpy/instruments/fd12p.py,sha256=aGYpkczdSl7FSmK1bByMnpUBD5GAl7RTKkopt0cpWas,6822
-cloudnetpy/instruments/galileo.py,sha256=BjWE15_S3tTCOmAM5k--oicI3wghKaO0hv9EUBxtbl8,4830
+cloudnetpy/instruments/galileo.py,sha256=vcY7mYcGD8YtMw8ioy9CNGu5yarQlwE-vfWIRSbTQG0,4745
 cloudnetpy/instruments/hatpro.py,sha256=G1fHsY9LTos4vHP5kFubjE5Wg2uTVFZpYDSD8VAo-zw,9590
 cloudnetpy/instruments/instruments.py,sha256=z8Osjww3iQRxKvzXdISl-5vV6gShtji8Db5k-ZzDQ-0,4843
 cloudnetpy/instruments/lufft.py,sha256=nIoEKuuFGKq2dLqkX7zW-HpAifefG472tZhKfXE1yoA,4212
-cloudnetpy/instruments/mira.py,sha256=IH88dnV5fdAQ-A04S23ROgNmT4GBAtzXQxCr_9fWj-Q,11634
+cloudnetpy/instruments/mira.py,sha256=mH53Wpq3JnmHqDa2Bo0t6REEf8OQ4MjkHYm8AEM2_S4,11140
 cloudnetpy/instruments/mrr.py,sha256=eeAzCp3CiHGauywjwvMUAFwZ4vBOZMcd3IlF8KsrLQo,5711
 cloudnetpy/instruments/nc_lidar.py,sha256=5gQG9PApnNPrHmS9_zanl8HEYIQuGRpbnzC3wfTcOyQ,1705
 cloudnetpy/instruments/nc_radar.py,sha256=HlaZeH5939R86ukF8K-P4Kfzb5-CpLB15LU2u94C5eI,7330
@@ -117,10 +117,10 @@ cloudnetpy/products/lwc.py,sha256=sl6Al2tuH3KkCBrPbWTmuz3jlD5UQJ4D6qBsn1tt2CQ,18
 cloudnetpy/products/mie_lu_tables.nc,sha256=It4fYpqJXlqOgL8jeZ-PxGzP08PMrELIDVe55y9ob58,16637951
 cloudnetpy/products/mwr_tools.py,sha256=8HPZpQMTojKZP1JS1S83IE0sxmbDE9bxlaWoqmGnUZE,6199
 cloudnetpy/products/product_tools.py,sha256=uu4l6reuGbPcW3TgttbaSrqIKbyYGhBVTdnC7opKvmg,11101
-cloudnetpy-1.77.2.dist-info/licenses/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
+cloudnetpy-1.78.0.dist-info/licenses/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
 docs/source/conf.py,sha256=IKiFWw6xhUd8NrCg0q7l596Ck1d61XWeVjIFHVSG9Og,1490
-cloudnetpy-1.77.2.dist-info/METADATA,sha256=soDh6zzIkRMSYIeYp6NiOTksTWZX0zttG8MnWJuFjNs,5796
-cloudnetpy-1.77.2.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-cloudnetpy-1.77.2.dist-info/entry_points.txt,sha256=HhY7LwCFk4qFgDlXx_Fy983ZTd831WlhtdPIzV-Y3dY,51
-cloudnetpy-1.77.2.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
-cloudnetpy-1.77.2.dist-info/RECORD,,
+cloudnetpy-1.78.0.dist-info/METADATA,sha256=4CkmH9P2aO02UADla12HiMWCqnPts65FVD6PrZYBre4,5796
+cloudnetpy-1.78.0.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+cloudnetpy-1.78.0.dist-info/entry_points.txt,sha256=HhY7LwCFk4qFgDlXx_Fy983ZTd831WlhtdPIzV-Y3dY,51
+cloudnetpy-1.78.0.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
+cloudnetpy-1.78.0.dist-info/RECORD,,