cloudnetpy 1.67.3__py3-none-any.whl → 1.67.5__py3-none-any.whl

cloudnetpy/concat_lib.py

@@ -1,7 +1,10 @@
 """Module for concatenating netCDF files."""
 
+import logging
 import shutil
+from collections.abc import Iterable
 from os import PathLike
+from pathlib import Path
 from typing import Literal
 
 import netCDF4
@@ -80,14 +83,14 @@ def update_nc(old_file: str, new_file: str) -> int:
 
 
 def concatenate_files(
-    filenames: list,
+    filenames: Iterable[PathLike | str],
     output_file: str,
     concat_dimension: str = "time",
     variables: list | None = None,
     new_attributes: dict | None = None,
     ignore: list | None = None,
     allow_difference: list | None = None,
-) -> None:
+) -> list:
     """Concatenate netCDF files in one dimension.
 
     Args:
@@ -101,6 +104,9 @@ def concatenate_files(
         allow_difference: Names of scalar variables that can differ from one file to
             another (value from the first file is saved).
 
+    Returns:
+        List of filenames that were successfully concatenated.
+
     Notes:
         Arrays without 'concat_dimension', scalars, and global attributes will be taken
         from the first file. Groups, possibly present in a NETCDF4 formatted file,
@@ -110,7 +116,7 @@ def concatenate_files(
     with _Concat(filenames, output_file, concat_dimension) as concat:
         concat.get_common_variables()
         concat.create_global_attributes(new_attributes)
-        concat.concat_data(variables, ignore, allow_difference)
+        return concat.concat_data(variables, ignore, allow_difference)
 
 
 class _Concat:
@@ -118,11 +124,11 @@ class _Concat:
 
     def __init__(
         self,
-        filenames: list,
+        filenames: Iterable[PathLike | str],
         output_file: str,
         concat_dimension: str = "time",
     ):
-        self.filenames = sorted(filenames)
+        self.filenames = sorted(map(Path, filenames), key=lambda f: f.name)
         self.concat_dimension = concat_dimension
         self.first_filename = self.filenames[0]
         self.first_file = netCDF4.Dataset(self.first_filename)
@@ -147,12 +153,22 @@ class _Concat:
         variables: list | None,
         ignore: list | None,
         allow_vary: list | None,
-    ) -> None:
+    ) -> list:
         """Concatenates data arrays."""
         self._write_initial_data(variables, ignore)
+        output = [self.first_filename]
         if len(self.filenames) > 1:
             for filename in self.filenames[1:]:
-                self._append_data(filename, allow_vary)
+                try:
+                    self._append_data(filename, allow_vary)
+                except RuntimeError as e:
+                    if "NetCDF: HDF error" in str(e):
+                        msg = f"Caught a NetCDF HDF error. Skipping file '{filename}'."
+                        logging.exception(msg)
+                        continue
+                    raise
+                output.append(filename)
+        return output
 
     def _write_initial_data(self, variables: list | None, ignore: list | None) -> None:
         for key in self.first_file.variables:
@@ -185,7 +201,7 @@ class _Concat:
             var[:] = array
             _copy_attributes(self.first_file[key], var)
 
-    def _append_data(self, filename: str, allow_vary: list | None) -> None:
+    def _append_data(self, filename: str | PathLike, allow_vary: list | None) -> None:
         with netCDF4.Dataset(filename) as file:
             auto_scale = False
             file.set_auto_scale(auto_scale)

cloudnetpy/instruments/copernicus.py

@@ -91,15 +91,19 @@ def copernicus2nc(
             copernicus.mask_corrupted_values()
             copernicus.mask_first_range_gates()
             copernicus.mask_invalid_data()
-            copernicus.add_time_and_range()
             copernicus.fix_range_offset(site_meta)
             copernicus.screen_negative_ranges()
             copernicus.add_radar_specific_variables()
             copernicus.add_nyquist_velocity(keymap)
             copernicus.add_site_geolocation()
-            valid_indices = copernicus.add_zenith_and_azimuth_angles()
+            valid_indices = copernicus.add_zenith_and_azimuth_angles(
+                elevation_threshold=1,
+                elevation_diff_threshold=0.1,
+                azimuth_diff_threshold=0.1,
+            )
             copernicus.screen_time_indices(valid_indices)
             copernicus.add_height()
+            copernicus.test_if_all_masked()
         attributes = output.add_time_attribute(ATTRIBUTES, copernicus.date)
         output.update_attributes(copernicus.data, attributes)
         return output.save_level1b(copernicus, output_file, uuid)

cloudnetpy/instruments/galileo.py

@@ -89,13 +89,17 @@ def galileo2nc(
             galileo.screen_by_snr(snr_limit=snr_limit)
             galileo.mask_clutter()
             galileo.mask_invalid_data()
-            galileo.add_time_and_range()
             galileo.add_radar_specific_variables()
             galileo.add_nyquist_velocity(keymap)
             galileo.add_site_geolocation()
-            valid_indices = galileo.add_zenith_and_azimuth_angles()
+            valid_indices = galileo.add_zenith_and_azimuth_angles(
+                elevation_threshold=1,
+                elevation_diff_threshold=0.1,
+                azimuth_diff_threshold=0.1,
+            )
             galileo.screen_time_indices(valid_indices)
             galileo.add_height()
+            galileo.test_if_all_masked()
         attributes = output.add_time_attribute(ATTRIBUTES, galileo.date)
         output.update_attributes(galileo.data, attributes)
         return output.save_level1b(galileo, output_file, uuid)

cloudnetpy/instruments/mira.py

@@ -80,7 +80,11 @@ def mira2nc(
             mira.add_time_and_range()
             mira.add_site_geolocation()
             mira.add_radar_specific_variables()
-            valid_indices = mira.add_zenith_and_azimuth_angles()
+            valid_indices = mira.add_zenith_and_azimuth_angles(
+                elevation_threshold=1,
+                elevation_diff_threshold=1e-6,
+                azimuth_diff_threshold=1e-3,
+            )
             mira.screen_time_indices(valid_indices)
             mira.add_height()
             mira.test_if_all_masked()

cloudnetpy/instruments/nc_radar.py

@@ -90,7 +90,12 @@ class NcRadar(DataSource, CloudnetInstrument):
         if len(ind) > 0:
             self.data["v"].data[:, ind] = ma.masked
 
-    def add_zenith_and_azimuth_angles(self) -> list:
+    def add_zenith_and_azimuth_angles(
+        self,
+        elevation_threshold: float,
+        elevation_diff_threshold: float,
+        azimuth_diff_threshold: float,
+    ) -> list:
         """Adds non-varying instrument zenith and azimuth angles and returns valid
         time indices.
         """
@@ -100,9 +105,9 @@ class NcRadar(DataSource, CloudnetInstrument):
         elevation_diff = ma.diff(elevation, prepend=elevation[1])
         azimuth_diff = ma.diff(azimuth, prepend=azimuth[1])
 
-        is_stable = np.abs(elevation - 90) < 1
-        is_stable &= np.abs(elevation_diff) < 1e-6
-        is_stable &= np.abs(azimuth_diff) < 1e-3
+        is_stable = np.abs(elevation - 90) < elevation_threshold
+        is_stable &= np.abs(elevation_diff) < elevation_diff_threshold
+        is_stable &= np.abs(azimuth_diff) < azimuth_diff_threshold
 
         # If scanning unit is broken, data are missing
         # (assume it's vertically pointing)

cloudnetpy/instruments/pollyxt.py

@@ -203,7 +203,12 @@ def _fetch_files_with_same_range(
     bsc_sums = [get_sum(f) for f in bsc_files]
     depol_sums = [get_sum(f) for f in depol_files]
     all_sums = bsc_sums + depol_sums
-    most_common_sum = Counter(all_sums).most_common(1)[0][0]
+
+    filtered_sums = [item for item in all_sums if item is not ma.masked]
+    if not filtered_sums:
+        return [], []
+
+    most_common_sum = Counter(filtered_sums).most_common(1)[0][0]
     valid_indices = [
         i
         for i, (bs, ds) in enumerate(zip(bsc_sums, depol_sums, strict=False))

cloudnetpy/metadata.py

@@ -121,6 +121,11 @@ COMMON_ATTRIBUTES = {
         standard_name="zenith_angle",
         comment="Angle to the local vertical. A value of zero is directly overhead.",
     ),
+    "ir_zenith_angle": MetaData(
+        long_name="Infrared sensor zenith angle",
+        units="degree",
+        comment="90=horizon, 0=zenith",
+    ),
     "azimuth_angle": MetaData(
         long_name="Azimuth angle",
         standard_name="sensor_azimuth_angle",

cloudnetpy/version.py

@@ -1,4 +1,4 @@
 MAJOR = 1
 MINOR = 67
-PATCH = 3
+PATCH = 5
 __version__ = f"{MAJOR}.{MINOR}.{PATCH}"

{cloudnetpy-1.67.3.dist-info → cloudnetpy-1.67.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudnetpy
-Version: 1.67.3
+Version: 1.67.5
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

{cloudnetpy-1.67.3.dist-info → cloudnetpy-1.67.5.dist-info}/RECORD

@@ -1,15 +1,15 @@
 cloudnetpy/__init__.py,sha256=X_FqY-4yg5GUj5Edo14SToLEos6JIsC3fN-v1FUgQoA,43
 cloudnetpy/cli.py,sha256=lHkeAErmAijI-Ugpd4DHRHfbZP4SXOake0LIY5Ovv_Q,20782
 cloudnetpy/cloudnetarray.py,sha256=Ol1ha4RPAmFZANL__U5CaMKX4oYMXYR6OnjoCZ9w3eo,7077
-cloudnetpy/concat_lib.py,sha256=lLjEC0ILPI1ghv8Wu9WVDiQoAkwwBKlW5jV8KoIde_8,11963
+cloudnetpy/concat_lib.py,sha256=jcLppqAmVHVkykcXBcpwUr8MS_k8v2Xl2xBLmVRE_DI,12624
 cloudnetpy/constants.py,sha256=jjW1eO5BwmLjtCKXKGXtKF_BoKpvSNrRfpgW43fWT_0,824
 cloudnetpy/datasource.py,sha256=FcWS77jz56gIzwnbafDLdj-HjAyu0P_VtY7gkeVZThU,7952
 cloudnetpy/exceptions.py,sha256=ns48useL9RN3mPh7CqIiLA19VI9OmVbyRsKTkwbThF8,1760
-cloudnetpy/metadata.py,sha256=DOGt7EQLS-AVJEszrUrpXr3gHVQv655FzeCzKrOPvoU,5477
+cloudnetpy/metadata.py,sha256=QhmwRDzIqT6qAceocwpNx_CZVHAb1MUVoAmihybVkIM,5631
 cloudnetpy/output.py,sha256=lq4YSeMT_d-j4rlQkKm9KIZ8boupTBBBKV1eUawpmCI,15672
 cloudnetpy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloudnetpy/utils.py,sha256=U0iMIKPiKLrLVAfs_u9pPuoWYW1RJHcM8dbLF9a4yIA,29796
-cloudnetpy/version.py,sha256=g8g1vACVDsZA7aoMeIfXx3LzdjPkjA1SXN6-nsmvJUM,72
+cloudnetpy/version.py,sha256=1kieuJxeVgwMXQUtOdo25ewj0bWtAJPCVKU_wHSvJ5M,72
 cloudnetpy/categorize/__init__.py,sha256=s-SJaysvVpVVo5kidiruWQO6p3gv2TXwY1wEHYO5D6I,44
 cloudnetpy/categorize/atmos_utils.py,sha256=RcmbKxm2COkE7WEya0mK3yX5rzUbrewRVh3ekm01RtM,10598
 cloudnetpy/categorize/attenuation.py,sha256=Y_-fzmQTltWTqIZTulJhovC7a6ifpMcaAazDJcnMIOc,990
@@ -39,16 +39,16 @@ cloudnetpy/instruments/ceilo.py,sha256=xrI7iYNftKvGZf-3C_ESUNsu-QhXV43iWkDuKp3bi
 cloudnetpy/instruments/ceilometer.py,sha256=pdmLVljsuciyKpaGxWxJ_f1IrJK-UrkBC0lSeuirLlU,12095
 cloudnetpy/instruments/cl61d.py,sha256=g6DNBFju3wYhLFl32DKmC8pUup7y-EupXoUU0fuoGGA,1990
 cloudnetpy/instruments/cloudnet_instrument.py,sha256=NQZ_FMXh8iyzXYSCKQSpIdp0MZFAh7WqKE8mZRmVbF4,4164
-cloudnetpy/instruments/copernicus.py,sha256=nmgqGOjVQFngj7BNbpcuCwA-W3yksvBbqn__iq7MyDk,6469
-cloudnetpy/instruments/galileo.py,sha256=yQBedd7dmDnwuWi1MtXOsg4-RyRx0uRAXumCY4YuH9k,4686
+cloudnetpy/instruments/copernicus.py,sha256=he15ncH6SDCidBhr0BkOJc6Rg8cMvLWN0vGmpui6nGQ,6611
+cloudnetpy/instruments/galileo.py,sha256=qABO3IUdRpGFDgqKsN0Pl4TQ-CWHQz8s13PxnwdIUnU,4828
 cloudnetpy/instruments/hatpro.py,sha256=DzCWzTJxTc5BSOgoeyM8RjYkSXX6NDi3QXgKRp0uxlI,8759
 cloudnetpy/instruments/instruments.py,sha256=cHP0RN-Z8Jl9yoDHdvaOflTrzuogDTGmd-nxSOk8Uq4,3568
 cloudnetpy/instruments/lufft.py,sha256=ugXF6pssHAAz1Y_hqPdpKuluAjxxHSR88xBmQuS6RlI,3705
-cloudnetpy/instruments/mira.py,sha256=HXfSRIWXrsSet3mSi0hiU-m-eEA6V_Pz0vjtiG8pbUM,10263
+cloudnetpy/instruments/mira.py,sha256=un7qcoJrIeZmKWmVFCV84rmYCZkyJtvYnYXEhkENrY4,10407
 cloudnetpy/instruments/mrr.py,sha256=eeAzCp3CiHGauywjwvMUAFwZ4vBOZMcd3IlF8KsrLQo,5711
 cloudnetpy/instruments/nc_lidar.py,sha256=5gQG9PApnNPrHmS9_zanl8HEYIQuGRpbnzC3wfTcOyQ,1705
-cloudnetpy/instruments/nc_radar.py,sha256=ZNNmPdSF4yTVhq8CpnfgecH1DuYy7psc4YTxp3BLWRw,6724
-cloudnetpy/instruments/pollyxt.py,sha256=TAhpwwV9EcFTg2Zb-Hjwc6PUetEkO4csGs74ghrpTe4,9899
+cloudnetpy/instruments/nc_radar.py,sha256=AjPn3mkq5a1mE7YzKtZnxX5suNju9NhUq-TDvs7T_uU,6911
+cloudnetpy/instruments/pollyxt.py,sha256=ra7sYQ_cmkC0T9TBYrMN6iiQEZimmWGdW9Ilv61JB-Q,10027
 cloudnetpy/instruments/radiometrics.py,sha256=ySG4a042XkgjMTG8d20oAPNvFvw9bMwwiqS3zv-JF_w,11825
 cloudnetpy/instruments/rpg.py,sha256=IozvBJ8_qXTPqtp58FQwRsoI5_aI3-kycpXugZkS0d4,17462
 cloudnetpy/instruments/rpg_reader.py,sha256=ThztFuVrWxhmWVAfZTfQDeUiKK1XMTbtv08IBe8GK98,11364
@@ -115,9 +115,9 @@ cloudnetpy/products/mie_lu_tables.nc,sha256=It4fYpqJXlqOgL8jeZ-PxGzP08PMrELIDVe5
 cloudnetpy/products/mwr_tools.py,sha256=rd7UC67O4fsIE5SaHVZ4qWvUJTj41ZGwgQWPwZzOM14,5377
 cloudnetpy/products/product_tools.py,sha256=uu4l6reuGbPcW3TgttbaSrqIKbyYGhBVTdnC7opKvmg,11101
 docs/source/conf.py,sha256=IKiFWw6xhUd8NrCg0q7l596Ck1d61XWeVjIFHVSG9Og,1490
-cloudnetpy-1.67.3.dist-info/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
-cloudnetpy-1.67.3.dist-info/METADATA,sha256=QrnsmPKlK0kGbF6XogT2gmffoZ2ZEhKTjGY5ldJty0I,5793
-cloudnetpy-1.67.3.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-cloudnetpy-1.67.3.dist-info/entry_points.txt,sha256=HhY7LwCFk4qFgDlXx_Fy983ZTd831WlhtdPIzV-Y3dY,51
-cloudnetpy-1.67.3.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
-cloudnetpy-1.67.3.dist-info/RECORD,,
+cloudnetpy-1.67.5.dist-info/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
+cloudnetpy-1.67.5.dist-info/METADATA,sha256=FEyHXhhROX3iv-iXHFwVrR7XtS0C3uCa7wfsr-sACNA,5793
+cloudnetpy-1.67.5.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+cloudnetpy-1.67.5.dist-info/entry_points.txt,sha256=HhY7LwCFk4qFgDlXx_Fy983ZTd831WlhtdPIzV-Y3dY,51
+cloudnetpy-1.67.5.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
+cloudnetpy-1.67.5.dist-info/RECORD,,