cloudnetpy 1.67.2__py3-none-any.whl → 1.67.4__py3-none-any.whl

cloudnetpy/instruments/copernicus.py

@@ -91,15 +91,19 @@ def copernicus2nc(
             copernicus.mask_corrupted_values()
             copernicus.mask_first_range_gates()
             copernicus.mask_invalid_data()
-            copernicus.add_time_and_range()
             copernicus.fix_range_offset(site_meta)
             copernicus.screen_negative_ranges()
             copernicus.add_radar_specific_variables()
             copernicus.add_nyquist_velocity(keymap)
             copernicus.add_site_geolocation()
-            valid_indices = copernicus.add_zenith_and_azimuth_angles()
+            valid_indices = copernicus.add_zenith_and_azimuth_angles(
+                elevation_threshold=1,
+                elevation_diff_threshold=0.1,
+                azimuth_diff_threshold=0.1,
+            )
             copernicus.screen_time_indices(valid_indices)
             copernicus.add_height()
+            copernicus.test_if_all_masked()
         attributes = output.add_time_attribute(ATTRIBUTES, copernicus.date)
         output.update_attributes(copernicus.data, attributes)
         return output.save_level1b(copernicus, output_file, uuid)

cloudnetpy/instruments/galileo.py

@@ -89,13 +89,17 @@ def galileo2nc(
             galileo.screen_by_snr(snr_limit=snr_limit)
             galileo.mask_clutter()
             galileo.mask_invalid_data()
-            galileo.add_time_and_range()
             galileo.add_radar_specific_variables()
             galileo.add_nyquist_velocity(keymap)
             galileo.add_site_geolocation()
-            valid_indices = galileo.add_zenith_and_azimuth_angles()
+            valid_indices = galileo.add_zenith_and_azimuth_angles(
+                elevation_threshold=1,
+                elevation_diff_threshold=0.1,
+                azimuth_diff_threshold=0.1,
+            )
             galileo.screen_time_indices(valid_indices)
             galileo.add_height()
+            galileo.test_if_all_masked()
         attributes = output.add_time_attribute(ATTRIBUTES, galileo.date)
         output.update_attributes(galileo.data, attributes)
         return output.save_level1b(galileo, output_file, uuid)

cloudnetpy/instruments/mira.py

@@ -80,7 +80,11 @@ def mira2nc(
             mira.add_time_and_range()
             mira.add_site_geolocation()
             mira.add_radar_specific_variables()
-            valid_indices = mira.add_zenith_and_azimuth_angles()
+            valid_indices = mira.add_zenith_and_azimuth_angles(
+                elevation_threshold=1,
+                elevation_diff_threshold=1e-6,
+                azimuth_diff_threshold=1e-3,
+            )
             mira.screen_time_indices(valid_indices)
             mira.add_height()
             mira.test_if_all_masked()

cloudnetpy/instruments/nc_radar.py

@@ -90,7 +90,12 @@ class NcRadar(DataSource, CloudnetInstrument):
         if len(ind) > 0:
             self.data["v"].data[:, ind] = ma.masked
 
-    def add_zenith_and_azimuth_angles(self) -> list:
+    def add_zenith_and_azimuth_angles(
+        self,
+        elevation_threshold: float,
+        elevation_diff_threshold: float,
+        azimuth_diff_threshold: float,
+    ) -> list:
         """Adds non-varying instrument zenith and azimuth angles and returns valid
         time indices.
         """
@@ -100,9 +105,9 @@ class NcRadar(DataSource, CloudnetInstrument):
         elevation_diff = ma.diff(elevation, prepend=elevation[1])
         azimuth_diff = ma.diff(azimuth, prepend=azimuth[1])
 
-        is_stable = np.abs(elevation - 90) < 1
-        is_stable &= np.abs(elevation_diff) < 1e-6
-        is_stable &= np.abs(azimuth_diff) < 1e-3
+        is_stable = np.abs(elevation - 90) < elevation_threshold
+        is_stable &= np.abs(elevation_diff) < elevation_diff_threshold
+        is_stable &= np.abs(azimuth_diff) < azimuth_diff_threshold
 
         # If scanning unit is broken, data are missing
         # (assume it's vertically pointing)

cloudnetpy/instruments/pollyxt.py

@@ -2,6 +2,7 @@
 
 import glob
 import logging
+from collections import Counter
 
 import netCDF4
 import numpy as np
@@ -105,12 +106,13 @@ class PollyXt(Ceilometer):
         if len(bsc_files) != len(depol_files):
             msg = "Inconsistent number of pollyxt bsc / depol files"
             raise InconsistentDataError(msg)
+        bsc_files, depol_files = _fetch_files_with_same_range(bsc_files, depol_files)
+        if not bsc_files:
+            msg = "No pollyxt files with same range found"
+            raise InconsistentDataError(msg)
         self._fetch_attributes(bsc_files[0])
-        self.data["range"] = _read_array_from_multiple_files(
-            bsc_files,
-            depol_files,
-            "height",
-        )
+        with netCDF4.Dataset(bsc_files[0], "r") as nc:
+            self.data["range"] = nc.variables["height"][:]
         calibration_factors: np.ndarray = np.array([])
         beta_channel = self._get_valid_beta_channel(bsc_files)
         bsc_key = f"attenuated_backscatter_{beta_channel}nm"
@@ -127,14 +129,22 @@ class PollyXt(Ceilometer):
                 except AssertionError as err:
                     logging.warning(
                         "Ignoring files '%s' and '%s': %s",
-                        nc_bsc,
-                        nc_depol,
+                        bsc_file,
+                        depol_file,
                         err,
                     )
                     continue
                 beta_raw = nc_bsc.variables[bsc_key][:]
                 depol_raw = nc_depol.variables["volume_depolarization_ratio_532nm"][:]
-                snr = nc_bsc.variables[f"SNR_{beta_channel}nm"][:]
+                try:
+                    snr = nc_bsc.variables[f"SNR_{beta_channel}nm"][:]
+                except KeyError:
+                    logging.warning(
+                        "Ignoring files '%s' and '%s'",
+                        bsc_file,
+                        depol_file,
+                    )
+                    continue
                 for array, key in zip(
                     [beta_raw, depol_raw, time, snr],
                     ["beta_raw", "depolarisation_raw", "time", "snr"],
@@ -183,17 +193,30 @@ class PollyXt(Ceilometer):
                 self.serial_number = nc.source.lower()
 
 
-def _read_array_from_multiple_files(files1: list, files2: list, key) -> np.ndarray:
-    array: np.ndarray = np.array([])
-    for ind, (file1, file2) in enumerate(zip(files1, files2, strict=True)):
-        with netCDF4.Dataset(file1, "r") as nc1, netCDF4.Dataset(file2, "r") as nc2:
-            array1 = _read_array_from_file_pair(nc1, nc2, key)
-            if ind == 0:
-                array = array1
-        assert_almost_equal(
-            array, array1, err_msg=f"Inconsistent variable '{key}'", decimal=2
-        )
-    return np.array(array)
+def _fetch_files_with_same_range(
+    bsc_files: list[str], depol_files: list[str]
+) -> tuple[list[str], list[str]]:
+    def get_sum(file: str):
+        with netCDF4.Dataset(file, "r") as nc:
+            return np.sum(np.round(nc.variables["height"][:]))
+
+    bsc_sums = [get_sum(f) for f in bsc_files]
+    depol_sums = [get_sum(f) for f in depol_files]
+    all_sums = bsc_sums + depol_sums
+    most_common_sum = Counter(all_sums).most_common(1)[0][0]
+    valid_indices = [
+        i
+        for i, (bs, ds) in enumerate(zip(bsc_sums, depol_sums, strict=False))
+        if bs == most_common_sum and ds == most_common_sum
+    ]
+    if len(valid_indices) != len(bsc_files):
+        n_ignored = len(bsc_files) - len(valid_indices)
+        msg = f"Ignoring {n_ignored} file(s) with different range"
+        logging.warning(msg)
+    return (
+        [bsc_files[i] for i in valid_indices],
+        [depol_files[i] for i in valid_indices],
+    )
 
 
 def _read_array_from_file_pair(

cloudnetpy/plotting/plotting.py

@@ -868,58 +868,62 @@ def generate_figure(
         Dimensions: Dimensions of a generated figure in pixels.
 
     """
-    if options is None:
-        options = PlotParameters()
+    fig = None
+    try:
+        if options is None:
+            options = PlotParameters()
 
-    with netCDF4.Dataset(filename) as file:
-        figure_data = FigureData(file, variables, options)
-        fig, axes = figure_data.initialize_figure()
+        with netCDF4.Dataset(filename) as file:
+            figure_data = FigureData(file, variables, options)
+            fig, axes = figure_data.initialize_figure()
 
-        for ax, variable, ind in zip(
-            axes, figure_data.variables, figure_data.indices, strict=True
-        ):
-            file_type = getattr(file, "cloudnet_file_type", None)
-            subplot = SubPlot(ax, variable, options, file_type)
-
-            if variable.name in ("tb", "irt") and ind is not None:
-                Plot1D(subplot).plot_tb(figure_data, ind)
-            elif variable.ndim == 1:
-                Plot1D(subplot).plot(figure_data)
-            elif variable.name in ("number_concentration", "fall_velocity"):
-                Plot2D(subplot).plot(figure_data)
-                subplot.set_yax(ylabel="Diameter (mm)", y_limits=(0, 10))
-            else:
-                Plot2D(subplot).plot(figure_data)
-                subplot.set_yax(y_limits=(0, figure_data.options.max_y))
+            for ax, variable, ind in zip(
+                axes, figure_data.variables, figure_data.indices, strict=True
+            ):
+                file_type = getattr(file, "cloudnet_file_type", None)
+                subplot = SubPlot(ax, variable, options, file_type)
 
-            subplot.set_xax()
+                if variable.name in ("tb", "irt") and ind is not None:
+                    Plot1D(subplot).plot_tb(figure_data, ind)
+                elif variable.ndim == 1:
+                    Plot1D(subplot).plot(figure_data)
+                elif variable.name in ("number_concentration", "fall_velocity"):
+                    Plot2D(subplot).plot(figure_data)
+                    subplot.set_yax(ylabel="Diameter (mm)", y_limits=(0, 10))
+                else:
+                    Plot2D(subplot).plot(figure_data)
+                    subplot.set_yax(y_limits=(0, figure_data.options.max_y))
 
-            if options.title:
-                subplot.add_title(ind)
+                subplot.set_xax()
 
-            if options.grid:
-                subplot.add_grid()
+                if options.title:
+                    subplot.add_title(ind)
 
-            if options.show_sources:
-                subplot.add_sources(figure_data)
+                if options.grid:
+                    subplot.add_grid()
 
-            if options.subtitle and variable == figure_data.variables[-1]:
-                figure_data.add_subtitle(fig)
+                if options.show_sources:
+                    subplot.add_sources(figure_data)
 
-    subplot.set_xlabel()
+                if options.subtitle and variable == figure_data.variables[-1]:
+                    figure_data.add_subtitle(fig)
 
-    if options.footer_text is not None:
-        subplot.show_footer(fig, ax)
+        subplot.set_xlabel()
 
-    if output_filename:
-        plt.savefig(output_filename, bbox_inches="tight")
+        if options.footer_text is not None:
+            subplot.show_footer(fig, ax)
 
-    if show:
-        plt.show()
+        if output_filename:
+            plt.savefig(output_filename, bbox_inches="tight")
 
-    plt.close()
+        if show:
+            plt.show()
+
+        return Dimensions(fig, axes)
 
-    return Dimensions(fig, axes)
+    finally:
+        if fig:
+            plt.close(fig)
 
 
 def lin2log(*args) -> list:

cloudnetpy/version.py

@@ -1,4 +1,4 @@
 MAJOR = 1
 MINOR = 67
-PATCH = 2
+PATCH = 4
 __version__ = f"{MAJOR}.{MINOR}.{PATCH}"

{cloudnetpy-1.67.2.dist-info → cloudnetpy-1.67.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudnetpy
-Version: 1.67.2
+Version: 1.67.4
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

{cloudnetpy-1.67.2.dist-info → cloudnetpy-1.67.4.dist-info}/RECORD

@@ -9,7 +9,7 @@ cloudnetpy/metadata.py,sha256=DOGt7EQLS-AVJEszrUrpXr3gHVQv655FzeCzKrOPvoU,5477
 cloudnetpy/output.py,sha256=lq4YSeMT_d-j4rlQkKm9KIZ8boupTBBBKV1eUawpmCI,15672
 cloudnetpy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloudnetpy/utils.py,sha256=U0iMIKPiKLrLVAfs_u9pPuoWYW1RJHcM8dbLF9a4yIA,29796
-cloudnetpy/version.py,sha256=hqEDsWOF5LYM_h9AA1H7x5U6QAtJ7SGqOc-kBd6vsrA,72
+cloudnetpy/version.py,sha256=bB1gVbPp01OH9vaIK4R6vM7yCkY9CJ9vS7zOVCzkf6o,72
 cloudnetpy/categorize/__init__.py,sha256=s-SJaysvVpVVo5kidiruWQO6p3gv2TXwY1wEHYO5D6I,44
 cloudnetpy/categorize/atmos_utils.py,sha256=RcmbKxm2COkE7WEya0mK3yX5rzUbrewRVh3ekm01RtM,10598
 cloudnetpy/categorize/attenuation.py,sha256=Y_-fzmQTltWTqIZTulJhovC7a6ifpMcaAazDJcnMIOc,990
@@ -39,16 +39,16 @@ cloudnetpy/instruments/ceilo.py,sha256=xrI7iYNftKvGZf-3C_ESUNsu-QhXV43iWkDuKp3bi
 cloudnetpy/instruments/ceilometer.py,sha256=pdmLVljsuciyKpaGxWxJ_f1IrJK-UrkBC0lSeuirLlU,12095
 cloudnetpy/instruments/cl61d.py,sha256=g6DNBFju3wYhLFl32DKmC8pUup7y-EupXoUU0fuoGGA,1990
 cloudnetpy/instruments/cloudnet_instrument.py,sha256=NQZ_FMXh8iyzXYSCKQSpIdp0MZFAh7WqKE8mZRmVbF4,4164
-cloudnetpy/instruments/copernicus.py,sha256=nmgqGOjVQFngj7BNbpcuCwA-W3yksvBbqn__iq7MyDk,6469
-cloudnetpy/instruments/galileo.py,sha256=yQBedd7dmDnwuWi1MtXOsg4-RyRx0uRAXumCY4YuH9k,4686
+cloudnetpy/instruments/copernicus.py,sha256=he15ncH6SDCidBhr0BkOJc6Rg8cMvLWN0vGmpui6nGQ,6611
+cloudnetpy/instruments/galileo.py,sha256=qABO3IUdRpGFDgqKsN0Pl4TQ-CWHQz8s13PxnwdIUnU,4828
 cloudnetpy/instruments/hatpro.py,sha256=DzCWzTJxTc5BSOgoeyM8RjYkSXX6NDi3QXgKRp0uxlI,8759
 cloudnetpy/instruments/instruments.py,sha256=cHP0RN-Z8Jl9yoDHdvaOflTrzuogDTGmd-nxSOk8Uq4,3568
 cloudnetpy/instruments/lufft.py,sha256=ugXF6pssHAAz1Y_hqPdpKuluAjxxHSR88xBmQuS6RlI,3705
-cloudnetpy/instruments/mira.py,sha256=HXfSRIWXrsSet3mSi0hiU-m-eEA6V_Pz0vjtiG8pbUM,10263
+cloudnetpy/instruments/mira.py,sha256=un7qcoJrIeZmKWmVFCV84rmYCZkyJtvYnYXEhkENrY4,10407
 cloudnetpy/instruments/mrr.py,sha256=eeAzCp3CiHGauywjwvMUAFwZ4vBOZMcd3IlF8KsrLQo,5711
 cloudnetpy/instruments/nc_lidar.py,sha256=5gQG9PApnNPrHmS9_zanl8HEYIQuGRpbnzC3wfTcOyQ,1705
-cloudnetpy/instruments/nc_radar.py,sha256=ZNNmPdSF4yTVhq8CpnfgecH1DuYy7psc4YTxp3BLWRw,6724
-cloudnetpy/instruments/pollyxt.py,sha256=lkiBu8ChvLd86eCkeIGxHKwraeDQFuVSgCVlsAYTYN8,9010
+cloudnetpy/instruments/nc_radar.py,sha256=AjPn3mkq5a1mE7YzKtZnxX5suNju9NhUq-TDvs7T_uU,6911
+cloudnetpy/instruments/pollyxt.py,sha256=TAhpwwV9EcFTg2Zb-Hjwc6PUetEkO4csGs74ghrpTe4,9899
 cloudnetpy/instruments/radiometrics.py,sha256=ySG4a042XkgjMTG8d20oAPNvFvw9bMwwiqS3zv-JF_w,11825
 cloudnetpy/instruments/rpg.py,sha256=IozvBJ8_qXTPqtp58FQwRsoI5_aI3-kycpXugZkS0d4,17462
 cloudnetpy/instruments/rpg_reader.py,sha256=ThztFuVrWxhmWVAfZTfQDeUiKK1XMTbtv08IBe8GK98,11364
@@ -101,7 +101,7 @@ cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py,sha256=Ra3r4V
 cloudnetpy/model_evaluation/tests/unit/test_tools.py,sha256=Ia_VrLdV2NstX5gbx_3AZTOAlrgLAy_xFZ8fHYVX0xI,3817
 cloudnetpy/plotting/__init__.py,sha256=lg9Smn4BI0dVBgnDLC3JVJ4GmwoSnO-qoSd4ApvwV6Y,107
 cloudnetpy/plotting/plot_meta.py,sha256=d7CIT4ltUsjKw0HNnOXPkaLdZbkSmTuq6AbKdbODfZE,16730
-cloudnetpy/plotting/plotting.py,sha256=E_UTnraLxYz8jEROOpQr0pFbntHpI8nL6gMcwgsyFj8,37109
+cloudnetpy/plotting/plotting.py,sha256=bO-y1xAW0UvHFDGy5-5RxHbslF8UVPvlp-ARDpFpz6M,37321
 cloudnetpy/products/__init__.py,sha256=2hRb5HG9hNrxH1if5laJkLeFeaZCd5W1q3hh4ewsX0E,273
 cloudnetpy/products/classification.py,sha256=KwAiBSgFwDqhM114NIgYiUjj8HoYc7gAlc8E1QgcSig,8207
 cloudnetpy/products/der.py,sha256=soypE7uSEP4uHUCCQVEhyXsKY6e9mzV9B_2S5GUizqk,12729
@@ -115,9 +115,9 @@ cloudnetpy/products/mie_lu_tables.nc,sha256=It4fYpqJXlqOgL8jeZ-PxGzP08PMrELIDVe5
 cloudnetpy/products/mwr_tools.py,sha256=rd7UC67O4fsIE5SaHVZ4qWvUJTj41ZGwgQWPwZzOM14,5377
 cloudnetpy/products/product_tools.py,sha256=uu4l6reuGbPcW3TgttbaSrqIKbyYGhBVTdnC7opKvmg,11101
 docs/source/conf.py,sha256=IKiFWw6xhUd8NrCg0q7l596Ck1d61XWeVjIFHVSG9Og,1490
-cloudnetpy-1.67.2.dist-info/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
-cloudnetpy-1.67.2.dist-info/METADATA,sha256=XbrV76_CKzMDg8wZw4QE4_2fTW51DP3zRUDq4UxvGlA,5793
-cloudnetpy-1.67.2.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-cloudnetpy-1.67.2.dist-info/entry_points.txt,sha256=HhY7LwCFk4qFgDlXx_Fy983ZTd831WlhtdPIzV-Y3dY,51
-cloudnetpy-1.67.2.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
-cloudnetpy-1.67.2.dist-info/RECORD,,
+cloudnetpy-1.67.4.dist-info/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
+cloudnetpy-1.67.4.dist-info/METADATA,sha256=DUMWfQX6VEeCONbm-gA6B7BxCGr7RWwYnIKHqNGXQgs,5793
+cloudnetpy-1.67.4.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+cloudnetpy-1.67.4.dist-info/entry_points.txt,sha256=HhY7LwCFk4qFgDlXx_Fy983ZTd831WlhtdPIzV-Y3dY,51
+cloudnetpy-1.67.4.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
+cloudnetpy-1.67.4.dist-info/RECORD,,