cloudnetpy 1.85.0__py3-none-any.whl → 1.85.2__py3-none-any.whl

cloudnetpy/instruments/mira.py

@@ -9,6 +9,7 @@ from os import PathLike
 from tempfile import NamedTemporaryFile, TemporaryDirectory
 from uuid import UUID
 
+import numpy as np
 from numpy import ma
 
 from cloudnetpy import concat_lib, output, utils
@@ -74,9 +75,7 @@ def mira2nc(
             mira.sort_timestamps()
             mira.remove_duplicate_timestamps()
             mira.linear_to_db(("Zh", "ldr", "SNR"))
-            n_profiles = utils.n_elements(mira.time, 5, "time")
-            valid_ind = utils.remove_masked_blocks(mira.data["Zh"][:], limit=n_profiles)
-            mira.screen_time_indices(valid_ind)
+            mira.screen_low_power()
 
             if "snr_limit" in site_meta and site_meta["snr_limit"] is not None:
                 snr_limit = site_meta["snr_limit"]
@@ -151,6 +150,24 @@ class Mira(NcRadar):
         time_stamps = self.getvar("time")
         return utils.seconds2date(float(time_stamps[0]), self.epoch).date()
 
+    def screen_low_power(self) -> None:
+        """Screen times with average transmit power close to zero."""
+        if "tpow" not in self.data:
+            logging.warning("Variable tpow is missing")
+            return
+        tpow = self.data["tpow"][:]
+        # Threshold for abnormally low power e.g. Limassol 2024-10-20. Average
+        # power should 30 to 60 W according to MIRA-35 data sheet. Based on a
+        # random sample, typical range is 15 to 25 W. In Lampedusa, the power is
+        # constantly as low as 1.9 W.
+        is_low = tpow < 1
+        n_removed = np.count_nonzero(is_low)
+        if n_removed > 0:
+            logging.warning(
+                "Filtering %s profiles due to low average transmit power", n_removed
+            )
+            self.screen_time_indices(~is_low)
+
     def screen_invalid_ldr(self) -> None:
         """Masks LDR in MIRA STSR mode data.
         Is there a better way to identify this mode?
@@ -284,6 +301,7 @@ def _get_keymap(filetype: str) -> dict[str, str]:
                 ("prf", "prf"),
                 ("rg0", "rg0"),
                 ("NyquistVelocity", "NyquistVelocity"),  # variable in some mmclx files
+                ("tpow", "tpow"),
             ]
         ),
     }
@@ -311,6 +329,9 @@ ATTRIBUTES = {
     "prf": MetaData(
         long_name="Pulse Repetition Frequency", units="Hz", dimensions=("time",)
     ),
+    "tpow": MetaData(
+        long_name="Average Transmit Power", units="W", dimensions=("time",)
+    ),
     "zenith_offset": MetaData(
         long_name="Zenith offset of the instrument",
         units="degrees",

cloudnetpy/instruments/rpg.py

@@ -83,6 +83,10 @@ def rpg2nc(
     valid_ind = fmcw.add_zenith_angle()
     fmcw.screen_time_indices(valid_ind)
     fmcw.add_height()
+    if len(np.unique(fmcw.data["time"][:].astype("f4"))) != len(fmcw.data["time"][:]):
+        msg = "Convert time to f8 to keep values unique in netCDF"
+        logging.info(msg)
+        fmcw.data["time"].data_type = "f8"
     attributes = output.add_time_attribute(RPG_ATTRIBUTES, fmcw.date)
     output.update_attributes(fmcw.data, attributes)
     output.save_level1b(fmcw, output_file, uuid)
@@ -137,19 +141,17 @@ def _reduce_header(header: dict) -> dict:
     """Removes duplicate header data. Otherwise, we would need n_files dimension."""
     reduced_header = {}
     for key, data in header.items():
+        # Handle outliers in latitude and longitude (e.g. Galati 2024-02-11):
+        if key in ("latitude", "longitude"):
+            reduced_header[key] = ma.median(data)
+            continue
         first_profile_value = data[0]
         is_identical_value = bool(
             np.isclose(data, first_profile_value, rtol=1e-2).all(),
         )
         if is_identical_value is False:
             msg = f"Inconsistent header: {key}: {data}"
-            if key in (
-                "latitude",
-                "longitude",
-                "sample_duration",
-                "calibration_interval",
-                "noise_threshold",
-            ):
+            if key in ("sample_duration", "calibration_interval", "noise_threshold"):
                 logging.warning(msg)
             else:
                 raise InconsistentDataError(msg)

cloudnetpy/products/epsilon.py

@@ -138,12 +138,12 @@ def _horizontal_wind_from_doppler_lidar_file(
         t = np.broadcast_to(time[:, None], mask.shape)[~mask]
         h = np.broadcast_to(height[None, :], mask.shape)[~mask]
 
-        if len(np.unique(t)) < 2 or len(np.unique(h)) < 2:
+        if len(np.unique(t)) < 3 or len(np.unique(h)) < 3:
             msg = "Not enough unique values for interpolation"
             raise ValidTimeStampError(msg)
 
-        interp_linear = LinearNDInterpolator(list(zip(t, h, strict=False)), V[~mask])
-        interp_nearest = NearestNDInterpolator(list(zip(t, h, strict=False)), V[~mask])
+        interp_linear = LinearNDInterpolator(list(zip(t, h, strict=True)), V[~mask])
+        interp_nearest = NearestNDInterpolator(list(zip(t, h, strict=True)), V[~mask])
         T, H = np.meshgrid(time, height, indexing="ij")
         V_linear = interp_linear(T, H)
         V_nearest = interp_nearest(T, H)

cloudnetpy/utils.py

@@ -1035,17 +1035,6 @@ def path_lengths_from_ground(height: npt.NDArray) -> npt.NDArray:
     return np.diff(height, prepend=0)
 
 
-def remove_masked_blocks(array: ma.MaskedArray, limit: int = 50) -> npt.NDArray:
-    """Filters out large blocks of completely masked profiles."""
-    if array.ndim == 1:
-        return np.array(not ma.all(array.mask))
-    masked_profiles = ma.all(array.mask, axis=1)
-    labeled_array, _ = ndimage.label(masked_profiles)
-    mask = np.bincount(labeled_array) < limit
-    mask[0] = True
-    return mask[labeled_array]
-
-
 def sha256sum(filename: str | PathLike) -> str:
     """Calculates hash of file using sha-256."""
     return _calc_hash_sum(filename, "sha256", is_base64=False)

cloudnetpy/version.py

@@ -1,4 +1,4 @@
 MAJOR = 1
 MINOR = 85
-PATCH = 0
+PATCH = 2
 __version__ = f"{MAJOR}.{MINOR}.{PATCH}"

{cloudnetpy-1.85.0.dist-info → cloudnetpy-1.85.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnetpy
-Version: 1.85.0
+Version: 1.85.2
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

{cloudnetpy-1.85.0.dist-info → cloudnetpy-1.85.2.dist-info}/RECORD

@@ -8,8 +8,8 @@ cloudnetpy/exceptions.py,sha256=ZB3aUwjVRznR0CcZ5sZHrB0yz13URDf52Ksv7G7C7EA,1817
 cloudnetpy/metadata.py,sha256=CFpXmdEkVPzvLPv2xHIR-aMMQ-TR26KfESYw-98j7sk,7213
 cloudnetpy/output.py,sha256=0bybnILsgKHWIuw2GYkqTz2iMCJDZLUN25IQ9o_v3Cg,14968
 cloudnetpy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cloudnetpy/utils.py,sha256=O5kEXMt03fFKabPxRiCfuahVfTqa4fWWZ11orQnQeXU,33530
-cloudnetpy/version.py,sha256=lLtOfbPTOgEcz4-gTcFPYaor-U52n66zRHoLKjGWJ30,72
+cloudnetpy/utils.py,sha256=x3hukn7wuOyF3ctG8EOhvLbSUHOVKLW9ez2LzTjeolU,33110
+cloudnetpy/version.py,sha256=n8Y4NlhyE8jS82m6d9Bxhu9NW4bStS8hn1r3KqZrqfA,72
 cloudnetpy/categorize/__init__.py,sha256=gtvzWr0IDRn2oA6yHBvinEhTGTuub-JkrOv93lBsgrE,61
 cloudnetpy/categorize/atmos_utils.py,sha256=uWc9TABVYPI0sn4H5Az9Jf6NVRaWyEKIi17f0pAJQxE,10679
 cloudnetpy/categorize/attenuation.py,sha256=Y_-fzmQTltWTqIZTulJhovC7a6ifpMcaAazDJcnMIOc,990
@@ -46,14 +46,14 @@ cloudnetpy/instruments/galileo.py,sha256=f_-GkRxhNaQPbI8HpOwSmoKfGqyjmD16A0ZFgwL
 cloudnetpy/instruments/hatpro.py,sha256=TGOqwW0TfoPEYk13MFvFzwgJGzm6MVE5AsPavcIoj3I,10248
 cloudnetpy/instruments/instruments.py,sha256=WZgH7HjzM9Ane1CSnYCSLidbST8hunUeSt2lPntq9As,4999
 cloudnetpy/instruments/lufft.py,sha256=G6KeJOeltLUlGCHHEk8ns2K7WJ9ImAr25rSB2JltawE,4286
-cloudnetpy/instruments/mira.py,sha256=XqmbytpeCJ2-hNugxdsXSBUDB8SAUc97_6lo5mHFG8E,11840
+cloudnetpy/instruments/mira.py,sha256=jJ17xJmXwqfvZ7hgJaongRrRxoQpivMEQYssgSQFYbA,12638
 cloudnetpy/instruments/mrr.py,sha256=z50VYLOBW2o7enU7FHZYNFQRW2goEQpeGe7-iCBRQtg,6020
 cloudnetpy/instruments/nc_lidar.py,sha256=PtZauDdI3bX3bv4gIVvV6N53e2Co-ehBL_tByHM9hj8,1713
 cloudnetpy/instruments/nc_radar.py,sha256=XFKxPLKivnhHTgjE5HFrxjWZ0oCifhDUAog051vkMiY,7533
 cloudnetpy/instruments/pollyxt.py,sha256=IFq_RJrhgJ79OVyuo48PwYQK_zZ6VZFB_S5bEirRyzs,10566
 cloudnetpy/instruments/radiometrics.py,sha256=QKfnrZlQ0sFcFjmv1ShnCMTJQv64w4akjK-JAIY4gCg,16116
 cloudnetpy/instruments/rain_e_h3.py,sha256=fjv3SgeUNx9GisYqLrBnX9AjnO17VtouyoPh12VE9uo,5465
-cloudnetpy/instruments/rpg.py,sha256=R1rUdeSADvB1IMkGOF1S0rUEJDGEI_19SPrmErZpn5M,18825
+cloudnetpy/instruments/rpg.py,sha256=d7qlfez1Pe326d4xkQrTk2xRxvs3g68LEGehErdoBoc,19122
 cloudnetpy/instruments/rpg_reader.py,sha256=NaOtTxXx20PozNTj1xNvmbsEsAxuplFXRzBiM1_-Zg4,11651
 cloudnetpy/instruments/toa5.py,sha256=CfmmBMv5iMGaWHIGBK01Rw24cuXC1R1RMNTXkmsm340,1760
 cloudnetpy/instruments/vaisala.py,sha256=tu7aljkMKep0uCWz-Sd-GuBXF_Yy421a4nHy0ffpMoc,4725
@@ -111,17 +111,17 @@ cloudnetpy/products/der.py,sha256=UXdAxmmwChVVWSI4QSGAXphfMnbymGRTtGdKWEvh-J4,13
 cloudnetpy/products/drizzle.py,sha256=0h1N_WVjC2GgIkAN-4ydOwl7WJn3psxeqmPHfX8WHhQ,11935
 cloudnetpy/products/drizzle_error.py,sha256=QN98Io9UsBoEYxKBqfwoS88OGBiK5U5RYnVQjyTWHCI,6220
 cloudnetpy/products/drizzle_tools.py,sha256=xYMB8Qxp-_wKzMv_XC6u6iMfRnEhEtmDpCHSQAbDToo,11201
-cloudnetpy/products/epsilon.py,sha256=a796W_OuHxbSiG7yL2pcTArnaMNwYG8eAA8sQ93dIrY,7930
+cloudnetpy/products/epsilon.py,sha256=EHrpfAbykQv4PYdi2yk7cQxOrg-kL-FhzGPVnPm98C0,7928
 cloudnetpy/products/ier.py,sha256=Eb5AK-6l5mN_7vWP1cxcXQzj886zAwDDsHXueUju0N0,6262
 cloudnetpy/products/iwc.py,sha256=pXl0xOFDD6AzGaAp_GzD2yapjOc7hXKTno9Q5G6HCOo,9826
 cloudnetpy/products/lwc.py,sha256=xsNiiG6dGKIkWaFk0xWTabc1bZ4ULf6SqcqHs7itAUk,19339
 cloudnetpy/products/mie_lu_tables.nc,sha256=It4fYpqJXlqOgL8jeZ-PxGzP08PMrELIDVe55y9ob58,16637951
 cloudnetpy/products/mwr_tools.py,sha256=MMWnp68U7bv157-CPB2VeTQvaR6zl7sexbBT_kJ_pn8,6734
 cloudnetpy/products/product_tools.py,sha256=eyqIw_0KhlpmmYQE69RpGdRIAOW7JVPlEgkTBp2kdps,11302
-cloudnetpy-1.85.0.dist-info/licenses/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
+cloudnetpy-1.85.2.dist-info/licenses/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
 docs/source/conf.py,sha256=IKiFWw6xhUd8NrCg0q7l596Ck1d61XWeVjIFHVSG9Og,1490
-cloudnetpy-1.85.0.dist-info/METADATA,sha256=RWZNjL4m1yNEhDoSpqzyUODXSsh3AULjma2QRrNd2HU,5836
-cloudnetpy-1.85.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cloudnetpy-1.85.0.dist-info/entry_points.txt,sha256=HhY7LwCFk4qFgDlXx_Fy983ZTd831WlhtdPIzV-Y3dY,51
-cloudnetpy-1.85.0.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
-cloudnetpy-1.85.0.dist-info/RECORD,,
+cloudnetpy-1.85.2.dist-info/METADATA,sha256=0Hi0m1LJX0OWv2eGf1x4wQY4qiiqb2UxmBiA5BYStLQ,5836
+cloudnetpy-1.85.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cloudnetpy-1.85.2.dist-info/entry_points.txt,sha256=HhY7LwCFk4qFgDlXx_Fy983ZTd831WlhtdPIzV-Y3dY,51
+cloudnetpy-1.85.2.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
+cloudnetpy-1.85.2.dist-info/RECORD,,