cloudnetpy 1.83.0__py3-none-any.whl → 1.84.0__py3-none-any.whl

cloudnetpy/categorize/categorize.py

@@ -93,7 +93,7 @@ def generate_categorize(
         if data.disdrometer is not None:
             data.disdrometer.interpolate_to_grid(time)
         if data.mwr is not None:
-            data.mwr.rebin_to_grid(time)
+            data.mwr.interpolate_to_grid(time)
         data.model.calc_attenuations(data.radar.radar_frequency)
         data.model.interpolate_to_common_height()
         model_gap_ind = data.model.interpolate_to_grid(time, height)

cloudnetpy/categorize/disdrometer.py

@@ -1,16 +1,12 @@
 """Mwr module, containing the :class:`Mwr` class."""
 
-import logging
 from os import PathLike
 
-import numpy as np
 import numpy.typing as npt
-from numpy import ma
-from scipy.interpolate import interp1d
 
-from cloudnetpy.categorize.lidar import get_gap_ind
 from cloudnetpy.datasource import DataSource
 from cloudnetpy.exceptions import DisdrometerDataError
+from cloudnetpy.utils import interpolate_1d
 
 
 class Disdrometer(DataSource):
@@ -28,7 +24,9 @@ class Disdrometer(DataSource):
 
     def interpolate_to_grid(self, time_grid: npt.NDArray) -> None:
         for key, array in self.data.items():
-            self.data[key].data = self._interpolate(array.data, time_grid)
+            self.data[key].data = interpolate_1d(
+                self.time, array.data, time_grid, max_time=1
+            )
 
     def _init_rainfall_rate(self) -> None:
         keys = ("rainfall_rate", "n_particles")
@@ -37,24 +35,3 @@ class Disdrometer(DataSource):
                 msg = f"variable {key} is missing"
                 raise DisdrometerDataError(msg)
             self.append_data(self.dataset.variables[key][:], key)
-
-    def _interpolate(self, y: ma.MaskedArray, x_new: npt.NDArray) -> npt.NDArray:
-        time = self.time
-        mask = ma.getmask(y)
-        if mask is not ma.nomask:
-            if np.all(mask):
-                return ma.masked_all(x_new.shape)
-            not_masked = ~mask
-            y = y[not_masked]
-            time = time[not_masked]
-        fun = interp1d(time, y, fill_value="extrapolate")
-        interpolated_array = ma.array(fun(x_new))
-        max_time = 1 / 60  # min -> fraction hour
-        mask_ind = get_gap_ind(time, x_new, max_time)
-
-        if len(mask_ind) > 0:
-            msg = f"Unable to interpolate disdrometer for {len(mask_ind)} time steps"
-            logging.warning(msg)
-            interpolated_array[mask_ind] = ma.masked
-
-        return interpolated_array

cloudnetpy/categorize/lidar.py

@@ -4,12 +4,11 @@ import logging
 from os import PathLike
 from typing import Literal
 
-import numpy as np
 import numpy.typing as npt
 from numpy import ma
 
 from cloudnetpy.datasource import DataSource
-from cloudnetpy.utils import interpolate_2d_nearest
+from cloudnetpy.utils import get_gap_ind, interpolate_2d_nearest
 
 
 class Lidar(DataSource):
@@ -70,13 +69,3 @@ class Lidar(DataSource):
         self.append_data(float(self.getvar("wavelength")), "lidar_wavelength")
         self.append_data(0.5, "beta_error")
         self.append_data(3.0, "beta_bias")
-
-
-def get_gap_ind(
-    grid: npt.NDArray, new_grid: npt.NDArray, threshold: float
-) -> list[int]:
-    return [
-        ind
-        for ind, value in enumerate(new_grid)
-        if np.min(np.abs(grid - value)) > threshold
-    ]

cloudnetpy/categorize/mwr.py

@@ -8,6 +8,7 @@ import numpy.typing as npt
 from cloudnetpy import utils
 from cloudnetpy.constants import G_TO_KG
 from cloudnetpy.datasource import DataSource
+from cloudnetpy.utils import interpolate_1d
 
 
 class Mwr(DataSource):
@@ -23,15 +24,11 @@ class Mwr(DataSource):
         self._init_lwp_data()
         self._init_lwp_error()
 
-    def rebin_to_grid(self, time_grid: npt.NDArray) -> None:
-        """Approximates lwp and its error in a grid using mean.
-
-        Args:
-            time_grid: 1D target time grid.
-
-        """
-        for array in self.data.values():
-            array.rebin_data(self.time, time_grid)
+    def interpolate_to_grid(self, time_grid: npt.NDArray, max_time: float = 1) -> None:
+        for key, array in self.data.items():
+            self.data[key].data = interpolate_1d(
+                self.time, array.data, time_grid, max_time=max_time
+            )
 
     def _init_lwp_data(self) -> None:
         lwp = self.dataset.variables["lwp"][:]

cloudnetpy/instruments/disdrometer/parsivel.py

@@ -205,6 +205,7 @@ TOA5_HEADERS = {
     "precipitation": "_rain_accum",
     "rain accum [mm]": "_rain_accum",
     "weatherCodeWaWa": "synop_WaWa",
+    "wawa": "synop_WaWa",
     "weather_code_wawa": "synop_WaWa",
     "radarReflectivity": "radar_reflectivity",
     "radar_reflectivity": "radar_reflectivity",

cloudnetpy/instruments/weather_station.py

@@ -70,6 +70,8 @@ def ws2nc(
         ws = LimassolWS(weather_station_file, site_meta)
     elif site_meta["name"] == "L'Aquila":
         ws = LAquilaWS(weather_station_file, site_meta)
+    elif site_meta["name"] == "Maïdo Observatory":
+        ws = MaidoWS(weather_station_file, site_meta)
     else:
         msg = "Unsupported site"
         raise ValueError(msg)
@@ -162,6 +164,26 @@ class WS(CSVFile):
 
 
 class PalaiseauWS(WS):
+    expected_header_identifiers: tuple[str, ...] = (
+        "DateTime(yyyy-mm-ddThh:mm:ssZ)",
+        "Windspeed(m/s)",
+        "Winddirection(deg",
+        "Airtemperature",
+        "Relativehumidity(%)",
+        "Pressure(hPa)",
+        "Precipitationrate(mm/min)",
+        "precipitation",
+    )
+    keys: tuple[str, ...] = (
+        "wind_speed",
+        "wind_direction",
+        "air_temperature",
+        "relative_humidity",
+        "air_pressure",
+        "rainfall_rate",
+        "rainfall_amount",
+    )
+
     def __init__(self, filenames: Sequence[str | PathLike], site_meta: dict) -> None:
         super().__init__(site_meta)
         self.filenames = filenames
@@ -175,16 +197,16 @@ class PalaiseauWS(WS):
             for row in data:
                 if not (columns := row.split()):
                     continue
-                if row.startswith("#"):
-                    header_row = "".join(columns)
-                    if header_row not in header:
-                        header.append(header_row)
-                else:
+                if re.match(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", columns[0]):
                     timestamp = datetime.datetime.strptime(
                         columns[0], "%Y-%m-%dT%H:%M:%SZ"
                     ).replace(tzinfo=datetime.timezone.utc)
                     values.append([timestamp] + [float(x) for x in columns[1:]])
                     timestamps.append(timestamp)
+                else:
+                    header_row = "".join(columns)
+                    if header_row not in header:
+                        header.append(header_row)
 
         self._validate_header(header)
         return {"time": timestamps, "values": values}
@@ -204,16 +226,9 @@ class PalaiseauWS(WS):
             ]
 
     def add_data(self) -> None:
-        keys = (
-            "wind_speed",
-            "wind_direction",
-            "air_temperature",
-            "relative_humidity",
-            "air_pressure",
-            "rainfall_rate",
-            "rainfall_amount",
-        )
-        for ind, key in enumerate(keys):
+        for ind, key in enumerate(self.keys):
+            if key.startswith("_"):
+                continue
             array = [row[ind + 1] for row in self._data["values"]]
             array_masked = ma.masked_invalid(array)
             self.data[key] = CloudnetArray(array_masked, key)
@@ -223,27 +238,46 @@ class PalaiseauWS(WS):
             self.data["rainfall_amount"][:] / 1000
         )  # mm -> m
 
-    @staticmethod
-    def _validate_header(header: list[str]) -> None:
-        expected_identifiers = [
-            "DateTime(yyyy-mm-ddThh:mm:ssZ)",
-            "Windspeed(m/s)",
-            "Winddirection(deg",
-            "Airtemperature",
-            "Relativehumidity(%)",
-            "Pressure(hPa)",
-            "Precipitationrate(mm/min)",
-            "precipitation",
-        ]
+    def _validate_header(self, header: list[str]) -> None:
         column_titles = [row for row in header if "Col." in row]
         error_msg = "Unexpected weather station file format"
-        if len(column_titles) != len(expected_identifiers):
+        if len(column_titles) != len(self.expected_header_identifiers):
             raise ValueError(error_msg)
-        for title, identifier in zip(column_titles, expected_identifiers, strict=True):
+        for title, identifier in zip(
+            column_titles, self.expected_header_identifiers, strict=True
+        ):
             if identifier not in title:
                 raise ValueError(error_msg)
 
 
+class MaidoWS(PalaiseauWS):
+    expected_header_identifiers = (
+        "DateTimeyyyy-mm-ddThh:mm:ssZ",
+        "Winddirection-average",
+        "Windspeed-maximumvalue(m/s)",
+        "Windspeed-average(m/s)",
+        "Pressure-average(hPa)",
+        "Relativehumidity-maximumvalue(%)",
+        "Relativehumidity-average(%)",
+        "Airtemperature-minimumvalue",
+        "Airtemperature-average",
+    )
+
+    keys = (
+        "wind_direction",
+        "_wind_speed_max",
+        "wind_speed",
+        "air_pressure",
+        "_relative_humidity_max",
+        "relative_humidity",
+        "_air_temperature_min",
+        "air_temperature",
+    )
+
+    def convert_rainfall_amount(self) -> None:
+        pass
+
+
 class BucharestWS(PalaiseauWS):
     def convert_rainfall_rate(self) -> None:
         rainfall_rate = self.data["rainfall_rate"][:]

cloudnetpy/utils.py

@@ -3,6 +3,7 @@
 import base64
 import datetime
 import hashlib
+import logging
 import os
 import re
 import textwrap
@@ -17,7 +18,12 @@ import numpy as np
 import numpy.typing as npt
 from numpy import ma
 from scipy import ndimage, stats
-from scipy.interpolate import RectBivariateSpline, RegularGridInterpolator, griddata
+from scipy.interpolate import (
+    RectBivariateSpline,
+    RegularGridInterpolator,
+    griddata,
+    interp1d,
+)
 
 from cloudnetpy.cloudnetarray import CloudnetArray
 from cloudnetpy.constants import SEC_IN_DAY, SEC_IN_HOUR, SEC_IN_MINUTE
@@ -413,6 +419,55 @@ def interpolate_2d_nearest(
     return fun((xx, yy)).T
 
 
+def interpolate_1d(
+    time: npt.NDArray, y: ma.MaskedArray, time_new: npt.NDArray, max_time: float
+) -> npt.NDArray:
+    """1D linear interpolation preserving the mask.
+
+    Args:
+        time: 1D array in fraction hour.
+        y: 1D masked array, data values.
+        time_new: 1D array, new time coordinates.
+        max_time: Maximum allowed gap in minutes. Values outside this gap will
+            be masked.
+    """
+    if np.max(time) > 24 or np.min(time) < 0:
+        msg = "Time vector must be in fraction hours between 0 and 24"
+        raise ValueError(msg)
+    if ma.is_masked(y):
+        if y.mask.all():
+            return ma.masked_all(time_new.shape)
+        time = time[~y.mask]
+        y = y[~y.mask]
+    fun = interp1d(time, y, fill_value=(y[0], y[-1]), bounds_error=False)
+    interpolated = ma.array(fun(time_new))
+    bad_idx = get_gap_ind(time, time_new, max_time / 60)
+
+    if len(bad_idx) > 0:
+        msg = f"Unable to interpolate for {len(bad_idx)} time steps"
+        logging.warning(msg)
+        interpolated[bad_idx] = ma.masked
+
+    return interpolated
+
+
+def get_gap_ind(
+    grid: npt.NDArray, new_grid: npt.NDArray, threshold: float
+) -> list[int]:
+    """Finds indices in new_grid that are too far from grid."""
+    if grid.size == 0:
+        return list(range(len(new_grid)))
+    idxs = np.searchsorted(grid, new_grid)
+    left_dist = np.where(idxs > 0, np.abs(new_grid - grid[idxs - 1]), np.inf)
+    right_dist = np.where(
+        idxs < len(grid),
+        np.abs(new_grid - grid[np.clip(idxs, 0, len(grid) - 1)]),
+        np.inf,
+    )
+    nearest = np.minimum(left_dist, right_dist)
+    return np.where(nearest > threshold)[0].tolist()
+
+
 def calc_relative_error(reference: npt.NDArray, array: npt.NDArray) -> npt.NDArray:
     """Calculates relative error (%)."""
     return ((array - reference) / reference) * 100

cloudnetpy/version.py

@@ -1,4 +1,4 @@
 MAJOR = 1
-MINOR = 83
+MINOR = 84
 PATCH = 0
 __version__ = f"{MAJOR}.{MINOR}.{PATCH}"

{cloudnetpy-1.83.0.dist-info → cloudnetpy-1.84.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnetpy
-Version: 1.83.0
+Version: 1.84.0
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

{cloudnetpy-1.83.0.dist-info → cloudnetpy-1.84.0.dist-info}/RECORD

@@ -8,24 +8,24 @@ cloudnetpy/exceptions.py,sha256=ZB3aUwjVRznR0CcZ5sZHrB0yz13URDf52Ksv7G7C7EA,1817
 cloudnetpy/metadata.py,sha256=CFpXmdEkVPzvLPv2xHIR-aMMQ-TR26KfESYw-98j7sk,7213
 cloudnetpy/output.py,sha256=0bybnILsgKHWIuw2GYkqTz2iMCJDZLUN25IQ9o_v3Cg,14968
 cloudnetpy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cloudnetpy/utils.py,sha256=Qv60_vxknB3f2S3EFtyoD2CBY3N6mgDRObNp2u1oYUc,31806
-cloudnetpy/version.py,sha256=eZHEAex9xrlQzhoqlO0MSLCFCdaJCCetQ0O35RSY7VM,72
+cloudnetpy/utils.py,sha256=O5kEXMt03fFKabPxRiCfuahVfTqa4fWWZ11orQnQeXU,33530
+cloudnetpy/version.py,sha256=muQUQb3xKBF1loNyy50A6CariaWU5ZNM_Ff1hDxBTqA,72
 cloudnetpy/categorize/__init__.py,sha256=gtvzWr0IDRn2oA6yHBvinEhTGTuub-JkrOv93lBsgrE,61
 cloudnetpy/categorize/atmos_utils.py,sha256=uWc9TABVYPI0sn4H5Az9Jf6NVRaWyEKIi17f0pAJQxE,10679
 cloudnetpy/categorize/attenuation.py,sha256=Y_-fzmQTltWTqIZTulJhovC7a6ifpMcaAazDJcnMIOc,990
-cloudnetpy/categorize/categorize.py,sha256=NtSnLblVBgUobCJkP5SscsQ0UJNoWjJQKuyl2ZyytV0,23221
+cloudnetpy/categorize/categorize.py,sha256=UZph73hVCU-UIxB2837oyLFKvTk8fdgLC-krgJDg8zk,23227
 cloudnetpy/categorize/classify.py,sha256=skA9K6Bxh9mFZ_fM4d78zt09BPDzfHLttXle6mFCMbw,8553
 cloudnetpy/categorize/containers.py,sha256=PIJwgQos3CxF9BG4hBNLTaZq252FTH0kdgagT31mFmc,5517
-cloudnetpy/categorize/disdrometer.py,sha256=XNL8kDtBAB12UmgRZ4ayxuVs8e3ghyLO0Hy5XpRgfMU,1966
+cloudnetpy/categorize/disdrometer.py,sha256=_Hc7EhoxIIsAn7HCrgSB6aPMelaIW6c4lLeMf0NOnVI,1117
 cloudnetpy/categorize/droplet.py,sha256=wnMN9rHNSMZLXNXuYEd-RAS_8eAIIo2vkE7pp3DSTKs,8725
 cloudnetpy/categorize/falling.py,sha256=ZscFGFPFz_Nlc7PwjVwFDSVOzMGOuCDVAFj7pLvYctQ,4475
 cloudnetpy/categorize/freezing.py,sha256=gigqpb4qfeQSlKXkrPUwCbMnMsxl74thJWSRW2iHJOg,3796
 cloudnetpy/categorize/insects.py,sha256=bAqm4kFRtU16RPttsRLedofPd-yfbALNqz26jKlMNUE,5357
 cloudnetpy/categorize/itu.py,sha256=ffXK27guyRS4d66VWQ2h4UEGjUIhGjPKbFmj7kh698c,10304
-cloudnetpy/categorize/lidar.py,sha256=CQsDEeQYiuQCfCmJQWrqQvCfmciN1NPZ6uRdt89CZLY,2685
+cloudnetpy/categorize/lidar.py,sha256=CkIbsMFHROYnhIc1e1ci61-POqeL8AOf6RCZicpNBVc,2452
 cloudnetpy/categorize/melting.py,sha256=vhc6zq3L4gp7oEPHMnQlT2m6YBE5-CS5gdTNA7gVHRg,6329
 cloudnetpy/categorize/model.py,sha256=iFakrXy3npbg4qUrpUGEBEdwBnmlWsMgogPCtfWl7sw,6805
-cloudnetpy/categorize/mwr.py,sha256=kzSivQuKrsqmFInDLlSM1R2wAG5j-tQebOi_1IwUW_I,1690
+cloudnetpy/categorize/mwr.py,sha256=QKc6f_h0mCVjzSa4BrwTtVAPx0Tn1VJ4jqGmIBke5II,1710
 cloudnetpy/categorize/radar.py,sha256=2mTDa9BLxQeaORm-YPQ1lJyjAKew6NYzjtUvjpIvBYU,16044
 cloudnetpy/categorize/attenuations/__init__.py,sha256=kIyQEZ6VVO6jJOAndrt7jNU15pm0Cavh5GnDjFmIG1M,1040
 cloudnetpy/categorize/attenuations/gas_attenuation.py,sha256=emr-RCxQT0i2N8k6eBNhRsmsCBPHJzQsWJfjC4fVSTo,975
@@ -56,10 +56,10 @@ cloudnetpy/instruments/rpg.py,sha256=R1rUdeSADvB1IMkGOF1S0rUEJDGEI_19SPrmErZpn5M
 cloudnetpy/instruments/rpg_reader.py,sha256=NaOtTxXx20PozNTj1xNvmbsEsAxuplFXRzBiM1_-Zg4,11651
 cloudnetpy/instruments/toa5.py,sha256=CfmmBMv5iMGaWHIGBK01Rw24cuXC1R1RMNTXkmsm340,1760
 cloudnetpy/instruments/vaisala.py,sha256=tu7aljkMKep0uCWz-Sd-GuBXF_Yy421a4nHy0ffpMoc,4725
-cloudnetpy/instruments/weather_station.py,sha256=FuaGILEkd4MxXMpLrNGXNUjuuTkMIBf-J7y9oepIsdM,27586
+cloudnetpy/instruments/weather_station.py,sha256=KD-pGpEEbeyma74DdNPd24PFxFCdZc1UMLE0Hm2PLO0,28509
 cloudnetpy/instruments/disdrometer/__init__.py,sha256=lyjwttWvFvuwYxEkusoAvgRcbBmglmOp5HJOpXUqLWo,93
 cloudnetpy/instruments/disdrometer/common.py,sha256=WCPRCfAlElUzZpllOSjjWrLG2jgkiRIy0rWz_omFoJQ,10815
-cloudnetpy/instruments/disdrometer/parsivel.py,sha256=1HIA52f1nGOvSd4SSTr2y3-JT3eKZWwdbMnIMRVvQ_U,25811
+cloudnetpy/instruments/disdrometer/parsivel.py,sha256=OZTLZgSOMfPF5cxY-ICSb0gvyEMm_3JspmRYYvb9hCo,25837
 cloudnetpy/instruments/disdrometer/thies.py,sha256=7gPYFzpa58ot6pTblhEnY5eRw5yrue42XwyacHQm14k,11205
 cloudnetpy/model_evaluation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloudnetpy/model_evaluation/file_handler.py,sha256=yAbJ3yOqufQGV3rahwOm9-k1swEDDq8V8n-zaNVaN54,6473
@@ -117,10 +117,10 @@ cloudnetpy/products/lwc.py,sha256=xsNiiG6dGKIkWaFk0xWTabc1bZ4ULf6SqcqHs7itAUk,19
 cloudnetpy/products/mie_lu_tables.nc,sha256=It4fYpqJXlqOgL8jeZ-PxGzP08PMrELIDVe55y9ob58,16637951
 cloudnetpy/products/mwr_tools.py,sha256=MMWnp68U7bv157-CPB2VeTQvaR6zl7sexbBT_kJ_pn8,6734
 cloudnetpy/products/product_tools.py,sha256=eyqIw_0KhlpmmYQE69RpGdRIAOW7JVPlEgkTBp2kdps,11302
-cloudnetpy-1.83.0.dist-info/licenses/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
+cloudnetpy-1.84.0.dist-info/licenses/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
 docs/source/conf.py,sha256=IKiFWw6xhUd8NrCg0q7l596Ck1d61XWeVjIFHVSG9Og,1490
-cloudnetpy-1.83.0.dist-info/METADATA,sha256=wDkaRhm_pjpqqiX-6MteJV6IvZeKpZ8XiqaxCehqnV4,5836
-cloudnetpy-1.83.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cloudnetpy-1.83.0.dist-info/entry_points.txt,sha256=HhY7LwCFk4qFgDlXx_Fy983ZTd831WlhtdPIzV-Y3dY,51
-cloudnetpy-1.83.0.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
-cloudnetpy-1.83.0.dist-info/RECORD,,
+cloudnetpy-1.84.0.dist-info/METADATA,sha256=b1T8fzF9ooTKaIjpb03WolZZKzdvlBxNP8IyO38aD7U,5836
+cloudnetpy-1.84.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cloudnetpy-1.84.0.dist-info/entry_points.txt,sha256=HhY7LwCFk4qFgDlXx_Fy983ZTd831WlhtdPIzV-Y3dY,51
+cloudnetpy-1.84.0.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
+cloudnetpy-1.84.0.dist-info/RECORD,,