cloudnetpy 1.89.2__tar.gz → 1.90.1__tar.gz

{cloudnetpy-1.89.2/cloudnetpy.egg-info → cloudnetpy-1.90.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnetpy
-Version: 1.89.2
+Version: 1.90.1
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/categorize/categorize.py

@@ -22,7 +22,6 @@ from cloudnetpy.categorize.mwr import Mwr
 from cloudnetpy.categorize.radar import Radar
 from cloudnetpy.datasource import DataSource
 from cloudnetpy.exceptions import DisdrometerDataError, ValidTimeStampError
-from cloudnetpy.instruments.rpg import RPG_ATTRIBUTES
 from cloudnetpy.metadata import COMMON_ATTRIBUTES, MetaData
 from cloudnetpy.products.product_tools import CategoryBits, QualityBits
 
@@ -436,7 +435,7 @@ CATEGORIZE_ATTRIBUTES = {
         units="m s-1",
         dimensions=("time", "height"),
     ),
-    "zdr": RPG_ATTRIBUTES["zdr"]._replace(dimensions=("time", "height")),
+    "zdr": COMMON_ATTRIBUTES["zdr"]._replace(dimensions=("time", "height")),
     "nyquist_velocity": COMMON_ATTRIBUTES["nyquist_velocity"]._replace(
         dimensions=("time", "height")
     ),

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/categorize/classify.py

@@ -86,7 +86,7 @@ def classify_measurements(data: Observations) -> ClassificationResult:
     bits.insect, insect_prob = insects.find_insects(obs, bits.melting, bits.droplet)
     bits.falling = falling.find_falling_hydrometeors(obs, bits.droplet, bits.insect)
     for _ in range(5):
-        _fix_undetected_melting_layer(bits)
+        _fix_undetected_melting_layer(bits, obs.lwp)
         _filter_insects(bits)
     bits.aerosol = _find_aerosols(obs, bits)
     _fix_super_cold_liquid(obs, bits)
@@ -223,10 +223,16 @@ def _find_aerosols(
     return is_beta & ~bits.falling & ~bits.droplet
 
 
-def _fix_undetected_melting_layer(bits: CategoryBits) -> None:
+def _fix_undetected_melting_layer(
+    bits: CategoryBits,
+    lwp: npt.NDArray,
+    lwp_threshold: float = 0.005,
+) -> None:
     drizzle_and_falling = _find_drizzle_and_falling(bits)
     transition = ma.diff(drizzle_and_falling, axis=1) == -1
-    bits.melting[:, 1:][transition] = True
+    masked = ma.getmaskarray(lwp)
+    allow = masked | (np.asarray(lwp) > lwp_threshold)
+    bits.melting[:, 1:][transition & allow[:, np.newaxis]] = True
 
 
 def _find_drizzle_and_falling(bits: CategoryBits) -> npt.NDArray:

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/categorize/insects.py

@@ -151,10 +151,7 @@ def _fill_missing_pixels(
     prob_from_ldr: npt.NDArray,
     prob_from_others: npt.NDArray,
 ) -> npt.NDArray:
-    prob_combined = np.copy(prob_from_ldr)
-    no_ldr = np.where(prob_from_ldr == 0)
-    prob_combined[no_ldr] = prob_from_others[no_ldr]
-    return prob_combined
+    return np.maximum(prob_from_ldr, prob_from_others)
 
 
 def _screen_insects(

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/categorize/lidar.py

@@ -59,10 +59,10 @@ class Lidar(DataSource):
     ) -> None:
         prefix = f"Unable to interpolate lidar for {len(ind)}"
         if dim == "time" and ind:
-            logging.warning("%s time steps", prefix)
+            logging.debug("%s time steps", prefix)
             data[ind, :] = ma.masked
         elif dim == "height" and ind:
-            logging.warning("%s altitudes", prefix)
+            logging.debug("%s altitudes", prefix)
             data[:, ind] = ma.masked
 
     def _add_meta(self) -> None:

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/cli.py

@@ -5,6 +5,7 @@ import datetime
 import gzip
 import hashlib
 import importlib
+import json
 import logging
 import re
 import shutil
@@ -136,7 +137,11 @@ def _process_categorize(
 
     try:
         logging.info("Processing categorize...")
-        generate_categorize(cast("CategorizeInput", input_files), cat_filepath)
+        generate_categorize(
+            cast("CategorizeInput", input_files),
+            cat_filepath,
+            options=args.options,
+        )
         logging.info("Processed categorize to %s", cat_filepath)
     except NameError:
         logging.info("No data available for this date.")
@@ -648,6 +653,16 @@ def main() -> None:
         help="Variables to plot (comma-separated), e.g. 'target_classification'",
         default=None,
     )
+    parser.add_argument(
+        "-o",
+        "--options",
+        type=json.loads,
+        help=(
+            "Options for categorize processing as a JSON string, "
+            "e.g. '{\"temperature_offset\": 3}'"
+        ),
+        default=None,
+    )
     args = parser.parse_args()
 
     logger = logging.getLogger()

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/constants.py

@@ -27,9 +27,11 @@ G_TO_KG: Final = 1e-3
 M_TO_KM: Final = 1e-3
 KG_TO_G: Final = 1e3
 M_TO_MM: Final = 1e3
+CM_TO_M: Final = 1e-2
 M_S_TO_MM_H: Final = SEC_IN_HOUR / MM_TO_M
 MM_H_TO_M_S: Final = 1 / M_S_TO_MM_H
 GHZ_TO_HZ: Final = 1e9
+HZ_TO_GHZ: Final = 1e-9
 HPA_TO_PA: Final = 100
 PA_TO_HPA: Final = 1 / HPA_TO_PA
 KM_H_TO_M_S: Final = 1000 / SEC_IN_HOUR

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/instruments/__init__.py

@@ -13,4 +13,5 @@ from .pollyxt import pollyxt2nc
 from .radiometrics import radiometrics2nc
 from .rain_e_h3 import rain_e_h32nc
 from .rpg import rpg2nc
+from .weather_radar import wr2nc
 from .weather_station import ws2nc

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/instruments/instruments.py

@@ -216,31 +216,38 @@ RD80 = Instrument(
 PLUVIO2 = Instrument(
     manufacturer="OTT HydroMet",
     domain="rain-gauge",
-    category="rain-gauge",
+    category="rain gauge",
     model="Pluvio2",
 )
 
 PLUVIO2S = Instrument(
     manufacturer="OTT HydroMet",
     domain="rain-gauge",
-    category="rain-gauge",
+    category="rain gauge",
     model="Pluvio2S",
 )
 
 THIES_PT = Instrument(
     manufacturer="Thies Clima",
     domain="rain-gauge",
-    category="rain-gauge",
+    category="rain gauge",
     model="Precipitation Transmitter",
 )
 
 RAIN_E_H3 = Instrument(
     manufacturer="LAMBRECHT meteo GmbH",
     domain="rain-gauge",
-    category="rain-gauge",
+    category="rain gauge",
     model="rain[e]H3",
 )
 
+WRM200 = Instrument(
+    manufacturer="Vaisala",
+    domain="weather-radar",
+    category="weather radar",
+    model="WRM200",
+)
+
 GENERIC_WEATHER_STATION = Instrument(
     domain="weather-station",
     category="weather station",

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/instruments/rain_e_h3.py

@@ -19,7 +19,7 @@ def rain_e_h32nc(
     uuid: str | UUID | None = None,
     date: str | datetime.date | None = None,
 ) -> UUID:
-    """Converts rain_e_h3 rain-gauge into Cloudnet Level 1b netCDF file.
+    """Converts rain_e_h3 rain gauge into Cloudnet Level 1b netCDF file.
 
     Args:
         input_file: Filename of rain_e_h3 CSV file.

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/instruments/rpg.py

@@ -471,12 +471,6 @@ RPG_ATTRIBUTES = {
         dimensions=("time", "range"),
     ),
     # STSR-mode radars
-    "zdr": MetaData(
-        long_name="Differential reflectivity", units="dB", dimensions=("time", "range")
-    ),
-    "rho_hv": MetaData(
-        long_name="Correlation coefficient", units="1", dimensions=("time", "range")
-    ),
     "phi_dp": MetaData(
         long_name="Differential phase", units="rad", dimensions=("time", "range")
     ),

cloudnetpy-1.90.1/cloudnetpy/instruments/weather_radar.py

@@ -0,0 +1,163 @@
+import datetime
+from collections import defaultdict
+from collections.abc import Iterable, Sequence
+from os import PathLike
+from uuid import UUID
+
+import netCDF4
+import numpy as np
+import numpy.typing as npt
+from numpy import ma
+
+from cloudnetpy import output, utils
+from cloudnetpy.cloudnetarray import CloudnetArray
+from cloudnetpy.constants import CM_TO_M, HZ_TO_GHZ, SPEED_OF_LIGHT
+from cloudnetpy.instruments import instruments
+from cloudnetpy.instruments.cloudnet_instrument import CloudnetInstrument
+
+
+def wr2nc(
+    input_files: str | PathLike | Sequence[str | PathLike],
+    output_file: str | PathLike,
+    site_meta: dict,
+    uuid: str | UUID | None = None,
+    date: str | datetime.date | None = None,
+) -> UUID:
+    """Converts OPERA HDF5 weather radar data into Cloudnet Level 1b netCDF file.
+
+    Args:
+        input_files: List of OPERA HDF5 files.
+        output_file: Output filename.
+        site_meta: Dictionary containing information about the site. Required
+            keys are `name`, `latitude`, `longitude` and `altitude`.
+        uuid: Set specific UUID for the file.
+        date: Expected date as YYYY-MM-DD.
+
+    Returns:
+        UUID of the generated file.
+    """
+    if isinstance(date, str):
+        date = datetime.date.fromisoformat(date)
+    uuid = utils.get_uuid(uuid)
+    if isinstance(input_files, str | PathLike):
+        input_files = [input_files]
+    wr = WeatherRadar(input_files, site_meta, date)
+    wr.sort_and_dedup_timestamps()
+    wr.convert_to_cloudnet_arrays()
+    wr.screen_noise()
+    wr.add_meta()
+    attributes = output.add_time_attribute({}, wr.date)
+    output.update_attributes(wr.data, attributes)
+    output.save_level1b(wr, output_file, uuid)
+    return uuid
+
+
+class WeatherRadar(CloudnetInstrument):
+    def __init__(
+        self,
+        filenames: Iterable[str | PathLike],
+        site_meta: dict,
+        expected_date: datetime.date | None = None,
+    ) -> None:
+        super().__init__()
+        self.site_meta = site_meta
+        self._read_data(filenames)
+        self._screen_time(expected_date)
+        self.instrument = instruments.WRM200
+
+    def _read_data(self, filenames: Iterable[str | PathLike]) -> None:
+        times = []
+        data = defaultdict(list)
+        for filename in filenames:
+            file_time, file_range, file_data, file_freq = _read_opera_h5(filename)
+            times.append(file_time)
+            for key, value in file_data.items():
+                data[key].append(value)
+        self.raw_time = np.array(times)
+        self.raw_range = file_range
+        self.raw_data = {key: ma.concatenate(value) for key, value in data.items()}
+        self.radar_frequency = file_freq
+
+    def _screen_time(self, expected_date: datetime.date | None = None) -> None:
+        if expected_date is None:
+            self.date = self.raw_time[0].date()
+        else:
+            is_valid = [dt.date() == expected_date for dt in self.raw_time]
+            self.raw_time = self.raw_time[is_valid]
+            for key in self.raw_data:
+                self.raw_data[key] = self.raw_data[key][is_valid]
+            self.date = expected_date
+
+    def sort_and_dedup_timestamps(self) -> None:
+        self.raw_time, time_ind = np.unique(self.raw_time, return_index=True)
+        for key in self.raw_data:
+            self.raw_data[key] = self.raw_data[key][time_ind]
+
+    def add_meta(self) -> None:
+        valid_keys = ("latitude", "longitude", "altitude")
+        for key, value in self.site_meta.items():
+            name = key.lower()
+            if name in valid_keys:
+                self.data[name] = CloudnetArray(float(value), name)
+
+    def convert_to_cloudnet_arrays(self) -> None:
+        epoch = datetime.datetime.combine(self.date, datetime.time())
+        hour = (self.raw_time - epoch) / datetime.timedelta(hours=1)
+        height = self.site_meta["altitude"] + self.raw_range
+        self.data["time"] = CloudnetArray(hour.astype(np.float32), "time")
+        self.data["range"] = CloudnetArray(self.raw_range, "range")
+        self.data["height"] = CloudnetArray(height, "height")
+        self.data["SNR"] = CloudnetArray(self.raw_data["SNR"], "SNR")
+        self.data["v"] = CloudnetArray(self.raw_data["VRADH"], "v")
+        self.data["width"] = CloudnetArray(self.raw_data["WRADH"], "width")
+        self.data["zdr"] = CloudnetArray(self.raw_data["ZDR"], "zdr")
+        self.data["rho_hv"] = CloudnetArray(self.raw_data["RHOHV"], "rho_hv")
+        self.data["radar_frequency"] = CloudnetArray(
+            self.radar_frequency, "radar_frequency"
+        )
+
+    def screen_noise(self) -> None:
+        is_noise = self.data["SNR"].data < -5
+        for cloudnet_array in self.data.values():
+            if cloudnet_array.data.ndim == 2:
+                cloudnet_array.mask_indices(is_noise)
+
+
+def _read_opera_h5(
+    file: str | PathLike,
+) -> tuple[datetime.datetime, npt.NDArray, dict[str, list], float]:
+    all_data = defaultdict(list)
+    with netCDF4.Dataset(file) as rootgrp:
+        date = rootgrp["what"].date
+        time = rootgrp["what"].time
+        dt = datetime.datetime.strptime(date + time, "%Y%m%d%H%M%S")
+
+        wavelength = rootgrp["how"].wavelength * CM_TO_M
+        frequency = HZ_TO_GHZ * SPEED_OF_LIGHT / wavelength
+
+        dataset = rootgrp["dataset1"]
+        nbins = dataset["where"].nbins
+        rstart = dataset["where"].nbins
+        rscale = dataset["where"].rscale
+        rng = rstart + rscale * np.arange(nbins)
+
+        grpnames = [group for group in dataset.groups if group.startswith("data")]
+        for grpname in grpnames:
+            grp = dataset[grpname]
+            quantity = grp["what"].quantity
+            is_db = quantity in ("ZDR", "SNR")
+            offset = grp["what"].offset
+            gain = grp["what"].gain
+            nodata = grp["what"].nodata
+            undetect = grp["what"].undetect
+            grpdata = grp["data"][:]
+            is_masked = (grpdata == nodata) | (grpdata == undetect)
+            grpdata = offset + gain * ma.masked_where(is_masked, grpdata)
+            if is_db:
+                grpdata = utils.db2lin(grpdata)
+            grpdata = ma.mean(grpdata, axis=0)
+            if is_db:
+                grpdata = utils.lin2db(grpdata)
+            all_data[quantity].append(grpdata)
+
+    return dt, rng, all_data, frequency

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/instruments/weather_station.py

@@ -399,7 +399,7 @@ class KenttarovaWS(WS):
 
 
 class HyytialaWS(WS):
-    """Hyytiälä rain-gauge variables: a = Pluvio400 and b = Pluvio200.
+    """Hyytiälä rain gauge variables: a = Pluvio400 and b = Pluvio200.
     E.g.
     - AaRNRT/mm = amount of non-real-time rain total (Pluvio400) [mm]
     - BbRT/mm = Bucket content in real-time (Pluvio200) [mm].

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/metadata.py

@@ -170,6 +170,12 @@ COMMON_ATTRIBUTES = {
     "SNR": MetaData(
         long_name="Signal-to-noise ratio", units="dB", dimensions=("time", "range")
     ),
+    "zdr": MetaData(
+        long_name="Differential reflectivity", units="dB", dimensions=("time", "range")
+    ),
+    "rho_hv": MetaData(
+        long_name="Correlation coefficient", units="1", dimensions=("time", "range")
+    ),
     "relative_humidity": MetaData(
         long_name="Relative humidity",
         standard_name="relative_humidity",

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/plotting/plotting.py

@@ -710,9 +710,9 @@ class Plot2D(Plot):
         cbar, clabel = _hide_segments()
         alt = self._screen_data_by_max_y(figure_data)
         image = self._ax.pcolorfast(
-            figure_data.time_including_gaps,
-            alt,
-            self._data.T[:-1, :-1],
+            _make_edges(figure_data.time_including_gaps),
+            _make_edges(alt),
+            self._data.T,
             cmap=ListedColormap(cbar),
             vmin=-0.5,
             vmax=len(cbar) - 0.5,
@@ -752,21 +752,12 @@ class Plot2D(Plot):
             "zorder": _get_zorder("data"),
         }
         image: Any
-        if getattr(figure_data.file, "cloudnet_file_type", "") == "model":
-            image = self._ax.pcolor(
-                figure_data.time_including_gaps,
-                alt,
-                self._data.T,
-                **pcolor_kwargs,
-                shading="nearest",
-            )
-        else:
-            image = self._ax.pcolorfast(
-                figure_data.time_including_gaps,
-                alt,
-                self._data.T[:-1, :-1],
-                **pcolor_kwargs,
-            )
+        image = self._ax.pcolorfast(
+            _make_edges(figure_data.time_including_gaps),
+            _make_edges(alt),
+            self._data.T,
+            **pcolor_kwargs,
+        )
         cbar = self._init_colorbar(image)
         cbar.set_label(str(self._plot_meta.clabel), fontsize=13)
 
@@ -1212,6 +1203,7 @@ def _get_max_gap_in_minutes(figure_data: FigureData) -> float:
         "epsilon-lidar": 75,
         "radar": 5,
         "cpr-simulation": 60,
+        "weather-radar": 20,
     }
     return max_allowed_gap.get(file_type, 10)
 
@@ -1225,6 +1217,14 @@ def _get_zorder(name: str) -> int:
     return zorder.get(name, -1)
 
 
+def _make_edges(centers: npt.NDArray) -> npt.NDArray:
+    edges = np.empty(len(centers) + 1)
+    edges[0] = centers[0] - (centers[1] - centers[0]) / 2
+    edges[1:-1] = (centers[:-1] + centers[1:]) / 2
+    edges[-1] = centers[-1] + (centers[-1] - centers[-2]) / 2
+    return edges
+
+
 def find_batches_of_ones(array: ndarray) -> list[tuple[int, int]]:
     """Find batches of ones in a binary array."""
     starts = np.where(np.diff(np.hstack(([0], array))) == 1)[0]

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/utils.py

@@ -504,7 +504,7 @@ def interpolate_1d(
 
     if len(bad_idx) > 0:
         msg = f"Unable to interpolate for {len(bad_idx)} time steps"
-        logging.warning(msg)
+        logging.debug(msg)
         interpolated[bad_idx] = ma.masked
 
     return interpolated

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy/version.py

@@ -1,4 +1,4 @@
 MAJOR = 1
-MINOR = 89
-PATCH = 2
+MINOR = 90
+PATCH = 1
 __version__ = f"{MAJOR}.{MINOR}.{PATCH}"

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1/cloudnetpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnetpy
-Version: 1.89.2
+Version: 1.90.1
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

{cloudnetpy-1.89.2 → cloudnetpy-1.90.1}/cloudnetpy.egg-info/SOURCES.txt

@@ -72,6 +72,7 @@ cloudnetpy/instruments/rpg.py
 cloudnetpy/instruments/rpg_reader.py
 cloudnetpy/instruments/toa5.py
 cloudnetpy/instruments/vaisala.py
+cloudnetpy/instruments/weather_radar.py
 cloudnetpy/instruments/weather_station.py
 cloudnetpy/instruments/disdrometer/__init__.py
 cloudnetpy/instruments/disdrometer/common.py