doppy 0.3.6__cp310-abi3-macosx_10_12_x86_64.whl → 0.4.0__cp310-abi3-macosx_10_12_x86_64.whl

doppy/defaults.py

@@ -1,2 +1,18 @@
+DEFAULT_BEAM_ENERGY = 1e-5
+DEFAULT_EFFECTIVE_DIAMETER = 25e-3
+
+
 class Halo:
-    wavelength = 1.565e-6
+    wavelength = 1.565e-6  # [m]
+    receiver_bandwidth = 50e6  # [Hz]
+    beam_energy = DEFAULT_BEAM_ENERGY
+    effective_diameter = DEFAULT_EFFECTIVE_DIAMETER
+
+
+class WindCube:
+    # https://doi.org/10.5194/essd-13-3539-2021
+    wavelength = 1.54e-6  # [m]
+    receiver_bandwidth = 55e6  # [Hz]
+    beam_energy = DEFAULT_BEAM_ENERGY
+    effective_diameter = 50e-3  # [m]
+    focus = 1e3  # [m]

doppy/product/stare.py

@@ -9,7 +9,7 @@ from typing import Sequence, Tuple, TypeAlias
 import numpy as np
 import numpy.typing as npt
 import scipy
-from scipy.ndimage import uniform_filter
+from scipy.ndimage import median_filter, uniform_filter
 from sklearn.cluster import KMeans
 
 import doppy
@@ -51,6 +51,46 @@ class Stare:
             )
         raise TypeError
 
+    @classmethod
+    def mask_nan(cls, x: npt.NDArray[np.float64]) -> npt.NDArray[np.bool_]:
+        return np.isnan(x)
+
+    @classmethod
+    def from_windcube_data(
+        cls,
+        data: Sequence[str]
+        | Sequence[Path]
+        | Sequence[bytes]
+        | Sequence[BufferedIOBase],
+    ) -> Stare:
+        raws = doppy.raw.WindCubeFixed.from_srcs(data)
+        raw = (
+            doppy.raw.WindCubeFixed.merge(raws).sorted_by_time().nan_profiles_removed()
+        )
+
+        wavelength = defaults.WindCube.wavelength
+        beta = _compute_beta(
+            snr=raw.cnr,
+            radial_distance=raw.radial_distance,
+            wavelength=wavelength,
+            beam_energy=defaults.WindCube.beam_energy,
+            receiver_bandwidth=defaults.WindCube.receiver_bandwidth,
+            focus=defaults.WindCube.focus,
+            effective_diameter=defaults.WindCube.effective_diameter,
+        )
+
+        mask = _compute_noise_mask_for_windcube(raw)
+        return cls(
+            time=raw.time,
+            radial_distance=raw.radial_distance,
+            elevation=raw.elevation,
+            beta=beta,
+            radial_velocity=raw.radial_velocity,
+            mask=mask,
+            wavelength=wavelength,
+            system_id=raw.system_id,
+        )
+
     @classmethod
     def from_halo_data(
         cls,
@@ -95,16 +135,21 @@ class Stare:
             raw, intensity_bg_corrected
         )
         wavelength = defaults.Halo.wavelength
+
         beta = _compute_beta(
-            intensity_noise_bias_corrected,
-            raw.radial_distance,
-            raw.header.focus_range,
-            wavelength,
+            snr=intensity_noise_bias_corrected - 1,
+            radial_distance=raw.radial_distance,
+            wavelength=wavelength,
+            beam_energy=defaults.Halo.beam_energy,
+            receiver_bandwidth=defaults.Halo.receiver_bandwidth,
+            focus=raw.header.focus_range,
+            effective_diameter=defaults.Halo.effective_diameter,
         )
+
         mask = _compute_noise_mask(
             intensity_noise_bias_corrected, raw.radial_velocity, raw.radial_distance
         )
-        return Stare(
+        return cls(
             time=raw.time,
             radial_distance=raw.radial_distance,
             elevation=raw.elevation,
@@ -177,6 +222,36 @@ class Stare:
             nc.add_attribute("doppy_version", doppy.__version__)
 
 
+def _compute_noise_mask_for_windcube(
+    raw: doppy.raw.WindCubeFixed,
+) -> npt.NDArray[np.bool_]:
+    if np.any(np.isnan(raw.cnr)) or np.any(np.isnan(raw.radial_velocity)):
+        raise ValueError("Unexpected nans in crn or radial_velocity")
+
+    mask = _mask_with_cnr_norm_dist(raw.cnr) | (np.abs(raw.radial_velocity) > 30)
+
+    cnr = raw.cnr.copy()
+    cnr[mask] = np.finfo(float).eps
+    cnr_filt = median_filter(cnr, size=(3, 3))
+    rel_diff = np.abs(cnr - cnr_filt) / np.abs(cnr)
+    diff_mask = rel_diff > 0.25
+
+    mask = mask | diff_mask
+
+    return np.array(mask, dtype=np.bool_)
+
+
+def _mask_with_cnr_norm_dist(cnr: npt.NDArray[np.float64]) -> npt.NDArray[np.bool_]:
+    th_trunc = -5.5
+    std_factor = 2
+    log_cnr = np.log(cnr)
+    log_cnr_trunc = log_cnr[log_cnr < th_trunc]
+    th_trunc_fit = np.percentile(log_cnr_trunc, 90)
+    log_cnr_for_fit = log_cnr_trunc[log_cnr_trunc < th_trunc_fit]
+    mean, std = scipy.stats.norm.fit(log_cnr_for_fit)
+    return np.array(np.log(cnr) < (mean + std_factor * std), dtype=np.bool_)
+
+
 def _compute_noise_mask(
     intensity: npt.NDArray[np.float64],
     radial_velocity: npt.NDArray[np.float64],
@@ -197,14 +272,19 @@ def _compute_noise_mask(
 
 
 def _compute_beta(
-    intensity: npt.NDArray[np.float64],
+    snr: npt.NDArray[np.float64],
     radial_distance: npt.NDArray[np.float64],
-    focus: float,
     wavelength: float,
+    beam_energy: float,
+    receiver_bandwidth: float,
+    focus: float,
+    effective_diameter: float,
 ) -> npt.NDArray[np.float64]:
     """
     Parameters
     ----------
+    snr
+        for halo: intensity - 1
     radial_distance
         distance from the instrument
     focus
@@ -236,22 +316,24 @@ def _compute_beta(
         doi: https://doi.org/10.5194/amt-13-2849-2020
     """
 
-    snr = intensity - 1
     h = scipy.constants.Planck
     c = scipy.constants.speed_of_light
     eta = 1
-    E = 1e-5
-    B = 5e7
+    E = beam_energy
+    B = receiver_bandwidth
     nu = c / wavelength
-    A_e = _compute_effective_receiver_energy(radial_distance, focus, wavelength)
+    A_e = _compute_effective_receiver_energy(
+        radial_distance, wavelength, focus, effective_diameter
+    )
     beta = 2 * h * nu * B * radial_distance**2 * snr / (eta * c * E * A_e)
     return np.array(beta, dtype=np.float64)
 
 
 def _compute_effective_receiver_energy(
     radial_distance: npt.NDArray[np.float64],
-    focus: float,
     wavelength: float,
+    focus: float,
+    effective_diameter: float,
 ) -> npt.NDArray[np.float64]:
     """
     NOTE
@@ -268,7 +350,7 @@ def _compute_effective_receiver_energy(
     wavelength
         laser wavelength
     """
-    D = 25e-3  # effective_diameter_of_gaussian_beam
+    D = effective_diameter
     return np.array(
         np.pi
         * D**2

doppy/raw/__init__.py

@@ -1,7 +1,7 @@
 from .halo_bg import HaloBg
 from .halo_hpl import HaloHpl
 from .halo_sys_params import HaloSysParams
-from .windcube import WindCube
+from .windcube import WindCube, WindCubeFixed
 from .wls70 import Wls70
 
-__all__ = ["HaloHpl", "HaloBg", "HaloSysParams", "WindCube", "Wls70"]
+__all__ = ["HaloHpl", "HaloBg", "HaloSysParams", "WindCube", "WindCubeFixed", "Wls70"]

doppy/raw/halo_hpl.py

@@ -4,7 +4,7 @@ import functools
 import io
 import re
 from dataclasses import dataclass
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from io import BufferedIOBase
 from os.path import commonprefix
 from pathlib import Path
@@ -309,7 +309,11 @@ def _raw_tuple2halo_hpl(
         resolution=float(header_dict["resolution"]),
         scan_type=str(header_dict["scan_type"]),
         focus_range=int(header_dict["focus_range"]),
-        start_time=datetime64(datetime.utcfromtimestamp(header_dict["start_time"])),
+        start_time=datetime64(
+            datetime.fromtimestamp(header_dict["start_time"], timezone.utc).replace(
+                tzinfo=None
+            )
+        ),
         system_id=str(header_dict["system_id"]),
         instrument_spectral_width=float(header_dict["instrument_spectral_width"])
         if header_dict["instrument_spectral_width"] is not None

doppy/raw/windcube.py

@@ -7,17 +7,104 @@ from typing import Sequence
 
 import numpy as np
 import numpy.typing as npt
-from netCDF4 import Dataset, num2date
+from netCDF4 import Dataset, Variable, num2date
 from numpy import datetime64
 
 from doppy.utils import merge_all_equal
 
 
+@dataclass
+class WindCubeFixed:
+    time: npt.NDArray[datetime64]  # dim: (time, )
+    radial_distance: npt.NDArray[np.float64]  # dim: (radial_distance,)
+    azimuth: npt.NDArray[np.float64]  # dim: (time, )
+    elevation: npt.NDArray[np.float64]  # dim: (time, )
+    cnr: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    relative_beta: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    radial_velocity: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    doppler_spectrum_width: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    radial_velocity_confidence: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    ray_accumulation_time: np.float64  # dim: ()
+    system_id: str
+
+    @classmethod
+    def from_srcs(
+        cls,
+        data: Sequence[str]
+        | Sequence[Path]
+        | Sequence[bytes]
+        | Sequence[BufferedIOBase],
+    ) -> list[WindCubeFixed]:
+        return [WindCubeFixed.from_fixed_src(src) for src in data]
+
+    @classmethod
+    def from_fixed_src(cls, data: str | Path | bytes | BufferedIOBase) -> WindCubeFixed:
+        data_bytes = _src_to_bytes(data)
+        nc = Dataset("inmemory.nc", "r", memory=data_bytes)
+        return _from_fixed_src(nc)
+
+    @classmethod
+    def merge(cls, raws: list[WindCubeFixed]) -> WindCubeFixed:
+        return WindCubeFixed(
+            time=np.concatenate([r.time for r in raws]),
+            radial_distance=_merge_radial_distance_for_fixed(
+                [r.radial_distance for r in raws]
+            ),
+            azimuth=np.concatenate([r.azimuth for r in raws]),
+            elevation=np.concatenate([r.elevation for r in raws]),
+            radial_velocity=np.concatenate([r.radial_velocity for r in raws]),
+            radial_velocity_confidence=np.concatenate(
+                [r.radial_velocity_confidence for r in raws]
+            ),
+            cnr=np.concatenate([r.cnr for r in raws]),
+            relative_beta=np.concatenate([r.relative_beta for r in raws]),
+            doppler_spectrum_width=np.concatenate(
+                [r.doppler_spectrum_width for r in raws]
+            ),
+            ray_accumulation_time=merge_all_equal(
+                "ray_accumulation_time", [r.ray_accumulation_time for r in raws]
+            ),
+            system_id=merge_all_equal("system_id", [r.system_id for r in raws]),
+        )
+
+    def __getitem__(
+        self,
+        index: int
+        | slice
+        | list[int]
+        | npt.NDArray[np.int64]
+        | npt.NDArray[np.bool_]
+        | tuple[slice, slice],
+    ) -> WindCubeFixed:
+        if isinstance(index, (int, slice, list, np.ndarray)):
+            return WindCubeFixed(
+                time=self.time[index],
+                radial_distance=self.radial_distance,
+                azimuth=self.azimuth[index],
+                elevation=self.elevation[index],
+                radial_velocity=self.radial_velocity[index],
+                radial_velocity_confidence=self.radial_velocity_confidence[index],
+                cnr=self.cnr[index],
+                relative_beta=self.relative_beta[index],
+                doppler_spectrum_width=self.doppler_spectrum_width[index],
+                ray_accumulation_time=self.ray_accumulation_time,
+                system_id=self.system_id,
+            )
+        raise TypeError
+
+    def sorted_by_time(self) -> WindCubeFixed:
+        sort_indices = np.argsort(self.time)
+        return self[sort_indices]
+
+    def nan_profiles_removed(self) -> WindCubeFixed:
+        return self[~np.all(np.isnan(self.cnr), axis=1)]
+
+
 @dataclass
 class WindCube:
     time: npt.NDArray[datetime64]  # dim: (time, )
-    radial_distance: npt.NDArray[np.int64]  # dim: (time, radial_distance)
-    height: npt.NDArray[np.int64]  # dim: (time,radial_distance)
+    radial_distance: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    height: npt.NDArray[np.float64]  # dim: (time,radial_distance)
     azimuth: npt.NDArray[np.float64]  # dim: (time, )
     elevation: npt.NDArray[np.float64]  # dim: (time, )
     cnr: npt.NDArray[np.float64]  # dim: (time, radial_distance)
@@ -128,6 +215,23 @@ def _merge_scan_index(index_list: list[npt.NDArray[np.int64]]) -> npt.NDArray[np
     return np.concatenate(new_index_list)
 
 
+def _merge_radial_distance_for_fixed(
+    radial_distance_list: list[npt.NDArray[np.float64]],
+) -> npt.NDArray[np.float64]:
+    if len(radial_distance_list) == 0:
+        raise ValueError("cannot merge empty list")
+    if not all(
+        np.allclose(arr.shape, radial_distance_list[0].shape)
+        for arr in radial_distance_list
+    ):
+        raise ValueError("Cannot merge radial distances with different shapes")
+    if not all(
+        np.allclose(arr, radial_distance_list[0]) for arr in radial_distance_list
+    ):
+        raise ValueError("Cannot merge radial distances")
+    return radial_distance_list[0]
+
+
 def _src_to_bytes(data: str | Path | bytes | BufferedIOBase) -> bytes:
     if isinstance(data, str):
         path = Path(data)
@@ -140,38 +244,127 @@ def _src_to_bytes(data: str | Path | bytes | BufferedIOBase) -> bytes:
         return data
     elif isinstance(data, BufferedIOBase):
         return data.read()
-    raise TypeError("Unsupported data type")
 
 
-def _from_vad_or_dbs_src(nc: Dataset) -> WindCube:
-    scan_index_list = []
+def _from_fixed_src(nc: Dataset) -> WindCubeFixed:
     time_list = []
     cnr_list = []
+    relative_beta_list = []
     radial_wind_speed_list = []
     radial_wind_speed_confidence_list = []
     azimuth_list = []
     elevation_list = []
     range_list = []
-    height_list = []
+    doppler_spectrum_width_list = []
+    ray_accumulation_time_list = []
+    time_reference = (
+        nc["time_reference"][:] if "time_reference" in nc.variables else None
+    )
+
+    expected_dimensions = ("time", "range")
+    for _, group in enumerate(
+        nc[group] for group in (nc.variables["sweep_group_name"][:])
+    ):
+        time_reference_ = time_reference
+        if time_reference is None and "time_reference" in group.variables:
+            time_reference_ = group["time_reference"][:]
+
+        time_list.append(_extract_datetime64_or_raise(group["time"], time_reference_))
+        radial_wind_speed_list.append(
+            _extract_float64_or_raise(group["radial_wind_speed"], expected_dimensions)
+        )
+        cnr_list.append(_extract_float64_or_raise(group["cnr"], expected_dimensions))
+        relative_beta_list.append(
+            _extract_float64_or_raise(group["relative_beta"], expected_dimensions)
+        )
+        radial_wind_speed_confidence_list.append(
+            _extract_float64_or_raise(
+                group["radial_wind_speed_ci"], expected_dimensions
+            )
+        )
+        azimuth_list.append(
+            _extract_float64_or_raise(group["azimuth"], expected_dimensions)
+        )
+        elevation_list.append(
+            _extract_float64_or_raise(group["elevation"], expected_dimensions)
+        )
+        range_list.append(
+            _extract_float64_or_raise(group["range"], (expected_dimensions[1],))
+        )
+        doppler_spectrum_width_list.append(
+            _extract_float64_or_raise(
+                group["doppler_spectrum_width"], expected_dimensions
+            )
+        )
+        ray_accumulation_time_list.append(
+            _extract_float64_or_raise(
+                group["ray_accumulation_time"], expected_dimensions
+            )
+        )
+
+    return WindCubeFixed(
+        time=np.concatenate(time_list),
+        radial_distance=np.concatenate(range_list),
+        azimuth=np.concatenate(azimuth_list),
+        elevation=np.concatenate(elevation_list),
+        radial_velocity=np.concatenate(radial_wind_speed_list),
+        radial_velocity_confidence=np.concatenate(radial_wind_speed_confidence_list),
+        cnr=np.concatenate(cnr_list),
+        relative_beta=np.concatenate(relative_beta_list),
+        doppler_spectrum_width=np.concatenate(doppler_spectrum_width_list),
+        ray_accumulation_time=merge_all_equal(
+            "ray_accumulation_time",
+            np.array(ray_accumulation_time_list, dtype=np.float64).tolist(),
+        ),
+        system_id=nc.instrument_name,
+    )
+
+
+def _from_vad_or_dbs_src(nc: Dataset) -> WindCube:
+    scan_index_list: list[npt.NDArray[np.int64]] = []
+    time_list: list[npt.NDArray[np.datetime64]] = []
+    cnr_list: list[npt.NDArray[np.float64]] = []
+    radial_wind_speed_list: list[npt.NDArray[np.float64]] = []
+    radial_wind_speed_confidence_list: list[npt.NDArray[np.float64]] = []
+    azimuth_list: list[npt.NDArray[np.float64]] = []
+    elevation_list: list[npt.NDArray[np.float64]] = []
+    range_list: list[npt.NDArray[np.float64]] = []
+    height_list: list[npt.NDArray[np.float64]] = []
+
     time_reference = (
         nc["time_reference"][:] if "time_reference" in nc.variables else None
     )
 
+    expected_dimensions = ("time", "gate_index")
     for i, group in enumerate(
         nc[group] for group in (nc.variables["sweep_group_name"][:])
     ):
-        time_list.append(_extract_datetime64_or_raise(group["time"], time_reference))
+        time_reference_ = time_reference
+        if time_reference is None and "time_reference" in group.variables:
+            time_reference_ = group["time_reference"][:]
+
+        time_list.append(_extract_datetime64_or_raise(group["time"], time_reference_))
         radial_wind_speed_list.append(
-            _extract_float64_or_raise(group["radial_wind_speed"])
+            _extract_float64_or_raise(group["radial_wind_speed"], expected_dimensions)
         )
-        cnr_list.append(_extract_float64_or_raise(group["cnr"]))
+        cnr_list.append(_extract_float64_or_raise(group["cnr"], expected_dimensions))
         radial_wind_speed_confidence_list.append(
-            _extract_float64_or_raise(group["radial_wind_speed_ci"])
+            _extract_float64_or_raise(
+                group["radial_wind_speed_ci"], expected_dimensions
+            )
+        )
+        azimuth_list.append(
+            _extract_float64_or_raise(group["azimuth"], expected_dimensions)
+        )
+        elevation_list.append(
+            _extract_float64_or_raise(group["elevation"], expected_dimensions)
+        )
+        range_list.append(
+            _extract_float64_or_raise(group["range"], expected_dimensions)
+        )
+        height_list.append(
+            _extract_float64_or_raise(group["measurement_height"], expected_dimensions)
         )
-        azimuth_list.append(_extract_float64_or_raise(group["azimuth"]))
-        elevation_list.append(_extract_float64_or_raise(group["elevation"]))
-        range_list.append(_extract_int64_or_raise(group["range"]))
-        height_list.append(_extract_int64_or_raise(group["measurement_height"]))
         scan_index_list.append(np.full(group["time"][:].shape, i, dtype=np.int64))
 
     return WindCube(
@@ -189,7 +382,7 @@ def _from_vad_or_dbs_src(nc: Dataset) -> WindCube:
 
 
 def _extract_datetime64_or_raise(
-    nc: Dataset, time_reference: str | None
+    nc: Variable[npt.NDArray[np.float64]], time_reference: str | None
 ) -> npt.NDArray[np.datetime64]:
     match nc.name:
         case "time":
@@ -207,18 +400,40 @@ def _extract_datetime64_or_raise(
             raise ValueError(f"Unexpected variable name {nc.name}")
 
 
-def _extract_float64_or_raise(nc: Dataset) -> npt.NDArray[np.float64]:
+def _dB_to_ratio(decibels: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]:
+    return 10 ** (0.1 * decibels)
+
+
+def _extract_float64_or_raise(
+    nc: Variable[npt.NDArray[np.float64]], expected_dimensions: tuple[str, ...]
+) -> npt.NDArray[np.float64]:
     match nc.name:
+        case "range" | "measurement_height":
+            if nc.dimensions != expected_dimensions:
+                raise ValueError(f"Unexpected dimensions for {nc.name}")
+            if nc.units != "m":
+                raise ValueError(f"Unexpected units for {nc.name}")
+            if nc[:].mask is not np.bool_(False):
+                raise ValueError
+            return np.array(nc[:].data, dtype=np.float64)
         case "cnr":
-            if nc.dimensions != ("time", "gate_index"):
+            if nc.dimensions != expected_dimensions:
                 raise ValueError(f"Unexpected dimensions for {nc.name}")
             if nc.units != "dB":
                 raise ValueError(f"Unexpected units for {nc.name}")
             if nc[:].mask is not np.bool_(False):
                 pass  # ignore that array contains masked values
+            return _dB_to_ratio(np.array(nc[:].data, dtype=np.float64))
+        case "relative_beta":
+            if nc.dimensions != expected_dimensions:
+                raise ValueError(f"Unexpected dimensions for {nc.name}")
+            if nc.units != "m-1 sr-1":
+                raise ValueError(f"Unexpected units for {nc.name}")
+            if nc[:].mask is not np.bool_(False):
+                pass  # ignore that array contains masked values
             return np.array(nc[:].data, dtype=np.float64)
         case "radial_wind_speed":
-            if nc.dimensions != ("time", "gate_index"):
+            if nc.dimensions != expected_dimensions:
                 raise ValueError(f"Unexpected dimensions for {nc.name}")
             if nc.units != "m s-1":
                 raise ValueError(f"Unexpected units for {nc.name}")
@@ -226,7 +441,7 @@ def _extract_float64_or_raise(nc: Dataset) -> npt.NDArray[np.float64]:
                 pass  # ignore that array contains masked values
             return np.array(nc[:].data, dtype=np.float64)
         case "radial_wind_speed_ci":
-            if nc.dimensions != ("time", "gate_index"):
+            if nc.dimensions != expected_dimensions:
                 raise ValueError(f"Unexpected dimensions for {nc.name}")
             if nc.units != "percent":
                 raise ValueError(f"Unexpected units for {nc.name}")
@@ -234,26 +449,28 @@ def _extract_float64_or_raise(nc: Dataset) -> npt.NDArray[np.float64]:
                 pass  # ignore that array contains masked values
             return np.array(nc[:].data, dtype=np.float64)
         case "azimuth" | "elevation":
-            if nc.dimensions != ("time",):
+            if nc.dimensions != (expected_dimensions[0],):
                 raise ValueError(f"Unexpected dimensions for {nc.name}")
             if nc.units != "degrees":
                 raise ValueError(f"Unexpected units for {nc.name}")
             if nc[:].mask is not np.bool_(False):
                 raise ValueError
             return np.array(nc[:].data, dtype=np.float64)
-        case _:
-            raise ValueError(f"Unexpected variable name {nc.name}")
-
-
-def _extract_int64_or_raise(nc: Dataset) -> npt.NDArray[np.int64]:
-    match nc.name:
-        case "range" | "measurement_height":
-            if nc.dimensions != ("time", "gate_index"):
+        case "doppler_spectrum_width":
+            if nc.dimensions != expected_dimensions:
                 raise ValueError(f"Unexpected dimensions for {nc.name}")
-            if nc.units != "m":
+            if nc.units != "m s-1":
                 raise ValueError(f"Unexpected units for {nc.name}")
             if nc[:].mask is not np.bool_(False):
-                raise ValueError
-            return np.array(nc[:].data, dtype=np.int64)
+                pass  # ignore that array contains masked values
+            return np.array(nc[:].data, dtype=np.float64)
+        case "ray_accumulation_time":
+            if nc.dimensions != ():
+                raise ValueError(f"Unexpected dimensions for {nc.name}")
+            if nc.units != "ms":
+                raise ValueError(f"Unexpected units for {nc.name}")
+            if nc[:].mask is not np.bool_(False):
+                raise ValueError(f"Variable {nc.name} contains masked values")
+            return np.array(nc[:].data, dtype=np.float64)
         case _:
             raise ValueError(f"Unexpected variable name {nc.name}")

doppy/raw/wls70.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timezone
 from io import BufferedIOBase
 from pathlib import Path
 from typing import Any, Sequence
@@ -166,7 +166,12 @@ def _raw_rs_to_wls70(
     cnr_threshold = float(info["cnr_threshold"])
     data = data.reshape(-1, len(cols))
     time_ts = data[:, 0]
-    time = np.array([datetime64(datetime.utcfromtimestamp(ts)) for ts in time_ts])
+    time = np.array(
+        [
+            datetime64(datetime.fromtimestamp(ts, timezone.utc).replace(tzinfo=None))
+            for ts in time_ts
+        ]
+    )
 
     position = data[:, 1]
     temperature = data[:, 2]

doppy/utils.py

@@ -1,9 +1,24 @@
-from typing import TypeVar
+from typing import TypeVar, cast
+
+import numpy as np
+from numpy.typing import NDArray
 
 T = TypeVar("T")
+NT = TypeVar("NT", bound=np.generic)
 
 
 def merge_all_equal(key: str, lst: list[T]) -> T:
     if len(set(lst)) != 1:
         raise ValueError(f"Cannot merge header key {key} values {lst}")
     return lst[0]
+
+
+def merge_all_close(key: str, lst: list[NDArray[NT]]) -> NT:
+    if len(lst) == 0:
+        raise ValueError(f"Cannot merge empty list for key {key}")
+    if any(arr.size == 0 for arr in lst):
+        raise ValueError(f"Cannot merge key {key}, one or more arrays are empty.")
+    arr = np.concatenate([arr.flatten() for arr in lst])
+    if not np.allclose(arr, arr[0]):
+        raise ValueError(f"Cannot merge key {key}, values are not close enough")
+    return cast(NT, arr[0])

{doppy-0.3.6.dist-info → doppy-0.4.0.dist-info}/METADATA

@@ -1,11 +1,12 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: doppy
-Version: 0.3.6
+Version: 0.4.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Intended Audience :: Science/Research
 Classifier: Operating System :: OS Independent
@@ -25,6 +26,7 @@ Requires-Dist: py-spy ; extra == 'dev'
 Requires-Dist: maturin ==1.4 ; extra == 'dev'
 Requires-Dist: release-version ; extra == 'dev'
 Requires-Dist: pre-commit ; extra == 'dev'
+Requires-Dist: xarray[io] ; extra == 'dev'
 Provides-Extra: dev
 License-File: LICENSE
 License-File: LICENSE

{doppy-0.3.6.dist-info → doppy-0.4.0.dist-info}/RECORD

@@ -1,29 +1,29 @@
-doppy-0.3.6.dist-info/METADATA,sha256=rIdxynFJ0EY_sbL0ATtBX4OTvQfS6zcsPeqTI6FG8eU,4182
-doppy-0.3.6.dist-info/WHEEL,sha256=-Y1xyYTN3M_8ej0ZmEausOjKRIhVhpqcBKtfhemr-pc,105
-doppy-0.3.6.dist-info/entry_points.txt,sha256=9b_Ca7vJoh6AwL3W8qAPh_UmJ_1Pa6hi-TDfCTDjvSk,43
-doppy-0.3.6.dist-info/licenses/LICENSE,sha256=V-0iroMNMI8ctnLgUau1kdFvwhkYhr9vi-5kWKxw2wc,1089
+doppy-0.4.0.dist-info/METADATA,sha256=bnTR2j3JN6ZfhwrYNbhIj0i6HdM-zpergm-CCBQa_5I,4276
+doppy-0.4.0.dist-info/WHEEL,sha256=J9wdJ6DlOtsCLbXTCN9C4tF9HRUjkxgtI13Jme_gk0Y,105
+doppy-0.4.0.dist-info/entry_points.txt,sha256=9b_Ca7vJoh6AwL3W8qAPh_UmJ_1Pa6hi-TDfCTDjvSk,43
+doppy-0.4.0.dist-info/licenses/LICENSE,sha256=V-0iroMNMI8ctnLgUau1kdFvwhkYhr9vi-5kWKxw2wc,1089
 doppy/options.py,sha256=73BDODO4OYHn2qOshhwz6u6G3J1kNd3uj6P0a3V4HBE,205
 doppy/__init__.py,sha256=Z9aEUlbPRWRUAoB8_-djkgrJuS4-6pjem4-mVSB6Z9I,191
 doppy/product/wind.py,sha256=hkSzAyi4tfPoGwUx8iE3l65oCDyzl_tRgtF2loxaEpY,16557
 doppy/product/__init__.py,sha256=C6s9cX20m9UwRsKo1lZH6TnYFfM5KmsX5MPUyShbgl4,235
 doppy/product/stare_depol.py,sha256=thrWzCpvdH3AiA6gmR37vrH_pDACNY2QTtqPteJ2s8Y,9653
-doppy/product/stare.py,sha256=OD9Ff0IUVvRVR-KRIfOiWIQq3sC5KKhYtlmnif81HcU,22625
+doppy/product/stare.py,sha256=TE6BrrXFuwv9-zLTYCCn3HNpNgjwdldbyU6qk1__FzA,25343
 doppy/bench.py,sha256=iVNYveMVGGRES2oe3Orsn31jQFCKTXOmxRFuFiJ8_OA,248
 doppy/netcdf.py,sha256=34TzB9UyypTwDpT2MfrRD14g2zGP3X-NuEU15mQGaUI,3925
-doppy/utils.py,sha256=qPtIYBJPaKKTmRWwJI93TFUuhJg7CAoecpyHCm5ZyxI,214
+doppy/utils.py,sha256=cY-tWrArrROX7lg_zJ2M4QJ6GUnU_BjhkUYL5aG9kbA,790
 doppy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 doppy/exceptions.py,sha256=OzdLXmKs3qZrvzwaI0pxjzpM2w9J5Of7dCo_Ekygecc,183
-doppy/defaults.py,sha256=-il4DWU1HPttlGFoVQTPC7R4pDZERoPvt3kw6xwMKrw,38
+doppy/defaults.py,sha256=-jR_xAVLf_soZNDu3uJ6mhOZa9L1tfKtt0k0tI6Rd9s,472
 doppy/data/cache.py,sha256=cxCZ7HyEQt2EKAGEiMqx3wJ2-5Y3hEAEPQ_XB4gKlkk,1160
 doppy/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 doppy/data/api.py,sha256=QaVKj304OPcu8OF5xgtduQzDis8Srn-I6UgR9qb2u9E,1863
 doppy/data/exceptions.py,sha256=6CS6OHIWq8CqlxiceEvC1j0EfWUYoIfM7dW88apQVn4,89
 doppy/raw/halo_sys_params.py,sha256=DeOIIRtVdO7HnNY3FaI4u035ScI65jfDgUrEccLepEo,3937
-doppy/raw/__init__.py,sha256=AMHyONuH0aUJUQz20EhlANaq9UjWJtSZf7kWUVx3ZjA,228
-doppy/raw/wls70.py,sha256=PMyr_JYTiKuYv1JpWHgVX6OA5zU83DOnr8vBCxFyFAw,7420
+doppy/raw/__init__.py,sha256=4pj4xSK9rA0XEfM79NsRBRYId7GaJRDCyPABxbEm63g,260
+doppy/raw/wls70.py,sha256=TePgyhmlWaUQCw1vkWh7uD7KT5EFMaDhNeVYoF2rspc,7510
 doppy/raw/halo_bg.py,sha256=8t9j-SUF1yJht3vrT6KAYJQyxcg3W-0zr8h0jAEhWes,5815
-doppy/raw/halo_hpl.py,sha256=YFTepAWXvIfAVGgLZCadsc1d-VaOh7j2RB3cpb6Sv6k,18486
-doppy/raw/windcube.py,sha256=LO5TXxWGdVxYD-kGwjaIebwEnEegB0AyxMXynkdrJqE,9952
+doppy/raw/halo_hpl.py,sha256=yzVPQEATnDVf1K3N-mGLXR4_RuT-E94OVXathT2bHes,18580
+doppy/raw/windcube.py,sha256=vgkoXVgcbI2Or4BsFCeq2STdjUP49h3mBQLoDXdBQHM,19074
 doppy/__main__.py,sha256=zrKQJVj0k0ypBQCGK65Czt9G9FZ_qx3ussw6Q9VJ14g,346
-doppy/rs.abi3.so,sha256=sJc44vpekTdry6zsIVs23pn1DrByFH2V2lE58_JsWE0,2682528
-doppy-0.3.6.dist-info/RECORD,,
+doppy/rs.abi3.so,sha256=qEol5mznGJTeLODcK2DUcCuEulCBdgy959ejQeIO2Tc,2695088
+doppy-0.4.0.dist-info/RECORD,,

{doppy-0.3.6.dist-info → doppy-0.4.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: maturin (1.7.4)
+Generator: maturin (1.7.7)
 Root-Is-Purelib: false
 Tag: cp310-abi3-macosx_10_12_x86_64