doppy 0.3.7__tar.gz → 0.4.0__tar.gz

{doppy-0.3.7 → doppy-0.4.0}/Cargo.lock

@@ -106,7 +106,7 @@ checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"
 
 [[package]]
 name = "doppy_rs"
-version = "0.3.7"
+version = "0.4.0"
 dependencies = [
  "doprs",
  "numpy",
@@ -115,7 +115,7 @@ dependencies = [
 
 [[package]]
 name = "doprs"
-version = "0.3.7"
+version = "0.4.0"
 dependencies = [
  "chrono",
  "rayon",

{doppy-0.3.7 → doppy-0.4.0}/Cargo.toml

@@ -4,6 +4,6 @@ resolver = "2"
 
 [workspace.package]
 edition = "2021"
-version = "0.3.7"
+version = "0.4.0"
 authors = ["Niko Leskinen <niko.leskinen@fmi.fi>"]
 license-file = "LICENSE"

{doppy-0.3.7 → doppy-0.4.0}/PKG-INFO

@@ -1,11 +1,12 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: doppy
-Version: 0.3.7
+Version: 0.4.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Intended Audience :: Science/Research
 Classifier: Operating System :: OS Independent
@@ -25,6 +26,7 @@ Requires-Dist: py-spy ; extra == 'dev'
 Requires-Dist: maturin ==1.4 ; extra == 'dev'
 Requires-Dist: release-version ; extra == 'dev'
 Requires-Dist: pre-commit ; extra == 'dev'
+Requires-Dist: xarray[io] ; extra == 'dev'
 Provides-Extra: dev
 License-File: LICENSE
 License-File: LICENSE

{doppy-0.3.7 → doppy-0.4.0}/pyproject.toml

@@ -6,16 +6,17 @@ build-backend = "maturin"
 name = "doppy"
 summary = "Doppler lidar processing"
 dynamic = ["version"]
-authors = [{name = "Niko Leskinen", email = "niko.leskinen@fmi.fi"}]
+authors = [{ name = "Niko Leskinen", email = "niko.leskinen@fmi.fi" }]
 requires-python = ">=3.10"
 readme = "README.md"
-license = {file = "LICENSE"}
+license = { file = "LICENSE" }
 classifiers = [
   "Development Status :: 4 - Beta",
   "Programming Language :: Python :: 3",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
   "License :: OSI Approved :: MIT License",
   "Intended Audience :: Science/Research",
   "Operating System :: OS Independent"
@@ -28,11 +29,21 @@ dependencies = [
   "typer",
   "matplotlib",
   "scikit-learn",
-  "scipy"
+  "scipy",
 ]
 
 [project.optional-dependencies]
-dev = ["mypy", "ruff", "pytest", "types-requests", "py-spy", "maturin==1.4", "release-version", "pre-commit"]
+dev = [
+  "mypy",
+  "ruff",
+  "pytest",
+  "types-requests",
+  "py-spy",
+  "maturin==1.4",
+  "release-version",
+  "pre-commit",
+  "xarray[io]",
+]
 
 [project.scripts]
 doppy = "doppy.__main__:app"

doppy-0.4.0/src/doppy/defaults.py

@@ -0,0 +1,18 @@
+DEFAULT_BEAM_ENERGY = 1e-5
+DEFAULT_EFFECTIVE_DIAMETER = 25e-3
+
+
+class Halo:
+    wavelength = 1.565e-6  # [m]
+    receiver_bandwidth = 50e6  # [Hz]
+    beam_energy = DEFAULT_BEAM_ENERGY
+    effective_diameter = DEFAULT_EFFECTIVE_DIAMETER
+
+
+class WindCube:
+    # https://doi.org/10.5194/essd-13-3539-2021
+    wavelength = 1.54e-6  # [m]
+    receiver_bandwidth = 55e6  # [Hz]
+    beam_energy = DEFAULT_BEAM_ENERGY
+    effective_diameter = 50e-3  # [m]
+    focus = 1e3  # [m]

{doppy-0.3.7 → doppy-0.4.0}/src/doppy/product/stare.py

@@ -9,7 +9,7 @@ from typing import Sequence, Tuple, TypeAlias
 import numpy as np
 import numpy.typing as npt
 import scipy
-from scipy.ndimage import uniform_filter
+from scipy.ndimage import median_filter, uniform_filter
 from sklearn.cluster import KMeans
 
 import doppy
@@ -51,6 +51,46 @@ class Stare:
             )
         raise TypeError
 
+    @classmethod
+    def mask_nan(cls, x: npt.NDArray[np.float64]) -> npt.NDArray[np.bool_]:
+        return np.isnan(x)
+
+    @classmethod
+    def from_windcube_data(
+        cls,
+        data: Sequence[str]
+        | Sequence[Path]
+        | Sequence[bytes]
+        | Sequence[BufferedIOBase],
+    ) -> Stare:
+        raws = doppy.raw.WindCubeFixed.from_srcs(data)
+        raw = (
+            doppy.raw.WindCubeFixed.merge(raws).sorted_by_time().nan_profiles_removed()
+        )
+
+        wavelength = defaults.WindCube.wavelength
+        beta = _compute_beta(
+            snr=raw.cnr,
+            radial_distance=raw.radial_distance,
+            wavelength=wavelength,
+            beam_energy=defaults.WindCube.beam_energy,
+            receiver_bandwidth=defaults.WindCube.receiver_bandwidth,
+            focus=defaults.WindCube.focus,
+            effective_diameter=defaults.WindCube.effective_diameter,
+        )
+
+        mask = _compute_noise_mask_for_windcube(raw)
+        return cls(
+            time=raw.time,
+            radial_distance=raw.radial_distance,
+            elevation=raw.elevation,
+            beta=beta,
+            radial_velocity=raw.radial_velocity,
+            mask=mask,
+            wavelength=wavelength,
+            system_id=raw.system_id,
+        )
+
     @classmethod
     def from_halo_data(
         cls,
@@ -95,16 +135,21 @@ class Stare:
             raw, intensity_bg_corrected
         )
         wavelength = defaults.Halo.wavelength
+
         beta = _compute_beta(
-            intensity_noise_bias_corrected,
-            raw.radial_distance,
-            raw.header.focus_range,
-            wavelength,
+            snr=intensity_noise_bias_corrected - 1,
+            radial_distance=raw.radial_distance,
+            wavelength=wavelength,
+            beam_energy=defaults.Halo.beam_energy,
+            receiver_bandwidth=defaults.Halo.receiver_bandwidth,
+            focus=raw.header.focus_range,
+            effective_diameter=defaults.Halo.effective_diameter,
         )
+
         mask = _compute_noise_mask(
             intensity_noise_bias_corrected, raw.radial_velocity, raw.radial_distance
         )
-        return Stare(
+        return cls(
             time=raw.time,
             radial_distance=raw.radial_distance,
             elevation=raw.elevation,
@@ -177,6 +222,36 @@ class Stare:
             nc.add_attribute("doppy_version", doppy.__version__)
 
 
+def _compute_noise_mask_for_windcube(
+    raw: doppy.raw.WindCubeFixed,
+) -> npt.NDArray[np.bool_]:
+    if np.any(np.isnan(raw.cnr)) or np.any(np.isnan(raw.radial_velocity)):
+        raise ValueError("Unexpected nans in crn or radial_velocity")
+
+    mask = _mask_with_cnr_norm_dist(raw.cnr) | (np.abs(raw.radial_velocity) > 30)
+
+    cnr = raw.cnr.copy()
+    cnr[mask] = np.finfo(float).eps
+    cnr_filt = median_filter(cnr, size=(3, 3))
+    rel_diff = np.abs(cnr - cnr_filt) / np.abs(cnr)
+    diff_mask = rel_diff > 0.25
+
+    mask = mask | diff_mask
+
+    return np.array(mask, dtype=np.bool_)
+
+
+def _mask_with_cnr_norm_dist(cnr: npt.NDArray[np.float64]) -> npt.NDArray[np.bool_]:
+    th_trunc = -5.5
+    std_factor = 2
+    log_cnr = np.log(cnr)
+    log_cnr_trunc = log_cnr[log_cnr < th_trunc]
+    th_trunc_fit = np.percentile(log_cnr_trunc, 90)
+    log_cnr_for_fit = log_cnr_trunc[log_cnr_trunc < th_trunc_fit]
+    mean, std = scipy.stats.norm.fit(log_cnr_for_fit)
+    return np.array(np.log(cnr) < (mean + std_factor * std), dtype=np.bool_)
+
+
 def _compute_noise_mask(
     intensity: npt.NDArray[np.float64],
     radial_velocity: npt.NDArray[np.float64],
@@ -197,14 +272,19 @@ def _compute_noise_mask(
 
 
 def _compute_beta(
-    intensity: npt.NDArray[np.float64],
+    snr: npt.NDArray[np.float64],
     radial_distance: npt.NDArray[np.float64],
-    focus: float,
     wavelength: float,
+    beam_energy: float,
+    receiver_bandwidth: float,
+    focus: float,
+    effective_diameter: float,
 ) -> npt.NDArray[np.float64]:
     """
     Parameters
     ----------
+    snr
+        for halo: intensity - 1
     radial_distance
         distance from the instrument
     focus
@@ -236,22 +316,24 @@ def _compute_beta(
         doi: https://doi.org/10.5194/amt-13-2849-2020
     """
 
-    snr = intensity - 1
     h = scipy.constants.Planck
     c = scipy.constants.speed_of_light
     eta = 1
-    E = 1e-5
-    B = 5e7
+    E = beam_energy
+    B = receiver_bandwidth
     nu = c / wavelength
-    A_e = _compute_effective_receiver_energy(radial_distance, focus, wavelength)
+    A_e = _compute_effective_receiver_energy(
+        radial_distance, wavelength, focus, effective_diameter
+    )
     beta = 2 * h * nu * B * radial_distance**2 * snr / (eta * c * E * A_e)
     return np.array(beta, dtype=np.float64)
 
 
 def _compute_effective_receiver_energy(
     radial_distance: npt.NDArray[np.float64],
-    focus: float,
     wavelength: float,
+    focus: float,
+    effective_diameter: float,
 ) -> npt.NDArray[np.float64]:
     """
     NOTE
@@ -268,7 +350,7 @@ def _compute_effective_receiver_energy(
     wavelength
         laser wavelength
     """
-    D = 25e-3  # effective_diameter_of_gaussian_beam
+    D = effective_diameter
     return np.array(
         np.pi
         * D**2

doppy-0.4.0/src/doppy/raw/__init__.py

@@ -0,0 +1,7 @@
+from .halo_bg import HaloBg
+from .halo_hpl import HaloHpl
+from .halo_sys_params import HaloSysParams
+from .windcube import WindCube, WindCubeFixed
+from .wls70 import Wls70
+
+__all__ = ["HaloHpl", "HaloBg", "HaloSysParams", "WindCube", "WindCubeFixed", "Wls70"]

{doppy-0.3.7 → doppy-0.4.0}/src/doppy/raw/halo_hpl.py

@@ -4,7 +4,7 @@ import functools
 import io
 import re
 from dataclasses import dataclass
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from io import BufferedIOBase
 from os.path import commonprefix
 from pathlib import Path
@@ -309,7 +309,11 @@ def _raw_tuple2halo_hpl(
         resolution=float(header_dict["resolution"]),
         scan_type=str(header_dict["scan_type"]),
         focus_range=int(header_dict["focus_range"]),
-        start_time=datetime64(datetime.utcfromtimestamp(header_dict["start_time"])),
+        start_time=datetime64(
+            datetime.fromtimestamp(header_dict["start_time"], timezone.utc).replace(
+                tzinfo=None
+            )
+        ),
         system_id=str(header_dict["system_id"]),
         instrument_spectral_width=float(header_dict["instrument_spectral_width"])
         if header_dict["instrument_spectral_width"] is not None

doppy-0.4.0/src/doppy/raw/windcube.py

@@ -0,0 +1,476 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from io import BufferedIOBase
+from pathlib import Path
+from typing import Sequence
+
+import numpy as np
+import numpy.typing as npt
+from netCDF4 import Dataset, Variable, num2date
+from numpy import datetime64
+
+from doppy.utils import merge_all_equal
+
+
+@dataclass
+class WindCubeFixed:
+    time: npt.NDArray[datetime64]  # dim: (time, )
+    radial_distance: npt.NDArray[np.float64]  # dim: (radial_distance,)
+    azimuth: npt.NDArray[np.float64]  # dim: (time, )
+    elevation: npt.NDArray[np.float64]  # dim: (time, )
+    cnr: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    relative_beta: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    radial_velocity: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    doppler_spectrum_width: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    radial_velocity_confidence: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    ray_accumulation_time: np.float64  # dim: ()
+    system_id: str
+
+    @classmethod
+    def from_srcs(
+        cls,
+        data: Sequence[str]
+        | Sequence[Path]
+        | Sequence[bytes]
+        | Sequence[BufferedIOBase],
+    ) -> list[WindCubeFixed]:
+        return [WindCubeFixed.from_fixed_src(src) for src in data]
+
+    @classmethod
+    def from_fixed_src(cls, data: str | Path | bytes | BufferedIOBase) -> WindCubeFixed:
+        data_bytes = _src_to_bytes(data)
+        nc = Dataset("inmemory.nc", "r", memory=data_bytes)
+        return _from_fixed_src(nc)
+
+    @classmethod
+    def merge(cls, raws: list[WindCubeFixed]) -> WindCubeFixed:
+        return WindCubeFixed(
+            time=np.concatenate([r.time for r in raws]),
+            radial_distance=_merge_radial_distance_for_fixed(
+                [r.radial_distance for r in raws]
+            ),
+            azimuth=np.concatenate([r.azimuth for r in raws]),
+            elevation=np.concatenate([r.elevation for r in raws]),
+            radial_velocity=np.concatenate([r.radial_velocity for r in raws]),
+            radial_velocity_confidence=np.concatenate(
+                [r.radial_velocity_confidence for r in raws]
+            ),
+            cnr=np.concatenate([r.cnr for r in raws]),
+            relative_beta=np.concatenate([r.relative_beta for r in raws]),
+            doppler_spectrum_width=np.concatenate(
+                [r.doppler_spectrum_width for r in raws]
+            ),
+            ray_accumulation_time=merge_all_equal(
+                "ray_accumulation_time", [r.ray_accumulation_time for r in raws]
+            ),
+            system_id=merge_all_equal("system_id", [r.system_id for r in raws]),
+        )
+
+    def __getitem__(
+        self,
+        index: int
+        | slice
+        | list[int]
+        | npt.NDArray[np.int64]
+        | npt.NDArray[np.bool_]
+        | tuple[slice, slice],
+    ) -> WindCubeFixed:
+        if isinstance(index, (int, slice, list, np.ndarray)):
+            return WindCubeFixed(
+                time=self.time[index],
+                radial_distance=self.radial_distance,
+                azimuth=self.azimuth[index],
+                elevation=self.elevation[index],
+                radial_velocity=self.radial_velocity[index],
+                radial_velocity_confidence=self.radial_velocity_confidence[index],
+                cnr=self.cnr[index],
+                relative_beta=self.relative_beta[index],
+                doppler_spectrum_width=self.doppler_spectrum_width[index],
+                ray_accumulation_time=self.ray_accumulation_time,
+                system_id=self.system_id,
+            )
+        raise TypeError
+
+    def sorted_by_time(self) -> WindCubeFixed:
+        sort_indices = np.argsort(self.time)
+        return self[sort_indices]
+
+    def nan_profiles_removed(self) -> WindCubeFixed:
+        return self[~np.all(np.isnan(self.cnr), axis=1)]
+
+
+@dataclass
+class WindCube:
+    time: npt.NDArray[datetime64]  # dim: (time, )
+    radial_distance: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    height: npt.NDArray[np.float64]  # dim: (time,radial_distance)
+    azimuth: npt.NDArray[np.float64]  # dim: (time, )
+    elevation: npt.NDArray[np.float64]  # dim: (time, )
+    cnr: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    radial_velocity: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    radial_velocity_confidence: npt.NDArray[np.float64]  # dim: (time, radial_distance)
+    scan_index: npt.NDArray[np.int64]
+    system_id: str
+
+    @classmethod
+    def from_vad_or_dbs_srcs(
+        cls,
+        data: Sequence[str]
+        | Sequence[Path]
+        | Sequence[bytes]
+        | Sequence[BufferedIOBase],
+    ) -> list[WindCube]:
+        return [WindCube.from_vad_or_dbs_src(src) for src in data]
+
+    @classmethod
+    def from_vad_or_dbs_src(cls, data: str | Path | bytes | BufferedIOBase) -> WindCube:
+        data_bytes = _src_to_bytes(data)
+        nc = Dataset("inmemory.nc", "r", memory=data_bytes)
+        return _from_vad_or_dbs_src(nc)
+
+    @classmethod
+    def merge(cls, raws: list[WindCube]) -> WindCube:
+        return WindCube(
+            scan_index=_merge_scan_index([r.scan_index for r in raws]),
+            time=np.concatenate([r.time for r in raws]),
+            height=np.concatenate([r.height for r in raws]),
+            radial_distance=np.concatenate([r.radial_distance for r in raws]),
+            azimuth=np.concatenate([r.azimuth for r in raws]),
+            elevation=np.concatenate([r.elevation for r in raws]),
+            radial_velocity=np.concatenate([r.radial_velocity for r in raws]),
+            radial_velocity_confidence=np.concatenate(
+                [r.radial_velocity_confidence for r in raws]
+            ),
+            cnr=np.concatenate([r.cnr for r in raws]),
+            system_id=merge_all_equal("system_id", [r.system_id for r in raws]),
+        )
+
+    def __getitem__(
+        self,
+        index: int
+        | slice
+        | list[int]
+        | npt.NDArray[np.int64]
+        | npt.NDArray[np.bool_]
+        | tuple[slice, slice],
+    ) -> WindCube:
+        if isinstance(index, (int, slice, list, np.ndarray)):
+            return WindCube(
+                time=self.time[index],
+                radial_distance=self.radial_distance[index],
+                height=self.height[index],
+                azimuth=self.azimuth[index],
+                elevation=self.elevation[index],
+                radial_velocity=self.radial_velocity[index],
+                radial_velocity_confidence=self.radial_velocity_confidence[index],
+                cnr=self.cnr[index],
+                scan_index=self.scan_index[index],
+                system_id=self.system_id,
+            )
+        raise TypeError
+
+    def sorted_by_time(self) -> WindCube:
+        sort_indices = np.argsort(self.time)
+        return self[sort_indices]
+
+    def non_strictly_increasing_timesteps_removed(self) -> WindCube:
+        if len(self.time) == 0:
+            return self
+        mask = np.ones_like(self.time, dtype=np.bool_)
+        latest_time = self.time[0]
+        for i, t in enumerate(self.time[1:], start=1):
+            if t <= latest_time:
+                mask[i] = False
+            else:
+                latest_time = t
+        return self[mask]
+
+    def reindex_scan_indices(self) -> WindCube:
+        new_indices = np.zeros_like(self.scan_index)
+        indexed = set()
+        j = 0
+        for i in self.scan_index:
+            if i in indexed:
+                continue
+            new_indices[i == self.scan_index] = j
+            indexed.add(i)
+            j += 1
+        self.scan_index = new_indices
+        return self
+
+
+def _merge_scan_index(index_list: list[npt.NDArray[np.int64]]) -> npt.NDArray[np.int64]:
+    if len(index_list) == 0:
+        raise ValueError("cannot merge empty list")
+
+    new_index_list = []
+    current_max = index_list[0].max()
+    new_index_list.append(index_list[0])
+
+    for index_arr in index_list[1:]:
+        new_arr = index_arr + current_max + 1
+        new_index_list.append(new_arr)
+        current_max += index_arr.max() + 1
+    return np.concatenate(new_index_list)
+
+
+def _merge_radial_distance_for_fixed(
+    radial_distance_list: list[npt.NDArray[np.float64]],
+) -> npt.NDArray[np.float64]:
+    if len(radial_distance_list) == 0:
+        raise ValueError("cannot merge empty list")
+    if not all(
+        np.allclose(arr.shape, radial_distance_list[0].shape)
+        for arr in radial_distance_list
+    ):
+        raise ValueError("Cannot merge radial distances with different shapes")
+    if not all(
+        np.allclose(arr, radial_distance_list[0]) for arr in radial_distance_list
+    ):
+        raise ValueError("Cannot merge radial distances")
+    return radial_distance_list[0]
+
+
+def _src_to_bytes(data: str | Path | bytes | BufferedIOBase) -> bytes:
+    if isinstance(data, str):
+        path = Path(data)
+        with path.open("rb") as f:
+            return f.read()
+    elif isinstance(data, Path):
+        with data.open("rb") as f:
+            return f.read()
+    elif isinstance(data, bytes):
+        return data
+    elif isinstance(data, BufferedIOBase):
+        return data.read()
+
+
+def _from_fixed_src(nc: Dataset) -> WindCubeFixed:
+    time_list = []
+    cnr_list = []
+    relative_beta_list = []
+    radial_wind_speed_list = []
+    radial_wind_speed_confidence_list = []
+    azimuth_list = []
+    elevation_list = []
+    range_list = []
+    doppler_spectrum_width_list = []
+    ray_accumulation_time_list = []
+    time_reference = (
+        nc["time_reference"][:] if "time_reference" in nc.variables else None
+    )
+
+    expected_dimensions = ("time", "range")
+    for _, group in enumerate(
+        nc[group] for group in (nc.variables["sweep_group_name"][:])
+    ):
+        time_reference_ = time_reference
+        if time_reference is None and "time_reference" in group.variables:
+            time_reference_ = group["time_reference"][:]
+
+        time_list.append(_extract_datetime64_or_raise(group["time"], time_reference_))
+        radial_wind_speed_list.append(
+            _extract_float64_or_raise(group["radial_wind_speed"], expected_dimensions)
+        )
+        cnr_list.append(_extract_float64_or_raise(group["cnr"], expected_dimensions))
+        relative_beta_list.append(
+            _extract_float64_or_raise(group["relative_beta"], expected_dimensions)
+        )
+        radial_wind_speed_confidence_list.append(
+            _extract_float64_or_raise(
+                group["radial_wind_speed_ci"], expected_dimensions
+            )
+        )
+        azimuth_list.append(
+            _extract_float64_or_raise(group["azimuth"], expected_dimensions)
+        )
+        elevation_list.append(
+            _extract_float64_or_raise(group["elevation"], expected_dimensions)
+        )
+        range_list.append(
+            _extract_float64_or_raise(group["range"], (expected_dimensions[1],))
+        )
+        doppler_spectrum_width_list.append(
+            _extract_float64_or_raise(
+                group["doppler_spectrum_width"], expected_dimensions
+            )
+        )
+        ray_accumulation_time_list.append(
+            _extract_float64_or_raise(
+                group["ray_accumulation_time"], expected_dimensions
+            )
+        )
+
+    return WindCubeFixed(
+        time=np.concatenate(time_list),
+        radial_distance=np.concatenate(range_list),
+        azimuth=np.concatenate(azimuth_list),
+        elevation=np.concatenate(elevation_list),
+        radial_velocity=np.concatenate(radial_wind_speed_list),
+        radial_velocity_confidence=np.concatenate(radial_wind_speed_confidence_list),
+        cnr=np.concatenate(cnr_list),
+        relative_beta=np.concatenate(relative_beta_list),
+        doppler_spectrum_width=np.concatenate(doppler_spectrum_width_list),
+        ray_accumulation_time=merge_all_equal(
+            "ray_accumulation_time",
+            np.array(ray_accumulation_time_list, dtype=np.float64).tolist(),
+        ),
+        system_id=nc.instrument_name,
+    )
+
+
+def _from_vad_or_dbs_src(nc: Dataset) -> WindCube:
+    scan_index_list: list[npt.NDArray[np.int64]] = []
+    time_list: list[npt.NDArray[np.datetime64]] = []
+    cnr_list: list[npt.NDArray[np.float64]] = []
+    radial_wind_speed_list: list[npt.NDArray[np.float64]] = []
+    radial_wind_speed_confidence_list: list[npt.NDArray[np.float64]] = []
+    azimuth_list: list[npt.NDArray[np.float64]] = []
+    elevation_list: list[npt.NDArray[np.float64]] = []
+    range_list: list[npt.NDArray[np.float64]] = []
+    height_list: list[npt.NDArray[np.float64]] = []
+
+    time_reference = (
+        nc["time_reference"][:] if "time_reference" in nc.variables else None
+    )
+
+    expected_dimensions = ("time", "gate_index")
+    for i, group in enumerate(
+        nc[group] for group in (nc.variables["sweep_group_name"][:])
+    ):
+        time_reference_ = time_reference
+        if time_reference is None and "time_reference" in group.variables:
+            time_reference_ = group["time_reference"][:]
+
+        time_list.append(_extract_datetime64_or_raise(group["time"], time_reference_))
+        radial_wind_speed_list.append(
+            _extract_float64_or_raise(group["radial_wind_speed"], expected_dimensions)
+        )
+        cnr_list.append(_extract_float64_or_raise(group["cnr"], expected_dimensions))
+        radial_wind_speed_confidence_list.append(
+            _extract_float64_or_raise(
+                group["radial_wind_speed_ci"], expected_dimensions
+            )
+        )
+        azimuth_list.append(
+            _extract_float64_or_raise(group["azimuth"], expected_dimensions)
+        )
+        elevation_list.append(
+            _extract_float64_or_raise(group["elevation"], expected_dimensions)
+        )
+        range_list.append(
+            _extract_float64_or_raise(group["range"], expected_dimensions)
+        )
+        height_list.append(
+            _extract_float64_or_raise(group["measurement_height"], expected_dimensions)
+        )
+        scan_index_list.append(np.full(group["time"][:].shape, i, dtype=np.int64))
+
+    return WindCube(
+        scan_index=np.concatenate(scan_index_list),
+        time=np.concatenate(time_list),
+        radial_distance=np.concatenate(range_list),
+        height=np.concatenate(height_list),
+        azimuth=np.concatenate(azimuth_list),
+        elevation=np.concatenate(elevation_list),
+        radial_velocity=np.concatenate(radial_wind_speed_list),
+        radial_velocity_confidence=np.concatenate(radial_wind_speed_confidence_list),
+        cnr=np.concatenate(cnr_list),
+        system_id=nc.instrument_name,
+    )
+
+
+def _extract_datetime64_or_raise(
+    nc: Variable[npt.NDArray[np.float64]], time_reference: str | None
+) -> npt.NDArray[np.datetime64]:
+    match nc.name:
+        case "time":
+            if nc.dimensions != ("time",):
+                raise ValueError
+
+            units = nc.units
+            if "time_reference" in nc.units:
+                if time_reference is not None:
+                    units = nc.units.replace("time_reference", time_reference)
+                else:
+                    raise ValueError("Unknown time_reference")
+            return np.array(num2date(nc[:], units=units), dtype="datetime64[us]")
+        case _:
+            raise ValueError(f"Unexpected variable name {nc.name}")
+
+
+def _dB_to_ratio(decibels: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]:
+    return 10 ** (0.1 * decibels)
+
+
+def _extract_float64_or_raise(
+    nc: Variable[npt.NDArray[np.float64]], expected_dimensions: tuple[str, ...]
+) -> npt.NDArray[np.float64]:
+    match nc.name:
+        case "range" | "measurement_height":
+            if nc.dimensions != expected_dimensions:
+                raise ValueError(f"Unexpected dimensions for {nc.name}")
+            if nc.units != "m":
+                raise ValueError(f"Unexpected units for {nc.name}")
+            if nc[:].mask is not np.bool_(False):
+                raise ValueError
+            return np.array(nc[:].data, dtype=np.float64)
+        case "cnr":
+            if nc.dimensions != expected_dimensions:
+                raise ValueError(f"Unexpected dimensions for {nc.name}")
+            if nc.units != "dB":
+                raise ValueError(f"Unexpected units for {nc.name}")
+            if nc[:].mask is not np.bool_(False):
+                pass  # ignore that array contains masked values
+            return _dB_to_ratio(np.array(nc[:].data, dtype=np.float64))
+        case "relative_beta":
+            if nc.dimensions != expected_dimensions:
+                raise ValueError(f"Unexpected dimensions for {nc.name}")
+            if nc.units != "m-1 sr-1":
+                raise ValueError(f"Unexpected units for {nc.name}")
+            if nc[:].mask is not np.bool_(False):
+                pass  # ignore that array contains masked values
+            return np.array(nc[:].data, dtype=np.float64)
+        case "radial_wind_speed":
+            if nc.dimensions != expected_dimensions:
+                raise ValueError(f"Unexpected dimensions for {nc.name}")
+            if nc.units != "m s-1":
+                raise ValueError(f"Unexpected units for {nc.name}")
+            if nc[:].mask is not np.bool_(False):
+                pass  # ignore that array contains masked values
+            return np.array(nc[:].data, dtype=np.float64)
+        case "radial_wind_speed_ci":
+            if nc.dimensions != expected_dimensions:
+                raise ValueError(f"Unexpected dimensions for {nc.name}")
+            if nc.units != "percent":
+                raise ValueError(f"Unexpected units for {nc.name}")
+            if nc[:].mask is not np.bool_(False):
+                pass  # ignore that array contains masked values
+            return np.array(nc[:].data, dtype=np.float64)
+        case "azimuth" | "elevation":
+            if nc.dimensions != (expected_dimensions[0],):
+                raise ValueError(f"Unexpected dimensions for {nc.name}")
+            if nc.units != "degrees":
+                raise ValueError(f"Unexpected units for {nc.name}")
+            if nc[:].mask is not np.bool_(False):
+                raise ValueError
+            return np.array(nc[:].data, dtype=np.float64)
+        case "doppler_spectrum_width":
+            if nc.dimensions != expected_dimensions:
+                raise ValueError(f"Unexpected dimensions for {nc.name}")
+            if nc.units != "m s-1":
+                raise ValueError(f"Unexpected units for {nc.name}")
+            if nc[:].mask is not np.bool_(False):
+                pass  # ignore that array contains masked values
+            return np.array(nc[:].data, dtype=np.float64)
+        case "ray_accumulation_time":
+            if nc.dimensions != ():
+                raise ValueError(f"Unexpected dimensions for {nc.name}")
+            if nc.units != "ms":
+                raise ValueError(f"Unexpected units for {nc.name}")
+            if nc[:].mask is not np.bool_(False):
+                raise ValueError(f"Variable {nc.name} contains masked values")
+            return np.array(nc[:].data, dtype=np.float64)
+        case _:
+            raise ValueError(f"Unexpected variable name {nc.name}")

{doppy-0.3.7 → doppy-0.4.0}/src/doppy/raw/wls70.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timezone
 from io import BufferedIOBase
 from pathlib import Path
 from typing import Any, Sequence
@@ -166,7 +166,12 @@ def _raw_rs_to_wls70(
     cnr_threshold = float(info["cnr_threshold"])
     data = data.reshape(-1, len(cols))
     time_ts = data[:, 0]
-    time = np.array([datetime64(datetime.utcfromtimestamp(ts)) for ts in time_ts])
+    time = np.array(
+        [
+            datetime64(datetime.fromtimestamp(ts, timezone.utc).replace(tzinfo=None))
+            for ts in time_ts
+        ]
+    )
 
     position = data[:, 1]
     temperature = data[:, 2]

doppy-0.4.0/src/doppy/utils.py

@@ -0,0 +1,24 @@
+from typing import TypeVar, cast
+
+import numpy as np
+from numpy.typing import NDArray
+
+T = TypeVar("T")
+NT = TypeVar("NT", bound=np.generic)
+
+
+def merge_all_equal(key: str, lst: list[T]) -> T:
+    if len(set(lst)) != 1:
+        raise ValueError(f"Cannot merge header key {key} values {lst}")
+    return lst[0]
+
+
+def merge_all_close(key: str, lst: list[NDArray[NT]]) -> NT:
+    if len(lst) == 0:
+        raise ValueError(f"Cannot merge empty list for key {key}")
+    if any(arr.size == 0 for arr in lst):
+        raise ValueError(f"Cannot merge key {key}, one or more arrays are empty.")
+    arr = np.concatenate([arr.flatten() for arr in lst])
+    if not np.allclose(arr, arr[0]):
+        raise ValueError(f"Cannot merge key {key}, values are not close enough")
+    return cast(NT, arr[0])

doppy-0.3.7/src/doppy/defaults.py

@@ -1,2 +0,0 @@
-class Halo:
-    wavelength = 1.565e-6

doppy-0.3.7/src/doppy/raw/__init__.py

@@ -1,7 +0,0 @@
-from .halo_bg import HaloBg
-from .halo_hpl import HaloHpl
-from .halo_sys_params import HaloSysParams
-from .windcube import WindCube
-from .wls70 import Wls70
-
-__all__ = ["HaloHpl", "HaloBg", "HaloSysParams", "WindCube", "Wls70"]

doppy-0.3.7/src/doppy/raw/windcube.py

@@ -1,263 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from io import BufferedIOBase
-from pathlib import Path
-from typing import Sequence
-
-import numpy as np
-import numpy.typing as npt
-from netCDF4 import Dataset, num2date
-from numpy import datetime64
-
-from doppy.utils import merge_all_equal
-
-
-@dataclass
-class WindCube:
-    time: npt.NDArray[datetime64]  # dim: (time, )
-    radial_distance: npt.NDArray[np.int64]  # dim: (time, radial_distance)
-    height: npt.NDArray[np.int64]  # dim: (time,radial_distance)
-    azimuth: npt.NDArray[np.float64]  # dim: (time, )
-    elevation: npt.NDArray[np.float64]  # dim: (time, )
-    cnr: npt.NDArray[np.float64]  # dim: (time, radial_distance)
-    radial_velocity: npt.NDArray[np.float64]  # dim: (time, radial_distance)
-    radial_velocity_confidence: npt.NDArray[np.float64]  # dim: (time, radial_distance)
-    scan_index: npt.NDArray[np.int64]
-    system_id: str
-
-    @classmethod
-    def from_vad_or_dbs_srcs(
-        cls,
-        data: Sequence[str]
-        | Sequence[Path]
-        | Sequence[bytes]
-        | Sequence[BufferedIOBase],
-    ) -> list[WindCube]:
-        return [WindCube.from_vad_or_dbs_src(src) for src in data]
-
-    @classmethod
-    def from_vad_or_dbs_src(cls, data: str | Path | bytes | BufferedIOBase) -> WindCube:
-        data_bytes = _src_to_bytes(data)
-        nc = Dataset("inmemory.nc", "r", memory=data_bytes)
-        return _from_vad_or_dbs_src(nc)
-
-    @classmethod
-    def merge(cls, raws: list[WindCube]) -> WindCube:
-        return WindCube(
-            scan_index=_merge_scan_index([r.scan_index for r in raws]),
-            time=np.concatenate([r.time for r in raws]),
-            height=np.concatenate([r.height for r in raws]),
-            radial_distance=np.concatenate([r.radial_distance for r in raws]),
-            azimuth=np.concatenate([r.azimuth for r in raws]),
-            elevation=np.concatenate([r.elevation for r in raws]),
-            radial_velocity=np.concatenate([r.radial_velocity for r in raws]),
-            radial_velocity_confidence=np.concatenate(
-                [r.radial_velocity_confidence for r in raws]
-            ),
-            cnr=np.concatenate([r.cnr for r in raws]),
-            system_id=merge_all_equal("system_id", [r.system_id for r in raws]),
-        )
-
-    def __getitem__(
-        self,
-        index: int
-        | slice
-        | list[int]
-        | npt.NDArray[np.int64]
-        | npt.NDArray[np.bool_]
-        | tuple[slice, slice],
-    ) -> WindCube:
-        if isinstance(index, (int, slice, list, np.ndarray)):
-            return WindCube(
-                time=self.time[index],
-                radial_distance=self.radial_distance[index],
-                height=self.height[index],
-                azimuth=self.azimuth[index],
-                elevation=self.elevation[index],
-                radial_velocity=self.radial_velocity[index],
-                radial_velocity_confidence=self.radial_velocity_confidence[index],
-                cnr=self.cnr[index],
-                scan_index=self.scan_index[index],
-                system_id=self.system_id,
-            )
-        raise TypeError
-
-    def sorted_by_time(self) -> WindCube:
-        sort_indices = np.argsort(self.time)
-        return self[sort_indices]
-
-    def non_strictly_increasing_timesteps_removed(self) -> WindCube:
-        if len(self.time) == 0:
-            return self
-        mask = np.ones_like(self.time, dtype=np.bool_)
-        latest_time = self.time[0]
-        for i, t in enumerate(self.time[1:], start=1):
-            if t <= latest_time:
-                mask[i] = False
-            else:
-                latest_time = t
-        return self[mask]
-
-    def reindex_scan_indices(self) -> WindCube:
-        new_indices = np.zeros_like(self.scan_index)
-        indexed = set()
-        j = 0
-        for i in self.scan_index:
-            if i in indexed:
-                continue
-            new_indices[i == self.scan_index] = j
-            indexed.add(i)
-            j += 1
-        self.scan_index = new_indices
-        return self
-
-
-def _merge_scan_index(index_list: list[npt.NDArray[np.int64]]) -> npt.NDArray[np.int64]:
-    if len(index_list) == 0:
-        raise ValueError("cannot merge empty list")
-
-    new_index_list = []
-    current_max = index_list[0].max()
-    new_index_list.append(index_list[0])
-
-    for index_arr in index_list[1:]:
-        new_arr = index_arr + current_max + 1
-        new_index_list.append(new_arr)
-        current_max += index_arr.max() + 1
-    return np.concatenate(new_index_list)
-
-
-def _src_to_bytes(data: str | Path | bytes | BufferedIOBase) -> bytes:
-    if isinstance(data, str):
-        path = Path(data)
-        with path.open("rb") as f:
-            return f.read()
-    elif isinstance(data, Path):
-        with data.open("rb") as f:
-            return f.read()
-    elif isinstance(data, bytes):
-        return data
-    elif isinstance(data, BufferedIOBase):
-        return data.read()
-    raise TypeError("Unsupported data type")
-
-
-def _from_vad_or_dbs_src(nc: Dataset) -> WindCube:
-    scan_index_list = []
-    time_list = []
-    cnr_list = []
-    radial_wind_speed_list = []
-    radial_wind_speed_confidence_list = []
-    azimuth_list = []
-    elevation_list = []
-    range_list = []
-    height_list = []
-    time_reference = (
-        nc["time_reference"][:] if "time_reference" in nc.variables else None
-    )
-
-    for i, group in enumerate(
-        nc[group] for group in (nc.variables["sweep_group_name"][:])
-    ):
-        time_reference_ = time_reference
-        if time_reference is None and "time_reference" in group.variables:
-            time_reference_ = group["time_reference"][:]
-
-        time_list.append(_extract_datetime64_or_raise(group["time"], time_reference_))
-        radial_wind_speed_list.append(
-            _extract_float64_or_raise(group["radial_wind_speed"])
-        )
-        cnr_list.append(_extract_float64_or_raise(group["cnr"]))
-        radial_wind_speed_confidence_list.append(
-            _extract_float64_or_raise(group["radial_wind_speed_ci"])
-        )
-        azimuth_list.append(_extract_float64_or_raise(group["azimuth"]))
-        elevation_list.append(_extract_float64_or_raise(group["elevation"]))
-        range_list.append(_extract_int64_or_raise(group["range"]))
-        height_list.append(_extract_int64_or_raise(group["measurement_height"]))
-        scan_index_list.append(np.full(group["time"][:].shape, i, dtype=np.int64))
-
-    return WindCube(
-        scan_index=np.concatenate(scan_index_list),
-        time=np.concatenate(time_list),
-        radial_distance=np.concatenate(range_list),
-        height=np.concatenate(height_list),
-        azimuth=np.concatenate(azimuth_list),
-        elevation=np.concatenate(elevation_list),
-        radial_velocity=np.concatenate(radial_wind_speed_list),
-        radial_velocity_confidence=np.concatenate(radial_wind_speed_confidence_list),
-        cnr=np.concatenate(cnr_list),
-        system_id=nc.instrument_name,
-    )
-
-
-def _extract_datetime64_or_raise(
-    nc: Dataset, time_reference: str | None
-) -> npt.NDArray[np.datetime64]:
-    match nc.name:
-        case "time":
-            if nc.dimensions != ("time",):
-                raise ValueError
-
-            units = nc.units
-            if "time_reference" in nc.units:
-                if time_reference is not None:
-                    units = nc.units.replace("time_reference", time_reference)
-                else:
-                    raise ValueError("Unknown time_reference")
-            return np.array(num2date(nc[:], units=units), dtype="datetime64[us]")
-        case _:
-            raise ValueError(f"Unexpected variable name {nc.name}")
-
-
-def _extract_float64_or_raise(nc: Dataset) -> npt.NDArray[np.float64]:
-    match nc.name:
-        case "cnr":
-            if nc.dimensions != ("time", "gate_index"):
-                raise ValueError(f"Unexpected dimensions for {nc.name}")
-            if nc.units != "dB":
-                raise ValueError(f"Unexpected units for {nc.name}")
-            if nc[:].mask is not np.bool_(False):
-                pass  # ignore that array contains masked values
-            return np.array(nc[:].data, dtype=np.float64)
-        case "radial_wind_speed":
-            if nc.dimensions != ("time", "gate_index"):
-                raise ValueError(f"Unexpected dimensions for {nc.name}")
-            if nc.units != "m s-1":
-                raise ValueError(f"Unexpected units for {nc.name}")
-            if nc[:].mask is not np.bool_(False):
-                pass  # ignore that array contains masked values
-            return np.array(nc[:].data, dtype=np.float64)
-        case "radial_wind_speed_ci":
-            if nc.dimensions != ("time", "gate_index"):
-                raise ValueError(f"Unexpected dimensions for {nc.name}")
-            if nc.units != "percent":
-                raise ValueError(f"Unexpected units for {nc.name}")
-            if nc[:].mask is not np.bool_(False):
-                pass  # ignore that array contains masked values
-            return np.array(nc[:].data, dtype=np.float64)
-        case "azimuth" | "elevation":
-            if nc.dimensions != ("time",):
-                raise ValueError(f"Unexpected dimensions for {nc.name}")
-            if nc.units != "degrees":
-                raise ValueError(f"Unexpected units for {nc.name}")
-            if nc[:].mask is not np.bool_(False):
-                raise ValueError
-            return np.array(nc[:].data, dtype=np.float64)
-        case _:
-            raise ValueError(f"Unexpected variable name {nc.name}")
-
-
-def _extract_int64_or_raise(nc: Dataset) -> npt.NDArray[np.int64]:
-    match nc.name:
-        case "range" | "measurement_height":
-            if nc.dimensions != ("time", "gate_index"):
-                raise ValueError(f"Unexpected dimensions for {nc.name}")
-            if nc.units != "m":
-                raise ValueError(f"Unexpected units for {nc.name}")
-            if nc[:].mask is not np.bool_(False):
-                raise ValueError
-            return np.array(nc[:].data, dtype=np.int64)
-        case _:
-            raise ValueError(f"Unexpected variable name {nc.name}")

doppy-0.3.7/src/doppy/utils.py

@@ -1,9 +0,0 @@
-from typing import TypeVar
-
-T = TypeVar("T")
-
-
-def merge_all_equal(key: str, lst: list[T]) -> T:
-    if len(set(lst)) != 1:
-        raise ValueError(f"Cannot merge header key {key} values {lst}")
-    return lst[0]