doppy 0.4.1__cp310-abi3-win_amd64.whl → 0.5.0__cp310-abi3-win_amd64.whl

doppy/bench.py

@@ -3,10 +3,11 @@ import time
 
 class Timer:
     def __init__(self):
-        self.start = None
+        self.start: float | None = None
 
     def __enter__(self):
         self.start = time.time()
 
     def __exit__(self, type, value, traceback):
-        print(f"Elapsed time: {time.time() - self.start:.2f}")
+        if isinstance(self.start, float):
+            print(f"Elapsed time: {time.time() - self.start:.2f}")

doppy/data/api.py

@@ -20,7 +20,7 @@ class Api:
         self.api_endpoint = "https://cloudnet.fmi.fi/api"
         self.cache = cache
 
-    def get(self, path: str, params: dict[str, str]) -> list:
+    def get(self, path: str, params: dict[str, str | list[str]]) -> list:
         res = self.session.get(
             f"{self.api_endpoint}/{path}", params=params, timeout=1800
         )

doppy/netcdf.py

@@ -13,8 +13,12 @@ NetCDFDataType: TypeAlias = Literal["f4", "f8", "i4", "i8", "u4", "u8"]
 
 
 class Dataset:
-    def __init__(self, filename: str | pathlib.Path) -> None:
-        self.nc = netCDF4.Dataset(filename, mode="w")
+    def __init__(
+        self,
+        filename: str | pathlib.Path,
+        format: Literal["NETCDF4", "NETCDF4_CLASSIC"] = "NETCDF4",
+    ) -> None:
+        self.nc = netCDF4.Dataset(filename, mode="w", format=format)
 
     def __enter__(self) -> Dataset:
         return self
@@ -27,8 +31,8 @@ class Dataset:
     ) -> None:
         self.close()
 
-    def add_dimension(self, dim: str) -> Dataset:
-        self.nc.createDimension(dim, None)
+    def add_dimension(self, dim: str, size: int | None = None) -> Dataset:
+        self.nc.createDimension(dim, size)
         return self
 
     def add_attribute(self, key: str, val: str) -> Dataset:

doppy/product/noise_utils.py

@@ -0,0 +1,106 @@
+import warnings
+
+import numpy as np
+import numpy.typing as npt
+import scipy
+
+
+def detect_wind_noise(
+    w: npt.NDArray[np.float64],
+    height: npt.NDArray[np.float64],
+    mask: npt.NDArray[np.bool_],
+    window: float = 150,
+    stride: int = 1,
+) -> npt.NDArray[np.bool_]:
+    """
+    Parameters
+    ----------
+    w
+        vertical velocity, dims: (time,height)
+
+    height
+        dims: (height,)
+
+    mask
+        old mask that still contains noisy velocity data,
+        mask[t,h] = True iff w[t,h] is noise
+
+    window
+        size of window used to compute rolling median in meters
+
+    stride
+        stride used to compute rolling median
+
+    Returns
+    -------
+    improved noise mask such that new_mask[t,h] = True iff w[t,h] is noise
+    """
+    warnings.simplefilter("ignore", RuntimeWarning)
+    v = _rolling_median_over_range(
+        height,
+        w,
+        mask,
+        window=window,  # meters
+        stride=stride,
+        fill_gaps=True,
+    )
+
+    th = 2
+    diff = np.abs(v - w)
+    new_mask = (diff > th) | mask
+    new_mask = _remove_one_hot(new_mask)
+    return np.array(new_mask, dtype=np.bool_)
+
+
+def _remove_one_hot(m: npt.NDArray[np.bool_]) -> npt.NDArray[np.bool_]:
+    if m.ndim != 2:
+        raise ValueError
+    if m.shape[1] < 3:
+        return m
+    x = ~m
+    y = np.full(x.shape, np.False_)
+    y[:, 0] = x[:, 0] & x[:, 1]
+    y[:, 1:-1] = x[:, 1:-1] & (x[:, 2:] | x[:, :-2])
+    y[:, -1] = x[:, -1] & x[:, -2]
+    return np.array(~y, dtype=np.bool_)
+
+
+def _rolling_median_over_range(
+    range_: npt.NDArray[np.float64],
+    arr: npt.NDArray[np.float64],
+    mask: npt.NDArray[np.bool_],
+    window: float,
+    stride: int = 1,
+    fill_gaps: bool = False,
+) -> npt.NDArray[np.float64]:
+    """
+    window
+        range window in meters
+    """
+    X = arr.T.copy()
+    X[mask.T] = np.nan
+
+    half_window = window / 2
+
+    i = 0
+    j = 0
+    n = len(range_)
+    med = np.full(X.shape, np.nan, dtype=np.float64)
+    for k in range(0, n, stride):
+        r = range_[k]
+        while i + 1 < n and r - range_[i + 1] >= half_window:
+            i += 1
+        while j + 1 < n and range_[j] - r < half_window:
+            j += 1
+        if i > k or j < k:
+            raise ValueError
+        med[k] = np.nanmedian(X[i : j + 1], axis=0)
+
+    if stride != 1 and fill_gaps:
+        ind = list(range(0, n, stride))
+        f_interp = scipy.interpolate.interp1d(
+            range_[ind], med[ind], axis=0, fill_value="extrapolate"
+        )
+        med_all = f_interp(range_)
+        return np.array(med_all.T.copy(), dtype=np.float64)
+    return np.array(med.T.copy(), dtype=np.float64)

doppy/product/stare.py

@@ -14,20 +14,35 @@ from sklearn.cluster import KMeans
 
 import doppy
 from doppy import defaults, options
+from doppy.product.noise_utils import detect_wind_noise
 
 SelectionGroupKeyType: TypeAlias = tuple[int,]
 
 
-@dataclass
+@dataclass(slots=True)
+class RayAccumulationTime:
+    # in seconds
+    value: float
+
+
+@dataclass(slots=True)
+class PulsesPerRay:
+    value: int
+
+
+@dataclass(slots=True)
 class Stare:
     time: npt.NDArray[np.datetime64]
     radial_distance: npt.NDArray[np.float64]
     elevation: npt.NDArray[np.float64]
     beta: npt.NDArray[np.float64]
+    snr: npt.NDArray[np.float64]
     radial_velocity: npt.NDArray[np.float64]
-    mask: npt.NDArray[np.bool_]
+    mask_beta: npt.NDArray[np.bool_]
+    mask_radial_velocity: npt.NDArray[np.bool_]
     wavelength: float
     system_id: str
+    ray_info: RayAccumulationTime | PulsesPerRay
 
     def __getitem__(
         self,
@@ -44,10 +59,13 @@ class Stare:
                 radial_distance=self.radial_distance,
                 elevation=self.elevation[index],
                 beta=self.beta[index],
+                snr=self.snr[index],
                 radial_velocity=self.radial_velocity[index],
-                mask=self.mask[index],
+                mask_beta=self.mask_beta[index],
+                mask_radial_velocity=self.mask_radial_velocity[index],
                 wavelength=self.wavelength,
                 system_id=self.system_id,
+                ray_info=self.ray_info,
             )
         raise TypeError
 
@@ -79,16 +97,23 @@ class Stare:
             effective_diameter=defaults.WindCube.effective_diameter,
         )
 
-        mask = _compute_noise_mask_for_windcube(raw)
+        mask_beta = _compute_noise_mask_for_windcube(raw)
+        mask_radial_velocity = detect_wind_noise(
+            raw.radial_velocity, raw.radial_distance, mask_beta
+        )
+
         return cls(
             time=raw.time,
             radial_distance=raw.radial_distance,
             elevation=raw.elevation,
             beta=beta,
+            snr=raw.cnr,
             radial_velocity=raw.radial_velocity,
-            mask=mask,
+            mask_beta=mask_beta,
+            mask_radial_velocity=mask_radial_velocity,
             wavelength=wavelength,
             system_id=raw.system_id,
+            ray_info=RayAccumulationTime(raw.ray_accumulation_time),
         )
 
     @classmethod
@@ -146,18 +171,25 @@ class Stare:
             effective_diameter=defaults.Halo.effective_diameter,
         )
 
-        mask = _compute_noise_mask(
+        mask_beta = _compute_noise_mask(
             intensity_noise_bias_corrected, raw.radial_velocity, raw.radial_distance
         )
+        mask_radial_velocity = detect_wind_noise(
+            raw.radial_velocity, raw.radial_distance, mask_beta
+        )
+
         return cls(
             time=raw.time,
             radial_distance=raw.radial_distance,
             elevation=raw.elevation,
             beta=beta,
+            snr=intensity_noise_bias_corrected - 1,
             radial_velocity=raw.radial_velocity,
-            mask=mask,
+            mask_beta=mask_beta,
+            mask_radial_velocity=mask_radial_velocity,
             wavelength=wavelength,
             system_id=raw.header.system_id,
+            ray_info=PulsesPerRay(raw.header.pulses_per_ray),
         )
 
     def write_to_netcdf(self, filename: str | Path) -> None:
@@ -200,7 +232,7 @@ class Stare:
                 units="sr-1 m-1",
                 data=self.beta,
                 dtype="f4",
-                mask=self.mask,
+                mask=self.mask_beta,
             )
             nc.add_variable(
                 name="v",
@@ -209,7 +241,7 @@ class Stare:
                 long_name="Doppler velocity",
                 data=self.radial_velocity,
                 dtype="f4",
-                mask=self.mask,
+                mask=self.mask_radial_velocity,
             )
             nc.add_scalar_variable(
                 name="wavelength",
@@ -218,6 +250,24 @@ class Stare:
                 data=self.wavelength,
                 dtype="f4",
             )
+            match self.ray_info:
+                case RayAccumulationTime(value):
+                    nc.add_scalar_variable(
+                        name="ray_accumulation_time",
+                        units="s",
+                        long_name="ray accumulation time",
+                        data=value,
+                        dtype="f4",
+                    )
+                case PulsesPerRay(value):
+                    nc.add_scalar_variable(
+                        name="pulses_per_ray",
+                        units="1",
+                        long_name="pulses per ray",
+                        data=value,
+                        dtype="u4",
+                    )
+
             nc.add_attribute("serial_number", self.system_id)
             nc.add_attribute("doppy_version", doppy.__version__)
 
@@ -232,7 +282,7 @@ def _compute_noise_mask_for_windcube(
 
     cnr = raw.cnr.copy()
     cnr[mask] = np.finfo(float).eps
-    cnr_filt = median_filter(cnr, size=(3, 3))
+    cnr_filt = np.array(median_filter(cnr, size=(3, 3)), dtype=np.float64)
     rel_diff = np.abs(cnr - cnr_filt) / np.abs(cnr)
     diff_mask = rel_diff > 0.25
 

doppy/product/stare_depol.py

@@ -10,7 +10,7 @@ import numpy.typing as npt
 
 import doppy
 from doppy import options
-from doppy.product.stare import Stare
+from doppy.product.stare import PulsesPerRay, RayAccumulationTime, Stare
 
 
 @dataclass
@@ -36,7 +36,9 @@ class StareDepol:
         sr-1 m-1.
     radial_velocity
         An array of radial velocities of the co-polarised signal, in m s-1.
-    mask
+    mask_beta
+        A boolean array indicating signal (True) or noise (False) data points.
+    mask_radial_velocity
         A boolean array indicating signal (True) or noise (False) data points.
     depolarisation
         An array of depolarisation ratios calculated as the ratio of
@@ -69,11 +71,13 @@ class StareDepol:
     beta: npt.NDArray[np.float64]
     beta_cross: npt.NDArray[np.float64]
     radial_velocity: npt.NDArray[np.float64]
-    mask: npt.NDArray[np.bool_]
+    mask_beta: npt.NDArray[np.bool_]
+    mask_radial_velocity: npt.NDArray[np.bool_]
     depolarisation: npt.NDArray[np.float64]
     polariser_bleed_through: float
     wavelength: float
     system_id: str
+    ray_info: RayAccumulationTime | PulsesPerRay
 
     def __init__(
         self,
@@ -138,11 +142,13 @@ class StareDepol:
         self.beta = co.beta
         self.beta_cross = cross_beta
         self.radial_velocity = co.radial_velocity
-        self.mask = co.mask
+        self.mask_beta = co.mask_beta
+        self.mask_radial_velocity = co.mask_radial_velocity
         self.depolarisation = depolarisation
         self.polariser_bleed_through = polariser_bleed_through
         self.wavelength = co.wavelength
         self.system_id = co.system_id
+        self.ray_info = co.ray_info
 
     @property
     def mask_depolarisation(self) -> npt.NDArray[np.bool_]:
@@ -224,7 +230,7 @@ class StareDepol:
                 units="sr-1 m-1",
                 data=self.beta,
                 dtype="f4",
-                mask=self.mask,
+                mask=self.mask_beta,
             )
             nc.add_variable(
                 name="v",
@@ -233,7 +239,7 @@ class StareDepol:
                 long_name="Doppler velocity",
                 data=self.radial_velocity,
                 dtype="f4",
-                mask=self.mask,
+                mask=self.mask_radial_velocity,
             )
             nc.add_scalar_variable(
                 name="wavelength",
@@ -256,7 +262,7 @@ class StareDepol:
                 units="1",
                 data=self.depolarisation,
                 dtype="f4",
-                mask=self.mask | self.mask_depolarisation,
+                mask=self.mask_beta | self.mask_depolarisation,
             )
             nc.add_variable(
                 name="beta_cross_raw",
@@ -271,7 +277,7 @@ class StareDepol:
                 dimensions=("time", "range"),
                 units="sr-1 m-1",
                 data=self.beta_cross,
-                mask=self.mask | self.mask_beta_cross,
+                mask=self.mask_beta | self.mask_beta_cross,
                 dtype="f4",
             )
             nc.add_scalar_variable(
@@ -281,5 +287,22 @@ class StareDepol:
                 data=self.polariser_bleed_through,
                 dtype="f4",
             )
+            match self.ray_info:
+                case RayAccumulationTime(value):
+                    nc.add_scalar_variable(
+                        name="ray_accumulation_time",
+                        units="s",
+                        long_name="ray accumulation time",
+                        data=value,
+                        dtype="f4",
+                    )
+                case PulsesPerRay(value):
+                    nc.add_scalar_variable(
+                        name="pulses_per_ray",
+                        units="1",
+                        long_name="pulses per ray",
+                        data=value,
+                        dtype="u4",
+                    )
             nc.add_attribute("serial_number", self.system_id)
             nc.add_attribute("doppy_version", doppy.__version__)

doppy/product/turbulence.py

@@ -0,0 +1,265 @@
+# type: ignore
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+import numpy.typing as npt
+from scipy.interpolate import RegularGridInterpolator
+
+
+@dataclass
+class HorizontalWind:
+    time: npt.NDArray[np.datetime64]
+    height: npt.NDArray[np.float64]  # Height in meters from reference
+    V: npt.NDArray[np.float64]  # Horizontal wind speed in m/s
+
+
+@dataclass
+class VerticalWind:
+    time: npt.NDArray[np.datetime64]
+    height: npt.NDArray[np.float64]  # Height in meters from reference
+    w: npt.NDArray[np.float64]  # Vertical wind speed in m/s
+    mask: npt.NDArray[np.bool_]  # mask[t,h] = True iff w[t,h] should be masked
+
+
+@dataclass
+class Options:
+    ray_accumulation_time: float  # in seconds
+    period: float = 600  # period for computing the variance in seconds
+    beam_divergence: float = 33e-6  # radians
+
+
+@dataclass
+class Turbulence:
+    time: npt.NDArray[np.datetime64]
+    height: npt.NDArray[np.float64]
+    turbulent_kinetic_energy_dissipation_rate: npt.NDArray[np.float64]
+    mask: npt.NDArray[np.bool_]
+
+    @classmethod
+    def from_winds(
+        cls, vert: VerticalWind, hori: HorizontalWind, options: Options
+    ) -> Turbulence:
+        V = _preprocess_horiontal_wind(vert, hori, options)
+        ls_low = _length_scale_low(V, vert.height, options)
+        res = _compute_variance(vert, options.period)
+        sampling_time = _sampling_time_in_seconds(res)
+        ls_up = V * sampling_time
+        dissipation_rate = _compute_dissipation_rate(res.variance, ls_low, ls_up)
+        mask = np.isnan(dissipation_rate) | vert.mask
+        return cls(
+            time=vert.time.copy(),
+            height=vert.height.copy(),
+            turbulent_kinetic_energy_dissipation_rate=dissipation_rate,
+            mask=mask,
+        )
+
+
+def _sampling_time_in_seconds(r: VarResult) -> npt.NDArray[np.float64]:
+    if not all(
+        (
+            t == np.dtype("datetime64[us]")
+            for t in (r.period_start.dtype, r.period_stop.dtype)
+        )
+    ):
+        raise ValueError("period times must be on datetime64[us]")
+    td = r.period_stop - r.period_start
+    td_in_seconds = td / np.timedelta64(1, "s")
+    return np.array(td_in_seconds, dtype=np.float64)
+
+
+@dataclass
+class VarResult:
+    variance: npt.NDArray[np.float64]
+    period_start: npt.NDArray[np.datetime64]
+    period_stop: npt.NDArray[np.datetime64]
+    nsamples: npt.NDArray[np.int64]
+
+
+def _compute_variance(vert: VerticalWind, period: float) -> VarResult:
+    # NOTE: numerically unstable
+
+    # To compute actual time window
+    next_valid = _next_valid_from_mask(vert.mask)
+    prev_valid = _prev_valid_from_mask(vert.mask)
+
+    X = vert.w.copy()
+    X[vert.mask] = 0
+    X2 = X**2
+    X_cumsum = X.cumsum(axis=0)
+    X2_cumsum = X2.cumsum(axis=0)
+
+    N_i = (~vert.mask).astype(int)
+    N_cumsum = N_i.cumsum(axis=0)
+
+    def N_func(i: int, j: int) -> npt.NDArray[np.float64]:
+        return N_cumsum[j] - N_cumsum[i] + N_i[i]
+
+    def S(i: int, j: int) -> npt.NDArray[np.float64]:
+        return X_cumsum[j] - X_cumsum[i] + X[i]
+
+    def S2(i: int, j: int) -> npt.NDArray[np.float64]:
+        return X2_cumsum[j] - X2_cumsum[i] + X2[i]
+
+    def var_ij(i: int, j: int) -> npt.NDArray[np.float64]:
+        N = N_func(i, j)
+        with np.errstate(invalid="ignore"):
+            return (S2(i, j) - S(i, j) ** 2 / N) / N
+
+    half_period = np.timedelta64(int(1e6 * period / 2), "us")
+    period_start = np.full(vert.w.shape, np.datetime64("NaT", "us"))
+    period_stop = np.full(vert.w.shape, np.datetime64("NaT", "us"))
+    var = np.full(vert.w.shape, np.nan, dtype=np.float64)
+    nsamples = np.zeros_like(vert.w, dtype=np.int64)
+    i = 0
+    j = 0
+    n = len(vert.time)
+    for k, t in enumerate(vert.time):
+        while i + 1 < n and t - vert.time[i + 1] >= half_period:
+            i += 1
+        while j + 1 < n and vert.time[j] - t < half_period:
+            j += 1
+        i_valid = next_valid[i]
+        i_inbound = (0 <= i_valid) & (i_valid < n)
+        j_valid = prev_valid[j]
+        j_inbound = (0 <= j_valid) & (j_valid < n)
+        period_start[k][i_inbound] = vert.time[i_valid[i_inbound]]
+        period_stop[k][j_inbound] = vert.time[j_valid[j_inbound]]
+        var[k] = var_ij(i, j)
+        nsamples[k] = N_func(i, j)
+    return VarResult(
+        variance=var,
+        period_start=period_start,
+        period_stop=period_stop,
+        nsamples=nsamples,
+    )
+
+
+def _length_scale_low(
+    V: npt.NDArray[np.float64], height: npt.NDArray[np.float64], opts: Options
+) -> npt.NDArray[np.float64]:
+    integration_time = opts.ray_accumulation_time
+    from_beam = 2 * height * np.sin(opts.beam_divergence / 2)
+    from_wind = V * integration_time
+    return np.array(from_wind + from_beam[np.newaxis, :], dtype=np.float64)
+
+
+def _preprocess_horiontal_wind(
+    vert: VerticalWind, hori: HorizontalWind, options: Options
+):
+    if np.isnan(hori.V).any():
+        raise ValueError("horizontal wind speed cannot contains NaNs")
+    trg_points = np.meshgrid(vert.time, vert.height, indexing="ij")
+    src_points = (hori.time, hori.height)
+    src_vals = hori.V
+
+    interp_nearest = RegularGridInterpolator(
+        src_points,
+        src_vals,
+        method="nearest",
+        bounds_error=False,
+        fill_value=None,
+    )
+    interp_linear = RegularGridInterpolator(
+        src_points, src_vals, method="linear", bounds_error=False
+    )
+    V_nearest = interp_nearest(trg_points)
+    V_linear = interp_linear(trg_points)
+    V = V_linear
+    V[np.isnan(V)] = V_nearest[np.isnan(V)]
+    if np.isnan(V).any():
+        raise ValueError("Unexpected NaNs")
+    V_rmean = _rolling_mean_over_time(vert.time, V, options.period)
+    return V_rmean
+
+
+def _rolling_mean_over_time(
+    time: npt.NDArray[np.datetime64], arr: npt.NDArray[np.float64], period: float
+) -> npt.NDArray[np.float64]:
+    if arr.ndim != 2:
+        raise ValueError("number of dims on arr should be 2")
+    if time.ndim != 1 or time.shape[0] != arr.shape[0]:
+        raise ValueError("time and arr dimensions do not match")
+    if time.dtype != np.dtype("datetime64[us]"):
+        raise TypeError(f"Invalid time type: {time.dtype}")
+
+    S = arr.cumsum(axis=0)
+
+    def rolling_mean(i: int, j: int) -> npt.NDArray[np.flaat64]:
+        return (S[j] - S[i] + arr[i]) / (j - i + 1)
+
+    half_period = np.timedelta64(int(period * 0.5e6), "us")
+    rol_mean = np.full(arr.shape, np.nan, dtype=np.float64)
+
+    i = 0
+    j = 0
+    n = len(time)
+    for k, t in enumerate(time):
+        while i + 1 < n and t - time[i + 1] >= half_period:
+            i += 1
+        while j + 1 < n and time[j] - t < half_period:
+            j += 1
+        rol_mean[k] = rolling_mean(i, j)
+    return rol_mean
+
+
+def _compute_dissipation_rate(
+    variance: npt.NDArray[np.float64],
+    length_scale_lower: npt.NDArray[np.float64],
+    length_scale_upper: npt.NDArray[np.float64],
+):
+    """
+    Parameters
+    ----------
+    variance, length_scale_lower, and length_scale_upper
+        dimensions: (time,range)
+    """
+    kolmogorov_constant = 0.55
+    with np.errstate(invalid="ignore"):
+        dr = (
+            2
+            * np.pi
+            * (2 / (3 * kolmogorov_constant)) ** (3 / 2)
+            * variance ** (3 / 2)
+            * (length_scale_upper ** (2 / 3) - length_scale_lower ** (2 / 3))
+            ** (-3 / 2)
+        )
+    return dr
+
+
+def _next_valid_from_mask(mask):
+    """
+    mask[t,v] (time,value)
+
+    returns N[t,v] = i where i = min { j | j >= t and mask[j,v] == False}
+    if the set is non empty and N[t,v] = len(mask) otherwise
+    """
+    n = len(mask)
+    N = np.full(mask.shape, n)
+    if mask.size == 0:
+        return N
+    N[-1][~mask[-1]] = n - 1
+
+    for t in reversed(range(n - 1)):
+        N[t][~mask[t]] = t
+        N[t][mask[t]] = N[t + 1][mask[t]]
+    return N
+
+
+def _prev_valid_from_mask(mask):
+    """
+    mask[t,v] (time,value)
+
+    returns N[t,v] = i where i = max { j | j <= t and mask[j,v] == False}
+    if the set is non empty and N[t,v] = -1 otherwise
+    """
+    n = len(mask)
+    N = np.full(mask.shape, -1)
+    if mask.size == 0:
+        return N
+    N[0][~mask[0]] = 0
+    for t in range(1, n):
+        N[t][~mask[t]] = t
+        N[t][mask[t]] = N[t - 1][mask[t]]
+    return N

doppy/product/wind.py

@@ -326,6 +326,13 @@ def _compute_wind(
 
     rmse (range,):
         Root-mean-square error of radial velocity fit for each range gate.
+
+    References
+    ----------
+    An assessment of the performance of a 1.5 µm Doppler lidar for
+    operational vertical wind profiling based on a 1-year trial
+        authors: E. Päschke, R. Leinweber, and V. Lehmann
+        doi: 10.5194/amt-8-2251-2015
     """
     elevation = np.deg2rad(raw.elevation)
     azimuth = np.deg2rad(raw.azimuth)
@@ -364,9 +371,9 @@ def _compute_mask(
     rmse (time,range)
     """
 
-    def neighbour_diff(X: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]:
+    def neighbour_diff(X: npt.NDArray[np.float64]) -> np.float64:
         mdiff = np.max(np.abs(X - X[len(X) // 2]))
-        return np.array(mdiff, dtype=np.float64)
+        return np.float64(mdiff)
 
     WIND_NEIGHBOUR_DIFFERENCE = 20
     neighbour_mask = np.any(

doppy/raw/halo_hpl.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import functools
-import io
 import re
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
@@ -16,6 +15,7 @@ from numpy import datetime64, timedelta64
 
 import doppy
 from doppy import exceptions
+from doppy.raw.utils import bytes_from_src
 from doppy.utils import merge_all_equal
 
 
@@ -35,30 +35,9 @@ class HaloHpl:
 
     @classmethod
     def from_srcs(
-        cls,
-        data: Sequence[str]
-        | Sequence[Path]
-        | Sequence[bytes]
-        | Sequence[BufferedIOBase],
+        cls, data: Sequence[str | bytes | Path | BufferedIOBase]
     ) -> list[HaloHpl]:
-        if not isinstance(data, (list, tuple)):
-            raise TypeError("data should be list or tuple")
-        if all(isinstance(src, bytes) for src in data):
-            data_bytes = data
-        elif all(isinstance(src, str) for src in data):
-            data_bytes = []
-            for src in data:
-                with Path(src).open("rb") as f:
-                    data_bytes.append(f.read())
-        elif all(isinstance(src, Path) for src in data):
-            data_bytes = []
-            for src in data:
-                with src.open("rb") as f:
-                    data_bytes.append(f.read())
-        elif all(isinstance(src, BufferedIOBase) for src in data):
-            data_bytes = [src.read() for src in data]
-        else:
-            raise TypeError("Unexpected types in data")
+        data_bytes = [bytes_from_src(src) for src in data]
         raw_dicts = doppy.rs.raw.halo_hpl.from_bytes_srcs(data_bytes)
         try:
             return [_raw_tuple2halo_hpl(r) for r in raw_dicts]
@@ -67,40 +46,12 @@ class HaloHpl:
 
     @classmethod
     def from_src(cls, data: str | Path | bytes | BufferedIOBase) -> HaloHpl:
-        if isinstance(data, str):
-            path = Path(data)
-            with path.open("rb") as f:
-                data_bytes = f.read()
-        elif isinstance(data, Path):
-            with data.open("rb") as f:
-                data_bytes = f.read()
-        elif isinstance(data, bytes):
-            data_bytes = data
-        elif isinstance(data, BufferedIOBase):
-            data_bytes = data.read()
-        else:
-            raise TypeError("Unsupported data type")
+        data_bytes = bytes_from_src(data)
         try:
             return _raw_tuple2halo_hpl(doppy.rs.raw.halo_hpl.from_bytes_src(data_bytes))
         except RuntimeError as err:
             raise exceptions.RawParsingError(err) from err
 
-    @classmethod
-    def _py_from_src(cls, data: str | Path | bytes | BufferedIOBase) -> HaloHpl:
-        if isinstance(data, str):
-            path = Path(data)
-            with path.open("rb") as f:
-                return _from_src(f)
-        elif isinstance(data, Path):
-            with data.open("rb") as f:
-                return _from_src(f)
-        elif isinstance(data, bytes):
-            return _from_src(io.BytesIO(data))
-        elif isinstance(data, BufferedIOBase):
-            return _from_src(data)
-        else:
-            raise TypeError("Unsupported data type")
-
     def __getitem__(
         self,
         index: int
@@ -187,7 +138,7 @@ class HaloHpl:
         return self[mask]
 
     def nans_removed(self) -> HaloHpl:
-        is_ok = ~np.isnan(self.intensity).any(axis=1)
+        is_ok = np.array(~np.isnan(self.intensity).any(axis=1), dtype=np.bool_)
         return self[is_ok]
 
 
@@ -381,7 +332,7 @@ def _convert_time(
     decimal_time: hours since beginning of the day of start_time
     """
     HOURS_TO_MICROSECONDS = 3600000000.0
-    start_of_day = datetime64(start_time, "D").astype("datetime64[us]")
+    start_of_day = start_time.astype("datetime64[D]").astype("datetime64[us]")
     delta_hours = (decimal_time * HOURS_TO_MICROSECONDS).astype("timedelta64[us]")
     return np.array(start_of_day + delta_hours, dtype=datetime64)
 

doppy/raw/halo_sys_params.py

@@ -20,8 +20,8 @@ class HaloSysParams:
     internal_relative_humidity: npt.NDArray[np.float64]  # dim: (time, )
     supply_voltage: npt.NDArray[np.float64]  # dim: (time, )
     acquisition_card_temperature: npt.NDArray[np.float64]  # dim: (time, )
-    pitch: npt.NDArray[np.float64]  # dim: (time, )
-    roll: npt.NDArray[np.float64]  # dim: (time, )
+    platform_pitch_angle: npt.NDArray[np.float64]  # dim: (time, ), unit: degrees
+    platform_roll_angle: npt.NDArray[np.float64]  # dim: (time, ), unit: degrees
 
     @classmethod
     def from_src(cls, data: str | Path | bytes | BufferedIOBase) -> HaloSysParams:
@@ -47,8 +47,8 @@ class HaloSysParams:
             np.concatenate(tuple(r.internal_relative_humidity for r in raws)),
             np.concatenate(tuple(r.supply_voltage for r in raws)),
             np.concatenate(tuple(r.acquisition_card_temperature for r in raws)),
-            np.concatenate(tuple(r.pitch for r in raws)),
-            np.concatenate(tuple(r.roll for r in raws)),
+            np.concatenate(tuple(r.platform_pitch_angle for r in raws)),
+            np.concatenate(tuple(r.platform_roll_angle for r in raws)),
         )
 
     def __getitem__(
@@ -62,8 +62,8 @@ class HaloSysParams:
                 self.internal_relative_humidity[index],
                 self.supply_voltage[index],
                 self.acquisition_card_temperature[index],
-                self.pitch[index],
-                self.roll[index],
+                self.platform_pitch_angle[index],
+                self.platform_roll_angle[index],
             )
         raise TypeError
 

doppy/raw/utils.py

@@ -0,0 +1,14 @@
+from io import BufferedIOBase
+from pathlib import Path
+
+
+def bytes_from_src(src: str | bytes | Path | BufferedIOBase) -> bytes:
+    if isinstance(src, (str, Path)):
+        with open(src, "rb") as f:
+            return f.read()
+    elif isinstance(src, bytes):
+        return src
+    elif isinstance(src, BufferedIOBase):
+        return src.read()
+    else:
+        raise TypeError(f"Unexpected type {type(src)} for src")

doppy/raw/windcube.py

@@ -24,7 +24,7 @@ class WindCubeFixed:
     radial_velocity: npt.NDArray[np.float64]  # dim: (time, radial_distance)
     doppler_spectrum_width: npt.NDArray[np.float64]  # dim: (time, radial_distance)
     radial_velocity_confidence: npt.NDArray[np.float64]  # dim: (time, radial_distance)
-    ray_accumulation_time: np.float64  # dim: ()
+    ray_accumulation_time: np.float64  # dim: (), unit: seconds
     system_id: str
 
     @classmethod
@@ -97,7 +97,7 @@ class WindCubeFixed:
         return self[sort_indices]
 
     def nan_profiles_removed(self) -> WindCubeFixed:
-        return self[~np.all(np.isnan(self.cnr), axis=1)]
+        return self[np.array(~np.all(np.isnan(self.cnr), axis=1), dtype=np.bool)]
 
 
 @dataclass
@@ -300,6 +300,7 @@ def _from_fixed_src(nc: Dataset) -> WindCubeFixed:
             _extract_float64_or_raise(
                 group["ray_accumulation_time"], expected_dimensions
             )
+            * 1e-3  # convert ms to s
         )
 
     return WindCubeFixed(
@@ -314,7 +315,7 @@ def _from_fixed_src(nc: Dataset) -> WindCubeFixed:
         doppler_spectrum_width=np.concatenate(doppler_spectrum_width_list),
         ray_accumulation_time=merge_all_equal(
             "ray_accumulation_time",
-            np.array(ray_accumulation_time_list, dtype=np.float64).tolist(),
+            list(np.array(ray_accumulation_time_list, dtype=np.float64)),
         ),
         system_id=nc.instrument_name,
     )

doppy/raw/wls70.py

@@ -12,6 +12,7 @@ from numpy import datetime64
 
 import doppy
 from doppy import exceptions
+from doppy.raw.utils import bytes_from_src
 from doppy.utils import merge_all_equal
 
 
@@ -37,30 +38,9 @@ class Wls70:
 
     @classmethod
     def from_srcs(
-        cls,
-        data: Sequence[str]
-        | Sequence[Path]
-        | Sequence[bytes]
-        | Sequence[BufferedIOBase],
+        cls, data: Sequence[str | bytes | Path | BufferedIOBase]
     ) -> list[Wls70]:
-        if not isinstance(data, (list, tuple)):
-            raise TypeError("data should be list or tuple")
-        if all(isinstance(src, bytes) for src in data):
-            data_bytes = data
-        elif all(isinstance(src, str) for src in data):
-            data_bytes = []
-            for src in data:
-                with Path(src).open("rb") as f:
-                    data_bytes.append(f.read())
-        elif all(isinstance(src, Path) for src in data):
-            data_bytes = []
-            for src in data:
-                with src.open("rb") as f:
-                    data_bytes.append(f.read())
-        elif all(isinstance(src, BufferedIOBase) for src in data):
-            data_bytes = [src.read() for src in data]
-        else:
-            raise TypeError("Unexpected types in data")
+        data_bytes = [bytes_from_src(src) for src in data]
         raws = doppy.rs.raw.wls70.from_bytes_srcs(data_bytes)
         try:
             return [_raw_rs_to_wls70(r) for r in raws]
@@ -69,19 +49,7 @@ class Wls70:
 
     @classmethod
     def from_src(cls, data: str | Path | bytes | BufferedIOBase) -> Wls70:
-        if isinstance(data, str):
-            path = Path(data)
-            with path.open("rb") as f:
-                data_bytes = f.read()
-        elif isinstance(data, Path):
-            with data.open("rb") as f:
-                data_bytes = f.read()
-        elif isinstance(data, bytes):
-            data_bytes = data
-        elif isinstance(data, BufferedIOBase):
-            data_bytes = data.read()
-        else:
-            raise TypeError("Unsupported data type")
+        data_bytes = bytes_from_src(data)
         try:
             return _raw_rs_to_wls70(doppy.rs.raw.wls70.from_bytes_src(data_bytes))
         except RuntimeError as err:

{doppy-0.4.1.dist-info → doppy-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: doppy
-Version: 0.4.1
+Version: 0.5.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
@@ -19,11 +19,12 @@ Requires-Dist: matplotlib
 Requires-Dist: scikit-learn
 Requires-Dist: scipy
 Requires-Dist: mypy ; extra == 'dev'
+Requires-Dist: pyright ; extra == 'dev'
 Requires-Dist: ruff ; extra == 'dev'
 Requires-Dist: pytest ; extra == 'dev'
 Requires-Dist: types-requests ; extra == 'dev'
 Requires-Dist: py-spy ; extra == 'dev'
-Requires-Dist: maturin ==1.4 ; extra == 'dev'
+Requires-Dist: maturin==1.8 ; extra == 'dev'
 Requires-Dist: release-version ; extra == 'dev'
 Requires-Dist: pre-commit ; extra == 'dev'
 Requires-Dist: xarray[io] ; extra == 'dev'

doppy-0.5.0.dist-info/RECORD

@@ -0,0 +1,32 @@
+doppy-0.5.0.dist-info/METADATA,sha256=WBN2SRmIlip8lZCFIOTt2XB3lQThw406jgHb8i7uGS0,4414
+doppy-0.5.0.dist-info/WHEEL,sha256=md9qofgGs0CN4-5nDhVd0IzxxzM8cFGCnkeRThUNGbI,95
+doppy-0.5.0.dist-info/entry_points.txt,sha256=9b_Ca7vJoh6AwL3W8qAPh_UmJ_1Pa6hi-TDfCTDjvSk,43
+doppy-0.5.0.dist-info/licenses/LICENSE,sha256=RIAxFjJLTw0wQ3_SM73JoTeppoD99DJJ72cjvVuRrW4,1110
+doppy/bench.py,sha256=X4yPXFPAM708ptzXKDYxVBOC0E3cMj_KxXzmD_Q9_RM,321
+doppy/data/api.py,sha256=IoLRodfITNHs4Pb2IvVvOjSirSeK-xNoSO4tsQYjsKc,1933
+doppy/data/cache.py,sha256=ERFzH-KjLLXV8fCePsewTepKZXmLwtXxeazmHiFdA80,1203
+doppy/data/exceptions.py,sha256=JOyekvUO-Ew4ZVezf3_IxZOrPN0IksfUILd8R2YcSts,95
+doppy/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+doppy/defaults.py,sha256=jZvGmHJkvOzFpwy97L6Sj9-bnF0-nvq0-zNNWxM_WpE,490
+doppy/exceptions.py,sha256=IUF8D_d-QOvtemUSEMg1raXDWA3dSXQ2MUBvlwgm1rU,197
+doppy/netcdf.py,sha256=1mydrGnBBb4IxKCeVrXbwsS0PoOCObww3PfJbVciFWA,4188
+doppy/options.py,sha256=uyIKM_G2GtbmV6Gve8o13eIShQqUwsnYZ41mhX2ypGE,218
+doppy/product/noise_utils.py,sha256=Bro3T8uMY9MmJcrXbOl_QKIU7OXVRUs_R6Norl5Tv3Q,2735
+doppy/product/stare.py,sha256=A_Xy6nIScnCUlEoziZ164aIW1HVqE7WogCcL5Ljpc_o,28009
+doppy/product/stare_depol.py,sha256=GkS-60Izk-rF9PnHPtOgAssr--K4ddzQHUlyLZ7JNkU,11026
+doppy/product/turbulence.py,sha256=IpqZ5ayxUrIX4J6C-3AcLTMem4OLHquKtvEC3zSEpkg,8516
+doppy/product/wind.py,sha256=SbVM_AyLJtIggOb3PHLTak6majHaBMFJHNX36wRti1k,17298
+doppy/product/__init__.py,sha256=xoBEXuhid-bvoof5Ogzpt1dKIhJgNMRyrAinCqUOUUI,241
+doppy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+doppy/raw/halo_bg.py,sha256=TrqJDUXa1atW2w4UDN9PPx2Mzrc7D9Q2Ndph0yB65p0,5988
+doppy/raw/halo_hpl.py,sha256=ap9WGBSHCYtLDZ89TaTXbHJHIwT6y3y-U0wO1kCR0o8,17304
+doppy/raw/halo_sys_params.py,sha256=9aZpAgaLFniADjYv7EO8wKtmBD6McJP2v7YUz94hrTM,5385
+doppy/raw/utils.py,sha256=pBeBk87_LMxop14_RK5Xj64iYBMu8AhH-IqBRoGgUzE,436
+doppy/raw/windcube.py,sha256=FBoiLHXNluN2HISe6OO5TE32BtZYTuvSwyvcYIafLtE,19626
+doppy/raw/wls70.py,sha256=lWSnuZlmdCGqrlYt6SUDFvPRBOOdtt6LiDiu_nhnPiU,6494
+doppy/raw/__init__.py,sha256=ycT-_mCiI19b2TH33L74-Eq_MYUTPe3B8NhwNodSx1E,267
+doppy/utils.py,sha256=2PwDiO8GyMjsHC-EYcTGyN2mEfPaCi3hhEn2twETOl0,814
+doppy/__init__.py,sha256=Af7_8p3oN1nTqS9fo0mVKVuiKf5CAEK69uQa32CSFBA,197
+doppy/__main__.py,sha256=38hIWWfanILuBBGorQiAaleSC4qYJoIxuzVBkxf7Dng,371
+doppy/rs.pyd,sha256=ZYCEafdpDXk_D8aBrwTA4lRvBXqY-4Y9GhAJIJtI0Sc,2164736
+doppy-0.5.0.dist-info/RECORD,,

{doppy-0.4.1.dist-info → doppy-0.5.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: maturin (1.7.8)
+Generator: maturin (1.8.2)
 Root-Is-Purelib: false
 Tag: cp310-abi3-win_amd64

doppy-0.4.1.dist-info/RECORD

@@ -1,29 +0,0 @@
-doppy-0.4.1.dist-info/METADATA,sha256=1GDlAOhuqFL8D874SwK8J86zaMKRaI9FJZnUHy951N4,4375
-doppy-0.4.1.dist-info/WHEEL,sha256=ff_aotEPDeP8au5CC0c1B8Qp-UHc3TEJkBgH6hZZ_jU,95
-doppy-0.4.1.dist-info/entry_points.txt,sha256=9b_Ca7vJoh6AwL3W8qAPh_UmJ_1Pa6hi-TDfCTDjvSk,43
-doppy-0.4.1.dist-info/licenses/LICENSE,sha256=RIAxFjJLTw0wQ3_SM73JoTeppoD99DJJ72cjvVuRrW4,1110
-doppy/bench.py,sha256=fLN2iS5mmoYH4qZjD80Vl1h9lp3C-KDfhj9fteWRPtM,260
-doppy/data/api.py,sha256=c3zbZI3IV7HnzhyFzJpUPOFT20eYFuQlJ5K_1ZEe1Pg,1921
-doppy/data/cache.py,sha256=ERFzH-KjLLXV8fCePsewTepKZXmLwtXxeazmHiFdA80,1203
-doppy/data/exceptions.py,sha256=JOyekvUO-Ew4ZVezf3_IxZOrPN0IksfUILd8R2YcSts,95
-doppy/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-doppy/defaults.py,sha256=jZvGmHJkvOzFpwy97L6Sj9-bnF0-nvq0-zNNWxM_WpE,490
-doppy/exceptions.py,sha256=IUF8D_d-QOvtemUSEMg1raXDWA3dSXQ2MUBvlwgm1rU,197
-doppy/netcdf.py,sha256=W5YPbsv6CQ2Ycy4sFaIyjfn9YjvDgygczm8j9E1_uAg,4054
-doppy/options.py,sha256=uyIKM_G2GtbmV6Gve8o13eIShQqUwsnYZ41mhX2ypGE,218
-doppy/product/stare.py,sha256=d5RnRD5tUWkbfKgdUhlpX13tXfY6BSkQ9uiANH_38b0,26122
-doppy/product/stare_depol.py,sha256=D0bVRZT4EP-pYHE84lioUxiuB3a2Cl6LvOgCE-YPJZY,9938
-doppy/product/wind.py,sha256=lGkvGf_QeOXtfT5kQEq5GuxKuSsLxTGVwRzoioF525o,17061
-doppy/product/__init__.py,sha256=xoBEXuhid-bvoof5Ogzpt1dKIhJgNMRyrAinCqUOUUI,241
-doppy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-doppy/raw/halo_bg.py,sha256=TrqJDUXa1atW2w4UDN9PPx2Mzrc7D9Q2Ndph0yB65p0,5988
-doppy/raw/halo_hpl.py,sha256=l9i_TTJ4BY5eE3qAEAiLKK-mXd-a2BseEyDUMvMEanI,19105
-doppy/raw/halo_sys_params.py,sha256=JBBQykXC9ozut_e4EvuTIAspUmjnB7FShCJoRiNEm54,5265
-doppy/raw/windcube.py,sha256=LB7Do9Bn4ygBy8tMBM9Rc1ZZKU_6WX88UHELKiTPJfQ,19550
-doppy/raw/wls70.py,sha256=J3ABVj_YrjSgmyRt6DGNoZUZIEZEzEoKaY5yiN9hE94,7717
-doppy/raw/__init__.py,sha256=ycT-_mCiI19b2TH33L74-Eq_MYUTPe3B8NhwNodSx1E,267
-doppy/utils.py,sha256=2PwDiO8GyMjsHC-EYcTGyN2mEfPaCi3hhEn2twETOl0,814
-doppy/__init__.py,sha256=Af7_8p3oN1nTqS9fo0mVKVuiKf5CAEK69uQa32CSFBA,197
-doppy/__main__.py,sha256=38hIWWfanILuBBGorQiAaleSC4qYJoIxuzVBkxf7Dng,371
-doppy/rs.pyd,sha256=2BsMpXq8hxnXADpS9lXWiLZN6nxiu5_5iFw1p4GhUQQ,2130944
-doppy-0.4.1.dist-info/RECORD,,