doppy 0.4.2__cp310-abi3-macosx_10_12_x86_64.whl → 0.5.0__cp310-abi3-macosx_10_12_x86_64.whl

doppy/bench.py

@@ -3,10 +3,11 @@ import time
 
 class Timer:
     def __init__(self):
-        self.start = None
+        self.start: float | None = None
 
     def __enter__(self):
         self.start = time.time()
 
     def __exit__(self, type, value, traceback):
-        print(f"Elapsed time: {time.time() - self.start:.2f}")
+        if isinstance(self.start, float):
+            print(f"Elapsed time: {time.time() - self.start:.2f}")

doppy/data/api.py

@@ -20,7 +20,7 @@ class Api:
         self.api_endpoint = "https://cloudnet.fmi.fi/api"
         self.cache = cache
 
-    def get(self, path: str, params: dict[str, str]) -> list:
+    def get(self, path: str, params: dict[str, str | list[str]]) -> list:
         res = self.session.get(
             f"{self.api_endpoint}/{path}", params=params, timeout=1800
         )

doppy/netcdf.py

@@ -13,8 +13,12 @@ NetCDFDataType: TypeAlias = Literal["f4", "f8", "i4", "i8", "u4", "u8"]
 
 
 class Dataset:
-    def __init__(self, filename: str | pathlib.Path) -> None:
-        self.nc = netCDF4.Dataset(filename, mode="w")
+    def __init__(
+        self,
+        filename: str | pathlib.Path,
+        format: Literal["NETCDF4", "NETCDF4_CLASSIC"] = "NETCDF4",
+    ) -> None:
+        self.nc = netCDF4.Dataset(filename, mode="w", format=format)
 
     def __enter__(self) -> Dataset:
         return self
@@ -27,8 +31,8 @@ class Dataset:
     ) -> None:
         self.close()
 
-    def add_dimension(self, dim: str) -> Dataset:
-        self.nc.createDimension(dim, None)
+    def add_dimension(self, dim: str, size: int | None = None) -> Dataset:
+        self.nc.createDimension(dim, size)
         return self
 
     def add_attribute(self, key: str, val: str) -> Dataset:

doppy/product/noise_utils.py

@@ -0,0 +1,106 @@
+import warnings
+
+import numpy as np
+import numpy.typing as npt
+import scipy
+
+
+def detect_wind_noise(
+    w: npt.NDArray[np.float64],
+    height: npt.NDArray[np.float64],
+    mask: npt.NDArray[np.bool_],
+    window: float = 150,
+    stride: int = 1,
+) -> npt.NDArray[np.bool_]:
+    """
+    Parameters
+    ----------
+    w
+        vertical velocity, dims: (time,height)
+
+    height
+        dims: (height,)
+
+    mask
+        old mask that still contains noisy velocity data,
+        mask[t,h] = True iff w[t,h] is noise
+
+    window
+        size of window used to compute rolling median in meters
+
+    stride
+        stride used to compute rolling median
+
+    Returns
+    -------
+    improved noise mask such that new_mask[t,h] = True iff w[t,h] is noise
+    """
+    warnings.simplefilter("ignore", RuntimeWarning)
+    v = _rolling_median_over_range(
+        height,
+        w,
+        mask,
+        window=window,  # meters
+        stride=stride,
+        fill_gaps=True,
+    )
+
+    th = 2
+    diff = np.abs(v - w)
+    new_mask = (diff > th) | mask
+    new_mask = _remove_one_hot(new_mask)
+    return np.array(new_mask, dtype=np.bool_)
+
+
+def _remove_one_hot(m: npt.NDArray[np.bool_]) -> npt.NDArray[np.bool_]:
+    if m.ndim != 2:
+        raise ValueError
+    if m.shape[1] < 3:
+        return m
+    x = ~m
+    y = np.full(x.shape, np.False_)
+    y[:, 0] = x[:, 0] & x[:, 1]
+    y[:, 1:-1] = x[:, 1:-1] & (x[:, 2:] | x[:, :-2])
+    y[:, -1] = x[:, -1] & x[:, -2]
+    return np.array(~y, dtype=np.bool_)
+
+
+def _rolling_median_over_range(
+    range_: npt.NDArray[np.float64],
+    arr: npt.NDArray[np.float64],
+    mask: npt.NDArray[np.bool_],
+    window: float,
+    stride: int = 1,
+    fill_gaps: bool = False,
+) -> npt.NDArray[np.float64]:
+    """
+    window
+        range window in meters
+    """
+    X = arr.T.copy()
+    X[mask.T] = np.nan
+
+    half_window = window / 2
+
+    i = 0
+    j = 0
+    n = len(range_)
+    med = np.full(X.shape, np.nan, dtype=np.float64)
+    for k in range(0, n, stride):
+        r = range_[k]
+        while i + 1 < n and r - range_[i + 1] >= half_window:
+            i += 1
+        while j + 1 < n and range_[j] - r < half_window:
+            j += 1
+        if i > k or j < k:
+            raise ValueError
+        med[k] = np.nanmedian(X[i : j + 1], axis=0)
+
+    if stride != 1 and fill_gaps:
+        ind = list(range(0, n, stride))
+        f_interp = scipy.interpolate.interp1d(
+            range_[ind], med[ind], axis=0, fill_value="extrapolate"
+        )
+        med_all = f_interp(range_)
+        return np.array(med_all.T.copy(), dtype=np.float64)
+    return np.array(med.T.copy(), dtype=np.float64)

doppy/product/stare.py

@@ -14,20 +14,35 @@ from sklearn.cluster import KMeans
 
 import doppy
 from doppy import defaults, options
+from doppy.product.noise_utils import detect_wind_noise
 
 SelectionGroupKeyType: TypeAlias = tuple[int,]
 
 
-@dataclass
+@dataclass(slots=True)
+class RayAccumulationTime:
+    # in seconds
+    value: float
+
+
+@dataclass(slots=True)
+class PulsesPerRay:
+    value: int
+
+
+@dataclass(slots=True)
 class Stare:
     time: npt.NDArray[np.datetime64]
     radial_distance: npt.NDArray[np.float64]
     elevation: npt.NDArray[np.float64]
     beta: npt.NDArray[np.float64]
+    snr: npt.NDArray[np.float64]
     radial_velocity: npt.NDArray[np.float64]
-    mask: npt.NDArray[np.bool_]
+    mask_beta: npt.NDArray[np.bool_]
+    mask_radial_velocity: npt.NDArray[np.bool_]
     wavelength: float
     system_id: str
+    ray_info: RayAccumulationTime | PulsesPerRay
 
     def __getitem__(
         self,
@@ -44,10 +59,13 @@ class Stare:
                 radial_distance=self.radial_distance,
                 elevation=self.elevation[index],
                 beta=self.beta[index],
+                snr=self.snr[index],
                 radial_velocity=self.radial_velocity[index],
-                mask=self.mask[index],
+                mask_beta=self.mask_beta[index],
+                mask_radial_velocity=self.mask_radial_velocity[index],
                 wavelength=self.wavelength,
                 system_id=self.system_id,
+                ray_info=self.ray_info,
             )
         raise TypeError
 
@@ -79,16 +97,23 @@ class Stare:
             effective_diameter=defaults.WindCube.effective_diameter,
         )
 
-        mask = _compute_noise_mask_for_windcube(raw)
+        mask_beta = _compute_noise_mask_for_windcube(raw)
+        mask_radial_velocity = detect_wind_noise(
+            raw.radial_velocity, raw.radial_distance, mask_beta
+        )
+
         return cls(
             time=raw.time,
             radial_distance=raw.radial_distance,
             elevation=raw.elevation,
             beta=beta,
+            snr=raw.cnr,
             radial_velocity=raw.radial_velocity,
-            mask=mask,
+            mask_beta=mask_beta,
+            mask_radial_velocity=mask_radial_velocity,
             wavelength=wavelength,
             system_id=raw.system_id,
+            ray_info=RayAccumulationTime(raw.ray_accumulation_time),
         )
 
     @classmethod
@@ -146,18 +171,25 @@ class Stare:
             effective_diameter=defaults.Halo.effective_diameter,
         )
 
-        mask = _compute_noise_mask(
+        mask_beta = _compute_noise_mask(
             intensity_noise_bias_corrected, raw.radial_velocity, raw.radial_distance
         )
+        mask_radial_velocity = detect_wind_noise(
+            raw.radial_velocity, raw.radial_distance, mask_beta
+        )
+
         return cls(
             time=raw.time,
             radial_distance=raw.radial_distance,
             elevation=raw.elevation,
             beta=beta,
+            snr=intensity_noise_bias_corrected - 1,
             radial_velocity=raw.radial_velocity,
-            mask=mask,
+            mask_beta=mask_beta,
+            mask_radial_velocity=mask_radial_velocity,
             wavelength=wavelength,
             system_id=raw.header.system_id,
+            ray_info=PulsesPerRay(raw.header.pulses_per_ray),
         )
 
     def write_to_netcdf(self, filename: str | Path) -> None:
@@ -200,7 +232,7 @@ class Stare:
                 units="sr-1 m-1",
                 data=self.beta,
                 dtype="f4",
-                mask=self.mask,
+                mask=self.mask_beta,
             )
             nc.add_variable(
                 name="v",
@@ -209,7 +241,7 @@ class Stare:
                 long_name="Doppler velocity",
                 data=self.radial_velocity,
                 dtype="f4",
-                mask=self.mask,
+                mask=self.mask_radial_velocity,
             )
             nc.add_scalar_variable(
                 name="wavelength",
@@ -218,6 +250,24 @@ class Stare:
                 data=self.wavelength,
                 dtype="f4",
             )
+            match self.ray_info:
+                case RayAccumulationTime(value):
+                    nc.add_scalar_variable(
+                        name="ray_accumulation_time",
+                        units="s",
+                        long_name="ray accumulation time",
+                        data=value,
+                        dtype="f4",
+                    )
+                case PulsesPerRay(value):
+                    nc.add_scalar_variable(
+                        name="pulses_per_ray",
+                        units="1",
+                        long_name="pulses per ray",
+                        data=value,
+                        dtype="u4",
+                    )
+
             nc.add_attribute("serial_number", self.system_id)
             nc.add_attribute("doppy_version", doppy.__version__)
 
@@ -232,7 +282,7 @@ def _compute_noise_mask_for_windcube(
 
     cnr = raw.cnr.copy()
     cnr[mask] = np.finfo(float).eps
-    cnr_filt = median_filter(cnr, size=(3, 3))
+    cnr_filt = np.array(median_filter(cnr, size=(3, 3)), dtype=np.float64)
     rel_diff = np.abs(cnr - cnr_filt) / np.abs(cnr)
     diff_mask = rel_diff > 0.25
 

doppy/product/stare_depol.py

@@ -10,7 +10,7 @@ import numpy.typing as npt
 
 import doppy
 from doppy import options
-from doppy.product.stare import Stare
+from doppy.product.stare import PulsesPerRay, RayAccumulationTime, Stare
 
 
 @dataclass
@@ -36,7 +36,9 @@ class StareDepol:
         sr-1 m-1.
     radial_velocity
         An array of radial velocities of the co-polarised signal, in m s-1.
-    mask
+    mask_beta
+        A boolean array indicating signal (True) or noise (False) data points.
+    mask_radial_velocity
         A boolean array indicating signal (True) or noise (False) data points.
     depolarisation
         An array of depolarisation ratios calculated as the ratio of
@@ -69,11 +71,13 @@ class StareDepol:
     beta: npt.NDArray[np.float64]
     beta_cross: npt.NDArray[np.float64]
     radial_velocity: npt.NDArray[np.float64]
-    mask: npt.NDArray[np.bool_]
+    mask_beta: npt.NDArray[np.bool_]
+    mask_radial_velocity: npt.NDArray[np.bool_]
     depolarisation: npt.NDArray[np.float64]
     polariser_bleed_through: float
     wavelength: float
     system_id: str
+    ray_info: RayAccumulationTime | PulsesPerRay
 
     def __init__(
         self,
@@ -138,11 +142,13 @@ class StareDepol:
         self.beta = co.beta
         self.beta_cross = cross_beta
         self.radial_velocity = co.radial_velocity
-        self.mask = co.mask
+        self.mask_beta = co.mask_beta
+        self.mask_radial_velocity = co.mask_radial_velocity
         self.depolarisation = depolarisation
         self.polariser_bleed_through = polariser_bleed_through
         self.wavelength = co.wavelength
         self.system_id = co.system_id
+        self.ray_info = co.ray_info
 
     @property
     def mask_depolarisation(self) -> npt.NDArray[np.bool_]:
@@ -224,7 +230,7 @@ class StareDepol:
                 units="sr-1 m-1",
                 data=self.beta,
                 dtype="f4",
-                mask=self.mask,
+                mask=self.mask_beta,
             )
             nc.add_variable(
                 name="v",
@@ -233,7 +239,7 @@ class StareDepol:
                 long_name="Doppler velocity",
                 data=self.radial_velocity,
                 dtype="f4",
-                mask=self.mask,
+                mask=self.mask_radial_velocity,
             )
             nc.add_scalar_variable(
                 name="wavelength",
@@ -256,7 +262,7 @@ class StareDepol:
                 units="1",
                 data=self.depolarisation,
                 dtype="f4",
-                mask=self.mask | self.mask_depolarisation,
+                mask=self.mask_beta | self.mask_depolarisation,
             )
             nc.add_variable(
                 name="beta_cross_raw",
@@ -271,7 +277,7 @@ class StareDepol:
                 dimensions=("time", "range"),
                 units="sr-1 m-1",
                 data=self.beta_cross,
-                mask=self.mask | self.mask_beta_cross,
+                mask=self.mask_beta | self.mask_beta_cross,
                 dtype="f4",
             )
             nc.add_scalar_variable(
@@ -281,5 +287,22 @@ class StareDepol:
                 data=self.polariser_bleed_through,
                 dtype="f4",
             )
+            match self.ray_info:
+                case RayAccumulationTime(value):
+                    nc.add_scalar_variable(
+                        name="ray_accumulation_time",
+                        units="s",
+                        long_name="ray accumulation time",
+                        data=value,
+                        dtype="f4",
+                    )
+                case PulsesPerRay(value):
+                    nc.add_scalar_variable(
+                        name="pulses_per_ray",
+                        units="1",
+                        long_name="pulses per ray",
+                        data=value,
+                        dtype="u4",
+                    )
             nc.add_attribute("serial_number", self.system_id)
             nc.add_attribute("doppy_version", doppy.__version__)

doppy/product/turbulence.py

@@ -0,0 +1,265 @@
+# type: ignore
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+import numpy.typing as npt
+from scipy.interpolate import RegularGridInterpolator
+
+
+@dataclass
+class HorizontalWind:
+    time: npt.NDArray[np.datetime64]
+    height: npt.NDArray[np.float64]  # Height in meters from reference
+    V: npt.NDArray[np.float64]  # Horizontal wind speed in m/s
+
+
+@dataclass
+class VerticalWind:
+    time: npt.NDArray[np.datetime64]
+    height: npt.NDArray[np.float64]  # Height in meters from reference
+    w: npt.NDArray[np.float64]  # Vertical wind speed in m/s
+    mask: npt.NDArray[np.bool_]  # mask[t,h] = True iff w[t,h] should be masked
+
+
+@dataclass
+class Options:
+    ray_accumulation_time: float  # in seconds
+    period: float = 600  # period for computing the variance in seconds
+    beam_divergence: float = 33e-6  # radians
+
+
+@dataclass
+class Turbulence:
+    time: npt.NDArray[np.datetime64]
+    height: npt.NDArray[np.float64]
+    turbulent_kinetic_energy_dissipation_rate: npt.NDArray[np.float64]
+    mask: npt.NDArray[np.bool_]
+
+    @classmethod
+    def from_winds(
+        cls, vert: VerticalWind, hori: HorizontalWind, options: Options
+    ) -> Turbulence:
+        V = _preprocess_horiontal_wind(vert, hori, options)
+        ls_low = _length_scale_low(V, vert.height, options)
+        res = _compute_variance(vert, options.period)
+        sampling_time = _sampling_time_in_seconds(res)
+        ls_up = V * sampling_time
+        dissipation_rate = _compute_dissipation_rate(res.variance, ls_low, ls_up)
+        mask = np.isnan(dissipation_rate) | vert.mask
+        return cls(
+            time=vert.time.copy(),
+            height=vert.height.copy(),
+            turbulent_kinetic_energy_dissipation_rate=dissipation_rate,
+            mask=mask,
+        )
+
+
+def _sampling_time_in_seconds(r: VarResult) -> npt.NDArray[np.float64]:
+    if not all(
+        (
+            t == np.dtype("datetime64[us]")
+            for t in (r.period_start.dtype, r.period_stop.dtype)
+        )
+    ):
+        raise ValueError("period times must be on datetime64[us]")
+    td = r.period_stop - r.period_start
+    td_in_seconds = td / np.timedelta64(1, "s")
+    return np.array(td_in_seconds, dtype=np.float64)
+
+
+@dataclass
+class VarResult:
+    variance: npt.NDArray[np.float64]
+    period_start: npt.NDArray[np.datetime64]
+    period_stop: npt.NDArray[np.datetime64]
+    nsamples: npt.NDArray[np.int64]
+
+
+def _compute_variance(vert: VerticalWind, period: float) -> VarResult:
+    # NOTE: numerically unstable
+
+    # To compute actual time window
+    next_valid = _next_valid_from_mask(vert.mask)
+    prev_valid = _prev_valid_from_mask(vert.mask)
+
+    X = vert.w.copy()
+    X[vert.mask] = 0
+    X2 = X**2
+    X_cumsum = X.cumsum(axis=0)
+    X2_cumsum = X2.cumsum(axis=0)
+
+    N_i = (~vert.mask).astype(int)
+    N_cumsum = N_i.cumsum(axis=0)
+
+    def N_func(i: int, j: int) -> npt.NDArray[np.float64]:
+        return N_cumsum[j] - N_cumsum[i] + N_i[i]
+
+    def S(i: int, j: int) -> npt.NDArray[np.float64]:
+        return X_cumsum[j] - X_cumsum[i] + X[i]
+
+    def S2(i: int, j: int) -> npt.NDArray[np.float64]:
+        return X2_cumsum[j] - X2_cumsum[i] + X2[i]
+
+    def var_ij(i: int, j: int) -> npt.NDArray[np.float64]:
+        N = N_func(i, j)
+        with np.errstate(invalid="ignore"):
+            return (S2(i, j) - S(i, j) ** 2 / N) / N
+
+    half_period = np.timedelta64(int(1e6 * period / 2), "us")
+    period_start = np.full(vert.w.shape, np.datetime64("NaT", "us"))
+    period_stop = np.full(vert.w.shape, np.datetime64("NaT", "us"))
+    var = np.full(vert.w.shape, np.nan, dtype=np.float64)
+    nsamples = np.zeros_like(vert.w, dtype=np.int64)
+    i = 0
+    j = 0
+    n = len(vert.time)
+    for k, t in enumerate(vert.time):
+        while i + 1 < n and t - vert.time[i + 1] >= half_period:
+            i += 1
+        while j + 1 < n and vert.time[j] - t < half_period:
+            j += 1
+        i_valid = next_valid[i]
+        i_inbound = (0 <= i_valid) & (i_valid < n)
+        j_valid = prev_valid[j]
+        j_inbound = (0 <= j_valid) & (j_valid < n)
+        period_start[k][i_inbound] = vert.time[i_valid[i_inbound]]
+        period_stop[k][j_inbound] = vert.time[j_valid[j_inbound]]
+        var[k] = var_ij(i, j)
+        nsamples[k] = N_func(i, j)
+    return VarResult(
+        variance=var,
+        period_start=period_start,
+        period_stop=period_stop,
+        nsamples=nsamples,
+    )
+
+
+def _length_scale_low(
+    V: npt.NDArray[np.float64], height: npt.NDArray[np.float64], opts: Options
+) -> npt.NDArray[np.float64]:
+    integration_time = opts.ray_accumulation_time
+    from_beam = 2 * height * np.sin(opts.beam_divergence / 2)
+    from_wind = V * integration_time
+    return np.array(from_wind + from_beam[np.newaxis, :], dtype=np.float64)
+
+
+def _preprocess_horiontal_wind(
+    vert: VerticalWind, hori: HorizontalWind, options: Options
+):
+    if np.isnan(hori.V).any():
+        raise ValueError("horizontal wind speed cannot contains NaNs")
+    trg_points = np.meshgrid(vert.time, vert.height, indexing="ij")
+    src_points = (hori.time, hori.height)
+    src_vals = hori.V
+
+    interp_nearest = RegularGridInterpolator(
+        src_points,
+        src_vals,
+        method="nearest",
+        bounds_error=False,
+        fill_value=None,
+    )
+    interp_linear = RegularGridInterpolator(
+        src_points, src_vals, method="linear", bounds_error=False
+    )
+    V_nearest = interp_nearest(trg_points)
+    V_linear = interp_linear(trg_points)
+    V = V_linear
+    V[np.isnan(V)] = V_nearest[np.isnan(V)]
+    if np.isnan(V).any():
+        raise ValueError("Unexpected NaNs")
+    V_rmean = _rolling_mean_over_time(vert.time, V, options.period)
+    return V_rmean
+
+
+def _rolling_mean_over_time(
+    time: npt.NDArray[np.datetime64], arr: npt.NDArray[np.float64], period: float
+) -> npt.NDArray[np.float64]:
+    if arr.ndim != 2:
+        raise ValueError("number of dims on arr should be 2")
+    if time.ndim != 1 or time.shape[0] != arr.shape[0]:
+        raise ValueError("time and arr dimensions do not match")
+    if time.dtype != np.dtype("datetime64[us]"):
+        raise TypeError(f"Invalid time type: {time.dtype}")
+
+    S = arr.cumsum(axis=0)
+
+    def rolling_mean(i: int, j: int) -> npt.NDArray[np.flaat64]:
+        return (S[j] - S[i] + arr[i]) / (j - i + 1)
+
+    half_period = np.timedelta64(int(period * 0.5e6), "us")
+    rol_mean = np.full(arr.shape, np.nan, dtype=np.float64)
+
+    i = 0
+    j = 0
+    n = len(time)
+    for k, t in enumerate(time):
+        while i + 1 < n and t - time[i + 1] >= half_period:
+            i += 1
+        while j + 1 < n and time[j] - t < half_period:
+            j += 1
+        rol_mean[k] = rolling_mean(i, j)
+    return rol_mean
+
+
+def _compute_dissipation_rate(
+    variance: npt.NDArray[np.float64],
+    length_scale_lower: npt.NDArray[np.float64],
+    length_scale_upper: npt.NDArray[np.float64],
+):
+    """
+    Parameters
+    ----------
+    variance, length_scale_lower, and length_scale_upper
+        dimensions: (time,range)
+    """
+    kolmogorov_constant = 0.55
+    with np.errstate(invalid="ignore"):
+        dr = (
+            2
+            * np.pi
+            * (2 / (3 * kolmogorov_constant)) ** (3 / 2)
+            * variance ** (3 / 2)
+            * (length_scale_upper ** (2 / 3) - length_scale_lower ** (2 / 3))
+            ** (-3 / 2)
+        )
+    return dr
+
+
+def _next_valid_from_mask(mask):
+    """
+    mask[t,v] (time,value)
+
+    returns N[t,v] = i where i = min { j | j >= t and mask[j,v] == False}
+    if the set is non empty and N[t,v] = len(mask) otherwise
+    """
+    n = len(mask)
+    N = np.full(mask.shape, n)
+    if mask.size == 0:
+        return N
+    N[-1][~mask[-1]] = n - 1
+
+    for t in reversed(range(n - 1)):
+        N[t][~mask[t]] = t
+        N[t][mask[t]] = N[t + 1][mask[t]]
+    return N
+
+
+def _prev_valid_from_mask(mask):
+    """
+    mask[t,v] (time,value)
+
+    returns N[t,v] = i where i = max { j | j <= t and mask[j,v] == False}
+    if the set is non empty and N[t,v] = -1 otherwise
+    """
+    n = len(mask)
+    N = np.full(mask.shape, -1)
+    if mask.size == 0:
+        return N
+    N[0][~mask[0]] = 0
+    for t in range(1, n):
+        N[t][~mask[t]] = t
+        N[t][mask[t]] = N[t - 1][mask[t]]
+    return N

doppy/product/wind.py

@@ -326,6 +326,13 @@ def _compute_wind(
 
     rmse (range,):
         Root-mean-square error of radial velocity fit for each range gate.
+
+    References
+    ----------
+    An assessment of the performance of a 1.5 µm Doppler lidar for
+    operational vertical wind profiling based on a 1-year trial
+        authors: E. Päschke, R. Leinweber, and V. Lehmann
+        doi: 10.5194/amt-8-2251-2015
     """
     elevation = np.deg2rad(raw.elevation)
     azimuth = np.deg2rad(raw.azimuth)
@@ -364,9 +371,9 @@ def _compute_mask(
     rmse (time,range)
     """
 
-    def neighbour_diff(X: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]:
+    def neighbour_diff(X: npt.NDArray[np.float64]) -> np.float64:
         mdiff = np.max(np.abs(X - X[len(X) // 2]))
-        return np.array(mdiff, dtype=np.float64)
+        return np.float64(mdiff)
 
     WIND_NEIGHBOUR_DIFFERENCE = 20
     neighbour_mask = np.any(

doppy/raw/halo_hpl.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import functools
-import io
 import re
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
@@ -16,6 +15,7 @@ from numpy import datetime64, timedelta64
 
 import doppy
 from doppy import exceptions
+from doppy.raw.utils import bytes_from_src
 from doppy.utils import merge_all_equal
 
 
@@ -35,30 +35,9 @@ class HaloHpl:
 
     @classmethod
     def from_srcs(
-        cls,
-        data: Sequence[str]
-        | Sequence[Path]
-        | Sequence[bytes]
-        | Sequence[BufferedIOBase],
+        cls, data: Sequence[str | bytes | Path | BufferedIOBase]
     ) -> list[HaloHpl]:
-        if not isinstance(data, (list, tuple)):
-            raise TypeError("data should be list or tuple")
-        if all(isinstance(src, bytes) for src in data):
-            data_bytes = data
-        elif all(isinstance(src, str) for src in data):
-            data_bytes = []
-            for src in data:
-                with Path(src).open("rb") as f:
-                    data_bytes.append(f.read())
-        elif all(isinstance(src, Path) for src in data):
-            data_bytes = []
-            for src in data:
-                with src.open("rb") as f:
-                    data_bytes.append(f.read())
-        elif all(isinstance(src, BufferedIOBase) for src in data):
-            data_bytes = [src.read() for src in data]
-        else:
-            raise TypeError("Unexpected types in data")
+        data_bytes = [bytes_from_src(src) for src in data]
         raw_dicts = doppy.rs.raw.halo_hpl.from_bytes_srcs(data_bytes)
         try:
             return [_raw_tuple2halo_hpl(r) for r in raw_dicts]
@@ -67,40 +46,12 @@ class HaloHpl:
 
     @classmethod
     def from_src(cls, data: str | Path | bytes | BufferedIOBase) -> HaloHpl:
-        if isinstance(data, str):
-            path = Path(data)
-            with path.open("rb") as f:
-                data_bytes = f.read()
-        elif isinstance(data, Path):
-            with data.open("rb") as f:
-                data_bytes = f.read()
-        elif isinstance(data, bytes):
-            data_bytes = data
-        elif isinstance(data, BufferedIOBase):
-            data_bytes = data.read()
-        else:
-            raise TypeError("Unsupported data type")
+        data_bytes = bytes_from_src(data)
         try:
             return _raw_tuple2halo_hpl(doppy.rs.raw.halo_hpl.from_bytes_src(data_bytes))
         except RuntimeError as err:
             raise exceptions.RawParsingError(err) from err
 
-    @classmethod
-    def _py_from_src(cls, data: str | Path | bytes | BufferedIOBase) -> HaloHpl:
-        if isinstance(data, str):
-            path = Path(data)
-            with path.open("rb") as f:
-                return _from_src(f)
-        elif isinstance(data, Path):
-            with data.open("rb") as f:
-                return _from_src(f)
-        elif isinstance(data, bytes):
-            return _from_src(io.BytesIO(data))
-        elif isinstance(data, BufferedIOBase):
-            return _from_src(data)
-        else:
-            raise TypeError("Unsupported data type")
-
     def __getitem__(
         self,
         index: int
@@ -187,7 +138,7 @@ class HaloHpl:
         return self[mask]
 
     def nans_removed(self) -> HaloHpl:
-        is_ok = ~np.isnan(self.intensity).any(axis=1)
+        is_ok = np.array(~np.isnan(self.intensity).any(axis=1), dtype=np.bool_)
         return self[is_ok]
 
 
@@ -381,7 +332,7 @@ def _convert_time(
     decimal_time: hours since beginning of the day of start_time
     """
     HOURS_TO_MICROSECONDS = 3600000000.0
-    start_of_day = datetime64(start_time, "D").astype("datetime64[us]")
+    start_of_day = start_time.astype("datetime64[D]").astype("datetime64[us]")
     delta_hours = (decimal_time * HOURS_TO_MICROSECONDS).astype("timedelta64[us]")
     return np.array(start_of_day + delta_hours, dtype=datetime64)
 

doppy/raw/utils.py

@@ -0,0 +1,14 @@
+from io import BufferedIOBase
+from pathlib import Path
+
+
+def bytes_from_src(src: str | bytes | Path | BufferedIOBase) -> bytes:
+    if isinstance(src, (str, Path)):
+        with open(src, "rb") as f:
+            return f.read()
+    elif isinstance(src, bytes):
+        return src
+    elif isinstance(src, BufferedIOBase):
+        return src.read()
+    else:
+        raise TypeError(f"Unexpected type {type(src)} for src")

doppy/raw/windcube.py

@@ -24,7 +24,7 @@ class WindCubeFixed:
     radial_velocity: npt.NDArray[np.float64]  # dim: (time, radial_distance)
     doppler_spectrum_width: npt.NDArray[np.float64]  # dim: (time, radial_distance)
     radial_velocity_confidence: npt.NDArray[np.float64]  # dim: (time, radial_distance)
-    ray_accumulation_time: np.float64  # dim: ()
+    ray_accumulation_time: np.float64  # dim: (), unit: seconds
     system_id: str
 
     @classmethod
@@ -97,7 +97,7 @@ class WindCubeFixed:
         return self[sort_indices]
 
     def nan_profiles_removed(self) -> WindCubeFixed:
-        return self[~np.all(np.isnan(self.cnr), axis=1)]
+        return self[np.array(~np.all(np.isnan(self.cnr), axis=1), dtype=np.bool)]
 
 
 @dataclass
@@ -300,6 +300,7 @@ def _from_fixed_src(nc: Dataset) -> WindCubeFixed:
             _extract_float64_or_raise(
                 group["ray_accumulation_time"], expected_dimensions
             )
+            * 1e-3  # convert ms to s
         )
 
     return WindCubeFixed(
@@ -314,7 +315,7 @@ def _from_fixed_src(nc: Dataset) -> WindCubeFixed:
         doppler_spectrum_width=np.concatenate(doppler_spectrum_width_list),
         ray_accumulation_time=merge_all_equal(
             "ray_accumulation_time",
-            np.array(ray_accumulation_time_list, dtype=np.float64).tolist(),
+            list(np.array(ray_accumulation_time_list, dtype=np.float64)),
         ),
         system_id=nc.instrument_name,
     )

doppy/raw/wls70.py

@@ -12,6 +12,7 @@ from numpy import datetime64
 
 import doppy
 from doppy import exceptions
+from doppy.raw.utils import bytes_from_src
 from doppy.utils import merge_all_equal
 
 
@@ -37,30 +38,9 @@ class Wls70:
 
     @classmethod
     def from_srcs(
-        cls,
-        data: Sequence[str]
-        | Sequence[Path]
-        | Sequence[bytes]
-        | Sequence[BufferedIOBase],
+        cls, data: Sequence[str | bytes | Path | BufferedIOBase]
     ) -> list[Wls70]:
-        if not isinstance(data, (list, tuple)):
-            raise TypeError("data should be list or tuple")
-        if all(isinstance(src, bytes) for src in data):
-            data_bytes = data
-        elif all(isinstance(src, str) for src in data):
-            data_bytes = []
-            for src in data:
-                with Path(src).open("rb") as f:
-                    data_bytes.append(f.read())
-        elif all(isinstance(src, Path) for src in data):
-            data_bytes = []
-            for src in data:
-                with src.open("rb") as f:
-                    data_bytes.append(f.read())
-        elif all(isinstance(src, BufferedIOBase) for src in data):
-            data_bytes = [src.read() for src in data]
-        else:
-            raise TypeError("Unexpected types in data")
+        data_bytes = [bytes_from_src(src) for src in data]
         raws = doppy.rs.raw.wls70.from_bytes_srcs(data_bytes)
         try:
             return [_raw_rs_to_wls70(r) for r in raws]
@@ -69,19 +49,7 @@ class Wls70:
 
     @classmethod
     def from_src(cls, data: str | Path | bytes | BufferedIOBase) -> Wls70:
-        if isinstance(data, str):
-            path = Path(data)
-            with path.open("rb") as f:
-                data_bytes = f.read()
-        elif isinstance(data, Path):
-            with data.open("rb") as f:
-                data_bytes = f.read()
-        elif isinstance(data, bytes):
-            data_bytes = data
-        elif isinstance(data, BufferedIOBase):
-            data_bytes = data.read()
-        else:
-            raise TypeError("Unsupported data type")
+        data_bytes = bytes_from_src(data)
         try:
             return _raw_rs_to_wls70(doppy.rs.raw.wls70.from_bytes_src(data_bytes))
         except RuntimeError as err:

{doppy-0.4.2.dist-info → doppy-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: doppy
-Version: 0.4.2
+Version: 0.5.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
@@ -19,11 +19,12 @@ Requires-Dist: matplotlib
 Requires-Dist: scikit-learn
 Requires-Dist: scipy
 Requires-Dist: mypy ; extra == 'dev'
+Requires-Dist: pyright ; extra == 'dev'
 Requires-Dist: ruff ; extra == 'dev'
 Requires-Dist: pytest ; extra == 'dev'
 Requires-Dist: types-requests ; extra == 'dev'
 Requires-Dist: py-spy ; extra == 'dev'
-Requires-Dist: maturin ==1.4 ; extra == 'dev'
+Requires-Dist: maturin==1.8 ; extra == 'dev'
 Requires-Dist: release-version ; extra == 'dev'
 Requires-Dist: pre-commit ; extra == 'dev'
 Requires-Dist: xarray[io] ; extra == 'dev'

doppy-0.5.0.dist-info/RECORD

@@ -0,0 +1,32 @@
+doppy-0.5.0.dist-info/METADATA,sha256=ziOeRuz0u6hFJuK0nrFj0Ud1weDkSLuuGx2PKNrmHIE,4315
+doppy-0.5.0.dist-info/WHEEL,sha256=VOPc4LnjY0yPhQSb3E0q8XFYhkjUnHylSjI09QJpAgM,105
+doppy-0.5.0.dist-info/entry_points.txt,sha256=9b_Ca7vJoh6AwL3W8qAPh_UmJ_1Pa6hi-TDfCTDjvSk,43
+doppy-0.5.0.dist-info/licenses/LICENSE,sha256=V-0iroMNMI8ctnLgUau1kdFvwhkYhr9vi-5kWKxw2wc,1089
+doppy/options.py,sha256=73BDODO4OYHn2qOshhwz6u6G3J1kNd3uj6P0a3V4HBE,205
+doppy/__init__.py,sha256=Z9aEUlbPRWRUAoB8_-djkgrJuS4-6pjem4-mVSB6Z9I,191
+doppy/product/turbulence.py,sha256=AcLHQFeyBla0pP5CQtvJbYKupuuy-dwZK0CFlowBl9k,8251
+doppy/product/wind.py,sha256=efBfYalILN2VvDiGpxP04mob-mCrdQFS9tdhReeyXiU,16787
+doppy/product/__init__.py,sha256=C6s9cX20m9UwRsKo1lZH6TnYFfM5KmsX5MPUyShbgl4,235
+doppy/product/stare_depol.py,sha256=gSlD13jGLRmHyTDADFs_MO6ECos6wwKF-TL7G7NwHAA,10718
+doppy/product/noise_utils.py,sha256=fP2bK2xfCUClQcDc_1FmLUUiU2d0KycrxkXLC5jmJAc,2629
+doppy/product/stare.py,sha256=RVHcW8eep91bH2GVobqDqsqswdRaFZ5SY4K_5fLSF3I,27180
+doppy/bench.py,sha256=xUsmwLCjNLXwO9l-_SCfq62jxXJ4eiwkS-WlCejcqZE,308
+doppy/netcdf.py,sha256=q2Wr9auTo3p_Exkt3_To-Z_drpIguiYkS3-32y1wcUk,4055
+doppy/utils.py,sha256=cY-tWrArrROX7lg_zJ2M4QJ6GUnU_BjhkUYL5aG9kbA,790
+doppy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+doppy/exceptions.py,sha256=OzdLXmKs3qZrvzwaI0pxjzpM2w9J5Of7dCo_Ekygecc,183
+doppy/defaults.py,sha256=-jR_xAVLf_soZNDu3uJ6mhOZa9L1tfKtt0k0tI6Rd9s,472
+doppy/data/cache.py,sha256=cxCZ7HyEQt2EKAGEiMqx3wJ2-5Y3hEAEPQ_XB4gKlkk,1160
+doppy/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+doppy/data/api.py,sha256=hUIqyCq1wFu89gamK7xK8R58eXAODWvLQHaaS5eu6OY,1875
+doppy/data/exceptions.py,sha256=6CS6OHIWq8CqlxiceEvC1j0EfWUYoIfM7dW88apQVn4,89
+doppy/raw/halo_sys_params.py,sha256=jclE4RbVg63iS3rVNYzvErKYAa3CI1q64xj3oai5tMc,5250
+doppy/raw/__init__.py,sha256=4pj4xSK9rA0XEfM79NsRBRYId7GaJRDCyPABxbEm63g,260
+doppy/raw/wls70.py,sha256=OfPczHZPKX4iqCcyjSv0nmkr1osy4pu6krNUi0-G0vM,6319
+doppy/raw/utils.py,sha256=EOWIaxAZIiRYa9dI1j50cSd4I058Zq7sz1HOeipZFTg,422
+doppy/raw/halo_bg.py,sha256=8t9j-SUF1yJht3vrT6KAYJQyxcg3W-0zr8h0jAEhWes,5815
+doppy/raw/halo_hpl.py,sha256=tK0H6LAOcoWqF4vvNu8VRCYq_spvcSTHRM4ju4gx5Lo,16828
+doppy/raw/windcube.py,sha256=HKAlU-TUVqA1sefRLOY5skM1twQaVe-MExlZoI40rK4,19149
+doppy/__main__.py,sha256=zrKQJVj0k0ypBQCGK65Czt9G9FZ_qx3ussw6Q9VJ14g,346
+doppy/rs.abi3.so,sha256=D5Ttp3gNjpYqW14v63O_7Dv9Nz6Qf9XUVRb6bZrsMyc,2716068
+doppy-0.5.0.dist-info/RECORD,,

{doppy-0.4.2.dist-info → doppy-0.5.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: maturin (1.7.8)
+Generator: maturin (1.8.2)
 Root-Is-Purelib: false
 Tag: cp310-abi3-macosx_10_12_x86_64

doppy-0.4.2.dist-info/RECORD

@@ -1,29 +0,0 @@
-doppy-0.4.2.dist-info/METADATA,sha256=p8gLRxyFiq40SMjnMrf-iMBTF3jkVMt0mwMPeVVc4q4,4276
-doppy-0.4.2.dist-info/WHEEL,sha256=MQ93xHb5jAFUhKISyBo7LLNSt0X2_WQfUUQCsGqgr6E,105
-doppy-0.4.2.dist-info/entry_points.txt,sha256=9b_Ca7vJoh6AwL3W8qAPh_UmJ_1Pa6hi-TDfCTDjvSk,43
-doppy-0.4.2.dist-info/licenses/LICENSE,sha256=V-0iroMNMI8ctnLgUau1kdFvwhkYhr9vi-5kWKxw2wc,1089
-doppy/options.py,sha256=73BDODO4OYHn2qOshhwz6u6G3J1kNd3uj6P0a3V4HBE,205
-doppy/__init__.py,sha256=Z9aEUlbPRWRUAoB8_-djkgrJuS4-6pjem4-mVSB6Z9I,191
-doppy/product/wind.py,sha256=hkSzAyi4tfPoGwUx8iE3l65oCDyzl_tRgtF2loxaEpY,16557
-doppy/product/__init__.py,sha256=C6s9cX20m9UwRsKo1lZH6TnYFfM5KmsX5MPUyShbgl4,235
-doppy/product/stare_depol.py,sha256=thrWzCpvdH3AiA6gmR37vrH_pDACNY2QTtqPteJ2s8Y,9653
-doppy/product/stare.py,sha256=TE6BrrXFuwv9-zLTYCCn3HNpNgjwdldbyU6qk1__FzA,25343
-doppy/bench.py,sha256=iVNYveMVGGRES2oe3Orsn31jQFCKTXOmxRFuFiJ8_OA,248
-doppy/netcdf.py,sha256=34TzB9UyypTwDpT2MfrRD14g2zGP3X-NuEU15mQGaUI,3925
-doppy/utils.py,sha256=cY-tWrArrROX7lg_zJ2M4QJ6GUnU_BjhkUYL5aG9kbA,790
-doppy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-doppy/exceptions.py,sha256=OzdLXmKs3qZrvzwaI0pxjzpM2w9J5Of7dCo_Ekygecc,183
-doppy/defaults.py,sha256=-jR_xAVLf_soZNDu3uJ6mhOZa9L1tfKtt0k0tI6Rd9s,472
-doppy/data/cache.py,sha256=cxCZ7HyEQt2EKAGEiMqx3wJ2-5Y3hEAEPQ_XB4gKlkk,1160
-doppy/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-doppy/data/api.py,sha256=QaVKj304OPcu8OF5xgtduQzDis8Srn-I6UgR9qb2u9E,1863
-doppy/data/exceptions.py,sha256=6CS6OHIWq8CqlxiceEvC1j0EfWUYoIfM7dW88apQVn4,89
-doppy/raw/halo_sys_params.py,sha256=jclE4RbVg63iS3rVNYzvErKYAa3CI1q64xj3oai5tMc,5250
-doppy/raw/__init__.py,sha256=4pj4xSK9rA0XEfM79NsRBRYId7GaJRDCyPABxbEm63g,260
-doppy/raw/wls70.py,sha256=TePgyhmlWaUQCw1vkWh7uD7KT5EFMaDhNeVYoF2rspc,7510
-doppy/raw/halo_bg.py,sha256=8t9j-SUF1yJht3vrT6KAYJQyxcg3W-0zr8h0jAEhWes,5815
-doppy/raw/halo_hpl.py,sha256=yzVPQEATnDVf1K3N-mGLXR4_RuT-E94OVXathT2bHes,18580
-doppy/raw/windcube.py,sha256=vgkoXVgcbI2Or4BsFCeq2STdjUP49h3mBQLoDXdBQHM,19074
-doppy/__main__.py,sha256=zrKQJVj0k0ypBQCGK65Czt9G9FZ_qx3ussw6Q9VJ14g,346
-doppy/rs.abi3.so,sha256=0eshPpHERdBkLZ_d5ZOgu_nKUDRWqmh7EOhpk6BcpSI,2692584
-doppy-0.4.2.dist-info/RECORD,,