doppy 0.5.9__cp310-abi3-macosx_10_12_x86_64.whl

doppy/__init__.py

@@ -0,0 +1,6 @@
+from doppy import data, netcdf, options, product, raw
+from doppy.rs import __version__
+
+from . import bench
+
+__all__ = ["raw", "options", "product", "netcdf", "bench", "data", "__version__"]

doppy/bench.py

@@ -0,0 +1,13 @@
+import time
+
+
+class Timer:
+    def __init__(self):
+        self.start: float | None = None
+
+    def __enter__(self):
+        self.start = time.time()
+
+    def __exit__(self, type, value, traceback):
+        if isinstance(self.start, float):
+            print(f"Elapsed time: {time.time() - self.start:.2f}")

doppy/data/api.py

@@ -0,0 +1,58 @@
+import gzip
+import io
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+from doppy.data import exceptions
+from doppy.data.cache import cached_record
+
+
+class Api:
+    def __init__(self, cache: bool = False) -> None:
+        retries = Retry(total=10, backoff_factor=0.2)
+        adapter = HTTPAdapter(max_retries=retries)
+        session = requests.Session()
+        session.mount("https://", adapter)
+        session.mount("http://", adapter)
+        self.session = session
+        self.api_endpoint = "https://cloudnet.fmi.fi/api"
+        self.cache = cache
+
+    def get(self, path: str, params: dict[str, str | list[str]]) -> list:
+        res = self.session.get(
+            f"{self.api_endpoint}/{path}", params=params, timeout=1800
+        )
+        if res.ok:
+            data = res.json()
+            if isinstance(data, list):
+                return data
+            raise exceptions.ApiRequestError(
+                f"Unexpected response type from api: {type(data)}"
+            )
+        raise exceptions.ApiRequestError(f"Api request error: {res.status_code}")
+
+    def get_raw_records(self, site: str, date: str) -> list:
+        return self.get(
+            "raw-files",
+            params={
+                "instrument": [
+                    "halo-doppler-lidar",
+                    "wls100s",
+                    "wls200s",
+                    "wls400s",
+                    "wls70",
+                ],
+                "site": site,
+                "date": date,
+            },
+        )
+
+    def get_record_content(self, rec: dict) -> io.BytesIO:
+        if self.cache:
+            return cached_record(rec, self.session)
+        content = self.session.get(rec["downloadUrl"]).content
+        if rec["filename"].endswith(".gz"):
+            content = gzip.decompress(content)
+        return io.BytesIO(content)

doppy/data/cache.py

@@ -0,0 +1,43 @@
+import gzip
+import logging
+import shutil
+from io import BytesIO
+from pathlib import Path
+
+from requests import Session
+
+
+def cached_record(
+    record: dict, session: Session, check_disk_usage: bool = True
+) -> BytesIO:
+    cache_dir = Path("cache")
+    path = cache_dir / record["uuid"]
+
+    if check_disk_usage:
+        HUNDRED_GIGABYTES_AS_BYTES = 100 * 1024 * 1024 * 1024
+
+        _, _, disk_free = shutil.disk_usage("./")
+        if disk_free < HUNDRED_GIGABYTES_AS_BYTES:
+            _clear_dir(cache_dir)
+
+    if path.is_file():
+        return BytesIO(path.read_bytes())
+    else:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        content = session.get(record["downloadUrl"]).content
+        if record["filename"].endswith(".gz"):
+            try:
+                content = gzip.decompress(content)
+            except EOFError:
+                logging.error(f"Failed to recompress {record['filename']}")
+                raise
+
+        with path.open("wb") as f:
+            f.write(content)
+        return BytesIO(content)
+
+
+def _clear_dir(path: Path) -> None:
+    for f in path.glob("**/*"):
+        if f.is_file():
+            f.unlink()

doppy/data/exceptions.py

@@ -0,0 +1,6 @@
+class ApiRequestError(Exception):
+    pass
+
+
+class CliArgumentError(Exception):
+    pass

doppy/defaults.py

@@ -0,0 +1,18 @@
+DEFAULT_BEAM_ENERGY = 1e-5
+DEFAULT_EFFECTIVE_DIAMETER = 25e-3
+
+
+class Halo:
+    wavelength = 1.565e-6  # [m]
+    receiver_bandwidth = 50e6  # [Hz]
+    beam_energy = DEFAULT_BEAM_ENERGY
+    effective_diameter = DEFAULT_EFFECTIVE_DIAMETER
+
+
+class WindCube:
+    # https://doi.org/10.5194/essd-13-3539-2021
+    wavelength = 1.54e-6  # [m]
+    receiver_bandwidth = 55e6  # [Hz]
+    beam_energy = DEFAULT_BEAM_ENERGY
+    effective_diameter = 50e-3  # [m]
+    focus = 1e3  # [m]

doppy/exceptions.py

@@ -0,0 +1,14 @@
+class DoppyException(Exception):
+    pass
+
+
+class RawParsingError(DoppyException):
+    pass
+
+
+class NoDataError(DoppyException):
+    pass
+
+
+class ShapeError(DoppyException):
+    pass

doppy/netcdf.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import pathlib
+import warnings
+from types import TracebackType
+from typing import Literal, TypeAlias
+
+import netCDF4
+import numpy as np
+import numpy.typing as npt
+
+NetCDFDataType: TypeAlias = Literal["f4", "f8", "i4", "i8", "u4", "u8"]
+
+
+class Dataset:
+    def __init__(
+        self,
+        filename: str | pathlib.Path,
+        format: Literal["NETCDF4", "NETCDF4_CLASSIC"] = "NETCDF4",
+    ) -> None:
+        self.nc = netCDF4.Dataset(filename, mode="w", format=format)
+
+    def __enter__(self) -> Dataset:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def add_dimension(self, dim: str, size: int | None = None) -> Dataset:
+        self.nc.createDimension(dim, size)
+        return self
+
+    def add_attribute(self, key: str, val: str) -> Dataset:
+        setattr(self.nc, key, val)
+        return self
+
+    def add_atribute(self, key: str, val: str) -> Dataset:
+        warnings.warn("Use add_attribute", DeprecationWarning, stacklevel=2)
+        return self.add_attribute(key, val)
+
+    def add_time(
+        self,
+        name: str,
+        dimensions: tuple[str, ...],
+        data: npt.NDArray[np.datetime64],
+        dtype: NetCDFDataType,
+        standard_name: str | None = None,
+        long_name: str | None = None,
+    ) -> Dataset:
+        time, units, calendar = _convert_time(data)
+        var = self.nc.createVariable(name, dtype, dimensions, compression="zlib")
+        var.units = units
+        var.calendar = calendar
+        var.axis = "T"
+        var[:] = time
+        if standard_name is not None:
+            var.standard_name = standard_name
+        if long_name is not None:
+            var.long_name = long_name
+        return self
+
+    def add_variable(
+        self,
+        name: str,
+        dimensions: tuple[str, ...],
+        units: str,
+        data: npt.NDArray[np.float64],
+        dtype: NetCDFDataType,
+        standard_name: str | None = None,
+        long_name: str | None = None,
+        mask: npt.NDArray[np.bool_] | None = None,
+    ) -> Dataset:
+        fill_value = netCDF4.default_fillvals[dtype] if mask is not None else None
+        var = self.nc.createVariable(
+            name, dtype, dimensions, fill_value=fill_value, compression="zlib"
+        )
+        var.units = units
+        if mask is not None:
+            var[:] = np.ma.masked_array(data, mask)  # type: ignore
+        else:
+            var[:] = data
+        if standard_name is not None:
+            var.standard_name = standard_name
+        if long_name is not None:
+            var.long_name = long_name
+        return self
+
+    def add_scalar_variable(
+        self,
+        name: str,
+        units: str,
+        data: np.float64 | np.int64 | float | int,
+        dtype: NetCDFDataType,
+        standard_name: str | None = None,
+        long_name: str | None = None,
+        mask: npt.NDArray[np.bool_] | None = None,
+    ) -> Dataset:
+        var = self.nc.createVariable(name, dtype)
+        var.units = units
+        var[:] = data
+        if standard_name is not None:
+            var.standard_name = standard_name
+        if long_name is not None:
+            var.long_name = long_name
+        return self
+
+    def close(self) -> None:
+        self.nc.close()
+
+
+def _convert_time(
+    time: npt.NDArray[np.datetime64],
+) -> tuple[npt.NDArray[np.float64], str, str]:
+    """
+    Parameters
+    ----------
+    time : npt.NDArray[np.datetime64["us"]]
+        Must be represented in UTC
+    """
+    if time.dtype != "<M8[us]":
+        raise TypeError("time must be datetime64[us]")
+    MICROSECONDS_TO_HOURS = 1 / (1e6 * 3600)
+    start_of_day = time.min().astype("datetime64[D]")
+    hours_since_start_of_day = (time - start_of_day).astype(
+        np.float64
+    ) * MICROSECONDS_TO_HOURS
+    units = f"hours since {np.datetime_as_string(start_of_day)} 00:00:00 +00:00"
+    calendar = "standard"
+    return hours_since_start_of_day, units, calendar

doppy/options.py

@@ -0,0 +1,13 @@
+from enum import Enum
+
+
+class BgCorrectionMethod(Enum):
+    FIT = "fit"
+    MEAN = "mean"
+    PRE_COMPUTED = "pre_computed"
+
+
+class BgFitMethod(Enum):
+    LIN = "lin"
+    EXP = "exp"
+    EXPLIN = "explin"

doppy/product/__init__.py

@@ -0,0 +1,6 @@
+from doppy.product.stare import Stare
+from doppy.product.stare_depol import StareDepol
+from doppy.product.wind import Options as WindOptions
+from doppy.product.wind import Wind
+
+__all__ = ["Stare", "StareDepol", "Wind", "WindOptions"]

doppy/product/noise_utils.py

@@ -0,0 +1,106 @@
+import warnings
+
+import numpy as np
+import numpy.typing as npt
+import scipy
+
+
+def detect_wind_noise(
+    w: npt.NDArray[np.float64],
+    height: npt.NDArray[np.float64],
+    mask: npt.NDArray[np.bool_],
+    window: float = 150,
+    stride: int = 1,
+) -> npt.NDArray[np.bool_]:
+    """
+    Parameters
+    ----------
+    w
+        vertical velocity, dims: (time,height)
+
+    height
+        dims: (height,)
+
+    mask
+        old mask that still contains noisy velocity data,
+        mask[t,h] = True iff w[t,h] is noise
+
+    window
+        size of window used to compute rolling median in meters
+
+    stride
+        stride used to compute rolling median
+
+    Returns
+    -------
+    improved noise mask such that new_mask[t,h] = True iff w[t,h] is noise
+    """
+    warnings.simplefilter("ignore", RuntimeWarning)
+    v = _rolling_median_over_range(
+        height,
+        w,
+        mask,
+        window=window,  # meters
+        stride=stride,
+        fill_gaps=True,
+    )
+
+    th = 2
+    diff = np.abs(v - w)
+    new_mask = (diff > th) | mask
+    new_mask = _remove_one_hot(new_mask)
+    return np.array(new_mask, dtype=np.bool_)
+
+
+def _remove_one_hot(m: npt.NDArray[np.bool_]) -> npt.NDArray[np.bool_]:
+    if m.ndim != 2:
+        raise ValueError
+    if m.shape[1] < 3:
+        return m
+    x = ~m
+    y = np.full(x.shape, np.False_)
+    y[:, 0] = x[:, 0] & x[:, 1]
+    y[:, 1:-1] = x[:, 1:-1] & (x[:, 2:] | x[:, :-2])
+    y[:, -1] = x[:, -1] & x[:, -2]
+    return np.array(~y, dtype=np.bool_)
+
+
+def _rolling_median_over_range(
+    range_: npt.NDArray[np.float64],
+    arr: npt.NDArray[np.float64],
+    mask: npt.NDArray[np.bool_],
+    window: float,
+    stride: int = 1,
+    fill_gaps: bool = False,
+) -> npt.NDArray[np.float64]:
+    """
+    window
+        range window in meters
+    """
+    X = arr.T.copy()
+    X[mask.T] = np.nan
+
+    half_window = window / 2
+
+    i = 0
+    j = 0
+    n = len(range_)
+    med = np.full(X.shape, np.nan, dtype=np.float64)
+    for k in range(0, n, stride):
+        r = range_[k]
+        while i + 1 < n and r - range_[i + 1] >= half_window:
+            i += 1
+        while j + 1 < n and range_[j] - r < half_window:
+            j += 1
+        if i > k or j < k:
+            raise ValueError
+        med[k] = np.nanmedian(X[i : j + 1], axis=0)
+
+    if stride != 1 and fill_gaps:
+        ind = list(range(0, n, stride))
+        f_interp = scipy.interpolate.interp1d(
+            range_[ind], med[ind], axis=0, fill_value="extrapolate"
+        )
+        med_all = f_interp(range_)
+        return np.array(med_all.T.copy(), dtype=np.float64)
+    return np.array(med.T.copy(), dtype=np.float64)