doppy 0.1.3__tar.gz → 0.2.0__tar.gz

{doppy-0.1.3 → doppy-0.2.0}/Cargo.lock

@@ -106,7 +106,7 @@ checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"
 
 [[package]]
 name = "doppy_rs"
-version = "0.1.3"
+version = "0.2.0"
 dependencies = [
  "doprs",
  "numpy",
@@ -115,7 +115,7 @@ dependencies = [
 
 [[package]]
 name = "doprs"
-version = "0.1.3"
+version = "0.2.0"
 dependencies = [
  "chrono",
  "rayon",

{doppy-0.1.3 → doppy-0.2.0}/Cargo.toml

@@ -4,6 +4,6 @@ resolver = "2"
 
 [workspace.package]
 edition = "2021"
-version = "0.1.3"
+version = "0.2.0"
 authors = ["Niko Leskinen <niko.leskinen@fmi.fi>"]
 license-file = "LICENSE"

{doppy-0.1.3 → doppy-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: doppy
-Version: 0.1.3
+Version: 0.2.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
@@ -13,7 +13,7 @@ Requires-Dist: requests
 Requires-Dist: urllib3
 Requires-Dist: numpy
 Requires-Dist: netcdf4
-Requires-Dist: typer[all]
+Requires-Dist: typer
 Requires-Dist: matplotlib
 Requires-Dist: scikit-learn
 Requires-Dist: scipy

doppy-0.2.0/crates/doppy_rs/src/raw/wls70.rs

@@ -0,0 +1,76 @@
+use numpy::{PyArray1, ToPyArray};
+use pyo3::exceptions::PyRuntimeError;
+use pyo3::prelude::*;
+use pyo3::types::{PyDict, PyList};
+
+type PyReturnType<'a> = (&'a PyDict, &'a PyList, &'a PyArray1<f64>);
+
+#[pymodule]
+pub fn wls70(_py: Python, m: &PyModule) -> PyResult<()> {
+    m.add_function(wrap_pyfunction!(from_filename_srcs, m)?)?;
+    m.add_function(wrap_pyfunction!(from_filename_src, m)?)?;
+    m.add_function(wrap_pyfunction!(from_bytes_srcs, m)?)?;
+    m.add_function(wrap_pyfunction!(from_bytes_src, m)?)?;
+    Ok(())
+}
+
+#[pyfunction]
+pub fn from_bytes_srcs<'a>(
+    py: Python<'a>,
+    contents: Vec<&'a [u8]>,
+) -> PyResult<Vec<PyReturnType<'a>>> {
+    let raws = doprs::raw::wls70::from_bytes_srcs(contents);
+    let mut result = Vec::new();
+    for raw in raws {
+        result.push(convert_to_python(py, raw)?);
+    }
+    Ok(result)
+}
+
+#[pyfunction]
+pub fn from_bytes_src<'a>(py: Python<'a>, content: &'a [u8]) -> PyResult<PyReturnType<'a>> {
+    let raw = match doprs::raw::wls70::from_bytes_src(content) {
+        Ok(raw) => raw,
+        Err(e) => {
+            return Err(PyRuntimeError::new_err(format!(
+                "Failed to read files: {}",
+                e
+            )))
+        }
+    };
+    convert_to_python(py, raw)
+}
+
+#[pyfunction]
+pub fn from_filename_srcs(py: Python, filenames: Vec<String>) -> PyResult<Vec<PyReturnType>> {
+    let raws = doprs::raw::wls70::from_filename_srcs(filenames);
+    let mut result = Vec::new();
+    for raw in raws {
+        result.push(convert_to_python(py, raw)?);
+    }
+    Ok(result)
+}
+
+#[pyfunction]
+pub fn from_filename_src(py: Python, filename: String) -> PyResult<PyReturnType> {
+    let raw = match doprs::raw::wls70::from_filename_src(filename) {
+        Ok(raw) => raw,
+        Err(e) => {
+            return Err(PyRuntimeError::new_err(format!(
+                "Failed to read files: {}",
+                e
+            )))
+        }
+    };
+    convert_to_python(py, raw)
+}
+
+fn convert_to_python(py: Python, raw: doprs::raw::wls70::Wls70) -> PyResult<PyReturnType> {
+    let info_dict = PyDict::new(py);
+    info_dict.set_item("altitude", raw.info.altitude.as_slice().to_pyarray(py))?;
+    Ok((
+        info_dict,
+        PyList::new(py, raw.data_columns),
+        raw.data.as_slice().to_pyarray(py),
+    ))
+}

{doppy-0.1.3 → doppy-0.2.0}/crates/doppy_rs/src/raw.rs

@@ -2,9 +2,11 @@ use pyo3::prelude::*;
 use pyo3::wrap_pymodule;
 
 pub mod halo_hpl;
+pub mod wls70;
 
 #[pymodule]
 pub fn raw(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_wrapped(wrap_pymodule!(halo_hpl::halo_hpl))?;
+    m.add_wrapped(wrap_pymodule!(wls70::wls70))?;
     Ok(())
 }

{doppy-0.1.3 → doppy-0.2.0}/crates/doprs/src/raw/error.rs

@@ -2,7 +2,7 @@ use std::fmt;
 
 #[derive(Debug, Clone)]
 pub struct RawParseError {
-    message: String,
+    pub message: String,
 }
 
 impl Default for RawParseError {

doppy-0.2.0/crates/doprs/src/raw/wls70.rs

@@ -0,0 +1,223 @@
+use std::fs::File;
+
+use chrono::{DateTime, NaiveDateTime, ParseError, Utc};
+use rayon::prelude::*;
+use std::io::{BufRead, Cursor, Read};
+
+use crate::raw::error::RawParseError;
+
+#[derive(Debug, Default, Clone)]
+pub struct Wls70 {
+    pub info: Info,
+    pub data_columns: Vec<String>,
+    pub data: Vec<f64>,
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct Info {
+    pub altitude: Vec<f64>,
+}
+
+pub fn from_file_src(mut file: &File) -> Result<Wls70, RawParseError> {
+    let mut content = vec![];
+    file.read_to_end(&mut content)?;
+    from_bytes_src(&content)
+}
+
+pub fn from_filename_src(filename: String) -> Result<Wls70, RawParseError> {
+    let file = File::open(filename)?;
+    from_file_src(&file)
+}
+
+pub fn from_filename_srcs(filenames: Vec<String>) -> Vec<Wls70> {
+    let results = filenames
+        .par_iter()
+        .filter_map(|filename| from_filename_src(filename.to_string()).ok())
+        .collect();
+    results
+}
+
+pub fn from_file_srcs(files: Vec<&File>) -> Vec<Wls70> {
+    let results = files
+        .par_iter()
+        .filter_map(|file| from_file_src(file).ok())
+        .collect();
+    results
+}
+
+pub fn from_bytes_srcs(contents: Vec<&[u8]>) -> Vec<Wls70> {
+    let results = contents
+        .par_iter()
+        .filter_map(|content| from_bytes_src(content).ok())
+        .collect();
+    results
+}
+
+enum Phase {
+    Info,
+    Data,
+}
+
+pub fn from_bytes_src(content: &[u8]) -> Result<Wls70, RawParseError> {
+    let cur = Cursor::new(content);
+
+    let mut info_str = Vec::new();
+    let mut header = Vec::new();
+    let mut data_str = Vec::new();
+
+    let mut phase = Phase::Info;
+
+    for line in cur.split(b'\n') {
+        let line = line?;
+        match phase {
+            Phase::Info => {
+                if line.starts_with(b"Timestamp\tPosition\tTemperature") {
+                    header.extend_from_slice(&line);
+                    header.push(b'\n');
+                    phase = Phase::Data;
+                } else {
+                    info_str.extend_from_slice(&line);
+                    info_str.push(b'\n');
+                }
+            }
+            Phase::Data => {
+                data_str.extend_from_slice(&line);
+                data_str.push(b'\n');
+            }
+        }
+    }
+    let info = parse_info(&info_str)?;
+    match parse_data(&data_str) {
+        Ok((data, ncols)) => {
+            let header_str: String = header.iter().map(|&c| c as char).collect();
+            let cols: Vec<_> = header_str
+                .split(|c| c == '\t')
+                .map(|s| s.trim().to_string())
+                .filter(|s| !s.is_empty())
+                .collect();
+            if ncols != (cols.len() as i64) {
+                return Err(RawParseError {
+                    message: "Number of columns on header and number of columns in data mismatch"
+                        .to_string(),
+                });
+            }
+            Ok(Wls70 {
+                info,
+                data_columns: cols,
+                data,
+            })
+        }
+        Err(e) => Err(e),
+    }
+}
+
+fn parse_info(info_str: &[u8]) -> Result<Info, RawParseError> {
+    let mut info = Info::default();
+    for line in info_str.split(|&b| b == b'\n') {
+        match line {
+            b if b.starts_with(b"Altitudes(m)=") => {
+                info.altitude = line
+                    .split(|&b| b == b'\t')
+                    .skip(1)
+                    .map(|part| {
+                        String::from_utf8(part.to_vec())
+                            .map_err(|_| RawParseError {
+                                message: "UTF-8 conversion error".into(),
+                            })
+                            .and_then(|s| {
+                                s.trim().parse::<f64>().map_err(|_| RawParseError {
+                                    message: "Parse float error".into(),
+                                })
+                            })
+                    })
+                    .collect::<Result<Vec<f64>, _>>()?;
+            }
+            _ => (),
+        }
+    }
+
+    Ok(info)
+}
+
+pub fn parse_data(data: &[u8]) -> Result<(Vec<f64>, i64), RawParseError> {
+    let mut ncols: i64 = -1;
+    let mut data_flat = vec![];
+    for line in data.split(|&b| b == b'\n') {
+        let parts: Vec<_> = line
+            .split(|&b| b == b'\t')
+            .filter(|part| !(part.is_empty() || part == b"\r"))
+            .collect();
+        if parts.is_empty() {
+            continue;
+        }
+        if ncols < 0 {
+            ncols = parts.len() as i64;
+        }
+        if ncols != parts.len() as i64 {
+            return Err(RawParseError {
+                message: "Unexpected number of columns".to_string(),
+            });
+        }
+        for (i, part) in parts.iter().enumerate() {
+            match i {
+                0 => {
+                    let date = String::from_utf8_lossy(part).trim().to_string();
+                    match datetime_to_timestamp(&date) {
+                        Ok(d) => {
+                            data_flat.push(d);
+                        }
+                        Err(_) => println!("Error with datetime"),
+                    }
+                }
+                3 => match String::from_utf8_lossy(part).trim() {
+                    "On" => {
+                        data_flat.push(1.);
+                    }
+                    "Off" => {
+                        data_flat.push(0.);
+                    }
+                    _ => {
+                        println!("Failed to read Wiper state");
+                        return Err(RawParseError {
+                            message: "Unexpected value for Wiper state".to_string(),
+                        });
+                    }
+                },
+                _ => match String::from_utf8_lossy(part).trim().parse::<f64>() {
+                    Ok(x) => {
+                        data_flat.push(x);
+                    }
+                    Err(_) => println!("Cannot parse float"),
+                },
+            }
+        }
+    }
+    if ncols < 1 || (data_flat.len() as i64) % ncols != 0 {
+        return Err(RawParseError {
+            message: "Unexpected number of columns".to_string(),
+        });
+    }
+    Ok((data_flat, ncols))
+}
+
+fn datetime_to_timestamp(s: &str) -> Result<f64, ParseError> {
+    let format = "%d/%m/%Y %H:%M:%S%.f";
+    let ndt = NaiveDateTime::parse_from_str(s, format)?;
+    let dt = DateTime::<Utc>::from_naive_utc_and_offset(ndt, Utc);
+    Ok(dt.timestamp() as f64 + dt.timestamp_subsec_millis() as f64 / 1000.0)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs::File;
+
+    #[test]
+    fn test_from_file_src() -> Result<(), Box<dyn std::error::Error>> {
+        let file_path = "../../data/palaiseau/2024-04-01/wlscerea_0a_windLz1Lb87R10s-HR_v01_20240401_000000.rtd";
+        let file = File::open(file_path)?;
+        assert!(from_file_src(&file).is_ok());
+
+        Ok(())
+    }
+}

{doppy-0.1.3 → doppy-0.2.0}/crates/doprs/src/raw.rs

@@ -1,2 +1,3 @@
 pub mod error;
 pub mod halo_hpl;
+pub mod wls70;

{doppy-0.1.3 → doppy-0.2.0}/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
   "urllib3",
   "numpy",
   "netCDF4",
-  "typer[all]",
+  "typer",
   "matplotlib",
   "scikit-learn",
   "scipy"

{doppy-0.1.3 → doppy-0.2.0}/src/doppy/data/api.py

@@ -37,7 +37,13 @@ class Api:
         return self.get(
             "raw-files",
             params={
-                "instrument": ["halo-doppler-lidar", "wls100s", "wls200s", "wls400s"],
+                "instrument": [
+                    "halo-doppler-lidar",
+                    "wls100s",
+                    "wls200s",
+                    "wls400s",
+                    "wls70",
+                ],
                 "site": site,
                 "date": date,
             },

{doppy-0.1.3 → doppy-0.2.0}/src/doppy/product/stare.py

@@ -50,6 +50,7 @@ class Stare:
             doppy.raw.HaloHpl.merge(_select_raws_for_stare(raws))
             .sorted_by_time()
             .non_strictly_increasing_timesteps_removed()
+            .nans_removed()
         )
 
         bgs = doppy.raw.HaloBg.from_srcs(data_bg)
@@ -65,6 +66,8 @@ class Stare:
             .non_strictly_increasing_timesteps_removed()
         )
         raw, intensity_bg_corrected = _correct_background(raw, bg, bg_correction_method)
+        if len(raw.time) == 0:
+            raise doppy.exceptions.NoDataError("No matching data and bg files")
         intensity_noise_bias_corrected = _correct_intensity_noise_bias(
             raw, intensity_bg_corrected
         )

{doppy-0.1.3 → doppy-0.2.0}/src/doppy/product/wind.py

@@ -54,6 +54,7 @@ class Wind:
             doppy.raw.HaloHpl.merge(_select_raws_for_wind(raws))
             .sorted_by_time()
             .non_strictly_increasing_timesteps_removed()
+            .nans_removed()
         )
         if len(raw.time) == 0:
             raise doppy.exceptions.NoDataError("No suitable data for the wind product")
@@ -148,6 +149,38 @@ class Wind:
             mask=mask,
         )
 
+    @classmethod
+    def from_wls70_data(
+        cls,
+        data: Sequence[str]
+        | Sequence[Path]
+        | Sequence[bytes]
+        | Sequence[BufferedIOBase],
+    ) -> Wind:
+        raws = doppy.raw.Wls70.from_srcs(data)
+
+        if len(raws) == 0:
+            raise doppy.exceptions.NoDataError("Wls70 data missing")
+
+        raw = (
+            doppy.raw.Wls70.merge(raws)
+            .sorted_by_time()
+            .non_strictly_increasing_timesteps_removed()
+        )
+        mask = (
+            np.isnan(raw.meridional_wind)
+            | np.isnan(raw.zonal_wind)
+            | np.isnan(raw.vertical_wind)
+        )
+        return Wind(
+            time=raw.time,
+            height=raw.altitude,
+            zonal_wind=raw.zonal_wind,
+            meridional_wind=raw.meridional_wind,
+            vertical_wind=raw.vertical_wind,
+            mask=mask,
+        )
+
 
 def _compute_wind(
     raw: doppy.raw.HaloHpl | doppy.raw.WindCube,

{doppy-0.1.3 → doppy-0.2.0}/src/doppy/raw/__init__.py

@@ -2,5 +2,6 @@ from .halo_bg import HaloBg
 from .halo_hpl import HaloHpl
 from .halo_sys_params import HaloSysParams
 from .windcube import WindCube
+from .wls70 import Wls70
 
-__all__ = ["HaloHpl", "HaloBg", "HaloSysParams", "WindCube"]
+__all__ = ["HaloHpl", "HaloBg", "HaloSysParams", "WindCube", "Wls70"]

{doppy-0.1.3 → doppy-0.2.0}/src/doppy/raw/halo_hpl.py

@@ -187,6 +187,10 @@ class HaloHpl:
                 latest_time = t
         return self[mask]
 
+    def nans_removed(self) -> HaloHpl:
+        is_ok = ~np.isnan(self.intensity).any(axis=1)
+        return self[is_ok]
+
 
 @dataclass(slots=True)
 class HaloHplHeader:

doppy-0.2.0/src/doppy/raw/wls70.py

@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from io import BufferedIOBase
+from pathlib import Path
+from typing import Sequence
+
+import numpy as np
+import numpy.typing as npt
+from numpy import datetime64
+
+import doppy
+from doppy import exceptions
+
+
+@dataclass
+class Wls70:
+    time: npt.NDArray[datetime64]  # dim: (time, )
+    altitude: npt.NDArray[np.float64]  # dim: (altitude, )
+    position: npt.NDArray[np.float64]  # dim: (time, )
+    temperature: npt.NDArray[np.float64]  # dim: (time, )
+    wiper: npt.NDArray[np.bool_]  # dim: (time, )
+    cnr: npt.NDArray[np.float64]  # dim: (time, altitude)
+    radial_velocity: npt.NDArray[np.float64]  # dim: (time, altitude)
+    radial_velocity_deviation: npt.NDArray[np.float64]  # dim: (time, altitude)
+    vh: npt.NDArray[np.float64]  # dim: (time, altitude)
+    wind_direction: npt.NDArray[np.float64]  # dim: (time, altitude)
+    zonal_wind: npt.NDArray[np.float64]  # u := zonal wind?, dim: (time, altitude)
+    meridional_wind: npt.NDArray[
+        np.float64
+    ]  # v := meridional wind?, dim: (time, altitude)
+    vertical_wind: npt.NDArray[np.float64]  # w := vertical wind?, dim: (time, altitude)
+
+    @classmethod
+    def from_srcs(
+        cls,
+        data: Sequence[str]
+        | Sequence[Path]
+        | Sequence[bytes]
+        | Sequence[BufferedIOBase],
+    ) -> list[Wls70]:
+        if not isinstance(data, (list, tuple)):
+            raise TypeError("data should be list or tuple")
+        if all(isinstance(src, bytes) for src in data):
+            data_bytes = data
+        elif all(isinstance(src, str) for src in data):
+            data_bytes = []
+            for src in data:
+                with Path(src).open("rb") as f:
+                    data_bytes.append(f.read())
+        elif all(isinstance(src, Path) for src in data):
+            data_bytes = []
+            for src in data:
+                with src.open("rb") as f:
+                    data_bytes.append(f.read())
+        elif all(isinstance(src, BufferedIOBase) for src in data):
+            data_bytes = [src.read() for src in data]
+        else:
+            raise TypeError("Unexpected types in data")
+        raws = doppy.rs.raw.wls70.from_bytes_srcs(data_bytes)
+        try:
+            return [_raw_rs_to_wls70(r) for r in raws]
+        except RuntimeError as err:
+            raise exceptions.RawParsingError(err) from err
+
+    @classmethod
+    def from_src(cls, data: str | Path | bytes | BufferedIOBase) -> Wls70:
+        if isinstance(data, str):
+            path = Path(data)
+            with path.open("rb") as f:
+                data_bytes = f.read()
+        elif isinstance(data, Path):
+            with data.open("rb") as f:
+                data_bytes = f.read()
+        elif isinstance(data, bytes):
+            data_bytes = data
+        elif isinstance(data, BufferedIOBase):
+            data_bytes = data.read()
+        else:
+            raise TypeError("Unsupported data type")
+        try:
+            return _raw_rs_to_wls70(doppy.rs.raw.wls70.from_bytes_src(data_bytes))
+        except RuntimeError as err:
+            raise exceptions.RawParsingError(err) from err
+
+    def __getitem__(
+        self,
+        index: int
+        | slice
+        | list[int]
+        | npt.NDArray[np.int64]
+        | npt.NDArray[np.bool_]
+        | tuple[slice, slice],
+    ) -> Wls70:
+        if isinstance(index, (int, slice, list, np.ndarray)):
+            return Wls70(
+                time=self.time[index],
+                altitude=self.altitude,
+                position=self.position[index],
+                temperature=self.temperature[index],
+                wiper=self.wiper[index],
+                cnr=self.cnr[index],
+                radial_velocity=self.radial_velocity[index],
+                radial_velocity_deviation=self.radial_velocity_deviation[index],
+                vh=self.vh[index],
+                wind_direction=self.wind_direction[index],
+                zonal_wind=self.zonal_wind[index],
+                meridional_wind=self.meridional_wind[index],
+                vertical_wind=self.vertical_wind[index],
+            )
+        raise TypeError
+
+    def sorted_by_time(self) -> Wls70:
+        sort_indices = np.argsort(self.time)
+        return self[sort_indices]
+
+    @classmethod
+    def merge(cls, raws: Sequence[Wls70]) -> Wls70:
+        return cls(
+            time=np.concatenate(tuple(r.time for r in raws)),
+            altitude=raws[0].altitude,
+            position=np.concatenate(tuple(r.position for r in raws)),
+            temperature=np.concatenate(tuple(r.temperature for r in raws)),
+            wiper=np.concatenate(tuple(r.wiper for r in raws)),
+            cnr=np.concatenate(tuple(r.cnr for r in raws)),
+            radial_velocity=np.concatenate(tuple(r.radial_velocity for r in raws)),
+            radial_velocity_deviation=np.concatenate(
+                tuple(r.radial_velocity_deviation for r in raws)
+            ),
+            vh=np.concatenate(tuple(r.vh for r in raws)),
+            wind_direction=np.concatenate(tuple(r.wind_direction for r in raws)),
+            zonal_wind=np.concatenate(tuple(r.zonal_wind for r in raws)),
+            meridional_wind=np.concatenate(tuple(r.meridional_wind for r in raws)),
+            vertical_wind=np.concatenate(tuple(r.vertical_wind for r in raws)),
+        )
+
+    def non_strictly_increasing_timesteps_removed(self) -> Wls70:
+        if len(self.time) == 0:
+            return self
+        mask = np.ones_like(self.time, dtype=np.bool_)
+        latest_time = self.time[0]
+        for i, t in enumerate(self.time[1:], start=1):
+            if t <= latest_time:
+                mask[i] = False
+            else:
+                latest_time = t
+        return self[mask]
+
+
+def _raw_rs_to_wls70(
+    raw_rs: tuple[
+        dict[str, npt.NDArray[np.float64]], list[str], npt.NDArray[np.float64]
+    ],
+) -> Wls70:
+    info, cols, data = raw_rs
+    altitude = info["altitude"]
+    data = data.reshape(-1, len(cols))
+    time_ts = data[:, 0]
+    time = np.array([datetime64(datetime.utcfromtimestamp(ts)) for ts in time_ts])
+
+    position = data[:, 1]
+    temperature = data[:, 2]
+    wiper = np.array(np.isclose(data[:, 3], 1), dtype=np.bool_)
+    cnr = data[:, 4::8]
+    rws = data[:, 5::8]
+    rwsd = data[:, 6::8]
+    vh = data[:, 7::8]
+    direction = data[:, 8::8]
+    u = data[:, 9::8]
+    v = data[:, 10::8]
+    w = data[:, 11::8]
+
+    return Wls70(
+        time=time,
+        altitude=altitude,
+        position=position,
+        temperature=temperature,
+        wiper=wiper,
+        cnr=cnr,
+        radial_velocity=rws,
+        radial_velocity_deviation=rwsd,
+        vh=vh,
+        wind_direction=direction,
+        zonal_wind=u,
+        meridional_wind=v,
+        vertical_wind=w,
+    )