disdrodb 0.2.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

disdrodb/l1/processing.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,35 +18,22 @@
 
 import xarray as xr
 
-from disdrodb.constants import DIAMETER_DIMENSION, VELOCITY_DIMENSION
-from disdrodb.l1.fall_velocity import get_raindrop_fall_velocity_from_ds
-from disdrodb.l1.filters import define_raindrop_spectrum_mask, filter_diameter_bins, filter_velocity_bins
+from disdrodb.constants import DIAMETER_DIMENSION, METEOROLOGICAL_VARIABLES, VELOCITY_DIMENSION
+from disdrodb.l1.classification import (
+    classify_raw_spectrum,
+    get_temperature,
+    map_precip_flag_to_precipitation_type,
+)
 from disdrodb.l1.resampling import add_sample_interval
 from disdrodb.l1_env.routines import load_env_dataset
-from disdrodb.l2.empirical_dsd import (  # TODO: maybe move out of L2
-    add_bins_metrics,
-    get_min_max_diameter,
-)
+from disdrodb.utils.manipulations import filter_diameter_bins
 from disdrodb.utils.time import ensure_sample_interval_in_seconds, infer_sample_interval
 from disdrodb.utils.writer import finalize_product
 
 
 def generate_l1(
     ds,
-    # Fall velocity option
-    fall_velocity_model="Beard1976",
-    # Diameter-Velocity Filtering Options
-    minimum_diameter=0,
-    maximum_diameter=10,
-    minimum_velocity=0,
-    maximum_velocity=12,
-    above_velocity_fraction=0.5,
-    above_velocity_tolerance=None,
-    below_velocity_fraction=0.5,
-    below_velocity_tolerance=None,
-    small_diameter_threshold=1,  # 2
-    small_velocity_threshold=2.5,  # 3
-    maintain_smallest_drops=True,
+    **kwargs,  # noqa
 ):
     """Generate DISDRODB L1 Dataset from DISDRODB L0C Dataset.
 
@@ -54,31 +41,6 @@ def generate_l1(
     ----------
     ds : xarray.Dataset
         DISDRODB L0C dataset.
-    fall_velocity_model : str, optional
-        Method to compute fall velocity.
-        The default method is ``"Beard1976"``.
-    minimum_diameter : float, optional
-        Minimum diameter for filtering. The default value is 0 mm.
-    maximum_diameter : float, optional
-        Maximum diameter for filtering. The default value is 10 mm.
-    minimum_velocity : float, optional
-        Minimum velocity for filtering. The default value is 0 m/s.
-    maximum_velocity : float, optional
-        Maximum velocity for filtering. The default value is 12 m/s.
-    above_velocity_fraction : float, optional
-        Fraction of drops above velocity threshold. The default value is 0.5.
-    above_velocity_tolerance : float or None, optional
-        Tolerance for above velocity filtering. The default value is ``None``.
-    below_velocity_fraction : float, optional
-        Fraction of drops below velocity threshold. The default value is 0.5.
-    below_velocity_tolerance : float or None, optional
-        Tolerance for below velocity filtering. The default value is ``None``.
-    small_diameter_threshold : float, optional
-        Threshold for small diameter drops. The default value is 1.
-    small_velocity_threshold : float, optional
-        Threshold for small velocity drops. The default value is 2.5.
-    maintain_smallest_drops : bool, optional
-        Whether to maintain the smallest drops. The default value is ``True``.
 
     Returns
     -------
@@ -95,7 +57,7 @@ def generate_l1(
     # - If not present, don't drop Parsivels first two bins
     sensor_name = attrs.get("sensor_name", "")
 
-    # ---------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     # Retrieve sample interval
     # --> sample_interval is a coordinate of L0C products
     if "sample_interval" in ds:
@@ -104,14 +66,14 @@ def generate_l1(
         # This line is not called in the DISDRODB processing chain !
         sample_interval = infer_sample_interval(ds, verbose=False)
 
-    # ---------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     # Retrieve ENV dataset or take defaults
     # - Used only for Beard fall velocity currently !
     # - It checks and includes default geolocation if missing
     # - For mobile disdrometer, infill missing geolocation with backward and forward filling
     ds_env = load_env_dataset(ds)
 
-    # ---------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     # Initialize L1 dataset
     ds_l1 = xr.Dataset()
 
@@ -122,77 +84,45 @@ def generate_l1(
     ds_l1 = add_sample_interval(ds_l1, sample_interval=sample_interval)
 
     # Add optional variables to L1 dataset
-    optional_variables = ["time_qc", "qc_resampling"]
+    optional_variables = ["qc_time", "qc_resampling", *METEOROLOGICAL_VARIABLES]
     for var in optional_variables:
         if var in ds:
             ds_l1[var] = ds[var]
 
-    # -------------------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     # Filter dataset by diameter and velocity bins
     if sensor_name in ["PARSIVEL", "PARSIVEL2"]:
         # - Remove first two bins because never reports data !
+        # - Could be removed also in L2E, but we save disk space here
         # - If not removed, can alter e.g. L2M model fitting
         ds_l1 = filter_diameter_bins(ds=ds_l1, minimum_diameter=0.2495)  # it includes the 0.2495-0.3745 bin
 
-    # - Filter diameter bins
-    ds_l1 = filter_diameter_bins(ds=ds_l1, minimum_diameter=minimum_diameter, maximum_diameter=maximum_diameter)
-    # - Filter velocity bins
-    if has_velocity_dimension:
-        ds_l1 = filter_velocity_bins(ds=ds_l1, minimum_velocity=minimum_velocity, maximum_velocity=maximum_velocity)
-
-    # -------------------------------------------------------------------------------------------
-    # Compute fall velocity
-    ds_l1["fall_velocity"] = get_raindrop_fall_velocity_from_ds(ds=ds_l1, ds_env=ds_env, model=fall_velocity_model)
-
-    # -------------------------------------------------------------------------------------------
-    # Define filtering mask according to fall velocity
+    # --------------------------------------------------------------------------
+    # If (diameter, velocity) spectrum is available, run hydrometeor classification
     if has_velocity_dimension:
-        mask = define_raindrop_spectrum_mask(
-            drop_number=ds_l1["raw_drop_number"],
-            fall_velocity=ds_l1["fall_velocity"],
-            above_velocity_fraction=above_velocity_fraction,
-            above_velocity_tolerance=above_velocity_tolerance,
-            below_velocity_fraction=below_velocity_fraction,
-            below_velocity_tolerance=below_velocity_tolerance,
-            small_diameter_threshold=small_diameter_threshold,
-            small_velocity_threshold=small_velocity_threshold,
-            maintain_smallest_drops=maintain_smallest_drops,
+        temperature, snow_temperature_upper_limit = get_temperature(ds)
+        temperature = temperature.compute() if temperature is not None else None
+        ds_hc = classify_raw_spectrum(
+            ds=ds_l1,
+            ds_env=ds_env,
+            sensor_name=sensor_name,
+            sample_interval=sample_interval,
+            temperature=temperature,
+            rain_temperature_lower_limit=-5,
+            snow_temperature_upper_limit=snow_temperature_upper_limit,
         )
+        ds_l1.update(ds_hc)
 
-    # -------------------------------------------------------------------------------------------
-    # Retrieve drop number and drop_counts arrays
-    if has_velocity_dimension:
-        drop_number = ds_l1["raw_drop_number"].where(mask, 0)  # 2D (diameter, velocity)
-        drop_counts = drop_number.sum(dim=VELOCITY_DIMENSION)  # 1D (diameter)
-        drop_counts_raw = ds_l1["raw_drop_number"].sum(dim=VELOCITY_DIMENSION)  # 1D (diameter)
+    # Otherwise, if no 2D spectrum, do nothing (temporary)
+    # --> Specialized QC for RD-80 or ODM-470 not yet implemented
     else:
-        drop_number = ds_l1["raw_drop_number"]  # 1D (diameter)
-        drop_counts = ds_l1["raw_drop_number"]  # 1D (diameter)
-        drop_counts_raw = ds_l1["raw_drop_number"]
-
-    # Add drop number and drop_counts
-    ds_l1["drop_number"] = drop_number
-    ds_l1["drop_counts"] = drop_counts
-
-    # -------------------------------------------------------------------------------------------
-    # Compute minimum and max drop diameter observed
-    min_drop_diameter, max_drop_diameter = get_min_max_diameter(drop_counts)
-
-    # Add drop statistics
-    ds_l1["Dmin"] = min_drop_diameter
-    ds_l1["Dmax"] = max_drop_diameter
-    ds_l1["N"] = drop_counts.sum(dim=DIAMETER_DIMENSION)
-    ds_l1["Nraw"] = drop_counts_raw.sum(dim=DIAMETER_DIMENSION)
-    ds_l1["Nremoved"] = ds_l1["Nraw"] - ds_l1["N"]
-
-    # Add bins statistics
-    ds_l1 = add_bins_metrics(ds_l1)
-
-    # -------------------------------------------------------------------------------------------
-    # Add quality flags
-    # TODO: snow_flags, insects_flag, ...
+        # If OceanRain ODM470 data, translate precip_flag to precipitation_type
+        if sensor_name == "ODM470" and "precip_flag" in ds:
+            ds_l1["precipitation_type"] = map_precip_flag_to_precipitation_type(ds["precip_flag"])
+        ds_l1["n_particles"] = ds_l1["raw_drop_number"].sum(dim=DIAMETER_DIMENSION)
+        pass
 
-    #### ----------------------------------------------------------------------------.
+    #### ----------------------------------------------------------------------.
     #### Finalize dataset
     # Add global attributes
     ds_l1.attrs = attrs

disdrodb/l1/resampling.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Utilities for temporal resampling."""
+
 import numpy as np
 import pandas as pd
 import xarray as xr
@@ -170,6 +171,9 @@ def resample_dataset(ds, sample_interval, temporal_resolution):
     - The function updates the dataset attributes and the sample_interval coordinate.
 
     """
+    from disdrodb.constants import METEOROLOGICAL_VARIABLES
+    from disdrodb.l1.classification import TEMPERATURE_VARIABLES
+
     # --------------------------------------------------------------------------.
     # Ensure sample interval in seconds
     sample_interval = int(ensure_sample_interval_in_seconds(sample_interval))
@@ -244,6 +248,7 @@ def resample_dataset(ds, sample_interval, temporal_resolution):
 
     # Retrieve variables to average/sum
     # - ATTENTION: it will not resample non-dimensional time coordinates of the dataset !
+    # - precip_flag used for OceanRain ODM470 data
     var_to_average = ["fall_velocity"]
     var_to_cumulate = [
         "raw_drop_number",
@@ -255,8 +260,9 @@ def resample_dataset(ds, sample_interval, temporal_resolution):
         "Nremoved",
         "qc_resampling",
     ]
-    var_to_min = ["Dmin"]
-    var_to_max = ["Dmax", "time_qc"]
+    var_to_min = ["Dmin", *TEMPERATURE_VARIABLES]
+    met_vars = set(METEOROLOGICAL_VARIABLES) - set(TEMPERATURE_VARIABLES)  # exclude air_temperature variable
+    var_to_max = ["Dmax", "qc_time", "precip_flag", *met_vars]
 
     # Retrieve available variables
     var_to_average = [var for var in var_to_average if var in ds]
@@ -305,6 +311,10 @@ def resample_dataset(ds, sample_interval, temporal_resolution):
     # Add attributes
     ds_resampled.attrs = attrs
 
+    # Add back crs coordinate
+    if "crs" in ds:
+        ds_resampled = ds_resampled.assign_coords({"crs": ds["crs"]})
+
     # Add accumulation_interval as new sample_interval coordinate
     ds_resampled = add_sample_interval(ds_resampled, sample_interval=accumulation_interval)
     return ds_resampled

disdrodb/l1_env/__init__.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

disdrodb/l1_env/routines.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Core functions for DISDRODB ENV production."""
+
 import numpy as np
 import xarray as xr
 
@@ -30,15 +31,15 @@ DEFAULT_GEOLOCATION = {
 
 
 def get_default_environment_dataset():
-    """Define defaults values for the ENV dataset."""
+    """Set International Standard Atmosphere values for the default ENV dataset."""
     ds_env = xr.Dataset()
-    ds_env["sea_level_air_pressure"] = 101_325  # Pa
+    ds_env["sea_level_air_pressure"] = 101_325  # Pa # sea level
     ds_env["gas_constant_dry_air"] = 287.04  # J kg⁻¹ K⁻¹
-    ds_env["lapse_rate"] = 0.0065  # K m⁻¹
+    ds_env["lapse_rate"] = 0.0065  # K m⁻¹  (6.5 deg/km)
     ds_env["relative_humidity"] = 0.95  # 0-1 !
-    ds_env["temperature"] = 20 + 273.15  # K
+    ds_env["temperature"] = 15 + 273.15  # K
     ds_env["water_density"] = 1000  # kg m⁻³   (T == 10 --> 999.7, T == 20 --> 998.2)
-    # get_water_density(temperature=temperature, air_pressure=air_pressure
+    # air density = 1.225 kg m⁻³ (if RH = 0) using retrieve_air_density(ds_env)
     return ds_env
 
 

disdrodb/l2/__init__.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Module for DISDRODB L2 production."""
+
 from disdrodb.l2.processing import generate_l2_radar, generate_l2e, generate_l2m
 
 __all__ = [

disdrodb/l2/empirical_dsd.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,11 +20,13 @@ The functions of this module expects xarray.DataArray objects as input.
 Zeros and NaN values input arrays are correctly processed.
 Infinite values should be removed beforehand or otherwise are propagated throughout the computations.
 """
+
 import numpy as np
 import xarray as xr
 
 from disdrodb.api.checks import check_sensor_name
 from disdrodb.constants import DIAMETER_DIMENSION, VELOCITY_DIMENSION
+from disdrodb.utils.time import ensure_sample_interval_in_seconds
 from disdrodb.utils.xarray import (
     remove_diameter_coordinates,
     remove_velocity_coordinates,
@@ -208,8 +210,26 @@ def add_bins_metrics(ds):
     return ds
 
 
-####-------------------------------------------------------------------------------------------------------------------.
-#### DSD Spectrum, Concentration, Moments
+####------------------------------------------------------------------------------------------.
+#### Sampling area and measurement interval
+def get_sampling_area(sensor_name):
+    """Return the sampling area in m2 of the disdrometer."""
+    check_sensor_name(sensor_name)
+    area_dict = {
+        "PARSIVEL": 0.0054,  # 54 cm2
+        "PARSIVEL2": 0.0054,  # 54 cm2
+        "LPM": 0.0045,  # 45 cm2
+        "LPM_V0": 0.0045,  # 45 cm2
+        "ODM470": 0.00264,  # 26.4 cm2
+        "PWS100": 0.004,  # 40 cm2
+        "RD80": 0.005,  # 50 cm2
+        "SWS250": 0.006504,  # 65.04 cm2  (Ricardo Reinoso Rondinel, personal communication)
+    }
+    # SWS250
+    # - Table 29 of the manual that the sample volume is 400cm3, path length?
+    # - Distance between the end of the hood heaters is 291 mm.
+    # - Adding a factor of 1.5 for better representation of the Tx-Rx distance: L= 436 mm.
+    return area_dict[sensor_name]
 
 
 def get_effective_sampling_area(sensor_name, diameter):
@@ -227,26 +247,22 @@ def get_effective_sampling_area(sensor_name, diameter):
         B = 30 / 1000  # Width of the Parsivel beam in m (30mm)
         sampling_area = L * (B - diameter / 2)  # d_eq
         return sampling_area
-    if sensor_name in ["LPM", "LPM_V0"]:
-        # Calculate sampling area for each diameter bin (S_i)
-        # L = 228 / 1000  # Length of the beam in m (228 mm)
-        # B = 20 / 1000  # Width of the beam in m (20 mm)
-        # sampling_area = L * (B - diameter / 2)
-        sampling_area = 0.0045  # m2
-        return sampling_area
+    return get_sampling_area(sensor_name)
+
+
+def get_effective_sampling_interval(ds, sensor_name):
+    """Return the effective sample interval in seconds of the disdrometer."""
+    sample_interval = ensure_sample_interval_in_seconds(ds["sample_interval"])  # s
+    # Adapt sample interval for sensor duty cycle
     if sensor_name == "PWS100":
-        sampling_area = 0.004  # m2
-        return sampling_area
-    if sensor_name == "RD80":
-        sampling_area = 0.005  # m2
-        return sampling_area
-    if sensor_name == "SWS250":
-        # Table 29 of the manual that the sample volume is 400cm3, path length?
-        # Distance between the end of the hood heaters is 291 mm.
-        # Adding a factor of 1.5 for better representation of the Tx-Rx distance: L= 436 mm.
-        sampling_area = 0.0091  # m2 #  0.006504 m2 maybe?
-        return sampling_area
-    raise NotImplementedError(f"Effective sampling area for {sensor_name} must yet to be specified in the software.")
+        # PWS100 has a duty cycle of 9s on, 1s off
+        # - 9 seconds out of 10 are used for counting drops, 1 second is used to process data
+        sample_interval = sample_interval * (9 / 10)
+    return sample_interval
+
+
+####-------------------------------------------------------------------------------------------------------------------.
+#### DSD Spectrum, Concentration, Moments
 
 
 def get_bin_dimensions(xr_obj):
@@ -307,15 +323,19 @@ def get_drop_number_concentration(drop_number, velocity, diameter_bin_width, sam
     # Ensure velocity is 2D (diameter, velocity)
     velocity = xr.ones_like(drop_number) * velocity
 
+    # Create a safe ratio: drop_number / velocity, but 0 where velocity is 0 or NaN
+    safe_ratio = drop_number / velocity
+    safe_ratio = safe_ratio.where((velocity != 0) & (~np.isnan(velocity)), 0)
+
     # Compute drop number concentration
     # - For disdrometer with velocity bins
     if VELOCITY_DIMENSION in drop_number.dims:
-        drop_number_concentration = (drop_number / velocity).sum(dim=VELOCITY_DIMENSION, skipna=False) / (
+        drop_number_concentration = safe_ratio.sum(dim=VELOCITY_DIMENSION, skipna=False) / (
             sampling_area * diameter_bin_width * sample_interval
         )
     # - For impact disdrometers
     else:
-        drop_number_concentration = (drop_number / velocity) / (sampling_area * diameter_bin_width * sample_interval)
+        drop_number_concentration = safe_ratio / (sampling_area * diameter_bin_width * sample_interval)
     return drop_number_concentration
 
 
@@ -1669,8 +1689,7 @@ def get_kinetic_energy_variables_from_drop_number(
     KEF = TKE / sample_interval * 3600
 
     # Compute Kinetic Energy per Rainfall Depth [J/m2/mm]
-    KED = KEF / R
-    KED = xr.where(R == 0, 0, KED)  # Ensure KED is 0 when R (and thus drop number is 0)
+    KED = xr.where(R == 0, 0, KEF / R)  # Ensure KED is 0 when R (and thus drop number is 0)
 
     # Create dataset
     dict_vars = {
@@ -1735,6 +1754,10 @@ def compute_integral_parameters(
         - KED: Kinetic Energy per unit rainfall Depth [J·m⁻²·mm⁻¹].
         - KEF: Kinetic Energy Flux [J·m⁻²·h⁻¹].
     """
+    # Ensure velocity does not contain NaN
+    # --> e.g. when D > 10 mm --> Replace to 0
+    velocity = velocity.fillna(0)
+
     # Initialize dataset
     ds = xr.Dataset()
 
@@ -1764,15 +1787,15 @@ def compute_integral_parameters(
             moment=moment,
         )
 
-    # Compute Liquid Water Content (LWC) (W) [g/m3]
-    # ds["W"] = get_liquid_water_content(
+    # Compute Liquid Water Content (LWC) [g/m3]
+    # ds["LWC"] = get_liquid_water_content(
     #     drop_number_concentration=drop_number_concentration,
     #     diameter=diameter,
     #     diameter_bin_width=diameter_bin_width,
     #     water_density=water_density,
     # )
 
-    ds["W"] = get_liquid_water_content_from_moments(moment_3=ds["M3"], water_density=water_density)
+    ds["LWC"] = get_liquid_water_content_from_moments(moment_3=ds["M3"], water_density=water_density)
 
     # Compute reflectivity in dBZ
     ds["Z"] = get_equivalent_reflectivity_factor(
@@ -1869,9 +1892,13 @@ def compute_spectrum_parameters(
         Dataset containing the following spectrum:
         - KE_spectrum : Kinetic Energy spectrum [J/m2/mm]
         - R_spectrum : Rain Rate spectrum [mm/h/mm]
-        - W_spectrum : Mass spectrum [g/m3/mm]
+        - LWC_spectrum : Mass spectrum [g/m3/mm]
         - Z_spectrum : Reflectivity spectrum [dBZ of mm6/m3/mm]
     """
+    # Ensure velocity does not contain NaN
+    # --> e.g. when D > 10 mm --> Replace to 0
+    velocity = velocity.fillna(0)
+
     # Initialize dataset
     ds = xr.Dataset()
     ds["KE_spectrum"] = get_kinetic_energy_spectrum(
@@ -1882,7 +1909,7 @@ def compute_spectrum_parameters(
         water_density=water_density,
     )
     ds["R_spectrum"] = get_rain_rate_spectrum(drop_number_concentration, velocity=velocity, diameter=diameter)
-    ds["W_spectrum"] = get_liquid_water_spectrum(
+    ds["LWC_spectrum"] = get_liquid_water_spectrum(
         drop_number_concentration,
         diameter=diameter,
         water_density=water_density,