disdrodb 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

disdrodb/l1/processing.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,35 +18,22 @@
 
 import xarray as xr
 
-from disdrodb.constants import DIAMETER_DIMENSION, VELOCITY_DIMENSION
-from disdrodb.l1.fall_velocity import get_raindrop_fall_velocity_from_ds
-from disdrodb.l1.filters import define_raindrop_spectrum_mask, filter_diameter_bins, filter_velocity_bins
+from disdrodb.constants import DIAMETER_DIMENSION, METEOROLOGICAL_VARIABLES, VELOCITY_DIMENSION
+from disdrodb.l1.classification import (
+    classify_raw_spectrum,
+    get_temperature,
+    map_precip_flag_to_precipitation_type,
+)
 from disdrodb.l1.resampling import add_sample_interval
 from disdrodb.l1_env.routines import load_env_dataset
-from disdrodb.l2.empirical_dsd import (  # TODO: maybe move out of L2
-    add_bins_metrics,
-    get_min_max_diameter,
-)
+from disdrodb.utils.manipulations import filter_diameter_bins
 from disdrodb.utils.time import ensure_sample_interval_in_seconds, infer_sample_interval
 from disdrodb.utils.writer import finalize_product
 
 
 def generate_l1(
     ds,
-    # Fall velocity option
-    fall_velocity_model="Beard1976",
-    # Diameter-Velocity Filtering Options
-    minimum_diameter=0,
-    maximum_diameter=10,
-    minimum_velocity=0,
-    maximum_velocity=12,
-    above_velocity_fraction=0.5,
-    above_velocity_tolerance=None,
-    below_velocity_fraction=0.5,
-    below_velocity_tolerance=None,
-    small_diameter_threshold=1,  # 2
-    small_velocity_threshold=2.5,  # 3
-    maintain_smallest_drops=True,
+    **kwargs,  # noqa
 ):
     """Generate DISDRODB L1 Dataset from DISDRODB L0C Dataset.
 
@@ -54,31 +41,6 @@ def generate_l1(
     ----------
     ds : xarray.Dataset
         DISDRODB L0C dataset.
-    fall_velocity_model : str, optional
-        Method to compute fall velocity.
-        The default method is ``"Beard1976"``.
-    minimum_diameter : float, optional
-        Minimum diameter for filtering. The default value is 0 mm.
-    maximum_diameter : float, optional
-        Maximum diameter for filtering. The default value is 10 mm.
-    minimum_velocity : float, optional
-        Minimum velocity for filtering. The default value is 0 m/s.
-    maximum_velocity : float, optional
-        Maximum velocity for filtering. The default value is 12 m/s.
-    above_velocity_fraction : float, optional
-        Fraction of drops above velocity threshold. The default value is 0.5.
-    above_velocity_tolerance : float or None, optional
-        Tolerance for above velocity filtering. The default value is ``None``.
-    below_velocity_fraction : float, optional
-        Fraction of drops below velocity threshold. The default value is 0.5.
-    below_velocity_tolerance : float or None, optional
-        Tolerance for below velocity filtering. The default value is ``None``.
-    small_diameter_threshold : float, optional
-        Threshold for small diameter drops. The default value is 1.
-    small_velocity_threshold : float, optional
-        Threshold for small velocity drops. The default value is 2.5.
-    maintain_smallest_drops : bool, optional
-        Whether to maintain the smallest drops. The default value is ``True``.
 
     Returns
     -------
@@ -95,7 +57,7 @@ def generate_l1(
     # - If not present, don't drop Parsivels first two bins
     sensor_name = attrs.get("sensor_name", "")
 
-    # ---------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     # Retrieve sample interval
     # --> sample_interval is a coordinate of L0C products
     if "sample_interval" in ds:
@@ -104,14 +66,14 @@ def generate_l1(
         # This line is not called in the DISDRODB processing chain !
         sample_interval = infer_sample_interval(ds, verbose=False)
 
-    # ---------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     # Retrieve ENV dataset or take defaults
     # - Used only for Beard fall velocity currently !
     # - It checks and includes default geolocation if missing
     # - For mobile disdrometer, infill missing geolocation with backward and forward filling
     ds_env = load_env_dataset(ds)
 
-    # ---------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     # Initialize L1 dataset
     ds_l1 = xr.Dataset()
 
@@ -122,77 +84,45 @@ def generate_l1(
     ds_l1 = add_sample_interval(ds_l1, sample_interval=sample_interval)
 
     # Add optional variables to L1 dataset
-    optional_variables = ["time_qc", "qc_resampling"]
+    optional_variables = ["qc_time", "qc_resampling", *METEOROLOGICAL_VARIABLES]
     for var in optional_variables:
         if var in ds:
             ds_l1[var] = ds[var]
 
-    # -------------------------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
     # Filter dataset by diameter and velocity bins
     if sensor_name in ["PARSIVEL", "PARSIVEL2"]:
         # - Remove first two bins because never reports data !
+        # - Could be removed also in L2E, but we save disk space here
         # - If not removed, can alter e.g. L2M model fitting
         ds_l1 = filter_diameter_bins(ds=ds_l1, minimum_diameter=0.2495)  # it includes the 0.2495-0.3745 bin
 
-    # - Filter diameter bins
-    ds_l1 = filter_diameter_bins(ds=ds_l1, minimum_diameter=minimum_diameter, maximum_diameter=maximum_diameter)
-    # - Filter velocity bins
-    if has_velocity_dimension:
-        ds_l1 = filter_velocity_bins(ds=ds_l1, minimum_velocity=minimum_velocity, maximum_velocity=maximum_velocity)
-
-    # -------------------------------------------------------------------------------------------
-    # Compute fall velocity
-    ds_l1["fall_velocity"] = get_raindrop_fall_velocity_from_ds(ds=ds_l1, ds_env=ds_env, model=fall_velocity_model)
-
-    # -------------------------------------------------------------------------------------------
-    # Define filtering mask according to fall velocity
+    # --------------------------------------------------------------------------
+    # If (diameter, velocity) spectrum is available, run hydrometeor classification
     if has_velocity_dimension:
-        mask = define_raindrop_spectrum_mask(
-            drop_number=ds_l1["raw_drop_number"],
-            fall_velocity=ds_l1["fall_velocity"],
-            above_velocity_fraction=above_velocity_fraction,
-            above_velocity_tolerance=above_velocity_tolerance,
-            below_velocity_fraction=below_velocity_fraction,
-            below_velocity_tolerance=below_velocity_tolerance,
-            small_diameter_threshold=small_diameter_threshold,
-            small_velocity_threshold=small_velocity_threshold,
-            maintain_smallest_drops=maintain_smallest_drops,
+        temperature, snow_temperature_upper_limit = get_temperature(ds)
+        temperature = temperature.compute() if temperature is not None else None
+        ds_hc = classify_raw_spectrum(
+            ds=ds_l1,
+            ds_env=ds_env,
+            sensor_name=sensor_name,
+            sample_interval=sample_interval,
+            temperature=temperature,
+            rain_temperature_lower_limit=-5,
+            snow_temperature_upper_limit=snow_temperature_upper_limit,
         )
+        ds_l1.update(ds_hc)
 
-    # -------------------------------------------------------------------------------------------
-    # Retrieve drop number and drop_counts arrays
-    if has_velocity_dimension:
-        drop_number = ds_l1["raw_drop_number"].where(mask, 0)  # 2D (diameter, velocity)
-        drop_counts = drop_number.sum(dim=VELOCITY_DIMENSION)  # 1D (diameter)
-        drop_counts_raw = ds_l1["raw_drop_number"].sum(dim=VELOCITY_DIMENSION)  # 1D (diameter)
+    # Otherwise, if no 2D spectrum, do nothing (temporary)
+    # --> Specialized QC for RD-80 or ODM-470 not yet implemented
     else:
-        drop_number = ds_l1["raw_drop_number"]  # 1D (diameter)
-        drop_counts = ds_l1["raw_drop_number"]  # 1D (diameter)
-        drop_counts_raw = ds_l1["raw_drop_number"]
-
-    # Add drop number and drop_counts
-    ds_l1["drop_number"] = drop_number
-    ds_l1["drop_counts"] = drop_counts
-
-    # -------------------------------------------------------------------------------------------
-    # Compute minimum and max drop diameter observed
-    min_drop_diameter, max_drop_diameter = get_min_max_diameter(drop_counts)
-
-    # Add drop statistics
-    ds_l1["Dmin"] = min_drop_diameter
-    ds_l1["Dmax"] = max_drop_diameter
-    ds_l1["N"] = drop_counts.sum(dim=DIAMETER_DIMENSION)
-    ds_l1["Nraw"] = drop_counts_raw.sum(dim=DIAMETER_DIMENSION)
-    ds_l1["Nremoved"] = ds_l1["Nraw"] - ds_l1["N"]
-
-    # Add bins statistics
-    ds_l1 = add_bins_metrics(ds_l1)
-
-    # -------------------------------------------------------------------------------------------
-    # Add quality flags
-    # TODO: snow_flags, insects_flag, ...
+        # If OceanRain ODM470 data, translate precip_flag to precipitation_type
+        if sensor_name == "ODM470" and "precip_flag" in ds:
+            ds_l1["precipitation_type"] = map_precip_flag_to_precipitation_type(ds["precip_flag"])
+        ds_l1["n_particles"] = ds_l1["raw_drop_number"].sum(dim=DIAMETER_DIMENSION)
+        pass
 
-    #### ----------------------------------------------------------------------------.
+    #### ----------------------------------------------------------------------.
     #### Finalize dataset
     # Add global attributes
     ds_l1.attrs = attrs

disdrodb/l1/resampling.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -170,6 +170,9 @@ def resample_dataset(ds, sample_interval, temporal_resolution):
     - The function updates the dataset attributes and the sample_interval coordinate.
 
     """
+    from disdrodb.constants import METEOROLOGICAL_VARIABLES
+    from disdrodb.l1.classification import TEMPERATURE_VARIABLES
+
     # --------------------------------------------------------------------------.
     # Ensure sample interval in seconds
     sample_interval = int(ensure_sample_interval_in_seconds(sample_interval))
@@ -244,6 +247,7 @@ def resample_dataset(ds, sample_interval, temporal_resolution):
 
     # Retrieve variables to average/sum
     # - ATTENTION: it will not resample non-dimensional time coordinates of the dataset !
+    # - precip_flag used for OceanRain ODM470 data
     var_to_average = ["fall_velocity"]
     var_to_cumulate = [
         "raw_drop_number",
@@ -255,8 +259,9 @@ def resample_dataset(ds, sample_interval, temporal_resolution):
         "Nremoved",
         "qc_resampling",
     ]
-    var_to_min = ["Dmin"]
-    var_to_max = ["Dmax", "time_qc"]
+    var_to_min = ["Dmin", *TEMPERATURE_VARIABLES]
+    met_vars = set(METEOROLOGICAL_VARIABLES) - set(TEMPERATURE_VARIABLES)  # exclude air_temperature variable
+    var_to_max = ["Dmax", "qc_time", "precip_flag", *met_vars]
 
     # Retrieve available variables
     var_to_average = [var for var in var_to_average if var in ds]

disdrodb/l1_env/__init__.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

disdrodb/l1_env/routines.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -30,15 +30,15 @@ DEFAULT_GEOLOCATION = {
 
 
 def get_default_environment_dataset():
-    """Define defaults values for the ENV dataset."""
+    """Set International Standard Atmosphere values for the default ENV dataset."""
     ds_env = xr.Dataset()
-    ds_env["sea_level_air_pressure"] = 101_325  # Pa
+    ds_env["sea_level_air_pressure"] = 101_325  # Pa # sea level
     ds_env["gas_constant_dry_air"] = 287.04  # J kg⁻¹ K⁻¹
-    ds_env["lapse_rate"] = 0.0065  # K m⁻¹
+    ds_env["lapse_rate"] = 0.0065  # K m⁻¹  (6.5 deg/km)
     ds_env["relative_humidity"] = 0.95  # 0-1 !
-    ds_env["temperature"] = 20 + 273.15  # K
+    ds_env["temperature"] = 15 + 273.15  # K
     ds_env["water_density"] = 1000  # kg m⁻³   (T == 10 --> 999.7, T == 20 --> 998.2)
-    # get_water_density(temperature=temperature, air_pressure=air_pressure
+    # air density = 1.225 kg m⁻³ (if RH = 0) using retrieve_air_density(ds_env)
     return ds_env
 
 

disdrodb/l2/__init__.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

disdrodb/l2/empirical_dsd.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -25,6 +25,7 @@ import xarray as xr
 
 from disdrodb.api.checks import check_sensor_name
 from disdrodb.constants import DIAMETER_DIMENSION, VELOCITY_DIMENSION
+from disdrodb.utils.time import ensure_sample_interval_in_seconds
 from disdrodb.utils.xarray import (
     remove_diameter_coordinates,
     remove_velocity_coordinates,
@@ -208,8 +209,26 @@ def add_bins_metrics(ds):
     return ds
 
 
-####-------------------------------------------------------------------------------------------------------------------.
-#### DSD Spectrum, Concentration, Moments
+####------------------------------------------------------------------------------------------.
+#### Sampling area and measurement interval
+def get_sampling_area(sensor_name):
+    """Return the sampling area in m2 of the disdrometer."""
+    check_sensor_name(sensor_name)
+    area_dict = {
+        "PARSIVEL": 0.0054,  # 54 cm2
+        "PARSIVEL2": 0.0054,  # 54 cm2
+        "LPM": 0.0045,  # 45 cm2
+        "LPM_V0": 0.0045,  # 45 cm2
+        "ODM470": 0.00264,  # 26.4 cm2
+        "PWS100": 0.004,  # 40 cm2
+        "RD80": 0.005,  # 50 cm2
+        "SWS250": 0.006504,  # 65.04 cm2  (Ricardo Reinoso Rondinel, personal communication)
+    }
+    # SWS250
+    # - Table 29 of the manual that the sample volume is 400cm3, path length?
+    # - Distance between the end of the hood heaters is 291 mm.
+    # - Adding a factor of 1.5 for better representation of the Tx-Rx distance: L= 436 mm.
+    return area_dict[sensor_name]
 
 
 def get_effective_sampling_area(sensor_name, diameter):
@@ -220,29 +239,29 @@ def get_effective_sampling_area(sensor_name, diameter):
     check_sensor_name(sensor_name)
     if sensor_name in ["PARSIVEL", "PARSIVEL2"]:
         # Calculate sampling area for each diameter bin (S_i)
+        # - Parsivel remove margin fallers !
+        # - The effective sampling area decreases with increasing drop diameter
+        # sampling_area = 0.0054 # m2
         L = 180 / 1000  # Length of the Parsivel beam in m (180 mm)
         B = 30 / 1000  # Width of the Parsivel beam in m (30mm)
-        sampling_area = L * (B - diameter / 2)
-        return sampling_area
-    if sensor_name == "LPM":
-        # Calculate sampling area for each diameter bin (S_i)
-        L = 228 / 1000  # Length of the Parsivel beam in m (228 mm)
-        B = 20 / 1000  # Width of the Parsivel beam in m (20 mm)
-        sampling_area = L * (B - diameter / 2)
+        sampling_area = L * (B - diameter / 2)  # d_eq
         return sampling_area
+    return get_sampling_area(sensor_name)
+
+
+def get_effective_sampling_interval(ds, sensor_name):
+    """Return the effective sample interval in seconds of the disdrometer."""
+    sample_interval = ensure_sample_interval_in_seconds(ds["sample_interval"])  # s
+    # Adapt sample interval for sensor duty cycle
     if sensor_name == "PWS100":
-        sampling_area = 0.004  # m2  # TODO: L * (B - diameter / 2) ?
-        return sampling_area
-    if sensor_name == "RD80":
-        sampling_area = 0.005  # m2
-        return sampling_area
-    if sensor_name == "SWS250":  # TODO: L * (B - diameter / 2) ?
-        # Table 29 of the manual that the sample volume is 400cm3, path length?
-        # Distance between the end of the hood heaters is 291 mm.
-        # Adding a factor of 1.5 for better representation of the Tx-Rx distance: L= 436 mm.
-        sampling_area = 0.0091  # m2
-        return sampling_area
-    raise NotImplementedError(f"Effective sampling area for {sensor_name} must yet to be specified in the software.")
+        # PWS100 has a duty cycle of 9s on, 1s off
+        # - 9 seconds out of 10 are used for counting drops, 1 second is used to process data
+        sample_interval = sample_interval * (9 / 10)
+    return sample_interval
+
+
+####-------------------------------------------------------------------------------------------------------------------.
+#### DSD Spectrum, Concentration, Moments
 
 
 def get_bin_dimensions(xr_obj):
@@ -303,15 +322,19 @@ def get_drop_number_concentration(drop_number, velocity, diameter_bin_width, sam
     # Ensure velocity is 2D (diameter, velocity)
     velocity = xr.ones_like(drop_number) * velocity
 
+    # Create a safe ratio: drop_number / velocity, but 0 where velocity is 0 or NaN
+    safe_ratio = drop_number / velocity
+    safe_ratio = safe_ratio.where((velocity != 0) & (~np.isnan(velocity)), 0)
+
     # Compute drop number concentration
     # - For disdrometer with velocity bins
     if VELOCITY_DIMENSION in drop_number.dims:
-        drop_number_concentration = (drop_number / velocity).sum(dim=VELOCITY_DIMENSION, skipna=False) / (
+        drop_number_concentration = safe_ratio.sum(dim=VELOCITY_DIMENSION, skipna=False) / (
             sampling_area * diameter_bin_width * sample_interval
         )
     # - For impact disdrometers
     else:
-        drop_number_concentration = (drop_number / velocity) / (sampling_area * diameter_bin_width * sample_interval)
+        drop_number_concentration = safe_ratio / (sampling_area * diameter_bin_width * sample_interval)
     return drop_number_concentration
 
 
@@ -1665,8 +1688,7 @@ def get_kinetic_energy_variables_from_drop_number(
     KEF = TKE / sample_interval * 3600
 
     # Compute Kinetic Energy per Rainfall Depth [J/m2/mm]
-    KED = KEF / R
-    KED = xr.where(R == 0, 0, KED)  # Ensure KED is 0 when R (and thus drop number is 0)
+    KED = xr.where(R == 0, 0, KEF / R)  # Ensure KED is 0 when R (and thus drop number is 0)
 
     # Create dataset
     dict_vars = {
@@ -1731,6 +1753,10 @@ def compute_integral_parameters(
         - KED: Kinetic Energy per unit rainfall Depth [J·m⁻²·mm⁻¹].
         - KEF: Kinetic Energy Flux [J·m⁻²·h⁻¹].
     """
+    # Ensure velocity does not contain NaN
+    # --> e.g. when D > 10 mm --> Replace to 0
+    velocity = velocity.fillna(0)
+
     # Initialize dataset
     ds = xr.Dataset()
 
@@ -1760,15 +1786,15 @@ def compute_integral_parameters(
             moment=moment,
         )
 
-    # Compute Liquid Water Content (LWC) (W) [g/m3]
-    # ds["W"] = get_liquid_water_content(
+    # Compute Liquid Water Content (LWC) [g/m3]
+    # ds["LWC"] = get_liquid_water_content(
     #     drop_number_concentration=drop_number_concentration,
     #     diameter=diameter,
     #     diameter_bin_width=diameter_bin_width,
     #     water_density=water_density,
     # )
 
-    ds["W"] = get_liquid_water_content_from_moments(moment_3=ds["M3"], water_density=water_density)
+    ds["LWC"] = get_liquid_water_content_from_moments(moment_3=ds["M3"], water_density=water_density)
 
     # Compute reflectivity in dBZ
     ds["Z"] = get_equivalent_reflectivity_factor(
@@ -1865,9 +1891,13 @@ def compute_spectrum_parameters(
         Dataset containing the following spectrum:
         - KE_spectrum : Kinetic Energy spectrum [J/m2/mm]
         - R_spectrum : Rain Rate spectrum [mm/h/mm]
-        - W_spectrum : Mass spectrum [g/m3/mm]
+        - LWC_spectrum : Mass spectrum [g/m3/mm]
         - Z_spectrum : Reflectivity spectrum [dBZ of mm6/m3/mm]
     """
+    # Ensure velocity does not contain NaN
+    # --> e.g. when D > 10 mm --> Replace to 0
+    velocity = velocity.fillna(0)
+
     # Initialize dataset
     ds = xr.Dataset()
     ds["KE_spectrum"] = get_kinetic_energy_spectrum(
@@ -1878,7 +1908,7 @@ def compute_spectrum_parameters(
         water_density=water_density,
     )
     ds["R_spectrum"] = get_rain_rate_spectrum(drop_number_concentration, velocity=velocity, diameter=diameter)
-    ds["W_spectrum"] = get_liquid_water_spectrum(
+    ds["LWC_spectrum"] = get_liquid_water_spectrum(
         drop_number_concentration,
         diameter=diameter,
         water_density=water_density,