disdrodb 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

disdrodb/l2/empirical_dsd.py

@@ -23,8 +23,8 @@ Infinite values should be removed beforehand or otherwise are propagated through
 import numpy as np
 import xarray as xr
 
-from disdrodb import DIAMETER_DIMENSION, VELOCITY_DIMENSION
 from disdrodb.api.checks import check_sensor_name
+from disdrodb.constants import DIAMETER_DIMENSION, VELOCITY_DIMENSION
 from disdrodb.utils.xarray import (
     remove_diameter_coordinates,
     remove_velocity_coordinates,
@@ -66,7 +66,7 @@ def get_drop_average_velocity(drop_number):
     ----------
     drop_number : xarray.DataArray
         Array of drop counts \\( n(D,v) \\) per diameter (and velocity, if available) bins
-        over the time integration period.
+        over the measurement interval.
         The DataArray must have the ``velocity_bin_center`` coordinate.
 
     Returns
@@ -80,6 +80,7 @@ def get_drop_average_velocity(drop_number):
         dim=VELOCITY_DIMENSION,
         skipna=False,
     )
+    average_velocity.name = "average_velocity"
     return average_velocity
 
 
@@ -138,6 +139,9 @@ def _compute_qc_bins_metrics(arr):
     return output
 
 
+BINS_METRICS = ["Nbins", "Nbins_missing", "Nbins_missing_fraction", "Nbins_missing_consecutive"]
+
+
 def compute_qc_bins_metrics(ds):
     """
     Compute quality-control metrics for drop-count bins along the diameter dimension.
@@ -191,11 +195,19 @@ def compute_qc_bins_metrics(ds):
     )
 
     # Assign meaningful labels to the qc 'metric' dimension
-    variables = ["Nbins", "Nbins_missing", "Nbins_missing_fraction", "Nbins_missing_consecutive"]
-    ds_qc_bins = da_qc_bins.assign_coords(metric=variables).to_dataset(dim="metric")
+    ds_qc_bins = da_qc_bins.assign_coords(metric=BINS_METRICS).to_dataset(dim="metric")
     return ds_qc_bins
 
 
+def add_bins_metrics(ds):
+    """Add bin metrics if missing."""
+    bins_metrics = BINS_METRICS
+    if not np.all(np.isin(bins_metrics, list(ds.data_vars))):
+        # Add bins statistics
+        ds.update(compute_qc_bins_metrics(ds))
+    return ds
+
+
 ####-------------------------------------------------------------------------------------------------------------------.
 #### DSD Spectrum, Concentration, Moments
 
@@ -252,7 +264,7 @@ def get_drop_number_concentration(drop_number, velocity, diameter_bin_width, sam
         Width of each diameter bin \\( \\Delta D \\) in millimeters (mm).
     drop_number : xarray.DataArray
         Array of drop counts \\(  n(D) or n(D,v) \\) per diameter (and velocity if available)
-        bins over the time integration period.
+        bins over the measurement interval.
     sample_interval : float or xarray.DataArray
         Time over which the drops are counted \\( \\Delta t \\) in seconds (s).
     sampling_area : float or xarray.DataArray
@@ -277,7 +289,7 @@ def get_drop_number_concentration(drop_number, velocity, diameter_bin_width, sam
     - \\( n(D,v) \\): Number of drops counted in diameter (and velocity) bins.
     - \\( A_{\text{eff}}(D) \\): Effective sampling area of the sensor for diameter \\( D \\) in square meters (m²).
     - \\( \\Delta D \\): Diameter bin width in millimeters (mm).
-    - \\( \\Delta t \\): Time integration period in seconds (s).
+    - \\( \\Delta t \\): Measurement interval in seconds (s).
     - \\( v(D) \\): Fall velocity of drops in diameter bin \\( D \\) in meters per second (m/s).
 
     The effective sampling area \\( A_{\text{eff}}(D) \\) depends on the sensor and may vary with drop diameter.
@@ -919,8 +931,7 @@ def get_min_max_diameter(drop_counts):
     return min_drop_diameter, max_drop_diameter
 
 
-def get_mode_diameter(drop_number_concentration, diameter):
-    """Get raindrop diameter with highest occurrence."""
+def _get_mode_diameter(drop_number_concentration, diameter):
     # If all NaN, set to 0 otherwise argmax fail when all NaN data
     idx_all_nan_mask = np.isnan(drop_number_concentration).all(dim=DIAMETER_DIMENSION)
     drop_number_concentration = drop_number_concentration.where(~idx_all_nan_mask, 0)
@@ -939,6 +950,43 @@ def get_mode_diameter(drop_number_concentration, diameter):
     return diameter_mode
 
 
+def get_mode_diameter(
+    drop_number_concentration,
+    diameter,
+):
+    """Get raindrop diameter with highest occurrence.
+
+    Parameters
+    ----------
+    drop_number_concentration : xarray.DataArray
+        The drop number concentration N(D) for each diameter bin, typically in units of
+        number per cubic meter per millimeter (m⁻³·mm⁻¹).
+    diameter : xarray.DataArray
+        The equivalent volume diameters D of the drops in each bin, in meters (m).
+
+    Returns
+    -------
+    xarray.DataArray
+        The diameter with the highest drop number concentration.
+    """
+    # Use map_blocks if working with Dask arrays
+    if hasattr(drop_number_concentration.data, "chunks"):
+        # Define the template for output
+        template = remove_diameter_coordinates(drop_number_concentration.isel({DIAMETER_DIMENSION: 0}))
+        diameter_mode = xr.map_blocks(
+            _get_mode_diameter,
+            drop_number_concentration,
+            kwargs={"diameter": diameter.compute()},
+            template=template,
+        )
+    else:
+        diameter_mode = _get_mode_diameter(
+            drop_number_concentration=drop_number_concentration,
+            diameter=diameter,
+        )
+    return diameter_mode
+
+
 ####-------------------------------------------------------------------------------------------------------------------.
 #### Mass Distribution Diameters
 
@@ -1369,7 +1417,7 @@ def get_normalized_intercept_parameter_from_moments(moment_3, moment_4):
         [m⁻³·mm³] (number per cubic meter times diameter cubed).
 
     moment_4 : float or array-like
-        The foruth moment of the drop size distribution, \\( M_3 \\), in units of
+        The fourth moment of the drop size distribution, \\( M_3 \\), in units of
         [m⁻³·mm4].
 
     Returns
@@ -1581,7 +1629,7 @@ def get_kinetic_energy_variables_from_drop_number(
     - \\( D_i \\) is the diameter of bin \\( i \\).
     - \\( v_j \\) is the velocity of bin \\( j \\).
     - \\( A \\) is the sampling area.
-    - \\( \\Delta t \\) is the time integration period in seconds.
+    - \\( \\Delta t \\) is the measurement interval in seconds.
     - \\( R \\) is the rainfall rate in mm/hr.
 
     """

disdrodb/l2/event.py

@@ -15,110 +15,17 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Functions for event definition."""
-import dask
+
 import numpy as np
 import pandas as pd
-import xarray as xr
 
 from disdrodb.api.info import get_start_end_time_from_filepaths
-from disdrodb.utils.time import acronym_to_seconds, ensure_sorted_by_time
-
-
-@dask.delayed
-def _delayed_open_dataset(filepath):
-    with dask.config.set(scheduler="synchronous"):
-        ds = xr.open_dataset(filepath, chunks={}, autoclose=True, decode_timedelta=False, cache=False)
-    return ds
-
-
-def identify_events(
-    filepaths,
-    parallel=False,
-    min_n_drops=5,
-    neighbor_min_size=2,
-    neighbor_time_interval="5MIN",
-    intra_event_max_time_gap="6H",
-    event_min_duration="5MIN",
-    event_min_size=3,
-):
-    """Return a list of rainy events.
-
-    Rainy timesteps are defined when N > min_n_drops.
-    Any rainy isolated timesteps (based on neighborhood criteria) is removed.
-    Then, consecutive rainy timesteps are grouped into the same event if the time gap between them does not
-    exceed `intra_event_max_time_gap`. Finally, events that do not meet minimum size or duration
-    requirements are filtered out.
-
-    Parameters
-    ----------
-    filepaths: list
-        List of L1C file paths.
-    parallel: bool
-        Whether to load the files in parallel.
-        Set parallel=True only in a multiprocessing environment.
-        The default is False.
-    neighbor_time_interval : str
-        The time interval around a given a timestep defining the neighborhood.
-        Only timesteps that fall within this time interval before or after a timestep are considered neighbors.
-    neighbor_min_size : int, optional
-        The minimum number of neighboring timesteps required within `neighbor_time_interval` for a
-        timestep to be considered non-isolated.  Isolated timesteps are removed !
-        - If `neighbor_min_size=0,  then no timestep is considered isolated and no filtering occurs.
-        - If `neighbor_min_size=1`, the timestep must have at least one neighbor within `neighbor_time_interval`.
-        - If `neighbor_min_size=2`, the timestep must have at least two timesteps within `neighbor_time_interval`.
-        Defaults to 1.
-    intra_event_max_time_gap: str
-        The maximum time interval between two timesteps to be considered part of the same event.
-        This parameters is used to group timesteps into events !
-    event_min_duration : str
-        The minimum duration an event must span. Events shorter than this duration are discarded.
-    event_min_size : int, optional
-        The minimum number of valid timesteps required for an event. Defaults to 1.
-
-    Returns
-    -------
-    list of dict
-        A list of events, where each event is represented as a dictionary with keys:
-        - "start_time": np.datetime64, start time of the event
-        - "end_time": np.datetime64, end time of the event
-        - "duration": np.timedelta64, duration of the event
-        - "n_timesteps": int, number of valid timesteps in the event
-    """
-    # Open datasets in parallel
-    if parallel:
-        list_ds = dask.compute([_delayed_open_dataset(filepath) for filepath in filepaths])[0]
-    else:
-        list_ds = [xr.open_dataset(filepath, chunks={}, cache=False, decode_timedelta=False) for filepath in filepaths]
-    # Filter dataset for requested variables
-    variables = ["time", "N"]
-    list_ds = [ds[variables] for ds in list_ds]
-    # Concat datasets
-    ds = xr.concat(list_ds, dim="time", compat="no_conflicts", combine_attrs="override")
-    # Read in memory the variable needed
-    ds = ds.compute()
-    # Close file on disk
-    _ = [ds.close() for ds in list_ds]
-    del list_ds
-    # Sort dataset by time
-    ds = ensure_sorted_by_time(ds)
-    # Define candidate timesteps to group into events
-    idx_valid = ds["N"].data > min_n_drops
-    timesteps = ds["time"].data[idx_valid]
-    # Define event list
-    event_list = group_timesteps_into_event(
-        timesteps=timesteps,
-        neighbor_min_size=neighbor_min_size,
-        neighbor_time_interval=neighbor_time_interval,
-        intra_event_max_time_gap=intra_event_max_time_gap,
-        event_min_duration=event_min_duration,
-        event_min_size=event_min_size,
-    )
-    return event_list
+from disdrodb.utils.time import ensure_timedelta_seconds_interval, temporal_resolution_to_seconds
 
 
 def group_timesteps_into_event(
     timesteps,
-    intra_event_max_time_gap,
+    event_max_time_gap,
     event_min_size=0,
     event_min_duration="0S",
     neighbor_min_size=0,
@@ -130,7 +37,7 @@ def group_timesteps_into_event(
     This function groups valid candidate timesteps into events by considering how they cluster
     in time. Any isolated timesteps (based on neighborhood criteria) are first removed. Then,
     consecutive timesteps are grouped into the same event if the time gap between them does not
-    exceed `intra_event_max_time_gap`. Finally, events that do not meet minimum size or duration
+    exceed `event_max_time_gap`. Finally, events that do not meet minimum size or duration
     requirements are filtered out.
 
     Please note that neighbor_min_size and neighbor_time_interval are very sensitive to the
@@ -150,7 +57,7 @@ def group_timesteps_into_event(
         - If `neighbor_min_size=1`, the timestep must have at least one neighbor within `neighbor_time_interval`.
         - If `neighbor_min_size=2`, the timestep must have at least two timesteps within `neighbor_time_interval`.
         Defaults to 1.
-    intra_event_max_time_gap: str
+    event_max_time_gap: str
         The maximum time interval between two timesteps to be considered part of the same event.
         This parameters is used to group timesteps into events !
     event_min_duration : str
@@ -168,9 +75,9 @@ def group_timesteps_into_event(
         - "n_timesteps": int, number of valid timesteps in the event
     """
     # Retrieve datetime arguments
-    neighbor_time_interval = pd.Timedelta(acronym_to_seconds(neighbor_time_interval), unit="seconds")
-    intra_event_max_time_gap = pd.Timedelta(acronym_to_seconds(intra_event_max_time_gap), unit="seconds")
-    event_min_duration = pd.Timedelta(acronym_to_seconds(event_min_duration), unit="seconds")
+    neighbor_time_interval = pd.Timedelta(temporal_resolution_to_seconds(neighbor_time_interval), unit="seconds")
+    event_max_time_gap = pd.Timedelta(temporal_resolution_to_seconds(event_max_time_gap), unit="seconds")
+    event_min_duration = pd.Timedelta(temporal_resolution_to_seconds(event_min_duration), unit="seconds")
 
     # Remove isolated timesteps
     timesteps = remove_isolated_timesteps(
@@ -180,8 +87,8 @@ def group_timesteps_into_event(
     )
 
     # Group timesteps into events
-    # - If two timesteps are separated by less than intra_event_max_time_gap, are considered the same event
-    events = group_timesteps_into_events(timesteps, intra_event_max_time_gap)
+    # - If two timesteps are separated by less than event_max_time_gap, are considered the same event
+    events = group_timesteps_into_events(timesteps, event_max_time_gap)
 
     # Define list of event
     event_list = [
@@ -270,7 +177,7 @@ def remove_isolated_timesteps(timesteps, neighbor_min_size, neighbor_time_interv
     return non_isolated_timesteps
 
 
-def group_timesteps_into_events(timesteps, intra_event_max_time_gap):
+def group_timesteps_into_events(timesteps, event_max_time_gap):
     """
     Group valid timesteps into events based on a maximum allowed dry interval.
 
@@ -278,7 +185,7 @@ def group_timesteps_into_events(timesteps, intra_event_max_time_gap):
     ----------
     timesteps : array-like of np.datetime64
         Sorted array of valid timesteps.
-    intra_event_max_time_gap : np.timedelta64
+    event_max_time_gap : np.timedelta64
         Maximum time interval allowed between consecutive valid timesteps for them
         to be considered part of the same event.
 
@@ -297,9 +204,9 @@ def group_timesteps_into_events(timesteps, intra_event_max_time_gap):
     # Compute differences between consecutive timesteps
     diffs = np.diff(timesteps)
 
-    # Identify the indices where the gap is larger than intra_event_max_time_gap
+    # Identify the indices where the gap is larger than event_max_time_gap
     # These indices represent boundaries between events
-    break_indices = np.where(diffs > intra_event_max_time_gap)[0] + 1
+    break_indices = np.where(diffs > event_max_time_gap)[0] + 1
 
     # Split the timesteps at the identified break points
     events = np.split(timesteps, break_indices)
@@ -311,7 +218,7 @@ def group_timesteps_into_events(timesteps, intra_event_max_time_gap):
     #     current_t = timesteps[i]
     #     previous_t = timesteps[i - 1]
 
-    #     if current_t - previous_t <= intra_event_max_time_gap:
+    #     if current_t - previous_t <= event_max_time_gap:
     #         current_event.append(current_t)
     #     else:
     #         events.append(current_event)
@@ -324,21 +231,23 @@ def group_timesteps_into_events(timesteps, intra_event_max_time_gap):
 ####-----------------------------------------------------------------------------------.
 
 
-def get_events_info(list_events, filepaths, accumulation_interval, rolling):
+def get_files_partitions(list_partitions, filepaths, sample_interval, accumulation_interval, rolling):  # noqa: ARG001
     """
     Provide information about the required files for each event.
 
-    For each event in `list_events`, this function identifies the file paths from `filepaths` that
+    For each event in `list_partitions`, this function identifies the file paths from `filepaths` that
     overlap with the event period, adjusted by the `accumulation_interval`. The event period is
     extended backward or forward based on the `rolling` parameter.
 
     Parameters
     ----------
-    list_events : list of dict
+    list_partitions : list of dict
         List of events, where each event is a dictionary containing at least 'start_time' and 'end_time'
         keys with `numpy.datetime64` values.
     filepaths : list of str
         List of file paths corresponding to data files.
+    sample_interval : numpy.timedelta64 or int
+        The sample interval of the input dataset.
     accumulation_interval : numpy.timedelta64 or int
         Time interval to adjust the event period for accumulation. If an integer is provided, it is
         assumed to be in seconds.
@@ -355,25 +264,23 @@ def get_events_info(list_events, filepaths, accumulation_interval, rolling):
         - 'filepaths': List of file paths overlapping with the adjusted event period.
 
     """
-    # Ensure accumulation_interval is numpy.timedelta64
-    if not isinstance(accumulation_interval, np.timedelta64):
-        accumulation_interval = np.timedelta64(accumulation_interval, "s")
+    # Ensure sample_interval and accumulation_interval is numpy.timedelta64
+    accumulation_interval = ensure_timedelta_seconds_interval(accumulation_interval)
+    sample_interval = ensure_timedelta_seconds_interval(sample_interval)
 
     # Retrieve file start_time and end_time
     files_start_time, files_end_time = get_start_end_time_from_filepaths(filepaths)
 
     # Retrieve information for each event
     event_info = []
-    for event_dict in list_events:
+    for event_dict in list_partitions:
         # Retrieve event time period
         event_start_time = event_dict["start_time"]
         event_end_time = event_dict["end_time"]
 
-        # Add buffer to account for accumulation interval
-        if rolling:  # backward
-            event_start_time = event_start_time - np.array(accumulation_interval, dtype="m8[s]")
-        else:  # aggregate forward
-            event_end_time = event_end_time + np.array(accumulation_interval, dtype="m8[s]")
+        # Adapt event_end_time if accumulation interval different from sample interval
+        if sample_interval != accumulation_interval:
+            event_end_time = event_end_time + accumulation_interval
 
         # Derive event filepaths
         overlaps = (files_start_time <= event_end_time) & (files_end_time >= event_start_time)