disdrodb 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

disdrodb/l1/resampling.py

@@ -15,12 +15,11 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Utilities for temporal resampling."""
-
-
+import numpy as np
 import pandas as pd
 import xarray as xr
 
-from disdrodb.utils.time import regularize_dataset
+from disdrodb.utils.time import ensure_sample_interval_in_seconds, regularize_dataset
 
 DEFAULT_ACCUMULATIONS = ["10s", "30s", "1min", "2min", "5min", "10min", "30min", "1hour"]
 
@@ -96,6 +95,24 @@ def define_window_size(sample_interval, accumulation_interval):
     return window_size
 
 
+def _resample(ds, variables, accumulation, op):
+    if not variables:
+        return {}
+    ds_subset = ds[variables]
+    if "time" in ds_subset.dims:
+        return getattr(ds_subset.resample({"time": accumulation}), op)(skipna=False)
+    return ds_subset
+
+
+def _rolling(ds, variables, window_size, op):
+    if not variables:
+        return {}
+    ds_subset = ds[variables]
+    if "time" in ds_subset.dims:
+        return getattr(ds_subset.rolling(time=window_size, center=False), op)(skipna=False)
+    return ds_subset
+
+
 def resample_dataset(ds, sample_interval, accumulation_interval, rolling=True):
     """
     Resample the dataset to a specified accumulation interval.
@@ -128,20 +145,61 @@ def resample_dataset(ds, sample_interval, accumulation_interval, rolling=True):
     - The function updates the dataset attributes and the sample_interval coordinate.
 
     """
-    # Retrieve attributes
-    attrs = ds.attrs.copy()
-
-    # TODO: here infill NaN with zero if necessary before regularizing !
+    # --------------------------------------------------------------------------.
+    # Ensure sample interval in seconds
+    sample_interval = int(ensure_sample_interval_in_seconds(sample_interval))
+
+    # --------------------------------------------------------------------------.
+    # Raise error if the accumulation_interval is less than the sample interval
+    if accumulation_interval < sample_interval:
+        raise ValueError("Expecting an accumulation_interval > sample interval.")
+    # Raise error if accumulation_interval is not multiple of sample_interval
+    if not accumulation_interval % sample_interval == 0:
+        raise ValueError("The accumulation_interval is not a multiple of sample interval.")
+
+    # --------------------------------------------------------------------------.
+    #### Preprocess the dataset
+    # Here we set NaN in the raw_drop_number to 0
+    # - We assume that NaN corresponds to 0
+    # - When we regularize, we infill with NaN
+    # - When we aggregate with sum, we don't skip NaN
+    # --> Aggregation with original missing timesteps currently results in NaN !
+
+    # Infill NaN values with zeros for drop_number and raw_drop_number
+    # - This might alter integrated statistics if NaN in spectrum does not actually correspond to 0 !
+    # - TODO: NaN should not be set as 0 !
+    for var in ["drop_number", "raw_drop_number"]:
+        if var in ds:
+            ds[var] = xr.where(np.isnan(ds[var]), 0, ds[var])
 
     # Ensure regular dataset without missing timesteps
+    # --> This adds NaN values for missing timesteps
     ds = regularize_dataset(ds, freq=f"{sample_interval}s")
 
+    # --------------------------------------------------------------------------.
+    # Define dataset attributes
+    attrs = ds.attrs.copy()
+    if rolling:
+        attrs["disdrodb_rolled_product"] = "True"
+    else:
+        attrs["disdrodb_rolled_product"] = "False"
+
+    if sample_interval == accumulation_interval:
+        attrs["disdrodb_aggregated_product"] = "False"
+        ds = add_sample_interval(ds, sample_interval=accumulation_interval)
+        ds.attrs = attrs
+        return ds
+
+    # --------------------------------------------------------------------------.
+    # Resample the dataset
+    attrs["disdrodb_aggregated_product"] = "True"
+
     # Initialize resample dataset
     ds_resampled = xr.Dataset()
 
     # Retrieve variables to average/sum
     var_to_average = ["fall_velocity"]
-    var_to_cumulate = ["raw_drop_number", "drop_number", "drop_counts", "N", "Nremoved"]
+    var_to_cumulate = ["raw_drop_number", "drop_number", "drop_counts", "N", "Nraw", "Nremoved"]
     var_to_min = ["Dmin"]
     var_to_max = ["Dmax"]
 
@@ -154,6 +212,7 @@ def resample_dataset(ds, sample_interval, accumulation_interval, rolling=True):
     # TODO Define custom processing
     # - quality_flag --> take worst
     # - skipna if less than fraction (to not waste lot of data when aggregating over i.e. hours)
+    # - Add tolerance on fraction of missing timesteps for large accumulation_intervals
 
     # Resample the dataset
     # - Rolling currently does not allow direct rolling forward.
@@ -163,74 +222,26 @@ def resample_dataset(ds, sample_interval, accumulation_interval, rolling=True):
     # - https://github.com/pydata/xarray/issues/8958
     if not rolling:
         # Resample
-        if len(var_to_average) > 0:
-            ds_resampled.update(
-                ds[var_to_average].resample({"time": pd.Timedelta(seconds=accumulation_interval)}).mean(skipna=False),
-            )
-        if len(var_to_cumulate) > 0:
-            ds_resampled.update(
-                ds[var_to_cumulate].resample({"time": pd.Timedelta(seconds=accumulation_interval)}).sum(skipna=False),
-            )
-        if len(var_to_min) > 0:
-            ds_resampled.update(
-                ds[var_to_min].resample({"time": pd.Timedelta(seconds=accumulation_interval)}).min(skipna=False),
-            )
-        if len(var_to_max) > 0:
-            ds_resampled.update(
-                ds[var_to_max].resample({"time": pd.Timedelta(seconds=accumulation_interval)}).max(skipna=False),
-            )
-
+        accumulation = pd.Timedelta(seconds=accumulation_interval)
+        ds_resampled.update(_resample(ds=ds, variables=var_to_average, accumulation=accumulation, op="mean"))
+        ds_resampled.update(_resample(ds=ds, variables=var_to_cumulate, accumulation=accumulation, op="sum"))
+        ds_resampled.update(_resample(ds=ds, variables=var_to_min, accumulation=accumulation, op="min"))
+        ds_resampled.update(_resample(ds=ds, variables=var_to_max, accumulation=accumulation, op="max"))
     else:
         # Roll and Resample
         window_size = define_window_size(sample_interval=sample_interval, accumulation_interval=accumulation_interval)
-        if len(var_to_average) > 0:
-            ds_resampled.update(ds[var_to_average].rolling({"time": window_size}, center=False).mean(skipna=False))
-        if len(var_to_cumulate) > 0:
-            ds_resampled.update(ds[var_to_cumulate].rolling({"time": window_size}, center=False).sum(skipna=False))
-
-        if len(var_to_min) > 0:
-            ds_resampled.update(ds[var_to_min].rolling({"time": window_size}, center=False).min(skipna=False))
-        if len(var_to_max) > 0:
-            ds_resampled.update(ds[var_to_max].rolling({"time": window_size}, center=False).max(skipna=False))
-        # Ensure time to correspond to the start time of the integration
+        ds_resampled.update(_rolling(ds=ds, variables=var_to_average, window_size=window_size, op="mean"))
+        ds_resampled.update(_rolling(ds=ds, variables=var_to_cumulate, window_size=window_size, op="sum"))
+        ds_resampled.update(_rolling(ds=ds, variables=var_to_min, window_size=window_size, op="min"))
+        ds_resampled.update(_rolling(ds=ds, variables=var_to_max, window_size=window_size, op="max"))
+        # Ensure time to correspond to the start time of the measurement period
         ds_resampled = ds_resampled.isel(time=slice(window_size - 1, None)).assign_coords(
             {"time": ds_resampled["time"].data[: -window_size + 1]},
         )
 
     # Add attributes
     ds_resampled.attrs = attrs
-    if rolling:
-        ds_resampled.attrs["rolled"] = "True"
-    else:
-        ds_resampled.attrs["rolled"] = "False"
 
     # Add accumulation_interval as new sample_interval coordinate
     ds_resampled = add_sample_interval(ds_resampled, sample_interval=accumulation_interval)
     return ds_resampled
-
-
-def get_possible_accumulations(sample_interval, accumulations=None):
-    """
-    Get a list of valid accumulation intervals based on the sampling time.
-
-    Parameters
-    ----------
-    - sample_interval (int): The inferred sampling time in seconds.
-    - accumulations (list of int or string): List of desired accumulation intervals.
-    If provide integers, specify accumulation in seconds.
-
-    Returns
-    -------
-    - list of int: Valid accumulation intervals in seconds.
-    """
-    # Select default accumulations
-    if accumulations is None:
-        accumulations = DEFAULT_ACCUMULATIONS
-
-    # Get accumulations in seconds
-    accumulations = [int(pd.Timedelta(acc).total_seconds()) if isinstance(acc, str) else acc for acc in accumulations]
-
-    # Filter candidate accumulations to include only those that are multiples of the sampling time
-    possible_accumulations = [acc for acc in accumulations if acc % sample_interval == 0]
-
-    return possible_accumulations

disdrodb/l1/routines.py

@@ -21,13 +21,14 @@
 import datetime
 import logging
 import os
+import shutil
 import time
 from typing import Optional
 
 import dask
 import xarray as xr
 
-# Directory
+from disdrodb.api.checks import check_station_inputs
 from disdrodb.api.create_directories import (
     create_logs_directory,
     create_product_directory,
@@ -38,7 +39,12 @@ from disdrodb.api.path import (
     define_l1_filename,
 )
 from disdrodb.api.search import get_required_product
-from disdrodb.configs import get_data_archive_dir, get_folder_partitioning, get_metadata_archive_dir
+from disdrodb.configs import (
+    get_data_archive_dir,
+    get_folder_partitioning,
+    get_metadata_archive_dir,
+    get_product_options,
+)
 from disdrodb.l1.processing import generate_l1
 from disdrodb.utils.decorators import delayed_if_parallel, single_threaded_if_parallel
 
@@ -55,42 +61,6 @@ from disdrodb.utils.writer import write_product
 logger = logging.getLogger(__name__)
 
 
-def get_l1_options():
-    """Get L1 options."""
-    # - TODO: from YAML
-    # - TODO: as function of sensor name
-
-    # minimum_diameter
-    # --> PWS100: 0 (0.05)
-    # --> PARSIVEL: 0.2495 (0.312)
-    # --> RD80: 0.313 (0.359)
-    # --> LPM: 0.125 (0.1875) (we currently discard first bin with default settings !)
-
-    # maximum_diameter
-    # LPM: 9 (10) mm
-    # RD80: 5.373 (5.6) mm
-    # OTT: 24.5 (26) mm
-    # PWS100: 27.2 (28.8) mm
-
-    l1_options = {
-        # Fall velocity option
-        "fall_velocity_method": "Beard1976",
-        # Diameter-Velocity Filtering Options
-        "minimum_diameter": 0.2495,  # OTT PARSIVEL first two bin no data !
-        "maximum_diameter": 10,
-        "minimum_velocity": 0,
-        "maximum_velocity": 12,
-        "above_velocity_fraction": 0.5,
-        "above_velocity_tolerance": None,
-        "below_velocity_fraction": 0.5,
-        "below_velocity_tolerance": None,
-        "small_diameter_threshold": 1,  # 2
-        "small_velocity_threshold": 2.5,  # 3
-        "maintain_smallest_drops": True,
-    }
-    return l1_options
-
-
 @delayed_if_parallel
 @single_threaded_if_parallel
 def _generate_l1(
@@ -153,16 +123,16 @@ def _generate_l1(
     # Log start processing
     msg = f"{product} processing of {filename} has started."
     log_info(logger=logger, msg=msg, verbose=verbose)
-
+    success_flag = False
     ##------------------------------------------------------------------------.
     # Retrieve L1 configurations
-    l1_options = get_l1_options()
+    l1_options = get_product_options("L1").get("product_options")
 
     ##------------------------------------------------------------------------.
     ### Core computation
     try:
         # Open the raw netCDF
-        with xr.open_dataset(filepath, chunks={}, decode_timedelta=False, cache=False) as ds:
+        with xr.open_dataset(filepath, chunks=-1, decode_timedelta=False, cache=False) as ds:
             ds = ds[["raw_drop_number"]].load()
 
         # Produce L1 dataset
@@ -175,7 +145,13 @@ def _generate_l1(
             folder_path = define_file_folder_path(ds, data_dir=data_dir, folder_partitioning=folder_partitioning)
             filepath = os.path.join(folder_path, filename)
             # Write to disk
-            write_product(ds, product=product, filepath=filepath, force=force)
+            write_product(ds, filepath=filepath, force=force)
+
+        ##--------------------------------------------------------------------.
+        #### - Define logger file final directory
+        if folder_partitioning != "":
+            log_dst_dir = define_file_folder_path(ds, data_dir=logs_dir, folder_partitioning=folder_partitioning)
+            os.makedirs(log_dst_dir, exist_ok=True)
 
         ##--------------------------------------------------------------------.
         # Clean environment
@@ -184,6 +160,7 @@ def _generate_l1(
         # Log end processing
         msg = f"{product} processing of {filename} has ended."
         log_info(logger=logger, msg=msg, verbose=verbose)
+        success_flag = True
 
     ##--------------------------------------------------------------------.
     # Otherwise log the error
@@ -195,6 +172,13 @@ def _generate_l1(
     # Close the file logger
     close_logger(logger)
 
+    # Move logger file to correct partitioning directory
+    if success_flag and folder_partitioning != "" and logger_filepath is not None:
+        # Move logger file to correct partitioning directory
+        dst_filepath = os.path.join(log_dst_dir, os.path.basename(logger_filepath))
+        shutil.move(logger_filepath, dst_filepath)
+        logger_filepath = dst_filepath
+
     # Return the logger file path
     return logger_filepath
 
@@ -260,6 +244,14 @@ def run_l1_station(
     # Retrieve DISDRODB Metadata Archive directory
     metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
 
+    # Check valid data_source, campaign_name, and station_name
+    check_station_inputs(
+        metadata_archive_dir=metadata_archive_dir,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+    )
+
     # Define logs directory
     logs_dir = create_logs_directory(
         product=product,
@@ -310,7 +302,7 @@ def run_l1_station(
     # If no data available, print error message and return None
     if flag_not_available_data:
         msg = (
-            f"{product} processing of {data_source} {campaign_name} {station_name}"
+            f"{product} processing of {data_source} {campaign_name} {station_name} "
             + f"has not been launched because of missing {required_product} data."
         )
         print(msg)

disdrodb/l1_env/routines.py

@@ -15,9 +15,10 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Core functions for DISDRODB ENV production."""
-
 import xarray as xr
 
+from disdrodb.constants import GEOLOCATION_COORDS
+
 
 def get_default_environment_dataset():
     """Define defaults values for the ENV dataset."""
@@ -30,9 +31,23 @@ def get_default_environment_dataset():
     return ds_env
 
 
+def _assign_geolocation(ds_src, dst_dst):
+
+    dict_coords = {coord: ds_src[coord] for coord in GEOLOCATION_COORDS if coord in ds_src}
+    dst_dst = dst_dst.assign_coords(dict_coords)
+    return dst_dst
+
+
 def load_env_dataset(ds):
     """Load the ENV dataset."""
-    # TODO - Retrieve relative_humidity and temperature from L1-ENV
+    # TODO: Retrieve relative_humidity and temperature from L1-ENV
     ds_env = get_default_environment_dataset()
-    ds_env = ds_env.assign_coords({"altitude": ds["altitude"], "latitude": ds["latitude"]})
+    # Compute water density
+    # get_water_density(
+    # temperature=temperature,
+    # air_pressure=air_pressure,
+    # )
+    # -->  (T == 10 --> 999.7, T == 20 --> 998.2
+    ds_env["water_density"] = 1000  # kg / m3 # TODO as function of ENV (temperature, ...) ?
+    ds_env = _assign_geolocation(ds_src=ds, dst_dst=ds_env)
     return ds_env

disdrodb/l2/__init__.py

@@ -15,3 +15,10 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Module for DISDRODB L2 production."""
+from disdrodb.l2.processing import generate_l2_radar, generate_l2e, generate_l2m
+
+__all__ = [
+    "generate_l2_radar",
+    "generate_l2e",
+    "generate_l2m",
+]

disdrodb/l2/empirical_dsd.py

@@ -23,8 +23,8 @@ Infinite values should be removed beforehand or otherwise are propagated through
 import numpy as np
 import xarray as xr
 
-from disdrodb import DIAMETER_DIMENSION, VELOCITY_DIMENSION
 from disdrodb.api.checks import check_sensor_name
+from disdrodb.constants import DIAMETER_DIMENSION, VELOCITY_DIMENSION
 from disdrodb.utils.xarray import (
     remove_diameter_coordinates,
     remove_velocity_coordinates,
@@ -66,7 +66,7 @@ def get_drop_average_velocity(drop_number):
     ----------
     drop_number : xarray.DataArray
         Array of drop counts \\( n(D,v) \\) per diameter (and velocity, if available) bins
-        over the time integration period.
+        over the measurement interval.
         The DataArray must have the ``velocity_bin_center`` coordinate.
 
     Returns
@@ -80,6 +80,7 @@ def get_drop_average_velocity(drop_number):
         dim=VELOCITY_DIMENSION,
         skipna=False,
     )
+    average_velocity.name = "average_velocity"
     return average_velocity
 
 
@@ -138,6 +139,9 @@ def _compute_qc_bins_metrics(arr):
     return output
 
 
+BINS_METRICS = ["Nbins", "Nbins_missing", "Nbins_missing_fraction", "Nbins_missing_consecutive"]
+
+
 def compute_qc_bins_metrics(ds):
     """
     Compute quality-control metrics for drop-count bins along the diameter dimension.
@@ -191,11 +195,19 @@ def compute_qc_bins_metrics(ds):
     )
 
     # Assign meaningful labels to the qc 'metric' dimension
-    variables = ["Nbins", "Nbins_missing", "Nbins_missing_fraction", "Nbins_missing_consecutive"]
-    ds_qc_bins = da_qc_bins.assign_coords(metric=variables).to_dataset(dim="metric")
+    ds_qc_bins = da_qc_bins.assign_coords(metric=BINS_METRICS).to_dataset(dim="metric")
     return ds_qc_bins
 
 
+def add_bins_metrics(ds):
+    """Add bin metrics if missing."""
+    bins_metrics = BINS_METRICS
+    if not np.all(np.isin(bins_metrics, list(ds.data_vars))):
+        # Add bins statistics
+        ds.update(compute_qc_bins_metrics(ds))
+    return ds
+
+
 ####-------------------------------------------------------------------------------------------------------------------.
 #### DSD Spectrum, Concentration, Moments
 
@@ -252,7 +264,7 @@ def get_drop_number_concentration(drop_number, velocity, diameter_bin_width, sam
         Width of each diameter bin \\( \\Delta D \\) in millimeters (mm).
     drop_number : xarray.DataArray
         Array of drop counts \\(  n(D) or n(D,v) \\) per diameter (and velocity if available)
-        bins over the time integration period.
+        bins over the measurement interval.
     sample_interval : float or xarray.DataArray
         Time over which the drops are counted \\( \\Delta t \\) in seconds (s).
     sampling_area : float or xarray.DataArray
@@ -277,7 +289,7 @@ def get_drop_number_concentration(drop_number, velocity, diameter_bin_width, sam
     - \\( n(D,v) \\): Number of drops counted in diameter (and velocity) bins.
     - \\( A_{\text{eff}}(D) \\): Effective sampling area of the sensor for diameter \\( D \\) in square meters (m²).
     - \\( \\Delta D \\): Diameter bin width in millimeters (mm).
-    - \\( \\Delta t \\): Time integration period in seconds (s).
+    - \\( \\Delta t \\): Measurement interval in seconds (s).
     - \\( v(D) \\): Fall velocity of drops in diameter bin \\( D \\) in meters per second (m/s).
 
     The effective sampling area \\( A_{\text{eff}}(D) \\) depends on the sensor and may vary with drop diameter.
@@ -919,8 +931,7 @@ def get_min_max_diameter(drop_counts):
     return min_drop_diameter, max_drop_diameter
 
 
-def get_mode_diameter(drop_number_concentration, diameter):
-    """Get raindrop diameter with highest occurrence."""
+def _get_mode_diameter(drop_number_concentration, diameter):
     # If all NaN, set to 0 otherwise argmax fail when all NaN data
     idx_all_nan_mask = np.isnan(drop_number_concentration).all(dim=DIAMETER_DIMENSION)
     drop_number_concentration = drop_number_concentration.where(~idx_all_nan_mask, 0)
@@ -939,6 +950,43 @@ def get_mode_diameter(drop_number_concentration, diameter):
     return diameter_mode
 
 
+def get_mode_diameter(
+    drop_number_concentration,
+    diameter,
+):
+    """Get raindrop diameter with highest occurrence.
+
+    Parameters
+    ----------
+    drop_number_concentration : xarray.DataArray
+        The drop number concentration N(D) for each diameter bin, typically in units of
+        number per cubic meter per millimeter (m⁻³·mm⁻¹).
+    diameter : xarray.DataArray
+        The equivalent volume diameters D of the drops in each bin, in meters (m).
+
+    Returns
+    -------
+    xarray.DataArray
+        The diameter with the highest drop number concentration.
+    """
+    # Use map_blocks if working with Dask arrays
+    if hasattr(drop_number_concentration.data, "chunks"):
+        # Define the template for output
+        template = remove_diameter_coordinates(drop_number_concentration.isel({DIAMETER_DIMENSION: 0}))
+        diameter_mode = xr.map_blocks(
+            _get_mode_diameter,
+            drop_number_concentration,
+            kwargs={"diameter": diameter.compute()},
+            template=template,
+        )
+    else:
+        diameter_mode = _get_mode_diameter(
+            drop_number_concentration=drop_number_concentration,
+            diameter=diameter,
+        )
+    return diameter_mode
+
+
 ####-------------------------------------------------------------------------------------------------------------------.
 #### Mass Distribution Diameters
 
@@ -1369,7 +1417,7 @@ def get_normalized_intercept_parameter_from_moments(moment_3, moment_4):
         [m⁻³·mm³] (number per cubic meter times diameter cubed).
 
     moment_4 : float or array-like
-        The foruth moment of the drop size distribution, \\( M_3 \\), in units of
+        The fourth moment of the drop size distribution, \\( M_3 \\), in units of
         [m⁻³·mm4].
 
     Returns
@@ -1581,7 +1629,7 @@ def get_kinetic_energy_variables_from_drop_number(
     - \\( D_i \\) is the diameter of bin \\( i \\).
     - \\( v_j \\) is the velocity of bin \\( j \\).
     - \\( A \\) is the sampling area.
-    - \\( \\Delta t \\) is the time integration period in seconds.
+    - \\( \\Delta t \\) is the measurement interval in seconds.
     - \\( R \\) is the rainfall rate in mm/hr.
 
     """