disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl

disdrodb/l1/resampling.py

@@ -0,0 +1,236 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Utilities for temporal resampling."""
+
+
+import pandas as pd
+import xarray as xr
+
+from disdrodb.utils.time import regularize_dataset
+
+DEFAULT_ACCUMULATIONS = ["10s", "30s", "1min", "2min", "5min", "10min", "30min", "1hour"]
+
+
+def add_sample_interval(ds, sample_interval):
+    """Add a sample_interval coordinate to the dataset.
+
+    Parameters
+    ----------
+    ds : xarray.Dataset
+        The input dataset to which the sample_interval coordinate will be added.
+    sample_interval : int or float
+        The dataset sample interval in seconds.
+
+    Returns
+    -------
+    xarray.Dataset
+        The dataset with the added sample interval coordinate.
+
+    Notes
+    -----
+    The function adds a new coordinate named 'sample_interval' to the dataset and
+    updates the 'measurement_interval' attribute.
+    """
+    # Add sample_interval coordinate
+    ds["sample_interval"] = sample_interval
+    ds["sample_interval"].attrs["description"] = "Sample interval"
+    ds["sample_interval"].attrs["long_name"] = "Sample interval"
+    ds["sample_interval"].attrs["units"] = "seconds"
+    ds = ds.set_coords("sample_interval")
+    # Update measurement_interval attribute
+    ds.attrs = ds.attrs.copy()
+    ds.attrs["measurement_interval"] = int(sample_interval)
+    return ds
+
+
+def define_window_size(sample_interval, accumulation_interval):
+    """
+    Calculate the rolling window size based on sampling and accumulation intervals.
+
+    Parameters
+    ----------
+    sampling_interval : int
+        The sampling interval in seconds.
+    accumulation_interval : int
+        The desired accumulation interval in seconds.
+
+    Returns
+    -------
+    int
+        The calculated window size as the number of sampling intervals required to cover the accumulation interval.
+
+    Raises
+    ------
+    ValueError
+        If the accumulation interval is not a multiple of the sampling interval.
+
+    Examples
+    --------
+    >>> define_window_size(60, 300)
+    5
+
+    >>> define_window_size(120, 600)
+    5
+    """
+    # Check compatitiblity
+    if accumulation_interval % sample_interval != 0:
+        raise ValueError("The accumulation interval must be a multiple of the sample interval.")
+
+    # Calculate the window size
+    window_size = accumulation_interval // sample_interval
+
+    return window_size
+
+
+def resample_dataset(ds, sample_interval, accumulation_interval, rolling=True):
+    """
+    Resample the dataset to a specified accumulation interval.
+
+    Parameters
+    ----------
+    ds : xarray.Dataset
+        The input dataset to be resampled.
+    sample_interval : int
+        The sample interval of the input dataset.
+    accumulation_interval : int
+        The interval in seconds over which to accumulate the data.
+    rolling : bool, optional
+        If True, apply a rolling window before resampling. Default is True.
+        If True, forward rolling is performed.
+        The output timesteps correspond to the starts of the periods over which
+        the resampling operation has been performed !
+
+    Returns
+    -------
+    xarray.Dataset
+        The resampled dataset with updated attributes.
+
+    Notes
+    -----
+    - The function regularizes the dataset (infill possible missing timesteps)
+      before performing the resampling operation.
+    - Variables are categorized into those to be averaged, accumulated, minimized, and maximized.
+    - Custom processing for quality flags and handling of NaNs is defined.
+    - The function updates the dataset attributes and the sample_interval coordinate.
+
+    """
+    # Retrieve attributes
+    attrs = ds.attrs.copy()
+
+    # TODO: here infill NaN with zero if necessary before regularizing !
+
+    # Ensure regular dataset without missing timesteps
+    ds = regularize_dataset(ds, freq=f"{sample_interval}s")
+
+    # Initialize resample dataset
+    ds_resampled = xr.Dataset()
+
+    # Retrieve variables to average/sum
+    var_to_average = ["fall_velocity"]
+    var_to_cumulate = ["raw_drop_number", "drop_number", "drop_counts", "N", "Nremoved"]
+    var_to_min = ["Dmin"]
+    var_to_max = ["Dmax"]
+
+    # Retrieve available variables
+    var_to_average = [var for var in var_to_average if var in ds]
+    var_to_cumulate = [var for var in var_to_cumulate if var in ds]
+    var_to_min = [var for var in var_to_min if var in ds]
+    var_to_max = [var for var in var_to_max if var in ds]
+
+    # TODO Define custom processing
+    # - quality_flag --> take worst
+    # - skipna if less than fraction (to not waste lot of data when aggregating over i.e. hours)
+
+    # Resample the dataset
+    # - Rolling currently does not allow direct rolling forward.
+    # - We currently use center=False which means search for data backward (right-aligned) !
+    # - We then drop the first 'window_size' NaN timesteps and we shift backward the timesteps.
+    # - https://github.com/pydata/xarray/issues/9773
+    # - https://github.com/pydata/xarray/issues/8958
+    if not rolling:
+        # Resample
+        if len(var_to_average) > 0:
+            ds_resampled.update(
+                ds[var_to_average].resample({"time": pd.Timedelta(seconds=accumulation_interval)}).mean(skipna=False),
+            )
+        if len(var_to_cumulate) > 0:
+            ds_resampled.update(
+                ds[var_to_cumulate].resample({"time": pd.Timedelta(seconds=accumulation_interval)}).sum(skipna=False),
+            )
+        if len(var_to_min) > 0:
+            ds_resampled.update(
+                ds[var_to_min].resample({"time": pd.Timedelta(seconds=accumulation_interval)}).min(skipna=False),
+            )
+        if len(var_to_max) > 0:
+            ds_resampled.update(
+                ds[var_to_max].resample({"time": pd.Timedelta(seconds=accumulation_interval)}).max(skipna=False),
+            )
+
+    else:
+        # Roll and Resample
+        window_size = define_window_size(sample_interval=sample_interval, accumulation_interval=accumulation_interval)
+        if len(var_to_average) > 0:
+            ds_resampled.update(ds[var_to_average].rolling({"time": window_size}, center=False).mean(skipna=False))
+        if len(var_to_cumulate) > 0:
+            ds_resampled.update(ds[var_to_cumulate].rolling({"time": window_size}, center=False).sum(skipna=False))
+
+        if len(var_to_min) > 0:
+            ds_resampled.update(ds[var_to_min].rolling({"time": window_size}, center=False).min(skipna=False))
+        if len(var_to_max) > 0:
+            ds_resampled.update(ds[var_to_max].rolling({"time": window_size}, center=False).max(skipna=False))
+        # Ensure time to correspond to the start time of the integration
+        ds_resampled = ds_resampled.isel(time=slice(window_size - 1, None)).assign_coords(
+            {"time": ds_resampled["time"].data[: -window_size + 1]},
+        )
+
+    # Add attributes
+    ds_resampled.attrs = attrs
+    if rolling:
+        ds_resampled.attrs["rolled"] = "True"
+    else:
+        ds_resampled.attrs["rolled"] = "False"
+
+    # Add accumulation_interval as new sample_interval coordinate
+    ds_resampled = add_sample_interval(ds_resampled, sample_interval=accumulation_interval)
+    return ds_resampled
+
+
+def get_possible_accumulations(sample_interval, accumulations=None):
+    """
+    Get a list of valid accumulation intervals based on the sampling time.
+
+    Parameters
+    ----------
+    - sample_interval (int): The inferred sampling time in seconds.
+    - accumulations (list of int or string): List of desired accumulation intervals.
+    If provide integers, specify accumulation in seconds.
+
+    Returns
+    -------
+    - list of int: Valid accumulation intervals in seconds.
+    """
+    # Select default accumulations
+    if accumulations is None:
+        accumulations = DEFAULT_ACCUMULATIONS
+
+    # Get accumulations in seconds
+    accumulations = [int(pd.Timedelta(acc).total_seconds()) if isinstance(acc, str) else acc for acc in accumulations]
+
+    # Filter candidate accumulations to include only those that are multiples of the sampling time
+    possible_accumulations = [acc for acc in accumulations if acc % sample_interval == 0]
+
+    return possible_accumulations

disdrodb/l1/routines.py

@@ -0,0 +1,358 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Implement DISDRODB L1 processing."""
+
+import datetime
+import logging
+import os
+import time
+from typing import Optional
+
+import dask
+import xarray as xr
+
+# Directory
+from disdrodb.api.create_directories import (
+    create_logs_directory,
+    create_product_directory,
+)
+from disdrodb.api.io import find_files
+from disdrodb.api.path import (
+    define_file_folder_path,
+    define_l1_filename,
+)
+from disdrodb.api.search import get_required_product
+from disdrodb.configs import get_data_archive_dir, get_folder_partitioning, get_metadata_archive_dir
+from disdrodb.l1.processing import generate_l1
+from disdrodb.utils.decorators import delayed_if_parallel, single_threaded_if_parallel
+
+# Logger
+from disdrodb.utils.logger import (
+    close_logger,
+    create_logger_file,
+    create_product_logs,
+    log_error,
+    log_info,
+)
+from disdrodb.utils.writer import write_product
+
+logger = logging.getLogger(__name__)
+
+
+def get_l1_options():
+    """Get L1 options."""
+    # - TODO: from YAML
+    # - TODO: as function of sensor name
+
+    # minimum_diameter
+    # --> PWS100: 0.05
+    # --> PARSIVEL: 0.2495
+    # --> RD80: 0.313
+    # --> LPM: 0.125 (we currently discard first bin with this setting)
+
+    # maximum_diameter
+    # LPM: 8 mm
+    # RD80: 5.6 mm
+    # OTT: 26 mm
+
+    l1_options = {
+        # Fall velocity option
+        "fall_velocity_method": "Beard1976",
+        # Diameter-Velocity Filtering Options
+        "minimum_diameter": 0.2495,  # OTT PARSIVEL first two bin no data !
+        "maximum_diameter": 10,
+        "minimum_velocity": 0,
+        "maximum_velocity": 12,
+        "above_velocity_fraction": 0.5,
+        "above_velocity_tolerance": None,
+        "below_velocity_fraction": 0.5,
+        "below_velocity_tolerance": None,
+        "small_diameter_threshold": 1,  # 2
+        "small_velocity_threshold": 2.5,  # 3
+        "maintain_smallest_drops": True,
+    }
+    return l1_options
+
+
+@delayed_if_parallel
+@single_threaded_if_parallel
+def _generate_l1(
+    filepath,
+    data_dir,
+    logs_dir,
+    campaign_name,
+    station_name,
+    # Processing options
+    force,
+    verbose,
+    parallel,  # this is used only to initialize the correct logger !
+):
+    """Generate the L1 product from the DISRODB L0C netCDF file.
+
+    Parameters
+    ----------
+    filepath : str
+        Path to the L0C netCDF file.
+    data_dir : str
+        Directory where the L1 netCDF file will be saved.
+    logs_dir : str
+        Directory where the log file will be saved.
+    campaign_name : str
+        Name of the campaign.
+    station_name : str
+        Name of the station.
+    force : bool
+        If True, overwrite existing files.
+    verbose : bool
+        Whether to verbose the processing.
+
+    Returns
+    -------
+    str
+        Path to the log file generated during processing.
+
+    Notes
+    -----
+    If an error occurs during processing, it is caught and logged,
+    but no error is raised to interrupt the execution.
+    """
+    # -----------------------------------------------------------------.
+    # Define product name
+    product = "L1"
+
+    # Define folder partitioning
+    folder_partitioning = get_folder_partitioning()
+
+    # -----------------------------------------------------------------.
+    # Create file logger
+    filename = os.path.basename(filepath)
+    logger, logger_filepath = create_logger_file(
+        logs_dir=logs_dir,
+        filename=filename,
+        parallel=parallel,
+    )
+
+    ##------------------------------------------------------------------------.
+    # Log start processing
+    msg = f"{product} processing of {filename} has started."
+    log_info(logger=logger, msg=msg, verbose=verbose)
+
+    ##------------------------------------------------------------------------.
+    # Retrieve L1 configurations
+    l1_options = get_l1_options()
+
+    ##------------------------------------------------------------------------.
+    ### Core computation
+    try:
+        # Open the raw netCDF
+        with xr.open_dataset(filepath, chunks={}, decode_timedelta=False, cache=False) as ds:
+            ds = ds[["raw_drop_number"]].load()
+
+        # Produce L1 dataset
+        ds = generate_l1(ds=ds, **l1_options)
+
+        # Write L1 netCDF4 dataset
+        if ds["time"].size > 1:
+            # Define filepath
+            filename = define_l1_filename(ds, campaign_name=campaign_name, station_name=station_name)
+            folder_path = define_file_folder_path(ds, data_dir=data_dir, folder_partitioning=folder_partitioning)
+            filepath = os.path.join(folder_path, filename)
+            # Write to disk
+            write_product(ds, product=product, filepath=filepath, force=force)
+
+        ##--------------------------------------------------------------------.
+        # Clean environment
+        del ds
+
+        # Log end processing
+        msg = f"{product} processing of {filename} has ended."
+        log_info(logger=logger, msg=msg, verbose=verbose)
+
+    ##--------------------------------------------------------------------.
+    # Otherwise log the error
+    except Exception as e:
+        error_type = str(type(e).__name__)
+        msg = f"{error_type}: {e}"
+        log_error(logger, msg, verbose=verbose)
+
+    # Close the file logger
+    close_logger(logger)
+
+    # Return the logger file path
+    return logger_filepath
+
+
+def run_l1_station(
+    # Station arguments
+    data_source,
+    campaign_name,
+    station_name,
+    # Processing options
+    force: bool = False,
+    verbose: bool = True,
+    parallel: bool = True,
+    debugging_mode: bool = False,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
+):
+    """
+    Run the L1 processing of a specific DISDRODB station when invoked from the terminal.
+
+    The L1 routines just filter the raw drop spectrum and compute basic statistics.
+    The L1 routine expects as input L0C files where each file has a unique sample interval.
+
+    This function is intended to be called through the ``disdrodb_run_l1_station``
+    command-line interface.
+
+    Parameters
+    ----------
+    data_source : str
+        The name of the institution (for campaigns spanning multiple countries) or
+        the name of the country (for campaigns or sensor networks within a single country).
+        Must be provided in UPPER CASE.
+    campaign_name : str
+        The name of the campaign. Must be provided in UPPER CASE.
+    station_name : str
+        The name of the station.
+    force : bool, optional
+        If ``True``, existing data in the destination directories will be overwritten.
+        If ``False`` (default), an error will be raised if data already exists in the destination directories.
+    verbose : bool, optional
+        If ``True`` (default), detailed processing information will be printed to the terminal.
+        If ``False``, less information will be displayed.
+    parallel : bool, optional
+        If ``True``, files will be processed in multiple processes simultaneously,
+        with each process using a single thread to avoid issues with the HDF/netCDF library.
+        If ``False`` (default), files will be processed sequentially in a single process,
+        and multi-threading will be automatically exploited to speed up I/O tasks.
+    debugging_mode : bool, optional
+        If ``True``, the amount of data processed will be reduced.
+        Only the first 3 files will be processed. The default value is ``False``.
+    data_archive_dir : str, optional
+        The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``.
+        If not specified, the path specified in the DISDRODB active configuration will be used.
+
+    """
+    # Define product
+    product = "L1"
+
+    # Define base directory
+    data_archive_dir = get_data_archive_dir(data_archive_dir)
+
+    # Retrieve DISDRODB Metadata Archive directory
+    metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
+
+    # Define logs directory
+    logs_dir = create_logs_directory(
+        product=product,
+        data_archive_dir=data_archive_dir,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+    )
+
+    # ------------------------------------------------------------------------.
+    # Start processing
+    if verbose:
+        t_i = time.time()
+        msg = f"{product} processing of station {station_name} has started."
+        log_info(logger=logger, msg=msg, verbose=verbose)
+
+    # ------------------------------------------------------------------------.
+    # Create directory structure
+    data_dir = create_product_directory(
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        product=product,
+        force=force,
+    )
+
+    # -------------------------------------------------------------------------.
+    # List files to process
+    required_product = get_required_product(product)
+    flag_not_available_data = False
+    try:
+        filepaths = find_files(
+            data_archive_dir=data_archive_dir,
+            data_source=data_source,
+            campaign_name=campaign_name,
+            station_name=station_name,
+            product=required_product,
+            # Processing options
+            debugging_mode=debugging_mode,
+        )
+    except Exception as e:
+        print(str(e))  # Case where no file paths available
+        flag_not_available_data = True
+
+    # -------------------------------------------------------------------------.
+    # If no data available, print error message and return None
+    if flag_not_available_data:
+        msg = (
+            f"{product} processing of {data_source} {campaign_name} {station_name}"
+            + f"has not been launched because of missing {required_product} data."
+        )
+        print(msg)
+        return
+
+    # -----------------------------------------------------------------.
+    # Generate L1 files
+    # - Loop over the L0 netCDF files and generate L1 files.
+    # - If parallel=True, it does that in parallel using dask.delayed
+    list_tasks = [
+        _generate_l1(
+            filepath=filepath,
+            data_dir=data_dir,
+            logs_dir=logs_dir,
+            campaign_name=campaign_name,
+            station_name=station_name,
+            # Processing options
+            force=force,
+            verbose=verbose,
+            parallel=parallel,
+        )
+        for filepath in filepaths
+    ]
+    list_logs = dask.compute(*list_tasks) if parallel else list_tasks
+
+    # -----------------------------------------------------------------.
+    # Define L1 summary logs
+    create_product_logs(
+        product=product,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        data_archive_dir=data_archive_dir,
+        # Logs list
+        list_logs=list_logs,
+    )
+
+    # ---------------------------------------------------------------------.
+    # End L1 processing
+    if verbose:
+        timedelta_str = str(datetime.timedelta(seconds=round(time.time() - t_i)))
+        msg = f"{product} processing of station {station_name} completed in {timedelta_str}"
+        log_info(logger=logger, msg=msg, verbose=verbose)
+
+
+####-------------------------------------------------------------------------------------------------------------------.

disdrodb/l1_env/__init__.py

@@ -0,0 +1,17 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Core functions for DISDRODB L1 ENV production."""

disdrodb/l1_env/routines.py

@@ -0,0 +1,38 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Core functions for DISDRODB ENV production."""
+
+import xarray as xr
+
+
+def get_default_environment_dataset():
+    """Define defaults values for the ENV dataset."""
+    ds_env = xr.Dataset()
+    ds_env["sea_level_air_pressure"] = 101_325
+    ds_env["gas_constant_dry_air"] = 287.04
+    ds_env["lapse_rate"] = 0.0065
+    ds_env["relative_humidity"] = 0.95  # Value between 0 and 1 !
+    ds_env["temperature"] = 20 + 273.15
+    return ds_env
+
+
+def load_env_dataset(ds):
+    """Load the ENV dataset."""
+    # TODO - Retrieve relative_humidity and temperature from L1-ENV
+    ds_env = get_default_environment_dataset()
+    ds_env = ds_env.assign_coords({"altitude": ds["altitude"], "latitude": ds["latitude"]})
+    return ds_env

disdrodb/l2/__init__.py

@@ -0,0 +1,17 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Module for DISDRODB L2 production."""