disdrodb 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

disdrodb/{l2 → utils}/event.py

@@ -15,110 +15,16 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Functions for event definition."""
-import dask
+
 import numpy as np
 import pandas as pd
-import xarray as xr
-
-from disdrodb.api.info import get_start_end_time_from_filepaths
-from disdrodb.utils.time import acronym_to_seconds, ensure_sorted_by_time
-
-
-@dask.delayed
-def _delayed_open_dataset(filepath):
-    with dask.config.set(scheduler="synchronous"):
-        ds = xr.open_dataset(filepath, chunks={}, autoclose=True, decode_timedelta=False, cache=False)
-    return ds
-
-
-def identify_events(
-    filepaths,
-    parallel=False,
-    min_n_drops=5,
-    neighbor_min_size=2,
-    neighbor_time_interval="5MIN",
-    intra_event_max_time_gap="6H",
-    event_min_duration="5MIN",
-    event_min_size=3,
-):
-    """Return a list of rainy events.
-
-    Rainy timesteps are defined when N > min_n_drops.
-    Any rainy isolated timesteps (based on neighborhood criteria) is removed.
-    Then, consecutive rainy timesteps are grouped into the same event if the time gap between them does not
-    exceed `intra_event_max_time_gap`. Finally, events that do not meet minimum size or duration
-    requirements are filtered out.
-
-    Parameters
-    ----------
-    filepaths: list
-        List of L1C file paths.
-    parallel: bool
-        Whether to load the files in parallel.
-        Set parallel=True only in a multiprocessing environment.
-        The default is False.
-    neighbor_time_interval : str
-        The time interval around a given a timestep defining the neighborhood.
-        Only timesteps that fall within this time interval before or after a timestep are considered neighbors.
-    neighbor_min_size : int, optional
-        The minimum number of neighboring timesteps required within `neighbor_time_interval` for a
-        timestep to be considered non-isolated.  Isolated timesteps are removed !
-        - If `neighbor_min_size=0,  then no timestep is considered isolated and no filtering occurs.
-        - If `neighbor_min_size=1`, the timestep must have at least one neighbor within `neighbor_time_interval`.
-        - If `neighbor_min_size=2`, the timestep must have at least two timesteps within `neighbor_time_interval`.
-        Defaults to 1.
-    intra_event_max_time_gap: str
-        The maximum time interval between two timesteps to be considered part of the same event.
-        This parameters is used to group timesteps into events !
-    event_min_duration : str
-        The minimum duration an event must span. Events shorter than this duration are discarded.
-    event_min_size : int, optional
-        The minimum number of valid timesteps required for an event. Defaults to 1.
 
-    Returns
-    -------
-    list of dict
-        A list of events, where each event is represented as a dictionary with keys:
-        - "start_time": np.datetime64, start time of the event
-        - "end_time": np.datetime64, end time of the event
-        - "duration": np.timedelta64, duration of the event
-        - "n_timesteps": int, number of valid timesteps in the event
-    """
-    # Open datasets in parallel
-    if parallel:
-        list_ds = dask.compute([_delayed_open_dataset(filepath) for filepath in filepaths])[0]
-    else:
-        list_ds = [xr.open_dataset(filepath, chunks={}, cache=False, decode_timedelta=False) for filepath in filepaths]
-    # Filter dataset for requested variables
-    variables = ["time", "N"]
-    list_ds = [ds[variables] for ds in list_ds]
-    # Concat datasets
-    ds = xr.concat(list_ds, dim="time", compat="no_conflicts", combine_attrs="override")
-    # Read in memory the variable needed
-    ds = ds.compute()
-    # Close file on disk
-    _ = [ds.close() for ds in list_ds]
-    del list_ds
-    # Sort dataset by time
-    ds = ensure_sorted_by_time(ds)
-    # Define candidate timesteps to group into events
-    idx_valid = ds["N"].data > min_n_drops
-    timesteps = ds["time"].data[idx_valid]
-    # Define event list
-    event_list = group_timesteps_into_event(
-        timesteps=timesteps,
-        neighbor_min_size=neighbor_min_size,
-        neighbor_time_interval=neighbor_time_interval,
-        intra_event_max_time_gap=intra_event_max_time_gap,
-        event_min_duration=event_min_duration,
-        event_min_size=event_min_size,
-    )
-    return event_list
+from disdrodb.utils.time import temporal_resolution_to_seconds
 
 
 def group_timesteps_into_event(
     timesteps,
-    intra_event_max_time_gap,
+    event_max_time_gap,
     event_min_size=0,
     event_min_duration="0S",
     neighbor_min_size=0,
@@ -130,7 +36,7 @@ def group_timesteps_into_event(
     This function groups valid candidate timesteps into events by considering how they cluster
     in time. Any isolated timesteps (based on neighborhood criteria) are first removed. Then,
     consecutive timesteps are grouped into the same event if the time gap between them does not
-    exceed `intra_event_max_time_gap`. Finally, events that do not meet minimum size or duration
+    exceed `event_max_time_gap`. Finally, events that do not meet minimum size or duration
     requirements are filtered out.
 
     Please note that neighbor_min_size and neighbor_time_interval are very sensitive to the
@@ -150,7 +56,7 @@ def group_timesteps_into_event(
         - If `neighbor_min_size=1`, the timestep must have at least one neighbor within `neighbor_time_interval`.
         - If `neighbor_min_size=2`, the timestep must have at least two timesteps within `neighbor_time_interval`.
         Defaults to 1.
-    intra_event_max_time_gap: str
+    event_max_time_gap: str
         The maximum time interval between two timesteps to be considered part of the same event.
         This parameters is used to group timesteps into events !
     event_min_duration : str
@@ -168,9 +74,9 @@ def group_timesteps_into_event(
         - "n_timesteps": int, number of valid timesteps in the event
     """
     # Retrieve datetime arguments
-    neighbor_time_interval = pd.Timedelta(acronym_to_seconds(neighbor_time_interval), unit="seconds")
-    intra_event_max_time_gap = pd.Timedelta(acronym_to_seconds(intra_event_max_time_gap), unit="seconds")
-    event_min_duration = pd.Timedelta(acronym_to_seconds(event_min_duration), unit="seconds")
+    neighbor_time_interval = pd.Timedelta(temporal_resolution_to_seconds(neighbor_time_interval), unit="seconds")
+    event_max_time_gap = pd.Timedelta(temporal_resolution_to_seconds(event_max_time_gap), unit="seconds")
+    event_min_duration = pd.Timedelta(temporal_resolution_to_seconds(event_min_duration), unit="seconds")
 
     # Remove isolated timesteps
     timesteps = remove_isolated_timesteps(
@@ -180,8 +86,8 @@ def group_timesteps_into_event(
     )
 
     # Group timesteps into events
-    # - If two timesteps are separated by less than intra_event_max_time_gap, are considered the same event
-    events = group_timesteps_into_events(timesteps, intra_event_max_time_gap)
+    # - If two timesteps are separated by less than event_max_time_gap, are considered the same event
+    events = group_timesteps_into_events(timesteps, event_max_time_gap)
 
     # Define list of event
     event_list = [
@@ -270,7 +176,7 @@ def remove_isolated_timesteps(timesteps, neighbor_min_size, neighbor_time_interv
     return non_isolated_timesteps
 
 
-def group_timesteps_into_events(timesteps, intra_event_max_time_gap):
+def group_timesteps_into_events(timesteps, event_max_time_gap):
     """
     Group valid timesteps into events based on a maximum allowed dry interval.
 
@@ -278,7 +184,7 @@ def group_timesteps_into_events(timesteps, intra_event_max_time_gap):
     ----------
     timesteps : array-like of np.datetime64
         Sorted array of valid timesteps.
-    intra_event_max_time_gap : np.timedelta64
+    event_max_time_gap : np.timedelta64
         Maximum time interval allowed between consecutive valid timesteps for them
         to be considered part of the same event.
 
@@ -297,9 +203,9 @@ def group_timesteps_into_events(timesteps, intra_event_max_time_gap):
     # Compute differences between consecutive timesteps
     diffs = np.diff(timesteps)
 
-    # Identify the indices where the gap is larger than intra_event_max_time_gap
+    # Identify the indices where the gap is larger than event_max_time_gap
     # These indices represent boundaries between events
-    break_indices = np.where(diffs > intra_event_max_time_gap)[0] + 1
+    break_indices = np.where(diffs > event_max_time_gap)[0] + 1
 
     # Split the timesteps at the identified break points
     events = np.split(timesteps, break_indices)
@@ -311,7 +217,7 @@ def group_timesteps_into_events(timesteps, intra_event_max_time_gap):
     #     current_t = timesteps[i]
     #     previous_t = timesteps[i - 1]
 
-    #     if current_t - previous_t <= intra_event_max_time_gap:
+    #     if current_t - previous_t <= event_max_time_gap:
     #         current_event.append(current_t)
     #     else:
     #         events.append(current_event)
@@ -322,67 +228,3 @@ def group_timesteps_into_events(timesteps, intra_event_max_time_gap):
 
 
 ####-----------------------------------------------------------------------------------.
-
-
-def get_events_info(list_events, filepaths, accumulation_interval, rolling):
-    """
-    Provide information about the required files for each event.
-
-    For each event in `list_events`, this function identifies the file paths from `filepaths` that
-    overlap with the event period, adjusted by the `accumulation_interval`. The event period is
-    extended backward or forward based on the `rolling` parameter.
-
-    Parameters
-    ----------
-    list_events : list of dict
-        List of events, where each event is a dictionary containing at least 'start_time' and 'end_time'
-        keys with `numpy.datetime64` values.
-    filepaths : list of str
-        List of file paths corresponding to data files.
-    accumulation_interval : numpy.timedelta64 or int
-        Time interval to adjust the event period for accumulation. If an integer is provided, it is
-        assumed to be in seconds.
-    rolling : bool
-        If True, adjust the event period backward by `accumulation_interval` (rolling backward).
-        If False, adjust forward (aggregate forward).
-
-    Returns
-    -------
-    list of dict
-        A list where each element is a dictionary containing:
-        - 'start_time': Adjusted start time of the event (`numpy.datetime64`).
-        - 'end_time': Adjusted end time of the event (`numpy.datetime64`).
-        - 'filepaths': List of file paths overlapping with the adjusted event period.
-
-    """
-    # Ensure accumulation_interval is numpy.timedelta64
-    if not isinstance(accumulation_interval, np.timedelta64):
-        accumulation_interval = np.timedelta64(accumulation_interval, "s")
-
-    # Retrieve file start_time and end_time
-    files_start_time, files_end_time = get_start_end_time_from_filepaths(filepaths)
-
-    # Retrieve information for each event
-    event_info = []
-    for event_dict in list_events:
-        # Retrieve event time period
-        event_start_time = event_dict["start_time"]
-        event_end_time = event_dict["end_time"]
-
-        # Add buffer to account for accumulation interval
-        if rolling:  # backward
-            event_start_time = event_start_time - np.array(accumulation_interval, dtype="m8[s]")
-        else:  # aggregate forward
-            event_end_time = event_end_time + np.array(accumulation_interval, dtype="m8[s]")
-
-        # Derive event filepaths
-        overlaps = (files_start_time <= event_end_time) & (files_end_time >= event_start_time)
-        event_filepaths = np.array(filepaths)[overlaps].tolist()
-
-        # Create dictionary
-        if len(event_filepaths) > 0:
-            event_info.append(
-                {"start_time": event_start_time, "end_time": event_end_time, "filepaths": event_filepaths},
-            )
-
-    return event_info

disdrodb/utils/logger.py

@@ -42,7 +42,7 @@ def create_logger_file(logs_dir, filename, parallel):
     format_type = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
     handler.setFormatter(logging.Formatter(format_type))
     logger.addHandler(handler)
-    logger.setLevel(logging.DEBUG)
+    logger.setLevel(logging.INFO)
 
     # Define logger filepath
     # - LogCaptureHandler of pytest does not have baseFilename attribute --> So set None
@@ -164,9 +164,16 @@ def _define_station_summary_log_file(list_logs, summary_filepath):
 
 
 def _define_station_problem_log_file(list_logs, problem_filepath):
-    # - Copy the log of files with warnings and error
-    list_keywords = ["ERROR"]  # "WARNING"
-    list_patterns = ["ValueError: Less than 5 timesteps available for day"]
+    # Copy the log of files with errors
+    list_keywords = ["ERROR"]
+    # Exclude lines with the following patterns
+    list_patterns = [
+        # Caused by no data with L2E and L2M filtering
+        "No timesteps with rain rate",
+        "No timesteps with N",
+        "No timesteps with Nbins",
+    ]
+    # Compile patterns to search, escaping any special regex characters
     re_keyword = re.compile("|".join(list_keywords))
     # Compile patterns to ignore, escaping any special regex characters
     re_patterns = re.compile("|".join(map(re.escape, list_patterns))) if list_patterns else None
@@ -221,7 +228,7 @@ def create_product_logs(
 
     The logs directory structure is the follow:
     /logs
-    - /files/<product_acronym>/<station> (same structure as data ... a log for each processed file)
+    - /files/<product_name>/<station> (same structure as data ... a log for each processed file)
     - /summary
       -->  SUMMARY.<PRODUCT_ACRONYM>.<CAMPAIGN_NAME>.<STATION_NAME>.log
     - /problems
@@ -269,7 +276,7 @@ def create_product_logs(
             # Product options
             **product_kwargs,
         )
-        list_logs = list_files(logs_dir, glob_pattern="*", recursive=True)
+        list_logs = list_files(logs_dir, recursive=True)
 
     # --------------------------------------------------------.
     # LogCaptureHandler of pytest does not have baseFilename attribute, so it returns None
@@ -332,5 +339,5 @@ def create_product_logs(
 
     # --------------------------------------------------------.
     # Remove /problem directory if empty !
-    if len(os.listdir(logs_problem_dir)) == 0:
+    if len(list_files(logs_problem_dir, glob_pattern="*.log")) == 0:
         os.rmdir(logs_problem_dir)

disdrodb/utils/manipulations.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Include functions helping for DISDRODB product manipulations."""
+
+import numpy as np
+
+from disdrodb.constants import DIAMETER_DIMENSION
+from disdrodb.utils.xarray import unstack_datarray_dimension
+
+
+def get_diameter_bin_edges(ds):
+    """Retrieve diameter bin edges."""
+    bin_edges = np.append(ds["diameter_bin_lower"].compute().data, ds["diameter_bin_upper"].compute().data[-1])
+    return bin_edges
+
+
+def convert_from_decibel(x):
+    """Convert dB to unit."""
+    return np.power(10.0, 0.1 * x)  # x/10
+
+
+def convert_to_decibel(x):
+    """Convert unit to dB."""
+    return 10 * np.log10(x)
+
+
+def unstack_radar_variables(ds):
+    """Unstack radar variables."""
+    from disdrodb.scattering import RADAR_VARIABLES
+
+    for var in RADAR_VARIABLES:
+        if var in ds:
+            ds_unstack = unstack_datarray_dimension(ds[var], dim="frequency", prefix="", suffix="_")
+            ds.update(ds_unstack)
+            ds = ds.drop_vars(var)
+    if "frequency" in ds.dims:
+        ds = ds.drop_dims("frequency")
+    return ds
+
+
+def get_diameter_coords_dict_from_bin_edges(diameter_bin_edges):
+    """Get dictionary with all relevant diameter coordinates."""
+    if np.size(diameter_bin_edges) < 2:
+        raise ValueError("Expecting at least 2 values defining bin edges.")
+    diameter_bin_center = diameter_bin_edges[:-1] + np.diff(diameter_bin_edges) / 2
+    diameter_bin_width = np.diff(diameter_bin_edges)
+    diameter_bin_lower = diameter_bin_edges[:-1]
+    diameter_bin_upper = diameter_bin_edges[1:]
+    coords_dict = {
+        "diameter_bin_center": (DIAMETER_DIMENSION, diameter_bin_center),
+        "diameter_bin_width": (DIAMETER_DIMENSION, diameter_bin_width),
+        "diameter_bin_lower": (DIAMETER_DIMENSION, diameter_bin_lower),
+        "diameter_bin_upper": (DIAMETER_DIMENSION, diameter_bin_upper),
+    }
+    return coords_dict
+
+
+def resample_drop_number_concentration(drop_number_concentration, diameter_bin_edges, method="linear"):
+    """Resample drop number concentration N(D) DataArray to high resolution diameter bins."""
+    diameters_bin_center = diameter_bin_edges[:-1] + np.diff(diameter_bin_edges) / 2
+
+    da = drop_number_concentration.interp(coords={"diameter_bin_center": diameters_bin_center}, method=method)
+    coords_dict = get_diameter_coords_dict_from_bin_edges(diameter_bin_edges)
+    da = da.assign_coords(coords_dict)
+    return da

disdrodb/utils/routines.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Utilities for DISDRODB processing routines."""
+import os
+import shutil
+import tempfile
+
+from disdrodb.api.io import find_files
+from disdrodb.api.path import define_file_folder_path, define_temporal_resolution
+from disdrodb.utils.logger import (
+    close_logger,
+    create_logger_file,
+    log_error,
+    log_info,
+)
+
+
+def is_possible_product(accumulation_interval, sample_interval, rolling):
+    """Assess if production is possible given the requested accumulation interval and source sample_interval."""
+    # Avoid rolling product generation at source sample interval
+    if rolling and accumulation_interval == sample_interval:
+        return False
+    # Avoid product generation if the accumulation_interval is less than the sample interval
+    if accumulation_interval < sample_interval:
+        return False
+    # Avoid producti generation if accumulation_interval is not multiple of sample_interval
+    return accumulation_interval % sample_interval == 0
+
+
+def try_get_required_filepaths(
+    product,
+    data_archive_dir,
+    data_source,
+    campaign_name,
+    station_name,
+    debugging_mode,
+    **product_kwargs,
+):
+    """Try to retrieve required filepaths for a product, or return None if unavailable."""
+    try:
+        filepaths = find_files(
+            data_archive_dir=data_archive_dir,
+            data_source=data_source,
+            campaign_name=campaign_name,
+            station_name=station_name,
+            product=product,
+            debugging_mode=debugging_mode,
+            **product_kwargs,
+        )
+        return filepaths
+    # If no files available, print informative message
+    except Exception as e:
+        temporal_resolution = ""
+        if "sample_interval" in product_kwargs and "rolling" in product_kwargs:
+            temporal_resolution = define_temporal_resolution(
+                seconds=product_kwargs["sample_interval"],
+                rolling=product_kwargs["rolling"],
+            )
+        print(str(e))
+        msg = (
+            f"{product} processing of {data_source} {campaign_name} {station_name} "
+            f"has not been launched because of missing {product} {temporal_resolution} data."
+        )
+        print(msg)
+        return None
+
+
+def run_product_generation(
+    product: str,
+    logs_dir: str,
+    logs_filename: str,
+    parallel: bool,
+    verbose: bool,
+    folder_partitioning: str,
+    core_func: callable,
+    core_func_kwargs: dict,
+    pass_logger=False,
+):
+    """
+    Generic wrapper for DISDRODB product generation.
+
+    Parameters
+    ----------
+    product : str
+        Product name (e.g., "L0A", "L0B", ...).
+
+    logs_dir : str
+        Logs directory.
+    logs_filename : str
+        Logs filename.
+    parallel : bool
+        Parallel flag (for logger).
+    verbose : bool
+        Verbose logging flag.
+    folder_partitioning : str
+        Partitioning scheme.
+    core_func : callable
+        Function with signature `core_func(logger)` that does the product-specific work.
+        Must return an xarray.Dataset or pandas.DataFrame (used to determine log subdir).
+    """
+    with tempfile.TemporaryDirectory() as tmpdir:
+        # Initialize log file
+        logger, tmp_logger_filepath = create_logger_file(
+            logs_dir=tmpdir,
+            filename=logs_filename,
+            parallel=parallel,
+        )
+
+        # Inform that product creation has started
+        log_info(logger, f"{product} processing of {logs_filename} has started.", verbose=verbose)
+
+        # Initialize object
+        obj = None  # if None, means the product creation failed
+
+        # Add logger to core_func_kwargs if specified
+        if pass_logger:
+            core_func_kwargs["logger"] = logger
+
+        # Try product creation
+        try:
+            # Run product creation
+            obj = core_func(**core_func_kwargs)
+
+            # Inform that product creation has ended
+            log_info(logger, f"{product} processing of {logs_filename} has ended.", verbose=verbose)
+
+        # Report error if the case
+        except Exception as e:
+            log_error(logger, f"{type(e).__name__}: {e}", verbose=verbose)
+
+        finally:
+            # Close logger
+            close_logger(logger)
+
+            # Move log file to final logs directory
+            success_flag = obj is not None
+            if success_flag:  # and "time" in obj and len(obj["time"]) > 0:
+                logs_dir = define_file_folder_path(obj, dir_path=logs_dir, folder_partitioning=folder_partitioning)
+            os.makedirs(logs_dir, exist_ok=True)
+            if tmp_logger_filepath is not None:  # (when running pytest, tmp_logger_filepath is None)
+                logger_filepath = os.path.join(logs_dir, os.path.basename(tmp_logger_filepath))
+                shutil.move(tmp_logger_filepath, logger_filepath)
+            else:
+                logger_filepath = None
+
+            # Free memory
+            del obj
+
+    # Return logger filepath
+    return logger_filepath