disdrodb 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

disdrodb/{l1/routines.py → routines/l1.py}

@@ -24,79 +24,46 @@ import os
 import time
 from typing import Optional
 
-import dask
 import xarray as xr
 
-# Directory
+from disdrodb.api.checks import check_station_inputs
 from disdrodb.api.create_directories import (
     create_logs_directory,
     create_product_directory,
 )
-from disdrodb.api.io import find_files
 from disdrodb.api.path import (
     define_file_folder_path,
     define_l1_filename,
 )
 from disdrodb.api.search import get_required_product
-from disdrodb.configs import get_data_archive_dir, get_folder_partitioning, get_metadata_archive_dir
+from disdrodb.configs import (
+    get_data_archive_dir,
+    get_folder_partitioning,
+    get_metadata_archive_dir,
+    get_product_options,
+)
 from disdrodb.l1.processing import generate_l1
+from disdrodb.utils.dask import execute_tasks_safely
 from disdrodb.utils.decorators import delayed_if_parallel, single_threaded_if_parallel
 
 # Logger
 from disdrodb.utils.logger import (
-    close_logger,
-    create_logger_file,
     create_product_logs,
-    log_error,
     log_info,
 )
+from disdrodb.utils.routines import run_product_generation, try_get_required_filepaths
 from disdrodb.utils.writer import write_product
 
 logger = logging.getLogger(__name__)
 
 
-def get_l1_options():
-    """Get L1 options."""
-    # - TODO: from YAML
-    # - TODO: as function of sensor name
-
-    # minimum_diameter
-    # --> PWS100: 0 (0.05)
-    # --> PARSIVEL: 0.2495 (0.312)
-    # --> RD80: 0.313 (0.359)
-    # --> LPM: 0.125 (0.1875) (we currently discard first bin with default settings !)
-
-    # maximum_diameter
-    # LPM: 9 (10) mm
-    # RD80: 5.373 (5.6) mm
-    # OTT: 24.5 (26) mm
-    # PWS100: 27.2 (28.8) mm
-
-    l1_options = {
-        # Fall velocity option
-        "fall_velocity_method": "Beard1976",
-        # Diameter-Velocity Filtering Options
-        "minimum_diameter": 0.2495,  # OTT PARSIVEL first two bin no data !
-        "maximum_diameter": 10,
-        "minimum_velocity": 0,
-        "maximum_velocity": 12,
-        "above_velocity_fraction": 0.5,
-        "above_velocity_tolerance": None,
-        "below_velocity_fraction": 0.5,
-        "below_velocity_tolerance": None,
-        "small_diameter_threshold": 1,  # 2
-        "small_velocity_threshold": 2.5,  # 3
-        "maintain_smallest_drops": True,
-    }
-    return l1_options
-
-
 @delayed_if_parallel
 @single_threaded_if_parallel
 def _generate_l1(
     filepath,
     data_dir,
     logs_dir,
+    logs_filename,
     campaign_name,
     station_name,
     # Processing options
@@ -104,7 +71,7 @@ def _generate_l1(
     verbose,
     parallel,  # this is used only to initialize the correct logger !
 ):
-    """Generate the L1 product from the DISRODB L0C netCDF file.
+    """Generate the L1 product from the DISDRODB L0C netCDF file.
 
     Parameters
     ----------
@@ -133,68 +100,64 @@ def _generate_l1(
     If an error occurs during processing, it is caught and logged,
     but no error is raised to interrupt the execution.
     """
-    # -----------------------------------------------------------------.
-    # Define product name
+    # Define product
     product = "L1"
-
     # Define folder partitioning
     folder_partitioning = get_folder_partitioning()
 
-    # -----------------------------------------------------------------.
-    # Create file logger
-    filename = os.path.basename(filepath)
-    logger, logger_filepath = create_logger_file(
-        logs_dir=logs_dir,
-        filename=filename,
-        parallel=parallel,
-    )
-
-    ##------------------------------------------------------------------------.
-    # Log start processing
-    msg = f"{product} processing of {filename} has started."
-    log_info(logger=logger, msg=msg, verbose=verbose)
-
-    ##------------------------------------------------------------------------.
-    # Retrieve L1 configurations
-    l1_options = get_l1_options()
+    # Define product processing function
+    def core(
+        filepath,
+        campaign_name,
+        station_name,
+        data_dir,
+        folder_partitioning,
+    ):
+        """Define L1 product processing."""
+        # Retrieve L1 configurations
+        l1_options = get_product_options("L1").get("product_options")  # TODO: MOVE OUTSIDE
 
-    ##------------------------------------------------------------------------.
-    ### Core computation
-    try:
         # Open the raw netCDF
-        with xr.open_dataset(filepath, chunks={}, decode_timedelta=False, cache=False) as ds:
+        with xr.open_dataset(filepath, chunks=-1, decode_timedelta=False, cache=False) as ds:
             ds = ds[["raw_drop_number"]].load()
 
         # Produce L1 dataset
         ds = generate_l1(ds=ds, **l1_options)
 
-        # Write L1 netCDF4 dataset
-        if ds["time"].size > 1:
-            # Define filepath
-            filename = define_l1_filename(ds, campaign_name=campaign_name, station_name=station_name)
-            folder_path = define_file_folder_path(ds, data_dir=data_dir, folder_partitioning=folder_partitioning)
-            filepath = os.path.join(folder_path, filename)
-            # Write to disk
-            write_product(ds, product=product, filepath=filepath, force=force)
-
-        ##--------------------------------------------------------------------.
-        # Clean environment
-        del ds
-
-        # Log end processing
-        msg = f"{product} processing of {filename} has ended."
-        log_info(logger=logger, msg=msg, verbose=verbose)
+        # Ensure at least 1 timestep available
+        if ds["time"].size <= 1:
+            return None
 
-    ##--------------------------------------------------------------------.
-    # Otherwise log the error
-    except Exception as e:
-        error_type = str(type(e).__name__)
-        msg = f"{error_type}: {e}"
-        log_error(logger, msg, verbose=verbose)
+        # Write L1 netCDF4 dataset
+        filename = define_l1_filename(ds, campaign_name=campaign_name, station_name=station_name)
+        folder_path = define_file_folder_path(ds, dir_path=data_dir, folder_partitioning=folder_partitioning)
+        filepath = os.path.join(folder_path, filename)
+        write_product(ds, filepath=filepath, force=force)
 
-    # Close the file logger
-    close_logger(logger)
+        # Return L1 dataset
+        return ds
 
+    # Define product processing function kwargs
+    core_func_kwargs = dict(  # noqa: C408
+        filepath=filepath,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        # Archiving options
+        data_dir=data_dir,
+        folder_partitioning=folder_partitioning,
+    )
+    # Run product generation
+    logger_filepath = run_product_generation(
+        product=product,
+        logs_dir=logs_dir,
+        logs_filename=logs_filename,
+        parallel=parallel,
+        verbose=verbose,
+        folder_partitioning=folder_partitioning,
+        core_func=core,
+        core_func_kwargs=core_func_kwargs,
+        pass_logger=False,
+    )
     # Return the logger file path
     return logger_filepath
 
@@ -260,6 +223,14 @@ def run_l1_station(
     # Retrieve DISDRODB Metadata Archive directory
     metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
 
+    # Check valid data_source, campaign_name, and station_name
+    check_station_inputs(
+        metadata_archive_dir=metadata_archive_dir,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+    )
+
     # Define logs directory
     logs_dir = create_logs_directory(
         product=product,
@@ -290,30 +261,18 @@ def run_l1_station(
 
     # -------------------------------------------------------------------------.
     # List files to process
+    # - If no data available, print error message and return None
     required_product = get_required_product(product)
-    flag_not_available_data = False
-    try:
-        filepaths = find_files(
-            data_archive_dir=data_archive_dir,
-            data_source=data_source,
-            campaign_name=campaign_name,
-            station_name=station_name,
-            product=required_product,
-            # Processing options
-            debugging_mode=debugging_mode,
-        )
-    except Exception as e:
-        print(str(e))  # Case where no file paths available
-        flag_not_available_data = True
-
-    # -------------------------------------------------------------------------.
-    # If no data available, print error message and return None
-    if flag_not_available_data:
-        msg = (
-            f"{product} processing of {data_source} {campaign_name} {station_name}"
-            + f"has not been launched because of missing {required_product} data."
-        )
-        print(msg)
+    filepaths = try_get_required_filepaths(
+        data_archive_dir=data_archive_dir,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        product=required_product,
+        # Processing options
+        debugging_mode=debugging_mode,
+    )
+    if filepaths is None:
         return
 
     # -----------------------------------------------------------------.
@@ -325,6 +284,7 @@ def run_l1_station(
             filepath=filepath,
             data_dir=data_dir,
             logs_dir=logs_dir,
+            logs_filename=os.path.basename(filepath),
             campaign_name=campaign_name,
             station_name=station_name,
             # Processing options
@@ -334,7 +294,7 @@ def run_l1_station(
         )
         for filepath in filepaths
     ]
-    list_logs = dask.compute(*list_tasks) if parallel else list_tasks
+    list_logs = execute_tasks_safely(list_tasks=list_tasks, parallel=parallel, logs_dir=logs_dir)
 
     # -----------------------------------------------------------------.
     # Define L1 summary logs