disdrodb 0.1.4__py3-none-any.whl → 0.2.0__py3-none-any.whl

disdrodb/routines/l1.py

@@ -24,13 +24,14 @@ import os
 import time
 from typing import Optional
 
-import xarray as xr
+import pandas as pd
 
 from disdrodb.api.checks import check_station_inputs
 from disdrodb.api.create_directories import (
     create_logs_directory,
     create_product_directory,
 )
+from disdrodb.api.io import open_netcdf_files
 from disdrodb.api.path import (
     define_file_folder_path,
     define_l1_filename,
@@ -40,9 +41,11 @@ from disdrodb.configs import (
     get_data_archive_dir,
     get_folder_partitioning,
     get_metadata_archive_dir,
-    get_product_options,
 )
 from disdrodb.l1.processing import generate_l1
+from disdrodb.l1.resampling import resample_dataset
+from disdrodb.metadata.reader import read_station_metadata
+from disdrodb.routines.options import L1ProcessingOptions
 from disdrodb.utils.dask import execute_tasks_safely
 from disdrodb.utils.decorators import delayed_if_parallel, single_threaded_if_parallel
 
@@ -52,20 +55,37 @@ from disdrodb.utils.logger import (
     log_info,
 )
 from disdrodb.utils.routines import run_product_generation, try_get_required_filepaths
+from disdrodb.utils.time import (
+    ensure_sample_interval_in_seconds,
+)
 from disdrodb.utils.writer import write_product
 
 logger = logging.getLogger(__name__)
 
 
+def define_l1_logs_filename(campaign_name, station_name, start_time, end_time, temporal_resolution):
+    """Define L1 logs filename."""
+    starting_time = pd.to_datetime(start_time).strftime("%Y%m%d%H%M%S")
+    ending_time = pd.to_datetime(end_time).strftime("%Y%m%d%H%M%S")
+    logs_filename = f"L1.{temporal_resolution}.{campaign_name}.{station_name}.s{starting_time}.e{ending_time}"
+    return logs_filename
+
+
 @delayed_if_parallel
 @single_threaded_if_parallel
 def _generate_l1(
-    filepath,
+    start_time,
+    end_time,
+    filepaths,
     data_dir,
     logs_dir,
     logs_filename,
+    folder_partitioning,
     campaign_name,
     station_name,
+    # L1 options
+    temporal_resolution,
+    product_options,
     # Processing options
     force,
     verbose,
@@ -107,29 +127,57 @@ def _generate_l1(
 
     # Define product processing function
     def core(
-        filepath,
+        filepaths,
+        start_time,
+        end_time,
         campaign_name,
         station_name,
+        # Processing options
+        # logger,
+        # verbose,
+        force,
+        # Product options
+        temporal_resolution,
+        product_options,
+        # Archiving arguments
         data_dir,
         folder_partitioning,
     ):
         """Define L1 product processing."""
-        # Retrieve L1 configurations
-        l1_options = get_product_options("L1").get("product_options")  # TODO: MOVE OUTSIDE
+        # Open the L0C netCDF files
+        ds = open_netcdf_files(
+            filepaths,
+            start_time=start_time,
+            end_time=end_time,
+            variables=["raw_drop_number", "time_qc"],
+            parallel=False,
+            compute=True,
+        )
+
+        # Define sample interval in seconds
+        sample_interval = ensure_sample_interval_in_seconds(ds["sample_interval"]).to_numpy().item()
 
-        # Open the raw netCDF
-        with xr.open_dataset(filepath, chunks=-1, decode_timedelta=False, cache=False) as ds:
-            ds = ds[["raw_drop_number"]].load()
+        # Resample dataset
+        ds = resample_dataset(
+            ds=ds,
+            sample_interval=sample_interval,
+            temporal_resolution=temporal_resolution,
+        )
 
         # Produce L1 dataset
-        ds = generate_l1(ds=ds, **l1_options)
+        ds = generate_l1(ds=ds, **product_options)
 
         # Ensure at least 1 timestep available
         if ds["time"].size <= 1:
             return None
 
         # Write L1 netCDF4 dataset
-        filename = define_l1_filename(ds, campaign_name=campaign_name, station_name=station_name)
+        filename = define_l1_filename(
+            ds,
+            campaign_name=campaign_name,
+            station_name=station_name,
+            temporal_resolution=temporal_resolution,
+        )
         folder_path = define_file_folder_path(ds, dir_path=data_dir, folder_partitioning=folder_partitioning)
         filepath = os.path.join(folder_path, filename)
         write_product(ds, filepath=filepath, force=force)
@@ -139,13 +187,23 @@ def _generate_l1(
 
     # Define product processing function kwargs
     core_func_kwargs = dict(  # noqa: C408
-        filepath=filepath,
+        filepaths=filepaths,
+        start_time=start_time,
+        end_time=end_time,
         campaign_name=campaign_name,
         station_name=station_name,
+        # Processing options
+        # verbose=verbose,
+        force=force,
+        # Product options
+        temporal_resolution=temporal_resolution,
+        product_options=product_options,
         # Archiving options
         data_dir=data_dir,
         folder_partitioning=folder_partitioning,
     )
+
+    # TODO: Inspect core arguments: pass logger, verbose, folder_partitioning if present?
     # Run product generation
     logger_filepath = run_product_generation(
         product=product,
@@ -231,15 +289,6 @@ def run_l1_station(
         station_name=station_name,
     )
 
-    # Define logs directory
-    logs_dir = create_logs_directory(
-        product=product,
-        data_archive_dir=data_archive_dir,
-        data_source=data_source,
-        campaign_name=campaign_name,
-        station_name=station_name,
-    )
-
     # ------------------------------------------------------------------------.
     # Start processing
     if verbose:
@@ -247,18 +296,6 @@ def run_l1_station(
         msg = f"{product} processing of station {station_name} has started."
         log_info(logger=logger, msg=msg, verbose=verbose)
 
-    # ------------------------------------------------------------------------.
-    # Create directory structure
-    data_dir = create_product_directory(
-        data_archive_dir=data_archive_dir,
-        metadata_archive_dir=metadata_archive_dir,
-        data_source=data_source,
-        campaign_name=campaign_name,
-        station_name=station_name,
-        product=product,
-        force=force,
-    )
-
     # -------------------------------------------------------------------------.
     # List files to process
     # - If no data available, print error message and return None
@@ -275,38 +312,114 @@ def run_l1_station(
     if filepaths is None:
         return
 
-    # -----------------------------------------------------------------.
-    # Generate L1 files
-    # - Loop over the L0 netCDF files and generate L1 files.
-    # - If parallel=True, it does that in parallel using dask.delayed
-    list_tasks = [
-        _generate_l1(
-            filepath=filepath,
-            data_dir=data_dir,
-            logs_dir=logs_dir,
-            logs_filename=os.path.basename(filepath),
+    # -------------------------------------------------------------------------.
+    # Read station metadata and retrieve sensor name
+    metadata = read_station_metadata(
+        metadata_archive_dir=metadata_archive_dir,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+    )
+    sensor_name = metadata["sensor_name"]
+
+    # -------------------------------------------------------------------------.
+    # Retrieve L1 processing options
+    l1_processing_options = L1ProcessingOptions(
+        sensor_name=sensor_name,
+        filepaths=filepaths,
+        parallel=parallel,
+    )
+
+    # -------------------------------------------------------------------------.
+    # Generate products for each temporal resolution
+    # temporal_resolution = "1MIN"
+    # temporal_resolution = "10MIN"
+    for temporal_resolution in l1_processing_options.temporal_resolutions:
+        # Print progress message
+        msg = f"Production of {product} {temporal_resolution} has started."
+        log_info(logger=logger, msg=msg, verbose=verbose)
+
+        # Retrieve event info
+        files_partitions = l1_processing_options.group_files_by_temporal_partitions(temporal_resolution)
+
+        # Retrieve folder partitioning (for files and logs)
+        folder_partitioning = l1_processing_options.get_folder_partitioning(temporal_resolution)
+
+        # Retrieve product options
+        product_options = l1_processing_options.get_product_options(temporal_resolution)
+        product_options = product_options.get("product_options")
+
+        # ------------------------------------------------------------------.
+        # Create product directory
+        data_dir = create_product_directory(
+            data_archive_dir=data_archive_dir,
+            metadata_archive_dir=metadata_archive_dir,
+            data_source=data_source,
             campaign_name=campaign_name,
             station_name=station_name,
-            # Processing options
+            product=product,
             force=force,
-            verbose=verbose,
-            parallel=parallel,
+            # Option for L1
+            temporal_resolution=temporal_resolution,
         )
-        for filepath in filepaths
-    ]
-    list_logs = execute_tasks_safely(list_tasks=list_tasks, parallel=parallel, logs_dir=logs_dir)
 
-    # -----------------------------------------------------------------.
-    # Define L1 summary logs
-    create_product_logs(
-        product=product,
-        data_source=data_source,
-        campaign_name=campaign_name,
-        station_name=station_name,
-        data_archive_dir=data_archive_dir,
-        # Logs list
-        list_logs=list_logs,
-    )
+        # Define logs directory
+        logs_dir = create_logs_directory(
+            product=product,
+            data_archive_dir=data_archive_dir,
+            data_source=data_source,
+            campaign_name=campaign_name,
+            station_name=station_name,
+            # Option for L1
+            temporal_resolution=temporal_resolution,
+        )
+
+        # ------------------------------------------------------------------.
+        # Generate files
+        # - Loop over the L0C netCDF files and generate L1 files.
+        # - If parallel=True, it does that in parallel using dask.delayed
+        list_tasks = [
+            _generate_l1(
+                start_time=event_info["start_time"],
+                end_time=event_info["end_time"],
+                filepaths=event_info["filepaths"],
+                data_dir=data_dir,
+                logs_dir=logs_dir,
+                logs_filename=define_l1_logs_filename(
+                    campaign_name=campaign_name,
+                    station_name=station_name,
+                    start_time=event_info["start_time"],
+                    end_time=event_info["end_time"],
+                    temporal_resolution=temporal_resolution,
+                ),
+                folder_partitioning=folder_partitioning,
+                campaign_name=campaign_name,
+                station_name=station_name,
+                # L1 product options
+                temporal_resolution=temporal_resolution,
+                product_options=product_options,
+                # Processing options
+                force=force,
+                verbose=verbose,
+                parallel=parallel,
+            )
+            for event_info in files_partitions
+        ]
+        list_logs = execute_tasks_safely(list_tasks=list_tasks, parallel=parallel, logs_dir=logs_dir)
+
+        # -----------------------------------------------------------------.
+        # Define product summary logs
+        create_product_logs(
+            product=product,
+            data_source=data_source,
+            campaign_name=campaign_name,
+            station_name=station_name,
+            data_archive_dir=data_archive_dir,
+            # Product options
+            temporal_resolution=temporal_resolution,
+            # Logs list
+            list_logs=list_logs,
+        )
 
     # ---------------------------------------------------------------------.
     # End L1 processing