disdrodb 0.1.5__py3-none-any.whl → 0.2.1__py3-none-any.whl

disdrodb/routines/l2.py

@@ -18,7 +18,6 @@
 
 import copy
 import datetime
-import json
 import logging
 import os
 import time
@@ -31,195 +30,51 @@ from disdrodb.api.create_directories import (
     create_logs_directory,
     create_product_directory,
 )
-from disdrodb.api.info import group_filepaths
 from disdrodb.api.io import open_netcdf_files
 from disdrodb.api.path import (
     define_file_folder_path,
     define_l2e_filename,
     define_l2m_filename,
-    define_temporal_resolution,
 )
 from disdrodb.api.search import get_required_product
 from disdrodb.configs import (
     get_data_archive_dir,
     get_metadata_archive_dir,
-    get_model_options,
-    get_product_options,
-    get_product_temporal_resolutions,
 )
-from disdrodb.l1.resampling import resample_dataset
 from disdrodb.l2.processing import (
     generate_l2_radar,
     generate_l2e,
     generate_l2m,
 )
 from disdrodb.metadata import read_station_metadata
+from disdrodb.routines.options import (
+    L2ProcessingOptions,
+    get_model_options,
+    get_product_temporal_resolutions,
+    is_possible_product,
+)
 from disdrodb.scattering.routines import precompute_scattering_tables
-from disdrodb.utils.archiving import define_temporal_partitions, get_files_partitions
 from disdrodb.utils.dask import execute_tasks_safely
 from disdrodb.utils.decorators import delayed_if_parallel, single_threaded_if_parallel
-from disdrodb.utils.list import flatten_list
-
-# Logger
 from disdrodb.utils.logger import (
     create_product_logs,
     log_info,
 )
 from disdrodb.utils.routines import (
-    is_possible_product,
     run_product_generation,
     try_get_required_filepaths,
 )
-from disdrodb.utils.time import (
-    ensure_sample_interval_in_seconds,
-    get_sampling_information,
-)
 from disdrodb.utils.writer import write_product
 
 logger = logging.getLogger(__name__)
 
 
-####----------------------------------------------------------------------------.
-
-
-class ProcessingOptions:
-    """Define L2 products processing options."""
-
-    # TODO: TO MOVE ELSEWHERE (AFTER L1 REFACTORING !)
-
-    def __init__(self, product, filepaths, parallel, temporal_resolutions=None):
-        """Define L2 products processing options."""
-        import disdrodb
-
-        # ---------------------------------------------------------------------.
-        # Define temporal resolutions for which to retrieve processing options
-        if temporal_resolutions is None:
-            temporal_resolutions = get_product_temporal_resolutions(product)
-        elif isinstance(temporal_resolutions, str):
-            temporal_resolutions = [temporal_resolutions]
-
-        # ---------------------------------------------------------------------.
-        # Get product options at various temporal resolutions
-        dict_product_options = {
-            temporal_resolution: get_product_options(product, temporal_resolution=temporal_resolution)
-            for temporal_resolution in temporal_resolutions
-        }
-
-        # ---------------------------------------------------------------------.
-        # Group filepaths by source sample intervals
-        # - Typically the sample interval is fixed and is just one
-        # - Some stations might change the sample interval along the years
-        # - For each sample interval, separated processing take place here after !
-        dict_filepaths = group_filepaths(filepaths, groups="sample_interval")
-
-        # ---------------------------------------------------------------------.
-        # Retrieve processing information for each temporal resolution
-        dict_folder_partitioning = {}
-        dict_files_partitions = {}
-        _cache_dict_list_partitions: dict[str, dict] = {}
-        for temporal_resolution in temporal_resolutions:
-
-            # -------------------------------------------------------------------------.
-            # Retrieve product options
-            product_options = dict_product_options[temporal_resolution].copy()
-
-            # Retrieve accumulation_interval and rolling option
-            accumulation_interval, rolling = get_sampling_information(temporal_resolution)
-
-            # Extract processing options
-            archive_options = product_options.pop("archive_options")
-
-            dict_product_options[temporal_resolution] = product_options
-            # -------------------------------------------------------------------------.
-            # Define folder partitioning
-            if "folder_partitioning" not in archive_options:
-                dict_folder_partitioning[temporal_resolution] = disdrodb.config.get("folder_partitioning")
-            else:
-                dict_folder_partitioning[temporal_resolution] = archive_options.pop("folder_partitioning")
-
-            # -------------------------------------------------------------------------.
-            # Define list of temporal partitions
-            # - [{start_time: np.datetime64, end_time: np.datetime64}, ....]
-            # - Either strategy: "event" or "time_block" or save_by_time_block"
-            # - "event" requires loading data into memory to identify events
-            #   --> Does some data filtering on what to process !
-            # - "time_block" does not require loading data into memory
-            #   --> Does not do data filtering on what to process !
-            # --> Here we cache dict_list_partitions so that we don't need to recompute
-            #     stuffs if processing options are the same
-            key = json.dumps(archive_options, sort_keys=True)
-            if key not in _cache_dict_list_partitions:
-                _cache_dict_list_partitions[key] = {
-                    sample_interval: define_temporal_partitions(filepaths, parallel=parallel, **archive_options)
-                    for sample_interval, filepaths in dict_filepaths.items()
-                }
-            dict_list_partitions = _cache_dict_list_partitions[key].copy()  # To avoid in-place replacement
-
-            # ------------------------------------------------------------------.
-            # Group filepaths by temporal partitions
-            # - This is done separately for each possible source sample interval
-            # - It groups filepaths by start_time and end_time provided by list_partitions
-            # - Here 'events' can also simply be period of times ('day', 'months', ...)
-            # - When aggregating/resampling/accumulating data, we need to load also
-            #   some data after the actual event end_time to ensure that the resampled dataset
-            #   contains the event_end_time
-            #   --> get_files_partitions adjust the event end_time to accounts for the required "border" data.
-            # - ATTENTION: get_files_partitions returns start_time and end_time as datetime objects !
-            files_partitions = [
-                get_files_partitions(
-                    list_partitions=list_partitions,
-                    filepaths=dict_filepaths[sample_interval],
-                    sample_interval=sample_interval,
-                    accumulation_interval=accumulation_interval,
-                    rolling=rolling,
-                )
-                for sample_interval, list_partitions in dict_list_partitions.items()
-                if product != "L2E"
-                or is_possible_product(
-                    accumulation_interval=accumulation_interval,
-                    sample_interval=sample_interval,
-                    rolling=rolling,
-                )
-            ]
-            files_partitions = flatten_list(files_partitions)
-            dict_files_partitions[temporal_resolution] = files_partitions
-
-        # ------------------------------------------------------------------.
-        # Keep only temporal_resolutions for which events could be defined
-        # - Remove e.g when not compatible accumulation_interval with source sample_interval
-        temporal_resolutions = [
-            temporal_resolution
-            for temporal_resolution in temporal_resolutions
-            if len(dict_files_partitions[temporal_resolution]) > 0
-        ]
-        # ------------------------------------------------------------------.
-        # Add attributes
-        self.temporal_resolutions = temporal_resolutions
-        self.dict_files_partitions = dict_files_partitions
-        self.dict_product_options = dict_product_options
-        self.dict_folder_partitioning = dict_folder_partitioning
-
-    def get_files_partitions(self, temporal_resolution):
-        """Return files partitions dictionary for a specific L2E product."""
-        return self.dict_files_partitions[temporal_resolution]
-
-    def get_product_options(self, temporal_resolution):
-        """Return product options dictionary for a specific L2E product."""
-        return self.dict_product_options[temporal_resolution]
-
-    def get_folder_partitioning(self, temporal_resolution):
-        """Return the folder partitioning for a specific L2E product."""
-        # to be used for logs and files !
-        return self.dict_folder_partitioning[temporal_resolution]
-
-
 ####----------------------------------------------------------------------------.
 #### L2E
 
 
-def define_l2e_logs_filename(campaign_name, station_name, start_time, end_time, accumulation_interval, rolling):
+def define_l2e_logs_filename(campaign_name, station_name, start_time, end_time, temporal_resolution):
     """Define L2E logs filename."""
-    temporal_resolution = define_temporal_resolution(seconds=accumulation_interval, rolling=rolling)
     starting_time = pd.to_datetime(start_time).strftime("%Y%m%d%H%M%S")
     ending_time = pd.to_datetime(end_time).strftime("%Y%m%d%H%M%S")
     logs_filename = f"L2E.{temporal_resolution}.{campaign_name}.{station_name}.s{starting_time}.e{ending_time}"
@@ -239,8 +94,7 @@ def _generate_l2e(
     campaign_name,
     station_name,
     # L2E options
-    accumulation_interval,
-    rolling,
+    temporal_resolution,
     product_options,
     # Processing options
     force,
@@ -254,42 +108,28 @@ def _generate_l2e(
     # Define product processing function
     def core(
         filepaths,
+        start_time,
+        end_time,
         campaign_name,
         station_name,
-        product_options,
         # Processing options
         logger,
         parallel,
         verbose,
         force,
-        # Resampling arguments
-        start_time,
-        end_time,
-        accumulation_interval,
-        rolling,
+        # Product options
+        temporal_resolution,
+        product_options,
         # Archiving arguments
         data_dir,
         folder_partitioning,
     ):
-        """Define L1 product processing."""
+        """Define L2E product processing."""
         # Copy to avoid in-place replacement (outside this function)
         product_options = product_options.copy()
 
         # Open the dataset over the period of interest
-        ds = open_netcdf_files(filepaths, start_time=start_time, end_time=end_time, parallel=False)
-        ds = ds.load()
-        ds.close()
-
-        # Resample dataset # TODO: in future to perform in L1
-        # - Define sample interval in seconds
-        sample_interval = ensure_sample_interval_in_seconds(ds["sample_interval"]).to_numpy().item()
-        # - Resample dataset
-        ds = resample_dataset(
-            ds=ds,
-            sample_interval=sample_interval,
-            accumulation_interval=accumulation_interval,
-            rolling=rolling,
-        )
+        ds = open_netcdf_files(filepaths, start_time=start_time, end_time=end_time, parallel=False, compute=True)
 
         # Extract L2E processing options
         l2e_options = product_options.get("product_options")
@@ -320,8 +160,7 @@ def _generate_l2e(
             ds,
             campaign_name=campaign_name,
             station_name=station_name,
-            sample_interval=accumulation_interval,
-            rolling=rolling,
+            temporal_resolution=temporal_resolution,
         )
         folder_path = define_file_folder_path(ds, dir_path=data_dir, folder_partitioning=folder_partitioning)
         filepath = os.path.join(folder_path, filename)
@@ -333,14 +172,14 @@ def _generate_l2e(
     # Define product processing function kwargs
     core_func_kwargs = dict(  # noqa: C408
         filepaths=filepaths,
+        start_time=start_time,
+        end_time=end_time,
+        # Station info
         campaign_name=campaign_name,
         station_name=station_name,
+        # Product options
+        temporal_resolution=temporal_resolution,
         product_options=product_options,
-        # Resampling arguments
-        start_time=start_time,
-        end_time=end_time,
-        accumulation_interval=accumulation_interval,
-        rolling=rolling,
         # Archiving arguments
         data_dir=data_dir,
         folder_partitioning=folder_partitioning,
@@ -449,57 +288,85 @@ def run_l2e_station(
         msg = f"{product} processing of station {station_name} has started."
         log_info(logger=logger, msg=msg, verbose=verbose)
 
-    # -------------------------------------------------------------------------.
-    # List files to process
-    # - If no data available, print error message and return None
-    required_product = get_required_product(product)
-    filepaths = try_get_required_filepaths(
-        data_archive_dir=data_archive_dir,
+    # ---------------------------------------------------------------------.
+    # Retrieve source sampling interval
+    # - If a station has varying measurement interval over time, choose the smallest one !
+    metadata = read_station_metadata(
+        metadata_archive_dir=metadata_archive_dir,
         data_source=data_source,
         campaign_name=campaign_name,
         station_name=station_name,
-        product=required_product,
-        # Processing options
-        debugging_mode=debugging_mode,
     )
-    if filepaths is None:
-        return
-
-    # -------------------------------------------------------------------------.
-    # Retrieve L2E processing options
-    l2e_processing_options = ProcessingOptions(product="L2E", filepaths=filepaths, parallel=parallel)
+    sample_interval = metadata["measurement_interval"]
+    if isinstance(sample_interval, list):
+        sample_interval = min(sample_interval)
 
-    # -------------------------------------------------------------------------.
+    # ---------------------------------------------------------------------.
     # Generate products for each temporal resolution
-    # rolling = False
-    # accumulation_interval = 60
+    # temporal_resolution = "1MIN"
     # temporal_resolution = "10MIN"
-    # folder_partitioning = ""
-    # product_options = l2e_processing_options.get_product_options(temporal_resolution)
+    temporal_resolutions = get_product_temporal_resolutions(product)
+    for temporal_resolution in temporal_resolutions:
 
-    for temporal_resolution in l2e_processing_options.temporal_resolutions:
-        # Print progress message
-        msg = f"Production of {product} {temporal_resolution} has started."
-        log_info(logger=logger, msg=msg, verbose=verbose)
+        # ------------------------------------------------------------------.
+        # Check if the product can be generated
+        if not is_possible_product(
+            temporal_resolution=temporal_resolution,
+            sample_interval=sample_interval,
+        ):
+            continue
 
-        # Retrieve event info
-        files_partitions = l2e_processing_options.get_files_partitions(temporal_resolution)
+        # ---------------------------------------------------------------------.
+        # List files to process
+        # - If no data available, print error message and try with other L2E accumulation intervals
+        required_product = get_required_product(product)
+        filepaths = try_get_required_filepaths(
+            data_archive_dir=data_archive_dir,
+            data_source=data_source,
+            campaign_name=campaign_name,
+            station_name=station_name,
+            product=required_product,
+            # Processing options
+            debugging_mode=debugging_mode,
+            # Product options
+            temporal_resolution=temporal_resolution,
+        )
+        if filepaths is None:
+            continue
+
+        # ---------------------------------------------------------------------.
+        # Retrieve L2E processing options
+        l2e_processing_options = L2ProcessingOptions(
+            product=product,
+            temporal_resolution=temporal_resolution,
+            filepaths=filepaths,
+            parallel=parallel,
+        )
+
+        # ---------------------------------------------------------------------.
+        # Retrieve files temporal partitions
+        files_partitions = l2e_processing_options.files_partitions
+
+        if len(files_partitions) == 0:
+            msg = (
+                f"{product} processing of {data_source} {campaign_name} {station_name} "
+                + f"has not been launched because of missing {required_product} {temporal_resolution} data."
+            )
+            log_info(logger=logger, msg=msg, verbose=verbose)
+            continue
 
         # Retrieve folder partitioning (for files and logs)
-        folder_partitioning = l2e_processing_options.get_folder_partitioning(temporal_resolution)
+        folder_partitioning = l2e_processing_options.folder_partitioning
 
         # Retrieve product options
-        product_options = l2e_processing_options.get_product_options(temporal_resolution)
-
-        # Retrieve accumulation_interval and rolling option
-        accumulation_interval, rolling = get_sampling_information(temporal_resolution)
+        product_options = l2e_processing_options.product_options
 
         # Precompute required scattering tables
         if product_options["radar_enabled"]:
             radar_options = product_options["radar_options"]
             precompute_scattering_tables(verbose=verbose, **radar_options)
 
-        # ------------------------------------------------------------------.
+        # ---------------------------------------------------------------------.
         # Create product directory
         data_dir = create_product_directory(
             data_archive_dir=data_archive_dir,
@@ -510,8 +377,7 @@ def run_l2e_station(
             product=product,
             force=force,
             # Option for L2E
-            sample_interval=accumulation_interval,
-            rolling=rolling,
+            temporal_resolution=temporal_resolution,
         )
 
         # Define logs directory
@@ -522,11 +388,10 @@ def run_l2e_station(
             campaign_name=campaign_name,
             station_name=station_name,
             # Option for L2E
-            sample_interval=accumulation_interval,
-            rolling=rolling,
+            temporal_resolution=temporal_resolution,
         )
 
-        # ------------------------------------------------------------------.
+        # ---------------------------------------------------------------------.
         # Generate files
         # - L2E product generation is optionally parallelized over events
         # - If parallel=True, it does that in parallel using dask.delayed
@@ -542,15 +407,13 @@ def run_l2e_station(
                     station_name=station_name,
                     start_time=event_info["start_time"],
                     end_time=event_info["end_time"],
-                    rolling=rolling,
-                    accumulation_interval=accumulation_interval,
+                    temporal_resolution=temporal_resolution,
                 ),
                 folder_partitioning=folder_partitioning,
                 campaign_name=campaign_name,
                 station_name=station_name,
                 # L2E options
-                rolling=rolling,
-                accumulation_interval=accumulation_interval,
+                temporal_resolution=temporal_resolution,
                 product_options=product_options,
                 # Processing options
                 force=force,
@@ -570,8 +433,7 @@ def run_l2e_station(
             station_name=station_name,
             data_archive_dir=data_archive_dir,
             # Product options
-            sample_interval=accumulation_interval,
-            rolling=rolling,
+            temporal_resolution=temporal_resolution,
             # Logs list
             list_logs=list_logs,
         )
@@ -586,9 +448,8 @@ def run_l2e_station(
 
 ####----------------------------------------------------------------------------.
 #### L2M
-def define_l2m_logs_filename(campaign_name, station_name, start_time, end_time, model_name, sample_interval, rolling):
+def define_l2m_logs_filename(campaign_name, station_name, start_time, end_time, model_name, temporal_resolution):
     """Define L2M logs filename."""
-    temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
     starting_time = pd.to_datetime(start_time).strftime("%Y%m%d%H%M%S")
     ending_time = pd.to_datetime(end_time).strftime("%Y%m%d%H%M%S")
     logs_filename = (
@@ -610,8 +471,7 @@ def _generate_l2m(
     campaign_name,
     station_name,
     # L2M options
-    sample_interval,
-    rolling,
+    temporal_resolution,
     model_name,
     product_options,
     # Processing options
@@ -636,14 +496,13 @@ def _generate_l2m(
         force,
         # Product options
         product_options,
-        sample_interval,
-        rolling,
+        temporal_resolution,
         model_name,
         # Archiving arguments
         data_dir,
         folder_partitioning,
     ):
-        """Define L1 product processing."""
+        """Define L2M product processing."""
         # Copy to avoid in-place replacement (outside this function)
         product_options = product_options.copy()
 
@@ -676,9 +535,14 @@ def _generate_l2m(
 
         ##------------------------------------------------------------------------.
         # Open the netCDF files
-        ds = open_netcdf_files(filepaths, start_time=start_time, end_time=end_time, variables=variables)
-        ds = ds.load()
-        ds.close()
+        ds = open_netcdf_files(
+            filepaths,
+            start_time=start_time,
+            end_time=end_time,
+            variables=variables,
+            parallel=False,
+            compute=True,
+        )
 
         # Produce L2M dataset
         ds = generate_l2m(
@@ -702,8 +566,7 @@ def _generate_l2m(
             ds,
             campaign_name=campaign_name,
             station_name=station_name,
-            sample_interval=sample_interval,
-            rolling=rolling,
+            temporal_resolution=temporal_resolution,
             model_name=model_name,
         )
         folder_path = define_file_folder_path(ds, dir_path=data_dir, folder_partitioning=folder_partitioning)
@@ -725,8 +588,7 @@ def _generate_l2m(
         force=force,
         # Product options
         product_options=product_options,
-        sample_interval=sample_interval,
-        rolling=rolling,
+        temporal_resolution=temporal_resolution,
         model_name=model_name,
         # Archiving arguments
         data_dir=data_dir,
@@ -838,19 +700,15 @@ def run_l2m_station(
     # Loop
     # temporal_resolution = "1MIN"
     # temporal_resolution = "10MIN"
-    temporal_resolutions = get_product_temporal_resolutions("L2M")
+    temporal_resolutions = get_product_temporal_resolutions(product)
     for temporal_resolution in temporal_resolutions:
 
-        # Retrieve accumulation_interval and rolling option
-        accumulation_interval, rolling = get_sampling_information(temporal_resolution)
-
         # ------------------------------------------------------------------.
-        # Avoid generation of rolling products for source sample interval !
-        if rolling and accumulation_interval == sample_interval:
-            continue
-
-        # Avoid product generation if the accumulation_interval is less than the sample interval
-        if accumulation_interval < sample_interval:
+        # Check if the product can be generated
+        if not is_possible_product(
+            temporal_resolution=temporal_resolution,
+            sample_interval=sample_interval,
+        ):
             continue
 
         # -----------------------------------------------------------------.
@@ -866,29 +724,28 @@ def run_l2m_station(
             # Processing options
             debugging_mode=debugging_mode,
             # Product options
-            sample_interval=accumulation_interval,
-            rolling=rolling,
+            temporal_resolution=temporal_resolution,
         )
         if filepaths is None:
             continue
 
         # -------------------------------------------------------------------------.
         # Retrieve L2M processing options
-        l2m_processing_options = ProcessingOptions(
-            product="L2M",
-            temporal_resolutions=temporal_resolution,
+        l2m_processing_options = L2ProcessingOptions(
+            product=product,
+            temporal_resolution=temporal_resolution,
             filepaths=filepaths,
             parallel=parallel,
         )
 
         # Retrieve folder partitioning (for files and logs)
-        folder_partitioning = l2m_processing_options.get_folder_partitioning(temporal_resolution)
+        folder_partitioning = l2m_processing_options.folder_partitioning
 
         # Retrieve product options
-        global_product_options = l2m_processing_options.get_product_options(temporal_resolution)
+        global_product_options = l2m_processing_options.product_options
 
         # Retrieve files temporal partitions
-        files_partitions = l2m_processing_options.get_files_partitions(temporal_resolution)
+        files_partitions = l2m_processing_options.files_partitions
 
         if len(files_partitions) == 0:
             msg = (
@@ -921,7 +778,7 @@ def run_l2m_station(
                 precompute_scattering_tables(verbose=verbose, **radar_options)
 
             # -----------------------------------------------------------------.
-            msg = f"Production of L2M_{model_name} for sample interval {accumulation_interval} s has started."
+            msg = f"Production of L2M_{model_name} for {temporal_resolution} has started."
             log_info(logger=logger, msg=msg, verbose=verbose)
             msg = f"Estimating {psd_model} parameters using {optimization}."
             log_info(logger=logger, msg=msg, verbose=verbose)
@@ -941,14 +798,13 @@ def run_l2m_station(
                     product=product,
                     force=force,
                     # Option for L2E
-                    sample_interval=accumulation_interval,
-                    rolling=rolling,
+                    temporal_resolution=temporal_resolution,
                     # Option for L2M
                     model_name=model_name,
                 )
             except Exception:
                 msg = (
-                    f"Production of L2M_{model_name} for sample interval {accumulation_interval} s has been "
+                    f"Production of L2M_{model_name} for {temporal_resolution} data has been "
                     + "skipped because the product already exists and force=False."
                 )
                 log_info(logger=logger, msg=msg, verbose=verbose)
@@ -963,8 +819,7 @@ def run_l2m_station(
                 campaign_name=campaign_name,
                 station_name=station_name,
                 # Option for L2E
-                sample_interval=accumulation_interval,
-                rolling=rolling,
+                temporal_resolution=temporal_resolution,
                 # Option for L2M
                 model_name=model_name,
             )
@@ -985,15 +840,13 @@ def run_l2m_station(
                         start_time=event_info["start_time"],
                         end_time=event_info["end_time"],
                         model_name=model_name,
-                        sample_interval=accumulation_interval,
-                        rolling=rolling,
+                        temporal_resolution=temporal_resolution,
                     ),
                     folder_partitioning=folder_partitioning,
                     campaign_name=campaign_name,
                     station_name=station_name,
                     # L2M options
-                    sample_interval=accumulation_interval,
-                    rolling=rolling,
+                    temporal_resolution=temporal_resolution,
                     model_name=model_name,
                     product_options=product_options,
                     # Processing options
@@ -1017,8 +870,7 @@ def run_l2m_station(
                 data_archive_dir=data_archive_dir,
                 # Product options
                 model_name=model_name,
-                sample_interval=accumulation_interval,
-                rolling=rolling,
+                temporal_resolution=temporal_resolution,
                 # Logs list
                 list_logs=list_logs,
             )