disdrodb 0.2.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

disdrodb/psd/models.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -24,6 +24,7 @@ Source code:
 - https://github.com/wolfidan/pyradsim/blob/master/pyradsim/psd.py
 
 """
+
 import importlib
 
 import dask.array

disdrodb/routines/__init__.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,9 +15,11 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """DISDRODB L0 software."""
+
 from disdrodb.routines.wrappers import (
     create_summary,
     create_summary_station,
+    run,
     run_l0,
     run_l0_station,
     run_l0a,
@@ -32,11 +34,13 @@ from disdrodb.routines.wrappers import (
     run_l2e_station,
     run_l2m,
     run_l2m_station,
+    run_station,
 )
 
 __all__ = [
     "create_summary",
     "create_summary_station",
+    "run",
     "run_l0",
     "run_l0_station",
     "run_l0a",
@@ -51,4 +55,5 @@ __all__ = [
     "run_l2e_station",
     "run_l2m",
     "run_l2m_station",
+    "run_station",
 ]

disdrodb/routines/l0.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -22,7 +20,6 @@ import datetime
 import logging
 import os
 import time
-from typing import Optional
 
 from disdrodb.api.checks import check_measurement_intervals, check_sensor_name, check_station_inputs
 from disdrodb.api.create_directories import (
@@ -50,6 +47,7 @@ from disdrodb.l0.l0b_nc_processing import sanitize_ds
 from disdrodb.l0.l0b_processing import generate_l0b
 from disdrodb.l0.l0c_processing import TOLERANCE_SECONDS, create_l0c_datasets
 from disdrodb.metadata import read_station_metadata
+from disdrodb.routines.options import L0CProcessingOptions
 from disdrodb.utils.archiving import group_files_by_time_block
 from disdrodb.utils.dask import execute_tasks_safely
 from disdrodb.utils.decorators import delayed_if_parallel, single_threaded_if_parallel
@@ -329,6 +327,7 @@ def _generate_l0c(
     # Processing info
     metadata,
     # Processing options
+    folder_partitioning,
     force,
     verbose,
     parallel,  # this is used only to initialize the correct logger !
@@ -336,8 +335,6 @@ def _generate_l0c(
     """Define L0C product processing."""
     # Define product
     product = "L0C"
-    # Define folder partitioning
-    folder_partitioning = get_folder_partitioning()
 
     # Define product processing function
     def core(
@@ -370,11 +367,16 @@ def _generate_l0c(
 
         # Write a dataset for each sample interval
         valid_datasets = []
-        for ds in dict_ds.values():  # (sample_interval, ds)
+        for sample_interval, ds in dict_ds.items():
             # Write L0C netCDF4 dataset
             if ds["time"].size > 1:
                 # Write L0C netCDF4 dataset
-                filename = define_l0c_filename(ds, campaign_name=campaign_name, station_name=station_name)
+                filename = define_l0c_filename(
+                    ds,
+                    campaign_name=campaign_name,
+                    station_name=station_name,
+                    sample_interval=sample_interval,
+                )
                 folder_path = define_file_folder_path(ds, dir_path=data_dir, folder_partitioning=folder_partitioning)
                 filepath = os.path.join(folder_path, filename)
                 write_product(ds, filepath=filepath, force=force)
@@ -428,8 +430,8 @@ def run_l0a_station(
     debugging_mode: bool = False,
     parallel: bool = True,
     # DISDRODB root directories
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
 ):
     """
     Run the L0A processing of a specific DISDRODB station when invoked from the terminal.
@@ -623,8 +625,8 @@ def run_l0b_station(
     parallel: bool = True,
     debugging_mode: bool = False,
     # DISDRODB root directories
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
 ):
     """
     Run the L0B processing of a specific DISDRODB station when invoked from the terminal.
@@ -804,8 +806,8 @@ def run_l0c_station(
     parallel: bool = True,
     debugging_mode: bool = False,
     # DISDRODB root directories
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
 ):
     """
     Run the L0C processing of a specific DISDRODB station when invoked from the terminal.
@@ -873,15 +875,6 @@ def run_l0c_station(
         station_name=station_name,
     )
 
-    # -----------------------------------------------------------------.
-    # Retrieve metadata
-    metadata = read_station_metadata(
-        metadata_archive_dir=metadata_archive_dir,
-        data_source=data_source,
-        campaign_name=campaign_name,
-        station_name=station_name,
-    )
-
     # ------------------------------------------------------------------------.
     # Start processing
     t_i = time.time()
@@ -925,10 +918,30 @@ def run_l0c_station(
     if filepaths is None:
         return
 
+    # -----------------------------------------------------------------.
+    # Read station metadata and sensor name
+    metadata = read_station_metadata(
+        metadata_archive_dir=metadata_archive_dir,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+    )
+    sensor_name = metadata["sensor_name"]
+
+    # -------------------------------------------------------------------------.
+    # Retrieve L0C processing options
+    l0c_processing_options = L0CProcessingOptions(sensor_name=sensor_name)
+    product_frequency = l0c_processing_options.product_frequency
+    folder_partitioning = l0c_processing_options.folder_partitioning
+
     # -------------------------------------------------------------------------.
     # Retrieve dictionary with the required files per time block
-    # TODO: allow customizing this in config file, but risk of out of memory !
-    list_event_info = group_files_by_time_block(filepaths=filepaths, freq="day", tolerance_seconds=TOLERANCE_SECONDS)
+    # - Default is "day". If "month", risk of going out of memory on small laptops
+    list_event_info = group_files_by_time_block(
+        filepaths=filepaths,
+        freq=product_frequency,
+        tolerance_seconds=TOLERANCE_SECONDS,
+    )
 
     # -----------------------------------------------------------------.
     # Generate L0C files
@@ -941,6 +954,7 @@ def run_l0c_station(
             data_dir=data_dir,
             logs_dir=logs_dir,
             logs_filename=event_info["start_time"].strftime("%Y%m%dT%H%M%S"),
+            folder_partitioning=folder_partitioning,
             # Processing options
             force=force,
             verbose=verbose,

disdrodb/routines/l1.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -22,7 +20,6 @@ import datetime
 import logging
 import os
 import time
-from typing import Optional
 
 import pandas as pd
 
@@ -42,6 +39,8 @@ from disdrodb.configs import (
     get_folder_partitioning,
     get_metadata_archive_dir,
 )
+from disdrodb.constants import METEOROLOGICAL_VARIABLES
+from disdrodb.l1.classification import TEMPERATURE_VARIABLES
 from disdrodb.l1.processing import generate_l1
 from disdrodb.l1.resampling import resample_dataset
 from disdrodb.metadata.reader import read_station_metadata
@@ -144,15 +143,23 @@ def _generate_l1(
         folder_partitioning,
     ):
         """Define L1 product processing."""
+        import dask
+
+        # Define variables to load
+        # - precip_flag used for OceanRain ODM470 data only
+        # - Missing variables in dataset are simply not selected
+        variables = ["raw_drop_number", "qc_time", "precip_flag", *TEMPERATURE_VARIABLES, *METEOROLOGICAL_VARIABLES]
+
         # Open the L0C netCDF files
-        ds = open_netcdf_files(
-            filepaths,
-            start_time=start_time,
-            end_time=end_time,
-            variables=["raw_drop_number", "time_qc"],
-            parallel=False,
-            compute=True,
-        )
+        with dask.config.set(scheduler="single-threaded"):  # synchronous
+            ds = open_netcdf_files(
+                filepaths,
+                start_time=start_time,
+                end_time=end_time,
+                variables=variables,
+                parallel=False,
+                compute=True,
+            )
 
         # Define sample interval in seconds
         sample_interval = ensure_sample_interval_in_seconds(ds["sample_interval"]).to_numpy().item()
@@ -231,8 +238,8 @@ def run_l1_station(
     parallel: bool = True,
     debugging_mode: bool = False,
     # DISDRODB root directories
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
 ):
     """
     Run the L1 processing of a specific DISDRODB station when invoked from the terminal.
@@ -313,7 +320,7 @@ def run_l1_station(
         return
 
     # -------------------------------------------------------------------------.
-    # Read station metadata and retrieve sensor name
+    # Read station metadata and sensor name
     metadata = read_station_metadata(
         metadata_archive_dir=metadata_archive_dir,
         data_source=data_source,
@@ -334,6 +341,7 @@ def run_l1_station(
     # Generate products for each temporal resolution
     # temporal_resolution = "1MIN"
     # temporal_resolution = "10MIN"
+    # temporal_resolution = "ROLL2MIN"
     for temporal_resolution in l1_processing_options.temporal_resolutions:
         # Print progress message
         msg = f"Production of {product} {temporal_resolution} has started."
@@ -347,7 +355,6 @@ def run_l1_station(
 
         # Retrieve product options
         product_options = l1_processing_options.get_product_options(temporal_resolution)
-        product_options = product_options.get("product_options")
 
         # ------------------------------------------------------------------.
         # Create product directory

disdrodb/routines/l2.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------.
-# Copyright (c) 2021-2023 DISDRODB developers
+# Copyright (c) 2021-2026 DISDRODB developers
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,7 +21,6 @@ import datetime
 import logging
 import os
 import time
-from typing import Optional
 
 import pandas as pd
 
@@ -215,8 +214,8 @@ def run_l2e_station(
     parallel: bool = True,
     debugging_mode: bool = False,
     # DISDRODB root directories
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
 ):
     """
     Generate the L2E product of a specific DISDRODB station when invoked from the terminal.
@@ -297,6 +296,7 @@ def run_l2e_station(
         campaign_name=campaign_name,
         station_name=station_name,
     )
+    sensor_name = metadata["sensor_name"]
     sample_interval = metadata["measurement_interval"]
     if isinstance(sample_interval, list):
         sample_interval = min(sample_interval)
@@ -305,7 +305,7 @@ def run_l2e_station(
     # Generate products for each temporal resolution
     # temporal_resolution = "1MIN"
     # temporal_resolution = "10MIN"
-    temporal_resolutions = get_product_temporal_resolutions(product)
+    temporal_resolutions = get_product_temporal_resolutions(product=product, sensor_name=sensor_name)
     for temporal_resolution in temporal_resolutions:
 
         # ------------------------------------------------------------------.
@@ -339,6 +339,7 @@ def run_l2e_station(
         l2e_processing_options = L2ProcessingOptions(
             product=product,
             temporal_resolution=temporal_resolution,
+            sensor_name=sensor_name,
             filepaths=filepaths,
             parallel=parallel,
         )
@@ -621,8 +622,8 @@ def run_l2m_station(
     parallel: bool = True,
     debugging_mode: bool = False,
     # DISDRODB root directories
-    data_archive_dir: Optional[str] = None,
-    metadata_archive_dir: Optional[str] = None,
+    data_archive_dir: str | None = None,
+    metadata_archive_dir: str | None = None,
 ):
     """
     Run the L2M processing of a specific DISDRODB station when invoked from the terminal.
@@ -692,6 +693,7 @@ def run_l2m_station(
         campaign_name=campaign_name,
         station_name=station_name,
     )
+    sensor_name = metadata["sensor_name"]
     sample_interval = metadata["measurement_interval"]
     if isinstance(sample_interval, list):
         sample_interval = min(sample_interval)
@@ -700,7 +702,7 @@ def run_l2m_station(
     # Loop
     # temporal_resolution = "1MIN"
     # temporal_resolution = "10MIN"
-    temporal_resolutions = get_product_temporal_resolutions(product)
+    temporal_resolutions = get_product_temporal_resolutions(product=product, sensor_name=sensor_name)
     for temporal_resolution in temporal_resolutions:
 
         # ------------------------------------------------------------------.
@@ -734,6 +736,7 @@ def run_l2m_station(
         l2m_processing_options = L2ProcessingOptions(
             product=product,
             temporal_resolution=temporal_resolution,
+            sensor_name=sensor_name,
             filepaths=filepaths,
             parallel=parallel,
         )
@@ -766,7 +769,7 @@ def run_l2m_station(
             # -----------------------------------------------------------------.
             # Retrieve product-model options
             product_options = copy.deepcopy(global_product_options)
-            model_options = get_model_options(product="L2M", model_name=model_name)
+            model_options = get_model_options(model_name=model_name)
             product_options["product_options"].update(model_options)
 
             psd_model = model_options["psd_model"]