disdrodb 0.0.21__py3-none-any.whl → 0.1.0__py3-none-any.whl

disdrodb/{l0/scripts → cli}/disdrodb_run_l0.py

@@ -14,34 +14,40 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L0 processing."""
 import sys
+from typing import Optional
 
 import click
 
-from disdrodb.l0.routines import (
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
     click_l0_archive_options,
-    click_l0_processing_options,
-    click_l0_stations_options,
+    click_metadata_archive_dir_option,
+    click_processing_options,
+    click_stations_options,
+    parse_archive_dir,
+    parse_arg_to_list,
 )
-from disdrodb.utils.scripts import click_base_dir_option, parse_arg_to_list, parse_base_dir
 
 sys.tracebacklimit = 0  # avoid full traceback error if occur
 
 
 @click.command()
-@click_l0_stations_options
+@click_stations_options
 @click_l0_archive_options
-@click_l0_processing_options
-@click_base_dir_option
+@click_processing_options
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
 def disdrodb_run_l0(
-    # L0 disdrodb stations options
-    data_sources: str = None,
-    campaign_names: str = None,
-    station_names: str = None,
+    # Stations options
+    data_sources: Optional[str] = None,
+    campaign_names: Optional[str] = None,
+    station_names: Optional[str] = None,
     # L0 archive options
     l0a_processing: bool = True,
     l0b_processing: bool = True,
-    l0b_concat: bool = True,
+    l0c_processing: bool = True,
     remove_l0a: bool = False,
     remove_l0b: bool = False,
     # Processing options
@@ -49,7 +55,9 @@ def disdrodb_run_l0(
     verbose: bool = True,
     parallel: bool = True,
     debugging_mode: bool = False,
-    base_dir: str = None,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
 ):
     """
     Run the L0 processing of DISDRODB stations.
@@ -60,9 +68,8 @@ def disdrodb_run_l0(
 
     Parameters
     ----------
-
-    base_dir : str
-        Base directory of DISDRODB
+    data_archive_dir : str
+        DISDRODB Data Archive directory
         Format: <...>/DISDRODB
     data_sources : str
         Name of data source(s) to process.
@@ -82,17 +89,14 @@ def disdrodb_run_l0(
     l0b_processing : bool
         Whether to launch processing to generate L0B netCDF4 file(s) from L0A data.
         The default is True.
-    l0b_concat : bool
-        Whether to concatenate all raw files into a single L0B netCDF file.
-        If l0b_concat=True, all raw files will be saved into a single L0B netCDF file.
-        If l0b_concat=False, each raw file will be converted into the corresponding L0B netCDF file.
-        The default is False.
+    l0c_processing : bool
+        Whether to launch processing to generate L0C netCDF4 file(s) from L0C data.
+        The default is True.
     remove_l0a : bool
         Whether to keep the L0A files after having generated the L0B netCDF products.
         The default is False.
     remove_l0b : bool
-         Whether to remove the L0B files after having concatenated all L0B netCDF files.
-         It takes places only if l0b_concat = True
+         Whether to remove the L0B files after having produced L0C netCDF files.
         The default is False.
     force : bool
         If True, overwrite existing data into destination directories.
@@ -113,29 +117,33 @@ def disdrodb_run_l0(
         For L0A, it processes just the first 3 raw data files.
         For L0B, it processes just the first 100 rows of 3 L0A files.
         The default is False.
-    base_dir : str
-        Base directory of DISDRODB
+    data_archive_dir : str
+        DISDRODB Data Archive directory
         Format: <...>/DISDRODB
         If not specified, uses path specified in the DISDRODB active configuration.
     """
-    from disdrodb.l0.routines import run_disdrodb_l0
+    from disdrodb.routines import run_l0
 
     # Parse data_sources, campaign_names and station arguments
-    base_dir = parse_base_dir(base_dir)
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
     data_sources = parse_arg_to_list(data_sources)
     campaign_names = parse_arg_to_list(campaign_names)
     station_names = parse_arg_to_list(station_names)
 
     # Run processing
-    run_disdrodb_l0(
-        base_dir=base_dir,
+    run_l0(
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+        # Stations options
         data_sources=data_sources,
         campaign_names=campaign_names,
         station_names=station_names,
         # L0 archive options
         l0a_processing=l0a_processing,
         l0b_processing=l0b_processing,
-        l0b_concat=l0b_concat,
+        l0c_processing=l0c_processing,
         remove_l0a=remove_l0a,
         remove_l0b=remove_l0b,
         # Processing options
@@ -144,4 +152,3 @@ def disdrodb_run_l0(
         debugging_mode=debugging_mode,
         parallel=parallel,
     )
-    return None

disdrodb/{l0/scripts → cli}/disdrodb_run_l0_station.py

@@ -14,18 +14,19 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L0 station processing."""
 import sys
+from typing import Optional
 
 import click
 
-from disdrodb.l0.routines import (
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
     click_l0_archive_options,
-    click_l0_processing_options,
-)
-from disdrodb.utils.scripts import (
-    click_base_dir_option,
+    click_metadata_archive_dir_option,
+    click_processing_options,
     click_station_arguments,
-    parse_base_dir,
+    parse_archive_dir,
 )
 
 sys.tracebacklimit = 0  # avoid full traceback error if occur
@@ -36,9 +37,10 @@ sys.tracebacklimit = 0  # avoid full traceback error if occur
 
 @click.command()
 @click_station_arguments
-@click_l0_processing_options
+@click_processing_options
 @click_l0_archive_options
-@click_base_dir_option
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
 def disdrodb_run_l0_station(
     # Station arguments
     data_source: str,
@@ -47,7 +49,7 @@ def disdrodb_run_l0_station(
     # L0 archive options
     l0a_processing: bool = True,
     l0b_processing: bool = True,
-    l0b_concat: bool = True,
+    l0c_processing: bool = True,
     remove_l0a: bool = False,
     remove_l0b: bool = False,
     # Processing options
@@ -55,9 +57,11 @@ def disdrodb_run_l0_station(
     verbose: bool = True,
     parallel: bool = True,
     debugging_mode: bool = False,
-    base_dir: str = None,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
 ):
-    """Run the L0 processing of a specific DISDRODB station from the terminal.
+    r"""Run the L0 processing of a specific DISDRODB station from the terminal.
 
     Parameters \n
     ---------- \n
@@ -74,19 +78,16 @@ def disdrodb_run_l0_station(
         The default is True.\n
     l0b_processing : bool \n
         Whether to launch processing to generate L0B netCDF4 file(s) from L0A data.\n
-        he default is True.\n
-    l0b_concat : bool \n
-        Whether to concatenate all raw files into a single L0B netCDF file.\n
-        If l0b_concat=True, all raw files will be saved into a single L0B netCDF file.\n
-        If l0b_concat=False, each raw file will be converted into the corresponding L0B netCDF file.\n
-        The default is False.\n
+        The default is True.\n
+    l0c_processing : bool
+        Whether to launch processing to generate L0C netCDF4 file(s) from L0C data.
+        The default is True.
     remove_l0a : bool \n
         Whether to keep the L0A files after having generated the L0B netCDF products.\n
         The default is False.\n
-    remove_l0b : bool \n
-         Whether to remove the L0B files after having concatenated all L0B netCDF files.\n
-         It takes places only if l0b_concat=True\n
-         The default is False.\n
+    remove_l0b : bool
+         Whether to remove the L0B files after having produced L0C netCDF files.
+        The default is False.
     force : bool \n
         If True, overwrite existing data into destination directories.\n
         If False, raise an error if there are already data into destination directories.\n
@@ -106,24 +107,27 @@ def disdrodb_run_l0_station(
         For L0A, it processes just the first 3 raw data files for each station.\n
         For L0B, it processes just the first 100 rows of 3 L0A files for each station.\n
         The default is False.\n
-    base_dir : str \n
-        Base directory of DISDRODB \n
+    data_archive_dir : str \n
+        DISDRODB Data Archive directory \n
         Format: <...>/DISDRODB \n
         If not specified, uses path specified in the DISDRODB active configuration. \n
     """
-    from disdrodb.l0.routines import run_disdrodb_l0_station
+    from disdrodb.routines import run_l0_station
 
-    base_dir = parse_base_dir(base_dir)
+    data_archive_dir = parse_archive_dir(data_archive_dir)
 
-    run_disdrodb_l0_station(
-        base_dir=base_dir,
+    run_l0_station(
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+        # Station arguments
         data_source=data_source,
         campaign_name=campaign_name,
         station_name=station_name,
         # L0 archive options
         l0a_processing=l0a_processing,
         l0b_processing=l0b_processing,
-        l0b_concat=l0b_concat,
+        l0c_processing=l0c_processing,
         remove_l0a=remove_l0a,
         remove_l0b=remove_l0b,
         # Processing options
@@ -132,5 +136,3 @@ def disdrodb_run_l0_station(
         debugging_mode=debugging_mode,
         parallel=parallel,
     )
-
-    return None

disdrodb/{l0/scripts → cli}/disdrodb_run_l0a.py

@@ -15,34 +15,42 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L0A processing."""
 import sys
+from typing import Optional
 
 import click
 
-from disdrodb.l0.routines import (
-    click_l0_processing_options,
-    click_l0_stations_options,
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_metadata_archive_dir_option,
+    click_processing_options,
+    click_stations_options,
+    parse_archive_dir,
+    parse_arg_to_list,
 )
-from disdrodb.utils.scripts import click_base_dir_option, parse_arg_to_list, parse_base_dir
 
 sys.tracebacklimit = 0  # avoid full traceback error if occur
 
 
 @click.command()
-@click_l0_stations_options
-@click_l0_processing_options
-@click_base_dir_option
+@click_stations_options
+@click_processing_options
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
 def disdrodb_run_l0a(
-    # L0 disdrodb stations options
-    data_sources: str = None,
-    campaign_names: str = None,
-    station_names: str = None,
+    # Stations options
+    data_sources: Optional[str] = None,
+    campaign_names: Optional[str] = None,
+    station_names: Optional[str] = None,
     # Processing options
     force: bool = False,
     verbose: bool = True,
     parallel: bool = True,
     debugging_mode: bool = False,
-    base_dir: str = None,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
 ):
     """
     Run the L0A processing of DISDRODB stations.
@@ -53,7 +61,6 @@ def disdrodb_run_l0a(
 
     Parameters
     ----------
-
     data_sources : str
         Name of data source(s) to process.
         The name(s) must be UPPER CASE.
@@ -84,22 +91,26 @@ def disdrodb_run_l0a(
         If True, it reduces the amount of data to process.
         It processes just the first 3 raw data files for each station.
         The default is False.
-    base_dir : str
-        Base directory of DISDRODB
+    data_archive_dir : str
+        DISDRODB Data Archive directory
         Format: <...>/DISDRODB
         If not specified, uses path specified in the DISDRODB active configuration.
     """
-    from disdrodb.l0.routines import run_disdrodb_l0a
+    from disdrodb.routines import run_l0a
 
     # Parse data_sources, campaign_names and station arguments
-    base_dir = parse_base_dir(base_dir)
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
     data_sources = parse_arg_to_list(data_sources)
     campaign_names = parse_arg_to_list(campaign_names)
     station_names = parse_arg_to_list(station_names)
 
     # Run processing
-    run_disdrodb_l0a(
-        base_dir=base_dir,
+    run_l0a(
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+        # Stations options
         data_sources=data_sources,
         campaign_names=campaign_names,
         station_names=station_names,
@@ -109,5 +120,3 @@ def disdrodb_run_l0a(
         debugging_mode=debugging_mode,
         parallel=parallel,
     )
-
-    return None

disdrodb/{l0/scripts → cli}/disdrodb_run_l0a_station.py

@@ -14,15 +14,18 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L0A station processing."""
 import sys
+from typing import Optional
 
 import click
 
-from disdrodb.l0.routines import click_l0_processing_options
-from disdrodb.utils.scripts import (
-    click_base_dir_option,
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_metadata_archive_dir_option,
+    click_processing_options,
     click_station_arguments,
-    parse_base_dir,
+    parse_archive_dir,
 )
 
 sys.tracebacklimit = 0  # avoid full traceback error if occur
@@ -33,8 +36,9 @@ sys.tracebacklimit = 0  # avoid full traceback error if occur
 
 @click.command()
 @click_station_arguments
-@click_l0_processing_options
-@click_base_dir_option
+@click_processing_options
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
 def disdrodb_run_l0a_station(
     # Station arguments
     data_source: str,
@@ -45,7 +49,9 @@ def disdrodb_run_l0a_station(
     verbose: bool = False,
     parallel: bool = True,
     debugging_mode: bool = False,
-    base_dir: str = None,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
 ):
     """
     Run the L0A processing of a specific DISDRODB station from the terminal.
@@ -78,37 +84,21 @@ def disdrodb_run_l0a_station(
         If True, it reduces the amount of data to process.
         It processes just the first 3 raw data files.
         The default is False.
-    base_dir : str
-        Base directory of DISDRODB.
+    data_archive_dir : str
+        DISDRODB Data Archive directory.
         Format: <...>/DISDRODB
         If not specified, uses path specified in the DISDRODB active configuration.
     """
-    import os
+    from disdrodb.l0.routines import run_l0a_station
+    from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
 
-    import dask
-    from dask.distributed import Client, LocalCluster
-
-    from disdrodb.l0.l0_processing import run_l0a_station
-
-    base_dir = parse_base_dir(base_dir)
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
 
     # -------------------------------------------------------------------------.
     # If parallel=True, set the dask environment
     if parallel:
-        # Set HDF5_USE_FILE_LOCKING to avoid going stuck with HDF
-        os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
-        # Retrieve the number of process to run
-        available_workers = os.cpu_count() - 2  # if not set, all CPUs
-        num_workers = dask.config.get("num_workers", available_workers)
-        # Create dask.distributed local cluster
-        cluster = LocalCluster(
-            n_workers=num_workers,
-            threads_per_worker=1,
-            processes=True,
-            # memory_limit='8GB',
-            # silence_logs=False,
-        )
-        Client(cluster)
+        cluster, client = initialize_dask_cluster()
     # -------------------------------------------------------------------------.
 
     run_l0a_station(
@@ -121,11 +111,12 @@ def disdrodb_run_l0a_station(
         verbose=verbose,
         debugging_mode=debugging_mode,
         parallel=parallel,
-        base_dir=base_dir,
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
     )
 
     # -------------------------------------------------------------------------.
     # Close the cluster
     if parallel:
-        cluster.close()
-    return None
+        close_dask_cluster(cluster, client)

disdrodb/{l0/scripts → cli}/disdrodb_run_l0b.py

@@ -15,37 +15,45 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L0B processing."""
 import sys
+from typing import Optional
 
 import click
 
-from disdrodb.l0.routines import (
-    click_l0_processing_options,
-    click_l0_stations_options,
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_metadata_archive_dir_option,
+    click_processing_options,
     click_remove_l0a_option,
+    click_stations_options,
+    parse_archive_dir,
+    parse_arg_to_list,
 )
-from disdrodb.utils.scripts import click_base_dir_option, parse_arg_to_list, parse_base_dir
 
 sys.tracebacklimit = 0  # avoid full traceback error if occur
 
 
 @click.command()
-@click_l0_stations_options
-@click_l0_processing_options
+@click_stations_options
+@click_processing_options
 @click_remove_l0a_option
-@click_base_dir_option
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
 def disdrodb_run_l0b(
-    # L0 disdrodb stations options
-    data_sources: str = None,
-    campaign_names: str = None,
-    station_names: str = None,
+    # Stations options
+    data_sources: Optional[str] = None,
+    campaign_names: Optional[str] = None,
+    station_names: Optional[str] = None,
     # Processing options
     force: bool = False,
     verbose: bool = True,
     parallel: bool = True,
     debugging_mode: bool = False,
     remove_l0a: bool = False,
-    base_dir: str = None,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
 ):
     """
     Run the L0B processing of DISDRODB stations.
@@ -56,7 +64,6 @@ def disdrodb_run_l0b(
 
     Parameters
     ----------
-
     data_sources : str
         Name of data source(s) to process.
         The name(s) must be UPPER CASE.
@@ -87,22 +94,26 @@ def disdrodb_run_l0b(
         If True, it reduces the amount of data to process.
         It processes just the first 100 rows of 3 L0A files for each station.
         The default is False.
-    base_dir : str
-        Base directory of DISDRODB
+    data_archive_dir : str
+        DISDRODB Data Archive directory
         Format: <...>/DISDRODB
         If not specified, uses path specified in the DISDRODB active configuration.
     """
-    from disdrodb.l0.routines import run_disdrodb_l0b
+    from disdrodb.routines import run_l0b
 
     # Parse data_sources, campaign_names and station arguments
-    base_dir = parse_base_dir(base_dir)
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
     data_sources = parse_arg_to_list(data_sources)
     campaign_names = parse_arg_to_list(campaign_names)
     station_names = parse_arg_to_list(station_names)
 
     # Run processing
-    run_disdrodb_l0b(
-        base_dir=base_dir,
+    run_l0b(
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+        # Stations options
         data_sources=data_sources,
         campaign_names=campaign_names,
         station_names=station_names,
@@ -113,5 +124,3 @@ def disdrodb_run_l0b(
         parallel=parallel,
         remove_l0a=remove_l0a,
     )
-
-    return None