disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl

disdrodb/{l0/scripts → cli}/disdrodb_run_l0b_station.py

@@ -14,15 +14,19 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L0B station processing."""
 import sys
+from typing import Optional
 
 import click
 
-from disdrodb.l0.routines import click_l0_processing_options, click_remove_l0a_option
-from disdrodb.utils.scripts import (
-    click_base_dir_option,
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_metadata_archive_dir_option,
+    click_processing_options,
+    click_remove_l0a_option,
     click_station_arguments,
-    parse_base_dir,
+    parse_archive_dir,
 )
 
 sys.tracebacklimit = 0  # avoid full traceback error if occur
@@ -33,9 +37,10 @@ sys.tracebacklimit = 0  # avoid full traceback error if occur
 
 @click.command()
 @click_station_arguments
-@click_l0_processing_options
+@click_processing_options
 @click_remove_l0a_option
-@click_base_dir_option
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
 def disdrodb_run_l0b_station(
     # Station arguments
     data_source: str,
@@ -47,13 +52,14 @@ def disdrodb_run_l0b_station(
     parallel: bool = True,
     debugging_mode: bool = False,
     remove_l0a: bool = False,
-    base_dir: str = None,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
 ):
     """Run the L0B processing of a specific DISDRODB station from the terminal.
 
     Parameters
     ----------
-
     data_source : str
         Institution name (when campaign data spans more than 1 country),
         or country (when all campaigns (or sensor networks) are inside a given country).
@@ -80,37 +86,21 @@ def disdrodb_run_l0b_station(
         If True, it reduces the amount of data to process.
         It processes just the first 100 rows of 3 L0A files.
         The default is False.
-    base_dir : str
-        Base directory of DISDRODB
+    data_archive_dir : str
+        DISDRODB Data Archive directory
         Format: <...>/DISDRODB
         If not specified, uses path specified in the DISDRODB active configuration.
     """
-    import os
-
-    import dask
-    from dask.distributed import Client, LocalCluster
-
-    from disdrodb.l0.l0_processing import run_l0b_station
+    from disdrodb.l0.routines import run_l0b_station
+    from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
 
-    base_dir = parse_base_dir(base_dir)
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
 
     # -------------------------------------------------------------------------.
     # If parallel=True, set the dask environment
     if parallel:
-        # Set HDF5_USE_FILE_LOCKING to avoid going stuck with HDF
-        os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
-        # Retrieve the number of process to run
-        available_workers = os.cpu_count() - 2  # if not set, all CPUs
-        num_workers = dask.config.get("num_workers", available_workers)
-        # Create dask.distributed local cluster
-        cluster = LocalCluster(
-            n_workers=num_workers,
-            threads_per_worker=1,
-            processes=True,
-            # memory_limit='8GB',
-            # silence_logs=False,
-        )
-        Client(cluster)
+        cluster, client = initialize_dask_cluster()
 
     # -------------------------------------------------------------------------.
     run_l0b_station(
@@ -124,11 +114,12 @@ def disdrodb_run_l0b_station(
         debugging_mode=debugging_mode,
         parallel=parallel,
         remove_l0a=remove_l0a,
-        base_dir=base_dir,
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
     )
 
     # -------------------------------------------------------------------------.
     # Close the cluster
     if parallel:
-        cluster.close()
-    return None
+        close_dask_cluster(cluster, client)

disdrodb/cli/disdrodb_run_l0c.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L0C processing."""
+import sys
+from typing import Optional
+
+import click
+
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_metadata_archive_dir_option,
+    click_processing_options,
+    click_remove_l0b_option,
+    click_stations_options,
+    parse_archive_dir,
+    parse_arg_to_list,
+)
+
+sys.tracebacklimit = 0  # avoid full traceback error if occur
+
+
+@click.command()
+@click_stations_options
+@click_processing_options
+@click_remove_l0b_option
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
+def disdrodb_run_l0c(
+    # Stations options
+    data_sources: Optional[str] = None,
+    campaign_names: Optional[str] = None,
+    station_names: Optional[str] = None,
+    # L0C processing options
+    remove_l0b: bool = False,
+    # Processing options
+    force: bool = False,
+    verbose: bool = True,
+    parallel: bool = True,
+    debugging_mode: bool = False,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
+):
+    """
+    Run the L0C processing of DISDRODB stations.
+
+    This function allows to launch the processing of many DISDRODB stations with a single command.
+    From the list of all available DISDRODB stations, it runs the processing of the
+    stations matching the provided data_sources, campaign_names and station_names.
+
+    Parameters
+    ----------
+    data_sources : str
+        Name of data source(s) to process.
+        The name(s) must be UPPER CASE.
+        If campaign_names and station are not specified, process all stations.
+        To specify multiple data sources, write i.e.: --data_sources 'GPM EPFL NCAR'
+    campaign_names : str
+        Name of the campaign(s) to process.
+        The name(s) must be UPPER CASE.
+        To specify multiple campaigns, write i.e.: --campaign_names 'IPEX IMPACTS'
+    station_names : str
+        Station names.
+        To specify multiple stations, write i.e.: --station_names 'station1 station2'
+    force : bool
+        If True, overwrite existing data into destination directories.
+        If False, raise an error if there are already data into destination directories.
+        The default is False.
+    verbose : bool
+        Whether to print detailed processing information into terminal.
+        The default is True.
+    parallel : bool
+        If True, the files are processed simultaneously in multiple processes.
+        Each process will use a single thread to avoid issues with the HDF/netCDF library.
+        By default, the number of process is defined with os.cpu_count().
+        However, you can customize it by typing: DASK_NUM_WORKERS=4 disdrodb_run_l0b
+        If False, the files are processed sequentially in a single process.
+        If False, multi-threading is automatically exploited to speed up I/0 tasks.
+    debugging_mode : bool
+        If True, it reduces the amount of data to process.
+        It processes just the first 100 rows of 3 L0A files for each station.
+        The default is False.
+    remove_l0b: bool, optional
+        Whether to remove the processed L0B files. The default value is ``False``.
+    data_archive_dir : str
+        DISDRODB Data Archive directory
+        Format: <...>/DISDRODB
+        If not specified, uses path specified in the DISDRODB active configuration.
+    """
+    from disdrodb.routines import run_l0c
+
+    # Parse data_sources, campaign_names and station arguments
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
+    data_sources = parse_arg_to_list(data_sources)
+    campaign_names = parse_arg_to_list(campaign_names)
+    station_names = parse_arg_to_list(station_names)
+
+    # Run processing
+    run_l0c(
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+        # Stations options
+        data_sources=data_sources,
+        campaign_names=campaign_names,
+        station_names=station_names,
+        # L0C processing options
+        remove_l0b=remove_l0b,
+        # Processing options
+        force=force,
+        verbose=verbose,
+        debugging_mode=debugging_mode,
+        parallel=parallel,
+    )

disdrodb/cli/disdrodb_run_l0c_station.py

@@ -0,0 +1,129 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L0C station processing."""
+import sys
+from typing import Optional
+
+import click
+
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_metadata_archive_dir_option,
+    click_processing_options,
+    click_remove_l0b_option,
+    click_station_arguments,
+    parse_archive_dir,
+)
+
+sys.tracebacklimit = 0  # avoid full traceback error if occur
+
+# -------------------------------------------------------------------------.
+# Click Command Line Interface decorator
+
+
+@click.command()
+@click_station_arguments
+@click_processing_options
+@click_remove_l0b_option
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
+def disdrodb_run_l0c_station(
+    # Station arguments
+    data_source: str,
+    campaign_name: str,
+    station_name: str,
+    # L0C processing options
+    remove_l0b: bool = False,
+    # Processing options
+    force: bool = False,
+    verbose: bool = True,
+    parallel: bool = True,
+    debugging_mode: bool = False,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
+):
+    """Run the L0C processing of a specific DISDRODB station from the terminal.
+
+    Parameters
+    ----------
+    data_source : str
+        Institution name (when campaign data spans more than 1 country),
+        or country (when all campaigns (or sensor networks) are inside a given country).
+        Must be UPPER CASE.
+    campaign_name : str
+        Campaign name. Must be UPPER CASE.
+    station_name : str
+        Station name
+    force : bool
+        If True, overwrite existing data into destination directories.
+        If False, raise an error if there are already data into destination directories.
+        The default is False.
+    verbose : bool
+        Whether to print detailed processing information into terminal.
+        The default is True.
+    parallel : bool
+        If True, the files are processed simultaneously in multiple processes.
+        Each process will use a single thread to avoid issues with the HDF/netCDF library.
+        By default, the number of process is defined with os.cpu_count().
+        However, you can customize it by typing: DASK_NUM_WORKERS=4 disdrodb_run_l0b_station
+        If False, the files are processed sequentially in a single process.
+        If False, multi-threading is automatically exploited to speed up I/0 tasks.
+    debugging_mode : bool
+        If True, it reduces the amount of data to process.
+        It processes just the first 100 rows of 3 L0A files.
+        The default is False.
+    remove_l0b: bool, optional
+        Whether to remove the processed L0B files. The default value is ``False``.
+    data_archive_dir : str
+        DISDRODB Data Archive directory
+        Format: <...>/DISDRODB
+        If not specified, uses path specified in the DISDRODB active configuration.
+    """
+    from disdrodb.l0.routines import run_l0c_station
+    from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
+
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
+
+    # -------------------------------------------------------------------------.
+    # If parallel=True, set the dask environment
+    if parallel:
+        cluster, client = initialize_dask_cluster()
+
+    # -------------------------------------------------------------------------.
+    run_l0c_station(
+        # Station arguments
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        # L0C processing options
+        remove_l0b=remove_l0b,
+        # Processing options
+        force=force,
+        verbose=verbose,
+        debugging_mode=debugging_mode,
+        parallel=parallel,
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+    )
+
+    # -------------------------------------------------------------------------.
+    # Close the cluster
+    if parallel:
+        close_dask_cluster(cluster, client)

disdrodb/cli/disdrodb_run_l1.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L1B processing."""
+import sys
+from typing import Optional
+
+import click
+
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_metadata_archive_dir_option,
+    click_processing_options,
+    click_stations_options,
+    parse_archive_dir,
+    parse_arg_to_list,
+)
+
+sys.tracebacklimit = 0  # avoid full traceback error if occur
+
+
+@click.command()
+@click_stations_options
+@click_processing_options
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
+def disdrodb_run_l1(
+    # Stations options
+    data_sources: Optional[str] = None,
+    campaign_names: Optional[str] = None,
+    station_names: Optional[str] = None,
+    # Processing options
+    force: bool = False,
+    verbose: bool = True,
+    parallel: bool = True,
+    debugging_mode: bool = False,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
+):
+    """
+    Run the L1 processing of DISDRODB stations.
+
+    This function allows to launch the processing of many DISDRODB stations with a single command.
+    From the list of all available DISDRODB stations, it runs the processing
+    of the stations matching the provided data_sources, campaign_names and station_names.
+
+    Parameters
+    ----------
+    data_sources : str
+        Name of data source(s) to process.
+        The name(s) must be UPPER CASE.
+        If campaign_names and station are not specified, process all stations.
+        To specify multiple data sources, write i.e.: --data_sources 'GPM EPFL NCAR'
+    campaign_names : str
+        Name of the campaign(s) to process.
+        The name(s) must be UPPER CASE.
+        To specify multiple campaigns, write i.e.: --campaign_names 'IPEX IMPACTS'
+    station_names : str
+        Station names.
+        To specify multiple stations, write i.e.: --station_names 'station1 station2'
+    force : bool
+        If True, overwrite existing data into destination directories.
+        If False, raise an error if there are already data into destination directories.
+        The default is False.
+    verbose : bool
+        Whether to print detailed processing information into terminal.
+        The default is False.
+    parallel : bool
+        If True, the files are processed simultaneously in multiple processes.
+        Each process will use a single thread.
+        By default, the number of process is defined with os.cpu_count().
+        However, you can customize it by typing: DASK_NUM_WORKERS=4 disdrodb_run_l0a
+        If False, the files are processed sequentially in a single process.
+        If False, multi-threading is automatically exploited to speed up I/0 tasks.
+    debugging_mode : bool
+        If True, it reduces the amount of data to process.
+        It processes just the first 3 raw data files for each station.
+        The default is False.
+    data_archive_dir : str
+        DISDRODB Data Archive directory
+        Format: <...>/DISDRODB
+        If not specified, uses path specified in the DISDRODB active configuration.
+    """
+    from disdrodb.routines import run_l1
+
+    # Parse data_sources, campaign_names and station arguments
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
+    data_sources = parse_arg_to_list(data_sources)
+    campaign_names = parse_arg_to_list(campaign_names)
+    station_names = parse_arg_to_list(station_names)
+
+    # Run processing
+    run_l1(
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+        # Station options
+        data_sources=data_sources,
+        campaign_names=campaign_names,
+        station_names=station_names,
+        # Processing options
+        force=force,
+        verbose=verbose,
+        debugging_mode=debugging_mode,
+        parallel=parallel,
+    )

disdrodb/cli/disdrodb_run_l1_station.py

@@ -0,0 +1,121 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L1 station processing."""
+import sys
+from typing import Optional
+
+import click
+
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_metadata_archive_dir_option,
+    click_processing_options,
+    click_station_arguments,
+    parse_archive_dir,
+)
+
+sys.tracebacklimit = 0  # avoid full traceback error if occur
+
+# -------------------------------------------------------------------------.
+# Click Command Line Interface decorator
+
+
+@click.command()
+@click_station_arguments
+@click_processing_options
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
+def disdrodb_run_l1_station(
+    # Station arguments
+    data_source: str,
+    campaign_name: str,
+    station_name: str,
+    # Processing options
+    force: bool = False,
+    verbose: bool = False,
+    parallel: bool = True,
+    debugging_mode: bool = False,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
+):
+    """
+    Run the L1 processing of a specific DISDRODB station from the terminal.
+
+    Parameters
+    ----------
+    data_source : str
+        Institution name (when campaign data spans more than 1 country),
+        or country (when all campaigns (or sensor networks) are inside a given country).
+        Must be UPPER CASE.
+    campaign_name : str
+        Campaign name. Must be UPPER CASE.
+    station_name : str
+        Station name
+    force : bool
+        If True, overwrite existing data into destination directories.
+        If False, raise an error if there are already data into destination directories.
+        The default is False.
+    verbose : bool
+        Whether to print detailed processing information into terminal.
+        The default is True.
+    parallel : bool
+        If True, the files are processed simultaneously in multiple processes.
+        Each process will use a single thread.
+        By default, the number of process is defined with os.cpu_count().
+        However, you can customize it by typing: DASK_NUM_WORKERS=4 disdrodb_run_l0a_station
+        If False, the files are processed sequentially in a single process.
+        If False, multi-threading is automatically exploited to speed up I/0 tasks.
+    debugging_mode : bool
+        If True, it reduces the amount of data to process.
+        It processes just the first 3 raw data files.
+        The default is False.
+    data_archive_dir : str
+        DISDRODB Data Archive directory.
+        Format: <...>/DISDRODB
+        If not specified, uses path specified in the DISDRODB active configuration.
+    """
+    from disdrodb.l1.routines import run_l1_station
+    from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
+
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
+    # -------------------------------------------------------------------------.
+    # If parallel=True, set the dask environment
+    if parallel:
+        cluster, client = initialize_dask_cluster()
+
+    # -------------------------------------------------------------------------.
+    run_l1_station(
+        # Station arguments
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        # Processing options
+        force=force,
+        verbose=verbose,
+        debugging_mode=debugging_mode,
+        parallel=parallel,
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+    )
+
+    # -------------------------------------------------------------------------.
+    # Close the cluster
+    if parallel:
+        close_dask_cluster(cluster, client)