disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl

disdrodb/cli/disdrodb_run_l2e.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L2E processing."""
+import sys
+from typing import Optional
+
+import click
+
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_metadata_archive_dir_option,
+    click_processing_options,
+    click_stations_options,
+    parse_archive_dir,
+    parse_arg_to_list,
+)
+
+sys.tracebacklimit = 0  # avoid full traceback error if occur
+
+
+@click.command()
+@click_stations_options
+@click_processing_options
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
+def disdrodb_run_l2e(
+    # Stations options
+    data_sources: Optional[str] = None,
+    campaign_names: Optional[str] = None,
+    station_names: Optional[str] = None,
+    # Processing options
+    force: bool = False,
+    verbose: bool = True,
+    parallel: bool = True,
+    debugging_mode: bool = False,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
+):
+    """
+    Run the L2E processing of DISDRODB stations.
+
+    This function allows to launch the processing of many DISDRODB stations with a single command.
+    From the list of all available DISDRODB stations, it runs the processing
+    of the stations matching the provided data_sources, campaign_names and station_names.
+
+    Parameters
+    ----------
+    data_sources : str
+        Name of data source(s) to process.
+        The name(s) must be UPPER CASE.
+        If campaign_names and station are not specified, process all stations.
+        To specify multiple data sources, write i.e.: --data_sources 'GPM EPFL NCAR'
+    campaign_names : str
+        Name of the campaign(s) to process.
+        The name(s) must be UPPER CASE.
+        To specify multiple campaigns, write i.e.: --campaign_names 'IPEX IMPACTS'
+    station_names : str
+        Station names.
+        To specify multiple stations, write i.e.: --station_names 'station1 station2'
+    force : bool
+        If True, overwrite existing data into destination directories.
+        If False, raise an error if there are already data into destination directories.
+        The default is False.
+    verbose : bool
+        Whether to print detailed processing information into terminal.
+        The default is False.
+    parallel : bool
+        If True, the files are processed simultaneously in multiple processes.
+        Each process will use a single thread.
+        By default, the number of process is defined with os.cpu_count().
+        However, you can customize it by typing: DASK_NUM_WORKERS=4 disdrodb_run_l0a
+        If False, the files are processed sequentially in a single process.
+        If False, multi-threading is automatically exploited to speed up I/0 tasks.
+    debugging_mode : bool
+        If True, it reduces the amount of data to process.
+        It processes just the first 3 raw data files for each station.
+        The default is False.
+    data_archive_dir : str
+        DISDRODB Data Archive directory
+        Format: <...>/DISDRODB
+        If not specified, uses path specified in the DISDRODB active configuration.
+    """
+    from disdrodb.routines import run_l2e
+
+    # Parse data_sources, campaign_names and station arguments
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
+    data_sources = parse_arg_to_list(data_sources)
+    campaign_names = parse_arg_to_list(campaign_names)
+    station_names = parse_arg_to_list(station_names)
+
+    # Run processing
+    run_l2e(
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+        # Stations options
+        data_sources=data_sources,
+        campaign_names=campaign_names,
+        station_names=station_names,
+        # Processing options
+        force=force,
+        verbose=verbose,
+        debugging_mode=debugging_mode,
+        parallel=parallel,
+    )

disdrodb/cli/disdrodb_run_l2e_station.py

@@ -0,0 +1,122 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L2E station processing."""
+import sys
+from typing import Optional
+
+import click
+
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_metadata_archive_dir_option,
+    click_processing_options,
+    click_station_arguments,
+    parse_archive_dir,
+)
+
+sys.tracebacklimit = 0  # avoid full traceback error if occur
+
+# -------------------------------------------------------------------------.
+# Click Command Line Interface decorator
+
+
+@click.command()
+@click_station_arguments
+@click_processing_options
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
+def disdrodb_run_l2e_station(
+    # Station arguments
+    data_source: str,
+    campaign_name: str,
+    station_name: str,
+    # Processing options
+    force: bool = False,
+    verbose: bool = False,
+    parallel: bool = True,
+    debugging_mode: bool = False,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
+):
+    """
+    Run the L2E processing of a specific DISDRODB station from the terminal.
+
+    Parameters
+    ----------
+    data_source : str
+        Institution name (when campaign data spans more than 1 country),
+        or country (when all campaigns (or sensor networks) are inside a given country).
+        Must be UPPER CASE.
+    campaign_name : str
+        Campaign name. Must be UPPER CASE.
+    station_name : str
+        Station name
+    force : bool
+        If True, overwrite existing data into destination directories.
+        If False, raise an error if there are already data into destination directories.
+        The default is False.
+    verbose : bool
+        Whether to print detailed processing information into terminal.
+        The default is True.
+    parallel : bool
+        If True, the files are processed simultaneously in multiple processes.
+        Each process will use a single thread.
+        By default, the number of process is defined with os.cpu_count().
+        However, you can customize it by typing: DASK_NUM_WORKERS=4 disdrodb_run_l0a_station
+        If False, the files are processed sequentially in a single process.
+        If False, multi-threading is automatically exploited to speed up I/0 tasks.
+    debugging_mode : bool
+        If True, it reduces the amount of data to process.
+        It processes just the first 3 raw data files.
+        The default is False.
+    data_archive_dir : str
+        DISDRODB Data Archive directory.
+        Format: <...>/DISDRODB
+        If not specified, uses path specified in the DISDRODB active configuration.
+    """
+    from disdrodb.l2.routines import run_l2e_station
+    from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
+
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
+
+    # -------------------------------------------------------------------------.
+    # If parallel=True, set the dask environment
+    if parallel:
+        cluster, client = initialize_dask_cluster()
+
+    # -------------------------------------------------------------------------.
+    run_l2e_station(
+        # Station arguments
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        # Processing options
+        force=force,
+        verbose=verbose,
+        debugging_mode=debugging_mode,
+        parallel=parallel,
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+    )
+
+    # -------------------------------------------------------------------------.
+    # Close the cluster
+    if parallel:
+        close_dask_cluster(cluster, client)

disdrodb/cli/disdrodb_run_l2m.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L2M processing."""
+import sys
+from typing import Optional
+
+import click
+
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_metadata_archive_dir_option,
+    click_processing_options,
+    click_stations_options,
+    parse_archive_dir,
+    parse_arg_to_list,
+)
+
+sys.tracebacklimit = 0  # avoid full traceback error if occur
+
+
+@click.command()
+@click_stations_options
+@click_processing_options
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
+def disdrodb_run_l2m(
+    # Stations options
+    data_sources: Optional[str] = None,
+    campaign_names: Optional[str] = None,
+    station_names: Optional[str] = None,
+    # Processing options
+    force: bool = False,
+    verbose: bool = True,
+    parallel: bool = True,
+    debugging_mode: bool = False,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
+):
+    """
+    Run the L2M processing of DISDRODB stations.
+
+    This function allows to launch the processing of many DISDRODB stations with a single command.
+    From the list of all available DISDRODB stations, it runs the processing
+    of the stations matching the provided data_sources, campaign_names and station_names.
+
+    Parameters
+    ----------
+    data_sources : str
+        Name of data source(s) to process.
+        The name(s) must be UPPER CASE.
+        If campaign_names and station are not specified, process all stations.
+        To specify multiple data sources, write i.e.: --data_sources 'GPM EPFL NCAR'
+    campaign_names : str
+        Name of the campaign(s) to process.
+        The name(s) must be UPPER CASE.
+        To specify multiple campaigns, write i.e.: --campaign_names 'IPEX IMPACTS'
+    station_names : str
+        Station names.
+        To specify multiple stations, write i.e.: --station_names 'station1 station2'
+    force : bool
+        If True, overwrite existing data into destination directories.
+        If False, raise an error if there are already data into destination directories.
+        The default is False.
+    verbose : bool
+        Whether to print detailed processing information into terminal.
+        The default is False.
+    parallel : bool
+        If True, the files are processed simultaneously in multiple processes.
+        Each process will use a single thread.
+        By default, the number of process is defined with os.cpu_count().
+        However, you can customize it by typing: DASK_NUM_WORKERS=4 disdrodb_run_l0a
+        If False, the files are processed sequentially in a single process.
+        If False, multi-threading is automatically exploited to speed up I/0 tasks.
+    debugging_mode : bool
+        If True, it reduces the amount of data to process.
+        It processes just the first 3 raw data files for each station.
+        The default is False.
+    data_archive_dir : str
+        DISDRODB Data Archive directory
+        Format: <...>/DISDRODB
+        If not specified, uses path specified in the DISDRODB active configuration.
+    """
+    from disdrodb.routines import run_l2m
+
+    # Parse data_sources, campaign_names and station arguments
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
+    data_sources = parse_arg_to_list(data_sources)
+    campaign_names = parse_arg_to_list(campaign_names)
+    station_names = parse_arg_to_list(station_names)
+
+    # Run processing
+    run_l2m(
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+        # Stations options
+        data_sources=data_sources,
+        campaign_names=campaign_names,
+        station_names=station_names,
+        # Processing options
+        force=force,
+        verbose=verbose,
+        debugging_mode=debugging_mode,
+        parallel=parallel,
+    )

disdrodb/cli/disdrodb_run_l2m_station.py

@@ -0,0 +1,122 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L2M station processing."""
+import sys
+from typing import Optional
+
+import click
+
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_metadata_archive_dir_option,
+    click_processing_options,
+    click_station_arguments,
+    parse_archive_dir,
+)
+
+sys.tracebacklimit = 0  # avoid full traceback error if occur
+
+# -------------------------------------------------------------------------.
+# Click Command Line Interface decorator
+
+
+@click.command()
+@click_station_arguments
+@click_processing_options
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
+def disdrodb_run_l2m_station(
+    # Station arguments
+    data_source: str,
+    campaign_name: str,
+    station_name: str,
+    # Processing options
+    force: bool = False,
+    verbose: bool = False,
+    parallel: bool = True,
+    debugging_mode: bool = False,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
+):
+    """
+    Run the L2M processing of a specific DISDRODB station from the terminal.
+
+    Parameters
+    ----------
+    data_source : str
+        Institution name (when campaign data spans more than 1 country),
+        or country (when all campaigns (or sensor networks) are inside a given country).
+        Must be UPPER CASE.
+    campaign_name : str
+        Campaign name. Must be UPPER CASE.
+    station_name : str
+        Station name
+    force : bool
+        If True, overwrite existing data into destination directories.
+        If False, raise an error if there are already data into destination directories.
+        The default is False.
+    verbose : bool
+        Whether to print detailed processing information into terminal.
+        The default is True.
+    parallel : bool
+        If True, the files are processed simultaneously in multiple processes.
+        Each process will use a single thread.
+        By default, the number of process is defined with os.cpu_count().
+        However, you can customize it by typing: DASK_NUM_WORKERS=4 disdrodb_run_l0a_station
+        If False, the files are processed sequentially in a single process.
+        If False, multi-threading is automatically exploited to speed up I/0 tasks.
+    debugging_mode : bool
+        If True, it reduces the amount of data to process.
+        It processes just the first 3 raw data files.
+        The default is False.
+    data_archive_dir : str
+        DISDRODB Data Archive directory.
+        Format: <...>/DISDRODB
+        If not specified, uses path specified in the DISDRODB active configuration.
+    """
+    from disdrodb.l2.routines import run_l2m_station
+    from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
+
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
+
+    # -------------------------------------------------------------------------.
+    # If parallel=True, set the dask environment
+    if parallel:
+        cluster, client = initialize_dask_cluster()
+
+    # -------------------------------------------------------------------------.
+    run_l2m_station(
+        # Station arguments
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        # Processing options
+        force=force,
+        verbose=verbose,
+        debugging_mode=debugging_mode,
+        parallel=parallel,
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+    )
+
+    # -------------------------------------------------------------------------.
+    # Close the cluster
+    if parallel:
+        close_dask_cluster(cluster, client)

disdrodb/cli/disdrodb_upload_archive.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Routines to upload data to the DISDRODB Decentralized Data Archive."""
+
+import sys
+from typing import Optional
+
+import click
+
+from disdrodb.data_transfer.upload_data import click_upload_archive_options, click_upload_options
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_metadata_archive_dir_option,
+    parse_archive_dir,
+    parse_arg_to_list,
+)
+
+sys.tracebacklimit = 0  # avoid full traceback error if occur
+
+
+@click.command()
+@click_upload_archive_options
+@click_upload_options
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
+def disdrodb_upload_archive(
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
+    # Stations options
+    data_sources: Optional[str] = None,
+    campaign_names: Optional[str] = None,
+    station_names: Optional[str] = None,
+    # Upload options
+    platform: Optional[str] = None,
+    force: bool = False,
+):
+    """Find all stations containing local data and upload them to a remote repository.
+
+    Parameters
+    ----------
+    platform: str, optional
+        Name of the remote platform.
+        The default platform is ``"sandbox.zenodo"`` (for testing purposes).
+        Switch to ``"zenodo"`` for final data dissemination.
+    force: bool, optional
+        If ``True``, upload even if data already exists on another remote location.
+        The default value is ``force=False``.
+    data_archive_dir : str (optional)
+        The directory path where the DISDRODB Data Archive is located.
+        The directory path must end with ``<...>/DISDRODB``.
+        If ``None``, it uses the ``data_archive_dir`` path specified
+        in the DISDRODB active configuration.
+
+    Other Parameters
+    ----------------
+    data_sources: str or list of str, optional
+        Data source name (eg: EPFL).
+        If not provided (``None``), all data sources will be uploaded.
+        The default value is ``data_source=None``.
+    campaign_names: str or list of str, optional
+        Campaign name (eg:  EPFL_ROOF_2012).
+        If not provided (``None``), all campaigns will be uploaded.
+        The default value is ``campaign_name=None``.
+    station_names: str or list of str, optional
+        Station name.
+        If not provided (``None``), all stations will be uploaded.
+        The default value is ``station_name=None``.
+    """
+    from disdrodb.data_transfer.upload_data import upload_archive
+
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
+    data_sources = parse_arg_to_list(data_sources)
+    campaign_names = parse_arg_to_list(campaign_names)
+    station_names = parse_arg_to_list(station_names)
+
+    upload_archive(
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+        # Stations options
+        data_sources=data_sources,
+        campaign_names=campaign_names,
+        station_names=station_names,
+        # Upload options
+        platform=platform,
+        force=force,
+    )

disdrodb/cli/disdrodb_upload_station.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Routines to upload station data to the DISDRODB Decentralized Data Archive."""
+
+import sys
+from typing import Optional
+
+import click
+
+from disdrodb.data_transfer.upload_data import click_upload_options
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_metadata_archive_dir_option,
+    click_station_arguments,
+    parse_archive_dir,
+)
+
+sys.tracebacklimit = 0  # avoid full traceback error if occur
+
+
+@click.command()
+@click_station_arguments
+@click_upload_options
+@click_data_archive_dir_option
+@click_metadata_archive_dir_option
+def disdrodb_upload_station(
+    # Station arguments
+    data_source: str,
+    campaign_name: str,
+    station_name: str,
+    # Upload options
+    platform: Optional[str] = None,
+    force: bool = False,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
+):
+    """
+    Upload data from a single DISDRODB station on a remote repository.
+
+    This function also automatically update the disdrodb_data url in the metadata file.
+
+    Parameters
+    ----------
+    data_source : str
+        The name of the institution (for campaigns spanning multiple countries) or
+        the name of the country (for campaigns or sensor networks within a single country).
+        Must be provided in UPPER CASE.
+    campaign_name : str
+        The name of the campaign. Must be provided in UPPER CASE.
+    station_name : str
+        The name of the station.
+    data_archive_dir : str (optional)
+        The directory path where the DISDRODB Data Archive is located.
+        The directory path must end with ``<...>/DISDRODB``.
+        If ``None``, it uses the ``data_archive_dir`` path specified
+        in the DISDRODB active configuration.
+    platform: str, optional
+        Name of the remote data storage platform.
+        The default platform is ``"sandbox.zenodo"`` (for testing purposes).
+        Switch to ``"zenodo"`` for final data dissemination.
+    force: bool, optional
+        If ``True``, upload the data and overwrite the ``disdrodb_data_url``.
+        The default value is ``force=False``.
+
+    """
+    from disdrodb.data_transfer.upload_data import upload_station
+
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+    metadata_archive_dir = parse_archive_dir(metadata_archive_dir)
+    upload_station(
+        # DISDRODB root directories
+        data_archive_dir=data_archive_dir,
+        metadata_archive_dir=metadata_archive_dir,
+        # Station argument
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        # Upload options
+        platform=platform,
+        force=force,
+    )