disdrodb 0.0.21__py3-none-any.whl → 0.1.0__py3-none-any.whl

disdrodb/utils/cli.py

@@ -0,0 +1,269 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""DISDRODB command-line-interface scripts utilities."""
+
+import click
+
+
+def _execute_cmd(cmd, raise_error=False):
+    """Execute command in the terminal, streaming output in python console."""
+    from subprocess import PIPE, CalledProcessError, Popen
+
+    with Popen(cmd, shell=True, stdout=PIPE, bufsize=1, universal_newlines=True) as p:
+        for line in p.stdout:
+            print(line, end="")
+
+    # Raise error if command didn't run successfully
+    if p.returncode != 0 and raise_error:
+        raise CalledProcessError(p.returncode, p.args)
+
+
+def _parse_empty_string_and_none(args):
+    """Utility to parse argument passed from the command line.
+
+    If ``args = ''``, returns None.
+    If ``args = 'None'`` returns None.
+    Otherwise return ``args``.
+    """
+    # If '', set to 'None'
+    args = None if args == "" else args
+    # - If multiple arguments, split by space
+    if isinstance(args, str) and args == "None":
+        args = None
+    return args
+
+
+def parse_arg_to_list(args):
+    """Utility to pass list to command line scripts.
+
+    If ``args = ''`` returns ``None``.
+    If ``args = 'None'`` returns ``None``.
+    If ``args = 'variable'`` returns ``[variable]``.
+    If ``args = 'variable1 variable2'`` returns ``[variable1, variable2]``.
+    """
+    # If '' or 'None' --> Set to None
+    args = _parse_empty_string_and_none(args)
+    # - If multiple arguments, split by space
+    if isinstance(args, str):
+        # - Split by space
+        list_args = args.split(" ")
+        # - Remove '' (deal with multi space)
+        args = [args for args in list_args if len(args) > 0]
+    return args
+
+
+def parse_archive_dir(archive_dir: str):
+    """Utility to parse archive directories provided by command line.
+
+    If ``archive_dir = 'None'`` returns ``None``.
+    If ``archive_dir = ''`` returns ``None``.
+    """
+    # If '', set to 'None'
+    return _parse_empty_string_and_none(archive_dir)
+
+
+def click_station_arguments(function: object):
+    """Click command line arguments for DISDRODB station processing.
+
+    Parameters
+    ----------
+    function : object
+        Function.
+    """
+    function = click.argument("station_name", metavar="<station>")(function)
+    function = click.argument("campaign_name", metavar="<CAMPAIGN_NAME>")(function)
+    function = click.argument("data_source", metavar="<DATA_SOURCE>")(function)
+    return function
+
+
+def click_data_archive_dir_option(function: object):
+    """Click command line argument for DISDRODB ``data_archive_dir``.
+
+    Parameters
+    ----------
+    function : object
+        Function.
+    """
+    function = click.option(
+        "--data_archive_dir",
+        type=str,
+        show_default=True,
+        default=None,
+        help="DISDRODB base directory",
+    )(function)
+    return function
+
+
+def click_metadata_archive_dir_option(function: object):
+    """Click command line argument for DISDRODB ``metadata_archive_dir``.
+
+    Parameters
+    ----------
+    function : object
+        Function.
+    """
+    function = click.option(
+        "--metadata_archive_dir",
+        type=str,
+        show_default=True,
+        default=None,
+        help="DISDRODB Metadata Archive Directory",
+    )(function)
+    return function
+
+
+def click_stations_options(function: object):
+    """Click command line options for DISDRODB archive L0 processing.
+
+    Parameters
+    ----------
+    function : object
+        Function.
+    """
+    function = click.option(
+        "--data_sources",
+        type=str,
+        show_default=True,
+        default="",
+        help="DISDRODB data sources to process",
+    )(function)
+    function = click.option(
+        "--campaign_names",
+        type=str,
+        show_default=True,
+        default="",
+        help="DISDRODB campaign names to process",
+    )(function)
+    function = click.option(
+        "--station_names",
+        type=str,
+        show_default=True,
+        default="",
+        help="DISDRODB station names to process",
+    )(function)
+    return function
+
+
+def click_processing_options(function: object):
+    """Click command line default parameters for L0 processing options.
+
+    Parameters
+    ----------
+    function : object
+        Function.
+    """
+    function = click.option(
+        "-p",
+        "--parallel",
+        type=bool,
+        show_default=True,
+        default=False,
+        help="Process files in parallel",
+    )(function)
+    function = click.option(
+        "-d",
+        "--debugging_mode",
+        type=bool,
+        show_default=True,
+        default=False,
+        help="Switch to debugging mode",
+    )(function)
+    function = click.option("-v", "--verbose", type=bool, show_default=True, default=True, help="Verbose")(function)
+    function = click.option(
+        "-f",
+        "--force",
+        type=bool,
+        show_default=True,
+        default=False,
+        help="Force overwriting",
+    )(function)
+    return function
+
+
+def click_remove_l0a_option(function: object):
+    """Click command line argument for ``remove_l0a``."""
+    function = click.option(
+        "--remove_l0a",
+        type=bool,
+        show_default=True,
+        default=False,
+        help="If true, remove the L0A files once the L0B processing is terminated.",
+    )(function)
+    return function
+
+
+def click_remove_l0b_option(function: object):
+    """Click command line argument for ``remove_l0b``."""
+    function = click.option(
+        "--remove_l0b",
+        type=bool,
+        show_default=True,
+        default=False,
+        help="If true, remove the L0B files once the L0C processing is terminated.",
+    )(function)
+    return function
+
+
+def click_l0_archive_options(function: object):
+    """Click command line arguments for L0 processing archiving of a station.
+
+    Parameters
+    ----------
+    function : object
+        Function.
+    """
+    function = click.option(
+        "--remove_l0b",
+        type=bool,
+        show_default=True,
+        default=False,
+        help="If true, remove all source L0B files once L0B concatenation is terminated.",
+    )(function)
+    function = click.option(
+        "--remove_l0a",
+        type=bool,
+        show_default=True,
+        default=False,
+        help="If true, remove the L0A files once the L0B processing is terminated.",
+    )(function)
+    function = click.option(
+        "-l0c",
+        "--l0c_processing",
+        type=bool,
+        show_default=True,
+        default=True,
+        help="Perform L0C processing.",
+    )(function)
+    function = click.option(
+        "-l0b",
+        "--l0b_processing",
+        type=bool,
+        show_default=True,
+        default=True,
+        help="Perform L0B processing.",
+    )(function)
+    function = click.option(
+        "-l0a",
+        "--l0a_processing",
+        type=bool,
+        show_default=True,
+        default=True,
+        help="Perform L0A processing.",
+    )(function)
+    return function

disdrodb/utils/compression.py

@@ -26,9 +26,11 @@ import tempfile
 import zipfile
 from typing import Optional
 
-from disdrodb.api.checks import check_base_dir
+from disdrodb.api.checks import check_data_archive_dir
 from disdrodb.api.path import define_station_dir
+from disdrodb.configs import get_data_archive_dir
 from disdrodb.utils.directories import list_files
+from disdrodb.utils.yaml import read_yaml
 
 COMPRESSION_OPTIONS = {
     "zip": ".zip",
@@ -38,17 +40,15 @@ COMPRESSION_OPTIONS = {
 
 
 def unzip_file(filepath: str, dest_path: str) -> None:
-    """Unzip a file into a directory
+    """Unzip a file into a directory.
 
     Parameters
-
     ----------
     filepath : str
-        Path of the file to unzip
+        Path of the file to unzip.
     dest_path : str
-        Path of the destination directory
+        Path of the destination directory.
     """
-
     with zipfile.ZipFile(filepath, "r") as zip_ref:
         zip_ref.extractall(dest_path)
 
@@ -59,21 +59,29 @@ def _zip_dir(dir_path: str) -> str:
     Parameters
     ----------
     dir_path : str
-        Path of the directory to zip
+        Path of the directory to zip.
 
     Returns
     -------
     str
-        Path of the zip archive
+        Path of the zip archive.
     """
-
     output_path_without_extension = os.path.join(tempfile.gettempdir(), os.path.basename(dir_path))
     output_path = output_path_without_extension + ".zip"
     shutil.make_archive(output_path_without_extension, "zip", dir_path)
     return output_path
 
 
-def archive_station_data(metadata_filepath: str) -> str:
+def check_consistent_station_name(metadata_filepath, station_name):
+    """Check consistent station_name between YAML file name and metadata key."""
+    # Check consistent station name
+    expected_station_name = os.path.basename(metadata_filepath).replace(".yml", "")
+    if station_name and str(station_name) != str(expected_station_name):
+        raise ValueError(f"Inconsistent station_name values in the {metadata_filepath} file. Download aborted.")
+    return station_name
+
+
+def archive_station_data(metadata_filepath: str, data_archive_dir: str) -> str:
     """Archive station data into a zip file for subsequent data upload.
 
     It create a zip file into a temporary directory !
@@ -83,26 +91,42 @@ def archive_station_data(metadata_filepath: str) -> str:
     metadata_filepath: str
         Metadata file path.
 
-    Returns
-    -------
-    station_zip_filepath
-        Filepath of the zip file containing the station's data.
     """
-    station_data_path = metadata_filepath.replace("metadata", "data")
-    station_data_path = os.path.splitext(station_data_path)[0]  # remove trailing ".yml"
-    station_zip_filepath = _zip_dir(station_data_path)
+    # Open metadata file
+    metadata_dict = read_yaml(metadata_filepath)
+    # Retrieve station information
+    data_archive_dir = get_data_archive_dir(data_archive_dir)
+    data_source = metadata_dict["data_source"]
+    campaign_name = metadata_dict["campaign_name"]
+    station_name = metadata_dict["station_name"]
+    station_name = check_consistent_station_name(metadata_filepath, station_name)
+    # Define the destination local filepath path
+    station_dir = define_station_dir(
+        data_archive_dir=data_archive_dir,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        product="RAW",
+    )
+    # Zip station directory
+    station_zip_filepath = _zip_dir(station_dir)
     return station_zip_filepath
 
 
 def compress_station_files(
-    base_dir: str, data_source: str, campaign_name: str, station_name: str, method: str = "gzip", skip: bool = True
+    data_archive_dir: str,
+    data_source: str,
+    campaign_name: str,
+    station_name: str,
+    method: str = "gzip",
+    skip: bool = True,
 ) -> None:
     """Compress each raw file of a station.
 
     Parameters
     ----------
-    base_dir : str
-        Base directory of DISDRODB
+    data_archive_dir : str
+        DISDRODB Data Archive directory
     data_source : str
         Name of data source of interest.
     campaign_name : str
@@ -110,19 +134,19 @@ def compress_station_files(
     station_name : str
         Station name of interest.
     method : str
-        Compression method. "zip", "gzip" or "bzip2".
+        Compression method. ``"zip"``, ``"gzip"`` or ``"bzip2"``.
     skip : bool
         Whether to raise an error if a file is already compressed.
-        If True, it does not raise an error and try to compress the other files.
-        If False, it raise an error and stop the compression routine.
-        THe default is True.
+        If ``True``, it does not raise an error and try to compress the other files.
+        If ``False``, it raise an error and stop the compression routine.
+        The default value is ``True``.
     """
     if method not in COMPRESSION_OPTIONS:
         raise ValueError(f"Invalid compression method {method}. Valid methods are {list(COMPRESSION_OPTIONS.keys())}")
 
-    base_dir = check_base_dir(base_dir)
+    data_archive_dir = check_data_archive_dir(data_archive_dir)
     station_dir = define_station_dir(
-        base_dir=base_dir,
+        data_archive_dir=data_archive_dir,
         product="RAW",
         data_source=data_source,
         campaign_name=campaign_name,
@@ -137,7 +161,7 @@ def compress_station_files(
     for filepath in filepaths:
         _ = _compress_file(filepath, method, skip=skip)
 
-    print("All files of {data_source} {campaign_name} {station_name} have been compressed.")
+    print(f"All files of {data_source} {campaign_name} {station_name} have been compressed.")
     print("Please now remember to update the glob_pattern of the reader ¨!")
 
 
@@ -151,11 +175,11 @@ def _compress_file(filepath: str, method: str, skip: bool) -> str:
     filepath : str
         Path of the file to compress.
     method : str
-        Compression method. None, "zip", "gzip" or "bzip2".
+        Compression method. ``None``, ``"zip"``, ``"gzip"`` or ``"bzip2"``.
     skip : bool
         Whether to raise an error if a file is already compressed.
-        If True, it does not raise an error return the input filepath.
-        If False, it raise an error.
+        If ``True``, it does not raise an error return the input filepath.
+        If ``False``, it raise an error.
 
     Returns
     -------
@@ -169,8 +193,7 @@ def _compress_file(filepath: str, method: str, skip: bool) -> str:
         if skip:
             print(f"File {filepath} is already compressed. Skipping.")
             return filepath
-        else:
-            raise ValueError(f"File {filepath} is already compressed !")
+        raise ValueError(f"File {filepath} is already compressed !")
 
     extension = COMPRESSION_OPTIONS[method]
     archive_name = os.path.basename(filepath) + extension
@@ -200,7 +223,7 @@ def _check_file_compression(filepath: str) -> Optional[str]:
     Returns
     -------
     Optional[str]
-        Compression method. None, "zip", "gzip" or "bzip2".
+        Compression method. ``None``, ``"zip"``, ``"gzip"`` or ``"bzip2"``.
 
     """
     magic_dict = {
@@ -229,7 +252,6 @@ def _compress_file_zip(filepath: str, compressed_filepath: str) -> None:
         Path of the compressed file.
 
     """
-
     with zipfile.ZipFile(compressed_filepath, "w", compression=zipfile.ZIP_DEFLATED) as zipf:
         zipf.write(filepath, os.path.basename(filepath))
 
@@ -246,10 +268,8 @@ def _compress_file_gzip(filepath: str, compressed_filepath: str) -> None:
         Path of the compressed file.
 
     """
-
-    with open(filepath, "rb") as f_in:
-        with gzip.open(compressed_filepath, "wb") as f_out:
-            f_out.writelines(f_in)
+    with open(filepath, "rb") as f_in, gzip.open(compressed_filepath, "wb") as f_out:
+        f_out.writelines(f_in)
 
 
 def _compress_file_bzip2(filepath: str, compressed_filepath: str) -> None:
@@ -264,7 +284,5 @@ def _compress_file_bzip2(filepath: str, compressed_filepath: str) -> None:
         Path of the compressed file.
 
     """
-
-    with open(filepath, "rb") as f_in:
-        with bz2.open(compressed_filepath, "wb") as f_out:
-            f_out.writelines(f_in)
+    with open(filepath, "rb") as f_in, bz2.open(compressed_filepath, "wb") as f_out:
+        f_out.writelines(f_in)

disdrodb/utils/dask.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Utilities for Dask Distributed computations."""
+import logging
+import os
+
+
+def initialize_dask_cluster():
+    """Initialize Dask Cluster."""
+    import dask
+    from dask.distributed import Client, LocalCluster
+
+    # Set HDF5_USE_FILE_LOCKING to avoid going stuck with HDF
+    os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+    # Retrieve the number of process to run
+    available_workers = os.cpu_count() - 2  # if not set, all CPUs
+    num_workers = dask.config.get("num_workers", available_workers)
+    # Silence dask warnings
+    dask.config.set({"logging.distributed": "error"})
+    # dask.config.set({"distributed.admin.system-monitor.gil.enabled": False})
+    # Create dask.distributed local cluster
+    cluster = LocalCluster(
+        n_workers=num_workers,
+        threads_per_worker=1,
+        processes=True,
+        # memory_limit='8GB',
+        # silence_logs=False,
+    )
+    client = Client(cluster)
+    return cluster, client
+
+
+def close_dask_cluster(cluster, client):
+    """Close Dask Cluster."""
+    logger = logging.getLogger()
+    # Backup current log level
+    original_level = logger.level
+    logger.setLevel(logging.CRITICAL + 1)  # Set level to suppress all logs
+    # Close cluster
+    # - Avoid log 'distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.'
+    try:
+        cluster.close()
+        client.close()
+    finally:
+        # Restore the original log level
+        logger.setLevel(original_level)

disdrodb/utils/decorators.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""DISDRODB decorators."""
+import functools
+import importlib
+
+import dask
+
+
+def delayed_if_parallel(function):
+    """Decorator to make the function delayed if its ``parallel`` argument is ``True``."""
+
+    @functools.wraps(function)
+    def wrapper(*args, **kwargs):
+        # Check if it must be a delayed function
+        parallel = kwargs.get("parallel")
+        # If parallel is True
+        if parallel:
+            # Enforce verbose to be False
+            kwargs["verbose"] = False
+            # Define the delayed task
+            result = dask.delayed(function)(*args, **kwargs)
+        else:
+            # Else run the function
+            result = function(*args, **kwargs)
+        return result
+
+    return wrapper
+
+
+def single_threaded_if_parallel(function):
+    """Decorator to make a function use a single threadon delayed if its ``parallel`` argument is ``True``."""
+
+    @functools.wraps(function)
+    def wrapper(*args, **kwargs):
+        # Check if it must be a delayed function
+        parallel = kwargs.get("parallel")
+        # If parallel is True
+        if parallel:
+            # Call function with single thread
+            # with dask.config.set(scheduler='single-threaded'):
+            with dask.config.set(scheduler="synchronous"):
+                result = function(*args, **kwargs)
+        else:
+            # Else run the function as usual
+            result = function(*args, **kwargs)
+        return result
+
+    return wrapper
+
+
+def check_software_availability(software, conda_package):
+    """A decorator to ensure that a software package is installed.
+
+    Parameters
+    ----------
+    software : str
+        The package name as recognized by Python's import system.
+    conda_package : str
+        The package name as recognized by conda-forge.
+    """
+
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            if not importlib.util.find_spec(software):
+                raise ImportError(
+                    f"The '{software}' package is required but not found.\n"
+                    "Please install it using conda:\n"
+                    f"    conda install -c conda-forge {conda_package}",
+                )
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+def check_pytmatrix_availability(func):
+    """Decorator to ensure that the 'pytmatrix' package is installed."""
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        if not importlib.util.find_spec("pytmatrix"):
+            raise ImportError(
+                "The 'pytmatrix' package is required but not found. \n"
+                "Please install the following software: \n"
+                "  conda install conda-forge gfortran \n"
+                "  conda install conda-forge meson \n"
+                "  pip install git+https://github.com/ltelab/pytmatrix-lte.git@main \n",
+            )
+        return func(*args, **kwargs)
+
+    return wrapper