disdrodb 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

disdrodb/cli/disdrodb_create_summary_station.py

@@ -0,0 +1,91 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Script to run the DISDRODB L0 station processing."""
+import sys
+from typing import Optional
+
+import click
+
+from disdrodb.utils.cli import (
+    click_data_archive_dir_option,
+    click_station_arguments,
+    parse_archive_dir,
+)
+
+sys.tracebacklimit = 0  # avoid full traceback error if occur
+
+# -------------------------------------------------------------------------.
+# Click Command Line Interface decorator
+
+
+@click.command()
+@click_station_arguments
+@click_data_archive_dir_option
+@click.option("-p", "--parallel", type=bool, show_default=True, default=False, help="Read files in parallel")
+def disdrodb_create_summary_station(
+    # Station arguments
+    data_source: str,
+    campaign_name: str,
+    station_name: str,
+    # Processing options:
+    parallel=False,
+    # DISDRODB root directories
+    data_archive_dir: Optional[str] = None,
+):
+    r"""Create summary figures and tables for a specific DISDRODB station.
+
+    Parameters \n
+    ---------- \n
+    data_source : str \n
+        Institution name (when campaign data spans more than 1 country),
+        or country (when all campaigns (or sensor networks) are inside a given country).\n
+        Must be UPPER CASE.\n
+    campaign_name : str \n
+        Campaign name. Must be UPPER CASE.\n
+    station_name : str \n
+        Station name \n
+    data_archive_dir : str \n
+        DISDRODB Data Archive directory \n
+        Format: <...>/DISDRODB \n
+        If not specified, uses path specified in the DISDRODB active configuration. \n
+    """
+    from disdrodb.summary.routines import create_station_summary
+    from disdrodb.utils.dask import close_dask_cluster, initialize_dask_cluster
+
+    data_archive_dir = parse_archive_dir(data_archive_dir)
+
+    # -------------------------------------------------------------------------.
+    # If parallel=True, set the dask environment
+    if parallel:
+        cluster, client = initialize_dask_cluster()
+
+    # -------------------------------------------------------------------------.
+    create_station_summary(
+        # Station arguments
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        # Options
+        parallel=parallel,
+        # DISDRODB root directory
+        data_archive_dir=data_archive_dir,
+    )
+
+    # -------------------------------------------------------------------------.
+    # Close the cluster
+    if parallel:
+        close_dask_cluster(cluster, client)

disdrodb/cli/disdrodb_run_l0.py

@@ -115,7 +115,7 @@ def disdrodb_run_l0(
     debugging_mode : bool
         If True, it reduces the amount of data to process.
         For L0A, it processes just the first 3 raw data files.
-        For L0B, it processes just the first 100 rows of 3 L0A files.
+        For L0B, it processes 100 rows sampled from 3 L0A files.
         The default is False.
     data_archive_dir : str
         DISDRODB Data Archive directory

disdrodb/cli/disdrodb_run_l0_station.py

@@ -105,7 +105,7 @@ def disdrodb_run_l0_station(
     debugging_mode : bool \n
         If True, it reduces the amount of data to process.\n
         For L0A, it processes just the first 3 raw data files for each station.\n
-        For L0B, it processes just the first 100 rows of 3 L0A files for each station.\n
+        For L0B, it processes 100 rows sampled from 3 L0A files for each station.\n
         The default is False.\n
     data_archive_dir : str \n
         DISDRODB Data Archive directory \n

disdrodb/cli/disdrodb_run_l0b.py

@@ -92,7 +92,7 @@ def disdrodb_run_l0b(
         If False, multi-threading is automatically exploited to speed up I/0 tasks.
     debugging_mode : bool
         If True, it reduces the amount of data to process.
-        It processes just the first 100 rows of 3 L0A files for each station.
+        It processes 100 rows sampled from 3 L0A files for each station.
         The default is False.
     data_archive_dir : str
         DISDRODB Data Archive directory

disdrodb/cli/disdrodb_run_l0b_station.py

@@ -84,7 +84,7 @@ def disdrodb_run_l0b_station(
         If False, multi-threading is automatically exploited to speed up I/0 tasks.
     debugging_mode : bool
         If True, it reduces the amount of data to process.
-        It processes just the first 100 rows of 3 L0A files.
+        It processes 100 rows sampled from 3 L0A files.
         The default is False.
     data_archive_dir : str
         DISDRODB Data Archive directory

disdrodb/cli/disdrodb_run_l0c.py

@@ -93,7 +93,7 @@ def disdrodb_run_l0c(
         If False, multi-threading is automatically exploited to speed up I/0 tasks.
     debugging_mode : bool
         If True, it reduces the amount of data to process.
-        It processes just the first 100 rows of 3 L0A files for each station.
+        It processes 100 rows sampled from 3 L0A files for each station.
         The default is False.
     remove_l0b: bool, optional
         Whether to remove the processed L0B files. The default value is ``False``.

disdrodb/cli/disdrodb_run_l0c_station.py

@@ -85,7 +85,7 @@ def disdrodb_run_l0c_station(
         If False, multi-threading is automatically exploited to speed up I/0 tasks.
     debugging_mode : bool
         If True, it reduces the amount of data to process.
-        It processes just the first 100 rows of 3 L0A files.
+        It processes 100 rows sampled from 3 L0A files.
         The default is False.
     remove_l0b: bool, optional
         Whether to remove the processed L0B files. The default value is ``False``.

disdrodb/cli/disdrodb_run_l2e_station.py

@@ -98,7 +98,7 @@ def disdrodb_run_l2e_station(
     # -------------------------------------------------------------------------.
     # If parallel=True, set the dask environment
     if parallel:
-        cluster, client = initialize_dask_cluster()
+        cluster, client = initialize_dask_cluster(minimum_memory="8GB")
 
     # -------------------------------------------------------------------------.
     run_l2e_station(

disdrodb/configs.py

@@ -19,6 +19,7 @@
 """DISDRODB Configuration File functions."""
 
 import os
+import shutil
 from typing import Optional
 
 from disdrodb.utils.yaml import read_yaml, write_yaml
@@ -32,9 +33,11 @@ def _define_config_filepath():
     return filepath
 
 
-def define_disdrodb_configs(
+def define_configs(
     data_archive_dir: Optional[str] = None,
     metadata_archive_dir: Optional[str] = None,
+    scattering_table_dir: Optional[str] = None,
+    configs_path: Optional[str] = None,
     folder_partitioning: Optional[str] = None,
     zenodo_token: Optional[str] = None,
     zenodo_sandbox_token: Optional[str] = None,
@@ -48,6 +51,10 @@ def define_disdrodb_configs(
         The directory path where the DISDRODB Data Archive is located.
     metadata_archive_dir : str
         The directory path where the DISDRODB Metadata Archive is located.
+    scattering_table_dir : str
+        The directory path where to store DISDRODB T-Matrix scattering tables.
+    configs_path : str
+        The directory path where the custom DISDRODB products configurations files are defined.
     folder_partitioning : str
         The folder partitioning scheme used in the DISDRODB Data Archive.
         Allowed values are:
@@ -69,7 +76,13 @@ def define_disdrodb_configs(
     The configuration file is used to run the various DISDRODB operations.
 
     """
-    from disdrodb.api.checks import check_data_archive_dir, check_folder_partitioning, check_metadata_archive_dir
+    import disdrodb
+    from disdrodb.api.checks import (
+        check_data_archive_dir,
+        check_folder_partitioning,
+        check_metadata_archive_dir,
+        check_scattering_table_dir,
+    )
 
     # Define path to .config_disdrodb.yaml file
     filepath = _define_config_filepath()
@@ -85,9 +98,16 @@ def define_disdrodb_configs(
     # Add DISDRODB Data Archive Directory
     if data_archive_dir is not None:
         config_dict["data_archive_dir"] = check_data_archive_dir(data_archive_dir)
+
     # Add DISDRODB Metadata Archive Directory
     if metadata_archive_dir is not None:
         config_dict["metadata_archive_dir"] = check_metadata_archive_dir(metadata_archive_dir)
+
+    # Add DISDRODB Scattering Table Directory
+    if scattering_table_dir is not None:
+        os.makedirs(scattering_table_dir, exist_ok=True)
+        config_dict["scattering_table_dir"] = check_scattering_table_dir(scattering_table_dir)
+
     # Add DISDRODB Folder Partitioning
     if folder_partitioning is not None:
         config_dict["folder_partitioning"] = check_folder_partitioning(folder_partitioning)
@@ -98,13 +118,21 @@ def define_disdrodb_configs(
     if zenodo_sandbox_token is not None:
         config_dict["zenodo_sandbox_token"] = zenodo_sandbox_token
 
+    if configs_path is not None:
+        config_dict["configs_path"] = configs_path
+
     # Write the DISDRODB config file
     write_yaml(config_dict, filepath, sort_keys=False)
 
     print(f"The DISDRODB config file has been {action_msg} successfully!")
 
+    # Now read the config file and set it as the active configuration
+    # - This avoid the need to restart a python session to take effect !
+    config_dict = read_configs()
+    disdrodb.config.update(config_dict)
+
 
-def read_disdrodb_configs() -> dict[str, str]:
+def read_configs() -> dict[str, str]:
     """
     Reads the DISDRODB configuration file and returns a dictionary with the configuration settings.
 
@@ -158,6 +186,19 @@ def get_metadata_archive_dir(metadata_archive_dir=None):
     return metadata_archive_dir
 
 
+def get_scattering_table_dir(scattering_table_dir=None):
+    """Return the directory where DISDRODB save pyTMatrix scattering tables."""
+    import disdrodb
+    from disdrodb.api.checks import check_scattering_table_dir
+
+    if scattering_table_dir is None:
+        scattering_table_dir = disdrodb.config.get("scattering_table_dir", None)
+    if scattering_table_dir is None:
+        raise ValueError("The directory where to save DISDRODB T-Matrix scattering tables is not specified.")
+    scattering_table_dir = check_scattering_table_dir(scattering_table_dir)  # ensure Path converted to str
+    return scattering_table_dir
+
+
 def get_folder_partitioning():
     """Return the folder partitioning."""
     import disdrodb
@@ -182,7 +223,7 @@ def get_zenodo_token(sandbox: bool):
         host = "zenodo.org"
         token_name = "zenodo_token"
 
-    # token = read_disdrodb_configs().get(token_name, None)
+    # token = read_configs().get(token_name, None)
     token = disdrodb.config.get(token_name, None)
 
     if token is None:
@@ -195,3 +236,107 @@ def get_zenodo_token(sandbox: bool):
         raise ValueError(f"Missing {token_name} in the DISDRODB config file !")
 
     return token
+
+
+def get_product_default_configs_path():
+    """Return the paths where DISDRODB products configuration files are stored."""
+    import disdrodb
+
+    configs_path = os.path.join(disdrodb.__root_path__, "disdrodb", "etc", "products")
+    return configs_path
+
+
+def check_availability_radar_simulations(options):
+    """Check radar simulations are possible for L2E and L2M products."""
+    import disdrodb
+
+    if "radar_enabled" in options and not disdrodb.is_pytmatrix_available():
+        options["radar_enabled"] = False
+    return options
+
+
+def copy_product_default_configs(configs_path):
+    """Copy the default DISDRODB products configuration directory to a custom location.
+
+    This function duplicates the entire directory of default product settings
+    (located at ``disdrodb/etc/products``) into the user-specified
+    ``configs_path``. Once copied, you can safely edit these files without
+    modifying the library's built-in defaults. To have DISDRODB use your
+    custom settings, point the global configuration at this new directory
+    (e.g by specifying ``configs_path`` with the ``disdrodb.define_configs`` function).
+
+    Parameters
+    ----------
+    configs_path:
+        Destination directory where the default product configuration files
+        will be copied. This directory must not already exist, and later
+        needs to be referenced in your DISDRODB global configuration.
+
+    Returns
+    -------
+    configs_path
+        The path to the newly created custom product configuration directory.
+
+    """
+    source_dir_path = get_product_default_configs_path()
+    if os.path.exists(configs_path):
+        raise FileExistsError(f"The {configs_path} directory already exists!")
+    configs_path = shutil.copytree(source_dir_path, configs_path)
+    return configs_path
+
+
+def get_product_options(product, temporal_resolution=None):
+    """Get options for DISDRODB products."""
+    import disdrodb
+    from disdrodb.api.checks import check_product
+
+    # Define configs path
+    if os.environ.get("PYTEST_CURRENT_TEST"):
+        configs_path = os.path.join(disdrodb.__root_path__, "disdrodb", "tests", "products")
+    else:
+        configs_path = disdrodb.config.get("configs_path", get_product_default_configs_path())
+
+    # Validate DISDRODB products configuration
+    validate_product_configuration(configs_path)
+
+    # Check product
+    check_product(product)
+
+    # Retrieve global product options
+    global_options = read_yaml(os.path.join(configs_path, product, "global.yaml"))
+    if temporal_resolution is None:
+        global_options = check_availability_radar_simulations(global_options)
+        return global_options
+
+    # If temporal resolutions are specified, drop 'temporal_resolutions' key
+    global_options.pop("temporal_resolutions", None)
+    custom_options_path = os.path.join(configs_path, product, f"{temporal_resolution}.yaml")
+    if not os.path.exists(custom_options_path):
+        return global_options
+    custom_options = read_yaml(custom_options_path)
+    options = global_options.copy()
+    options.update(custom_options)
+    options = check_availability_radar_simulations(options)
+    return options
+
+
+def get_product_temporal_resolutions(product):
+    """Get DISDRODB L2 product temporal aggregations."""
+    # Check only L2E and L2M
+    return get_product_options(product)["temporal_resolutions"]
+
+
+def get_model_options(product, model_name):
+    """Get DISDRODB L2M model options."""
+    import disdrodb
+
+    configs_path = disdrodb.config.get("configs_path", get_product_default_configs_path())
+    model_options_path = os.path.join(configs_path, product, f"{model_name}.yaml")
+    model_options = read_yaml(model_options_path)
+    return model_options
+
+
+def validate_product_configuration(configs_path):
+    """Validate the DISDRODB products configuration files."""
+    # TODO: Implement validation of DISDRODB products configuration files with pydantic
+    pass

disdrodb/constants.py

@@ -0,0 +1,61 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""DISDRODB constants."""
+import importlib
+
+ARCHIVE_VERSION = "V0"
+SOFTWARE_VERSION = "V" + importlib.metadata.version("disdrodb")
+CONVENTIONS = "CF-1.10, ACDD-1.3"
+
+# Define coordinates names
+DIAMETER_COORDS = ["diameter_bin_center", "diameter_bin_width", "diameter_bin_lower", "diameter_bin_upper"]
+VELOCITY_COORDS = ["velocity_bin_center", "velocity_bin_width", "velocity_bin_lower", "velocity_bin_upper"]
+GEOLOCATION_COORDS = ["longitude", "latitude", "altitude"]
+VELOCITY_DIMENSION = "velocity_bin_center"
+DIAMETER_DIMENSION = "diameter_bin_center"
+COORDINATES = [
+    "diameter_bin_center",
+    "diameter_bin_width",
+    "diameter_bin_upper",
+    "velocity_bin_lower",
+    "velocity_bin_center",
+    "velocity_bin_width",
+    "velocity_bin_upper",
+    "latitude",
+    "longitude",
+    "altitude",
+    "time",
+    "sample_interval",
+]
+OPTICAL_SENSORS = ["PARSIVEL", "PARSIVEL2", "LPM", "PWS100"]
+IMPACT_SENSORS = ["RD80"]
+
+PRODUCTS = ["RAW", "L0A", "L0B", "L0C", "L1", "L2E", "L2M"]
+
+PRODUCTS_ARGUMENTS = {
+    "L2E": ["rolling", "sample_interval"],
+    "L2M": ["rolling", "sample_interval", "model_name"],
+}
+
+PRODUCTS_REQUIREMENTS = {
+    "L0A": "RAW",
+    "L0B": "L0A",
+    "L0C": "L0B",
+    "L1": "L0C",
+    "L2E": "L1",
+    "L2M": "L2E",
+}

disdrodb/data_transfer/download_data.py

@@ -275,9 +275,9 @@ def download_station_data(metadata_filepath: str, data_archive_dir: str, force:
         raise ValueError(f"Invalid disdrodb_data_url '{disdrodb_data_url}' for station {station_name}")
 
     # Download files
-    # - Option 1: Zip file from Zenodo containing all station raw data
-    if disdrodb_data_url.startswith("https://zenodo.org/"):
-        download_zenodo_zip_file(url=disdrodb_data_url, dst_dir=station_dir, force=force)
+    # - Option 1: Download Zip file containing all station raw data
+    if disdrodb_data_url.startswith("https://zenodo.org/") or disdrodb_data_url.startswith("https://cloudnet.fmi.fi/"):
+        download_zip_file(url=disdrodb_data_url, dst_dir=station_dir, force=force)
     # - Option 2: Recursive download from a web server via HTTP or HTTPS.
     elif disdrodb_data_url.startswith("http"):
         download_web_server_data(url=disdrodb_data_url, dst_dir=station_dir, force=force, verbose=True)
@@ -373,7 +373,7 @@ def build_webserver_wget_command(url: str, cut_dirs: int, dst_dir: str, force: b
       - url
     """
     cmd = ["wget"]
-    if verbose:
+    if not verbose:
         cmd.append("-q")
     cmd += [
         "-r",
@@ -397,7 +397,7 @@ def build_webserver_wget_command(url: str, cut_dirs: int, dst_dir: str, force: b
 #### Download from Zenodo
 
 
-def download_zenodo_zip_file(url, dst_dir, force):
+def download_zip_file(url, dst_dir, force):
     """Download zip file from zenodo and extract station raw data."""
     # Download zip file
     zip_filepath = _download_file_from_url(url, dst_dir=dst_dir, force=force)