disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl

disdrodb/configs.py

@@ -1,11 +1,25 @@
 #!/usr/bin/env python3
-"""
-Created on Thu Nov  2 15:39:01 2023
 
-@author: ghiggi
-"""
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""DISDRODB Configuration File functions."""
+
 import os
-from typing import Dict
+from typing import Optional
 
 from disdrodb.utils.yaml import read_yaml, write_yaml
 
@@ -18,14 +32,31 @@ def _define_config_filepath():
     return filepath
 
 
-def define_disdrodb_configs(base_dir: str = None, zenodo_token: str = None, zenodo_sandbox_token: str = None):
+def define_disdrodb_configs(
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
+    folder_partitioning: Optional[str] = None,
+    zenodo_token: Optional[str] = None,
+    zenodo_sandbox_token: Optional[str] = None,
+):
     """
     Defines the DISDRODB configuration file with the given credentials and base directory.
 
     Parameters
     ----------
-    base_dir : str
-        The base directory where DISDRODB Metadata Archive is located.
+    data_archive_dir : str
+        The directory path where the DISDRODB Data Archive is located.
+    metadata_archive_dir : str
+        The directory path where the DISDRODB Metadata Archive is located.
+    folder_partitioning : str
+        The folder partitioning scheme used in the DISDRODB Data Archive.
+        Allowed values are:
+        - "": No additional subdirectories, files are saved directly in <station_dir>.
+        - "year": Files are stored under a subdirectory for the year (<station_dir>/2025).
+        - "year/month": Files are stored under subdirectories by year and month (<station_dir>/2025/04).
+        - "year/month/day": Files are stored under subdirectories by year, month and day (<station_dir>/2025/04/01).
+        - "year/month_name": Files are stored under subdirectories by year and month name (<station_dir>/2025/April).
+        - "year/quarter": Files are stored under subdirectories by year and quarter (<station_dir>/2025/Q2).
     zenodo__token: str
         Zenodo Access Token. It is required to upload stations data to Zenodo.
     zenodo_sandbox_token: str
@@ -33,11 +64,13 @@ def define_disdrodb_configs(base_dir: str = None, zenodo_token: str = None, zeno
 
     Notes
     -----
-    This function write or update the DISDRODB config YAML file
-    The DISDRODB config YAML file is located in the user's home directory at ~/.config_disdrodb.yml
+    This function write or update the DISDRODB config YAML file.
+    The DISDRODB config YAML file is located in the user's home directory at ``~/.config_disdrodb.yml``.
     The configuration file is used to run the various DISDRODB operations.
 
     """
+    from disdrodb.api.checks import check_data_archive_dir, check_folder_partitioning, check_metadata_archive_dir
+
     # Define path to .config_disdrodb.yaml file
     filepath = _define_config_filepath()
 
@@ -49,9 +82,15 @@ def define_disdrodb_configs(base_dir: str = None, zenodo_token: str = None, zeno
         config_dict = {}
         action_msg = "written"
 
-    # Add DISDRODB Base Directory
-    if base_dir is not None:
-        config_dict["base_dir"] = base_dir
+    # Add DISDRODB Data Archive Directory
+    if data_archive_dir is not None:
+        config_dict["data_archive_dir"] = check_data_archive_dir(data_archive_dir)
+    # Add DISDRODB Metadata Archive Directory
+    if metadata_archive_dir is not None:
+        config_dict["metadata_archive_dir"] = check_metadata_archive_dir(metadata_archive_dir)
+    # Add DISDRODB Folder Partitioning
+    if folder_partitioning is not None:
+        config_dict["folder_partitioning"] = check_folder_partitioning(folder_partitioning)
 
     # Add Zenodo Access Tokens
     if zenodo_token is not None:
@@ -63,10 +102,9 @@ def define_disdrodb_configs(base_dir: str = None, zenodo_token: str = None, zeno
     write_yaml(config_dict, filepath, sort_keys=False)
 
     print(f"The DISDRODB config file has been {action_msg} successfully!")
-    return
 
 
-def read_disdrodb_configs() -> Dict[str, str]:
+def read_disdrodb_configs() -> dict[str, str]:
     """
     Reads the DISDRODB configuration file and returns a dictionary with the configuration settings.
 
@@ -78,12 +116,12 @@ def read_disdrodb_configs() -> Dict[str, str]:
     Raises
     ------
     ValueError
-        If the configuration file has not been defined yet. Use `disdrodb.define_configs()` to
+        If the configuration file has not been defined yet. Use ``disdrodb.define_configs()`` to
         specify the configuration file path and settings.
 
     Notes
     -----
-    This function reads the YAML configuration file located at ~/.config_disdrodb.yml.
+    This function reads the YAML configuration file located at ``~/.config_disdrodb.yml``.
     """
     # Define path where .config_disdrodb.yaml file should be located
     filepath = _define_config_filepath()
@@ -94,20 +132,47 @@ def read_disdrodb_configs() -> Dict[str, str]:
     return config_dict
 
 
-def get_base_dir(base_dir=None):
+def get_data_archive_dir(data_archive_dir=None):
     """Return the DISDRODB base directory."""
     import disdrodb
+    from disdrodb.api.checks import check_data_archive_dir
+
+    if data_archive_dir is None:
+        data_archive_dir = disdrodb.config.get("data_archive_dir", None)
+    if data_archive_dir is None:
+        raise ValueError("The DISDRODB Data Archive Directory is not specified.")
+    data_archive_dir = check_data_archive_dir(data_archive_dir)  # ensure Path converted to str
+    return data_archive_dir
+
+
+def get_metadata_archive_dir(metadata_archive_dir=None):
+    """Return the DISDRODB Metadata Archive Directory."""
+    import disdrodb
+    from disdrodb.api.checks import check_metadata_archive_dir
+
+    if metadata_archive_dir is None:
+        metadata_archive_dir = disdrodb.config.get("metadata_archive_dir", None)
+    if metadata_archive_dir is None:
+        raise ValueError("The DISDRODB Metadata Archive Directory is not specified.")
+    metadata_archive_dir = check_metadata_archive_dir(metadata_archive_dir)  # ensure Path converted to str
+    return metadata_archive_dir
+
+
+def get_folder_partitioning():
+    """Return the folder partitioning."""
+    import disdrodb
+    from disdrodb.api.checks import check_folder_partitioning
 
-    if base_dir is None:
-        base_dir = disdrodb.config.get("base_dir", None)
-    if base_dir is None:
-        raise ValueError("The DISDRODB Base Directory is not specified.")
-    base_dir = str(base_dir)  # convert Path to str
-    return base_dir
+    # Get the folder partitioning
+    folder_partitioning = disdrodb.config.get("folder_partitioning", None)
+    if folder_partitioning is None:
+        raise ValueError("The folder partitioning is not specified.")
+    check_folder_partitioning(folder_partitioning)
+    return folder_partitioning
 
 
 def get_zenodo_token(sandbox: bool):
-    """Return the Zenodo Access Token."""
+    """Return the Zenodo access token."""
     import disdrodb
 
     if sandbox:

disdrodb/data_transfer/__init__.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Routines to download and upload data to the DISDRODB Decentralized Data Archive."""
+
+from disdrodb.data_transfer.upload_data import upload_station, upload_archive  # noqa
+from disdrodb.data_transfer.download_data import download_station, download_archive  # noqa

disdrodb/data_transfer/download_data.py

@@ -18,16 +18,17 @@
 # -----------------------------------------------------------------------------.
 """Routines to download data from the DISDRODB Decentralized Data Archive."""
 
+import logging
 import os
 import shutil
-from typing import List, Optional, Union
+from typing import Optional, Union
 
 import click
 import pooch
 import tqdm
 
-from disdrodb.api.path import define_metadata_filepath
-from disdrodb.configs import get_base_dir
+from disdrodb.api.path import define_metadata_filepath, define_station_dir
+from disdrodb.configs import get_data_archive_dir, get_metadata_archive_dir
 from disdrodb.metadata import get_list_metadata
 from disdrodb.utils.compression import unzip_file
 from disdrodb.utils.directories import is_empty_directory
@@ -83,7 +84,6 @@ def click_download_options(function: object):
     function : object
         Function.
     """
-
     function = click.option(
         "-f",
         "--force",
@@ -97,51 +97,56 @@ def click_download_options(function: object):
 
 
 def download_archive(
-    data_sources: Optional[Union[str, List[str]]] = None,
-    campaign_names: Optional[Union[str, List[str]]] = None,
-    station_names: Optional[Union[str, List[str]]] = None,
+    data_sources: Optional[Union[str, list[str]]] = None,
+    campaign_names: Optional[Union[str, list[str]]] = None,
+    station_names: Optional[Union[str, list[str]]] = None,
     force: bool = False,
-    base_dir: Optional[str] = None,
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
 ):
-    """Get all YAML files that contain the 'disdrodb_data_url' key
-    and download the data locally.
+    """Download DISDRODB stations with the ``disdrodb_data_url`` in the metadata.
 
     Parameters
     ----------
     data_sources : str or list of str, optional
         Data source name (eg : EPFL).
-        If not provided (None), all data sources will be downloaded.
-        The default is data_source=None.
+        If not provided (``None``), all data sources will be downloaded.
+        The default value is ``data_source=None``.
     campaign_names : str or list of str, optional
         Campaign name (eg :  EPFL_ROOF_2012).
-        If not provided (None), all campaigns will be downloaded.
-        The default is campaign_name=None.
+        If not provided (``None``), all campaigns will be downloaded.
+        The default value is ``campaign_name=None``.
     station_names : str or list of str, optional
         Station name.
-        If not provided (None), all stations will be downloaded.
-        The default is station_name=None.
+        If not provided (``None``), all stations will be downloaded.
+        The default value is ``station_name=None``.
     force : bool, optional
-        If True, overwrite the already existing raw data file.
-        The default is False.
-    base_dir : str (optional)
-        Base directory of DISDRODB. Format: <...>/DISDRODB
-        If None (the default), the disdrodb config variable 'dir' is used.
+        If ``True``, overwrite the already existing raw data file.
+        The default value is ``False``.
+    data_archive_dir : str (optional)
+        DISDRODB Data Archive directory. Format: ``<...>/DISDRODB``.
+        If ``None`` (the default), the disdrodb config variable ``data_archive_dir`` is used.
     """
-    # Retrieve the requested metadata
-    base_dir = get_base_dir(base_dir)
+    # Retrieve the DISDRODB Metadata and Data Archive Directories
+    data_archive_dir = get_data_archive_dir(data_archive_dir)
+    metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
+
+    # Select only metadata_filepaths with specified disdrodb_data_url
     metadata_filepaths = get_list_metadata(
-        base_dir=base_dir,
+        metadata_archive_dir=metadata_archive_dir,
         data_sources=data_sources,
         campaign_names=campaign_names,
         station_names=station_names,
-        with_stations_data=False,
+        product=None,  # --> Search in DISDRODB Metadata Archive
+        available_data=True,  # --> Select only metadata with disdrodb_data_url
+        raise_error_if_empty=False,  # Do not raise error if no matching metadata file found
+        invalid_fields_policy="raise",  # Raise error if invalid filtering criteria are specified
     )
 
-    # Select only metadata_filepaths with disdrodb_data_url
-    metadata_filepaths = _select_metadata_with_remote_data_url(metadata_filepaths)
+    # Return early if no data to download
     if len(metadata_filepaths) == 0:
-        print("No available remote data to download.")
-        return None
+        print("No available data to download from the online DISDRODB Decentralized Archive.")
+        return
 
     # Try to download the data
     # - It will download data only if the disdrodb_data_url is specified !
@@ -155,11 +160,12 @@ def download_archive(
                 data_source=data_source,
                 campaign_name=campaign_name,
                 station_name=station_name,
-                base_dir=base_dir,
+                data_archive_dir=data_archive_dir,
+                metadata_archive_dir=metadata_archive_dir,
                 force=force,
             )
-        except Exception as e:
-            print(f" - Download error: {e}")
+        except Exception:
+            print(" - Download error: {e}")
             print(" ")
 
 
@@ -168,7 +174,8 @@ def download_station(
     campaign_name: str,
     station_name: str,
     force: bool = False,
-    base_dir: Optional[str] = None,
+    data_archive_dir: Optional[str] = None,
+    metadata_archive_dir: Optional[str] = None,
 ) -> None:
     """
     Download data of a single DISDRODB station from the DISDRODB remote repository.
@@ -183,48 +190,35 @@ def download_station(
         The name of the campaign. Must be provided in UPPER CASE.
     station_name : str
         The name of the station.
-    base_dir : str, optional
+    data_archive_dir : str, optional
         The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``.
         If not specified, the path specified in the DISDRODB active configuration will be used.
     force: bool, optional
-        If True, overwrite the already existing raw data file.
-        The default is False.
-    base_dir : str (optional)
-        Base directory of DISDRODB. Format: <...>/DISDRODB
-        If None (the default), the disdrodb config variable 'dir' is used.
+        If ``True``, overwrite the already existing raw data file.
+        The default value is ``False``.
+    data_archive_dir : str (optional)
+        DISDRODB Data Archive directory. Format: ``<...>/DISDRODB``.
+        If ``None`` (the default), the disdrodb config variable ``data_archive_dir`` is used.
     """
     print(f"Start download of {data_source} {campaign_name} {station_name} station data")
+    # Retrieve the DISDRODB Metadata and Data Archive Directories
+    data_archive_dir = get_data_archive_dir(data_archive_dir)
+    metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
     # Define metadata_filepath
     metadata_filepath = define_metadata_filepath(
+        metadata_archive_dir=metadata_archive_dir,
         data_source=data_source,
         campaign_name=campaign_name,
         station_name=station_name,
-        base_dir=base_dir,
-        product="RAW",
         check_exists=True,
     )
     # Download data
-    _download_station_data(metadata_filepath, force=force)
+    _download_station_data(metadata_filepath, data_archive_dir=data_archive_dir, force=force)
 
 
 def _is_valid_disdrodb_data_url(disdrodb_data_url):
     """Check if it is a valid disdrodb_data_url."""
-    if isinstance(disdrodb_data_url, str) and len(disdrodb_data_url) > 10:
-        return True
-    else:
-        return False
-
-
-def _has_disdrodb_data_url(metadata_filepath):
-    """Check the metadata has a valid disdrodb_data_url."""
-    metadata_dict = read_yaml(metadata_filepath)
-    disdrodb_data_url = metadata_dict.get("disdrodb_data_url", "")
-    return _is_valid_disdrodb_data_url(disdrodb_data_url)
-
-
-def _select_metadata_with_remote_data_url(metadata_filepaths: List[str]) -> List[str]:
-    """Select metadata files that have a remote data url specified."""
-    return [fpath for fpath in metadata_filepaths if _has_disdrodb_data_url(fpath)]
+    return isinstance(disdrodb_data_url, str) and len(disdrodb_data_url) > 10
 
 
 def _extract_station_files(zip_filepath, station_dir):
@@ -234,7 +228,7 @@ def _extract_station_files(zip_filepath, station_dir):
         os.remove(zip_filepath)
 
 
-def _download_station_data(metadata_filepath: str, force: bool = False) -> None:
+def _download_station_data(metadata_filepath: str, data_archive_dir: str, force: bool = False) -> None:
     """Download and unzip the station data .
 
     Parameters
@@ -242,50 +236,44 @@ def _download_station_data(metadata_filepath: str, force: bool = False) -> None:
     metadata_filepaths : str
         Metadata file path.
     force : bool, optional
-        force download, by default False
+        If ``True``, delete existing files and redownload it. The default value is ``False``.
 
     """
-    disdrodb_data_url, station_dir = _get_station_url_and_dir_path(metadata_filepath)
+    # Open metadata file
+    metadata_dict = read_yaml(metadata_filepath)
+    # Retrieve station information
+    data_archive_dir = get_data_archive_dir(data_archive_dir)
+    data_source = metadata_dict["data_source"]
+    campaign_name = metadata_dict["campaign_name"]
+    station_name = metadata_dict["station_name"]
+    station_name = check_consistent_station_name(metadata_filepath, station_name)
+    # Define the destination local filepath path
+    station_dir = define_station_dir(
+        data_archive_dir=data_archive_dir,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        product="RAW",
+    )
+    # Check DISDRODB data url
+    disdrodb_data_url = metadata_dict.get("disdrodb_data_url", None)
+    if not _is_valid_disdrodb_data_url(disdrodb_data_url):
+        raise ValueError(f"Invalid disdrodb_data_url '{disdrodb_data_url}' for station {station_name}")
     # Download file
     zip_filepath = _download_file_from_url(disdrodb_data_url, dst_dir=station_dir, force=force)
     # Extract the stations files from the downloaded station.zip file
     _extract_station_files(zip_filepath, station_dir=station_dir)
 
 
-def _get_valid_station_name(metadata_filepath, metadata_dict):
+def check_consistent_station_name(metadata_filepath, station_name):
     """Check consistent station_name between YAML file name and metadata key."""
     # Check consistent station name
     expected_station_name = os.path.basename(metadata_filepath).replace(".yml", "")
-    station_name = metadata_dict.get("station_name")
     if station_name and str(station_name) != str(expected_station_name):
         raise ValueError(f"Inconsistent station_name values in the {metadata_filepath} file. Download aborted.")
     return station_name
 
 
-def _get_station_url_and_dir_path(metadata_filepath: str) -> tuple:
-    """Return the station's remote url and the local destination directory path.
-
-    Parameters
-    ----------
-    metadata_filepath : str
-        Path to the metadata YAML file.
-
-    Returns
-    -------
-    disdrodb_data_url, station_dir
-        Tuple containing the remote url and the DISDRODB station directory path.
-    """
-    metadata_dict = read_yaml(metadata_filepath)
-    station_name = _get_valid_station_name(metadata_filepath, metadata_dict)
-    disdrodb_data_url = metadata_dict.get("disdrodb_data_url", None)
-    if not _is_valid_disdrodb_data_url(disdrodb_data_url):
-        raise ValueError(f"Invalid disdrodb_data_url '{disdrodb_data_url}' for station {station_name}")
-    # Define the destination local filepath path
-    data_dir = os.path.dirname(metadata_filepath).replace("metadata", "data")
-    station_dir = os.path.join(data_dir, station_name)
-    return disdrodb_data_url, station_dir
-
-
 def _download_file_from_url(url: str, dst_dir: str, force: bool = False) -> str:
     """Download station zip file into the DISDRODB station data directory.
 
@@ -296,7 +284,7 @@ def _download_file_from_url(url: str, dst_dir: str, force: bool = False) -> str:
     dst_dir : str
         Local directory where to download the file (DISDRODB station data directory).
     force : bool, optional
-        Overwrite the raw data file if already existing, by default False.
+        Overwrite the raw data file if already existing. The default value is ``False``.
 
     Returns
     -------
@@ -314,12 +302,21 @@ def _download_file_from_url(url: str, dst_dir: str, force: bool = False) -> str:
             os.makedirs(dst_dir)  # station directory
         else:
             raise ValueError(
-                f"There are already raw files within {dst_dir}. Download is suspended. "
-                "Use force=True to force the download and overwrite existing raw files."
+                f"There are already raw files within the DISDRODB Data Archive at {dst_dir}. Download is suspended. "
+                "Use force=True to force the download and overwrite existing raw files.",
             )
 
     os.makedirs(dst_dir, exist_ok=True)
 
+    # Grab Pooch's logger and remember its current level
+    logger = pooch.get_logger()
+    orig_level = logger.level
+    # Silence INFO messages (including the SHA256 print)
+    logger.setLevel(logging.WARNING)
+    # Define pooch downloader
     downloader = pooch.HTTPDownloader(progressbar=True)
+    # Download the file
     pooch.retrieve(url=url, known_hash=None, path=dst_dir, fname=dst_filename, downloader=downloader, progressbar=tqdm)
+    # Restore the previous logging level
+    logger.setLevel(orig_level)
     return dst_filepath