disdrodb 0.0.21__py3-none-any.whl → 0.1.0__py3-none-any.whl

disdrodb/metadata/download.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Routine to download the DISDRODB Metadata Archive from GitHub."""
+import io
+import os
+import shutil
+import urllib.request
+import zipfile
+
+
+def download_metadata_archive(directory_path, force=False):
+    """Download the DISDRODB Metadata Archive to the specified directory.
+
+    Parameters
+    ----------
+    directory_path : str
+        The directory path where the DISDRODB-METADATA directory will be downloaded.
+    force : bool, optional
+        If ``True``, the existing DISDRODB-METADATA directory will be removed
+        and a new one will be downloaded. The default value is ``False``.
+
+    Returns
+    -------
+    metadata_archive_dir
+        The DISDRODB Metadata Archive directory path.
+    """
+    # Define DISDRODB Metadata Archive GitHub URL
+    archive_zip_url = "https://github.com/ltelab/DISDRODB-METADATA/archive/refs/heads/main.zip"
+
+    # Download archive to disk
+    resp = urllib.request.urlopen(archive_zip_url)
+    archive_data = resp.read()
+
+    # Unpack archive
+    with zipfile.ZipFile(io.BytesIO(archive_data)) as zf:
+        zf.extractall(directory_path)
+
+    # Check the archive has been download
+    extracted_dir = os.path.join(directory_path, "DISDRODB-METADATA-main")
+    if not os.path.isdir(extracted_dir):
+        raise ValueError(
+            "The DISDRODB Metadata Archive hosted on GitHub could not be downloaded!",
+        )
+
+    # Define target directory for the metadata archive
+    target_dir = os.path.join(directory_path, "DISDRODB-METADATA")
+
+    # Handle existing target directory
+    if os.path.exists(target_dir):
+        if force:
+            shutil.rmtree(target_dir)
+        else:
+            raise FileExistsError(
+                f"A DISDRODB Metadata Archive already exists at '{target_dir}'. Use force=True to update it.",
+            )
+
+    # Rename extracted directory to target
+    shutil.move(extracted_dir, target_dir)
+
+    # Define metadata archive directory
+    metadata_archive_dir = os.path.join(target_dir, "DISDRODB")
+
+    print("The DISDRODB Metadata Archive has been download successfully.")
+    print(f"The DISDRODB Metadata Archive directory path is {metadata_archive_dir}")
+    return metadata_archive_dir

disdrodb/metadata/geolocation.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""Metadata tools to verify/complete geolocation information."""
+import time
+
+import numpy as np
+import requests
+
+
+def infer_altitude(latitude, longitude, dem="aster30m"):
+    """Infer station altitude using a Digital Elevation Model (DEM).
+
+    This function uses the OpenTopoData API to infer the altitude of a given
+    location specified by latitude and longitude.
+    By default, it uses the ASTER DEM at 30m resolution.
+
+    Parameters
+    ----------
+    latitude : float
+        The latitude of the location for which to infer the altitude.
+    longitude : float
+        The longitude of the location for which to infer the altitude.
+    dem : str, optional
+        The DEM to use for altitude inference. Options are "aster30m" (default),
+        "srtm30", and "mapzen".
+
+    Returns
+    -------
+    elevation : float
+        The inferred altitude of the specified location.
+
+    Raises
+    ------
+    ValueError
+        If the altitude retrieval fails.
+
+    Notes
+    -----
+    - The OpenTopoData API has a limit of 1000 calls per day.
+    - Each request can include up to 100 locations.
+    - The API allows a maximum of 1 call per second.
+
+    References
+    ----------
+    https://www.opentopodata.org/api/
+    """
+    import requests
+
+    url = f"https://api.opentopodata.org/v1/{dem}?locations={latitude},{longitude}"
+    r = requests.get(url)
+
+    data = r.json()
+    if data["status"] == "OK":
+        elevation = data["results"][0]["elevation"]
+    else:
+        raise ValueError("Altitude retrieval failed.")
+    return elevation
+
+
+def infer_altitudes(lats, lons, dem="aster30m"):
+    """
+    Infer altitude of a given location using OpenTopoData API.
+
+    Parameters
+    ----------
+    lats : list or array-like
+        List or array of latitude coordinates.
+    lons : list or array-like
+        List or array of longitude coordinates.
+    dem : str, optional
+        Digital Elevation Model (DEM) to use for altitude inference.
+        The default DEM is "aster30m".
+
+    Returns
+    -------
+    elevations : numpy.ndarray
+        Array of inferred altitudes corresponding to the input coordinates.
+
+    Raises
+    ------
+    ValueError
+        If the latitude and longitude arrays do not have the same length.
+        If altitude retrieval fails for any block of coordinates.
+
+    Notes
+    -----
+    - The OpenTopoData API has a limit of 1000 calls per day.
+    - Each request can include up to 100 locations.
+    - The API allows a maximum of 1 call per second.
+    - The API requests are made in blocks of up to 100 coordinates,
+    with a 2-second delay between requests.
+    """
+    # Check that lats and lons have the same length
+    if len(lats) != len(lons):
+        raise ValueError("Latitude and longitude arrays must have the same length.")
+
+    # Maximum number of locations per API request
+    max_locations = 100
+    elevations = []
+
+    # Total number of coordinates
+    total_coords = len(lats)
+
+    # Loop over the coordinates in blocks of max_locations
+    for i in range(0, total_coords, max_locations):
+
+        # Wait 2 seconds before another API request
+        time.sleep(2)
+
+        # Get the block of coordinates
+        block_lats = lats[i : i + max_locations]
+        block_lons = lons[i : i + max_locations]
+
+        # Create the list_coords string in the format "lat1,lon1|lat2,lon2|..."
+        list_coords = "|".join([f"{lat},{lon}" for lat, lon in zip(block_lats, block_lons)])
+
+        # Define API URL
+        url = f"https://api.opentopodata.org/v1/{dem}?locations={list_coords}&interpolation=nearest"
+
+        # Retrieve info
+        r = requests.get(url)
+        data = r.json()
+
+        # Parse info
+        if data.get("status") == "OK":
+            elevations.extend([result["elevation"] for result in data["results"]])
+        else:
+            raise ValueError(f"Altitude retrieval failed for block starting at index {i}.")
+    elevations = np.array(elevations).astype(float)
+    return elevations

disdrodb/metadata/info.py

@@ -18,41 +18,49 @@
 # -----------------------------------------------------------------------------.
 """Test Metadata Info Extraction."""
 import os
+from typing import Optional
 
 from disdrodb.api.info import (
     infer_campaign_name_from_path,
     infer_data_source_from_path,
 )
-from disdrodb.configs import get_base_dir
+from disdrodb.configs import get_metadata_archive_dir
 from disdrodb.metadata.reader import read_station_metadata
 from disdrodb.metadata.search import get_list_metadata
 
 
-def get_archive_metadata_key_value(key: str, return_tuple: bool = True, base_dir: str = None):
+def get_archive_metadata_key_value(key: str, return_tuple: bool = True, metadata_archive_dir: Optional[str] = None):
     """Return the values of a metadata key for all the archive.
 
     Parameters
     ----------
-    base_dir : str
+    data_archive_dir : str
         Path to the disdrodb directory.
     key : str
         Metadata key.
     return_tuple : bool, optional
-       If True, returns a tuple of values with station, campaign and data source name.
-       If False, returns a list of values without station, campaign and data source name.
-       The default is True.
-    base_dir : str (optional)
-       Base directory of DISDRODB. Format: <...>/DISDRODB
-       If None (the default), the disdrodb config variable 'dir' is used.
+       If ``True``, returns a tuple (``data_source``,``campaign_name``,``station_name``, ``key_value``)
+       If ``False``, returns a list of the key values.
+       The default value is ``True``.
+    metadata_archive_dir : str (optional)
+        The directory path where the DISDRODB Metadata Archive is located.
+        The directory path must end with ``<...>/DISDRODB``.
+        If ``None``, it uses the ``metadata_archive_dir`` path specified
+        in the DISDRODB active configuration.
 
     Returns
     -------
     list or tuple
         List or tuple of values of the metadata key.
     """
-    base_dir = get_base_dir(base_dir)
+    metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
     list_metadata_paths = get_list_metadata(
-        base_dir=base_dir, data_sources=None, campaign_names=None, station_names=None, with_stations_data=False
+        metadata_archive_dir=metadata_archive_dir,
+        data_sources=None,
+        campaign_names=None,
+        station_names=None,
+        product=None,  # --> Search in DISDRODB Metadata Archive
+        available_data=False,  # --> Select all metadata matching the filtering criteria
     )
     list_info = []
     for filepath in list_metadata_paths:
@@ -60,8 +68,7 @@ def get_archive_metadata_key_value(key: str, return_tuple: bool = True, base_dir
         campaign_name = infer_campaign_name_from_path(filepath)
         station_name = os.path.basename(filepath).replace(".yml", "")
         metadata = read_station_metadata(
-            base_dir=base_dir,
-            product="RAW",
+            metadata_archive_dir=metadata_archive_dir,
             data_source=data_source,
             campaign_name=campaign_name,
             station_name=station_name,

disdrodb/metadata/manipulation.py

@@ -40,7 +40,7 @@ def add_missing_metadata_keys(metadata):
 
 
 def sort_metadata_dictionary(metadata):
-    """Sort the keys of the metadata dictionary by valid_metadata_keys list order."""
+    """Sort the keys of the metadata dictionary by ``valid_metadata_keys`` list order."""
     from disdrodb.metadata.standards import get_valid_metadata_keys
 
     list_metadata_keys = get_valid_metadata_keys()

disdrodb/metadata/reader.py

@@ -18,12 +18,13 @@
 # -----------------------------------------------------------------------------.
 """Routines to read the DISDRODB Metadata."""
 
+import pandas as pd
 
 from disdrodb.api.path import define_metadata_filepath
 from disdrodb.utils.yaml import read_yaml
 
 
-def read_station_metadata(data_source, campaign_name, station_name, base_dir=None, product="RAW"):
+def read_station_metadata(data_source, campaign_name, station_name, metadata_archive_dir=None):
     """Open the station metadata YAML file into a dictionary.
 
     Parameters
@@ -36,12 +37,10 @@ def read_station_metadata(data_source, campaign_name, station_name, base_dir=Non
         The name of the campaign. Must be provided in UPPER CASE.
     station_name : str
         The name of the station.
-    base_dir : str, optional
-        The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``.
+    metadata_archive_dir : str, optional
+        The directory path where the DISDRODB Metadata Archive is located.
         If not specified, the path specified in the DISDRODB active configuration will be used.
-    product : str, optional
-        The DISDRODB product in which to search for the metadata file.
-        The default is "RAW".
+        Expected path format: ``<...>/DISDRODB``.
 
     Returns
     -------
@@ -51,12 +50,64 @@ def read_station_metadata(data_source, campaign_name, station_name, base_dir=Non
     """
     # Retrieve metadata filepath
     metadata_filepath = define_metadata_filepath(
-        base_dir=base_dir,
+        metadata_archive_dir=metadata_archive_dir,
         data_source=data_source,
         campaign_name=campaign_name,
         station_name=station_name,
-        product=product,
         check_exists=True,
     )
+    # Open the metadata file
     metadata_dict = read_yaml(metadata_filepath)
     return metadata_dict
+
+
+def read_metadata_archive(
+    metadata_archive_dir=None,
+    data_sources=None,
+    campaign_names=None,
+    station_names=None,
+    available_data=False,
+):
+    """Read the DISDRODB Metadata Archive Database.
+
+    Parameters
+    ----------
+    metadata_archive_dir : str or Path-like, optional
+        Path to the root of the DISDRODB Metadata Archive. If None, the
+        default metadata base directory is used. Default is None.
+    data_sources : str or sequence of str, optional
+        One or more data source identifiers to filter stations by. If None,
+        no filtering on data source is applied. The default is is None.
+    campaign_names : str or sequence of str, optional
+        One or more campaign names to filter stations by. If None, no filtering
+        on campaign is applied. The default is is None.
+    station_names : str or sequence of str, optional
+        One or more station names to include. If None, all stations matching
+        other filters are considered. The default is is None.
+    available_data: bool, optional
+        If True, only information of stations with data available in the online
+        DISDRODB Decentralized Data Archive are returned.
+        If False (the default), all stations present in the DISDRODB Metadata Archive
+        matching the filtering criteria are returned,
+
+    Returns
+    -------
+    pandas.DataFrame
+
+    """
+    from disdrodb.configs import get_metadata_archive_dir
+    from disdrodb.metadata.search import get_list_metadata
+
+    metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir=metadata_archive_dir)
+
+    list_metadata_paths = get_list_metadata(
+        metadata_archive_dir=metadata_archive_dir,
+        data_sources=data_sources,
+        campaign_names=campaign_names,
+        station_names=station_names,
+        product=None,  # --> Search in DISDRODB Metadata Archive
+        available_data=available_data,
+    )
+    list_metadata = [read_yaml(fpath) for fpath in list_metadata_paths]
+    df = pd.DataFrame(list_metadata)
+    return df

disdrodb/metadata/search.py

@@ -18,150 +18,94 @@
 # -----------------------------------------------------------------------------.
 """Routines to manipulate the DISDRODB Metadata Archive."""
 
-import glob
-import os
 
 from disdrodb.api.path import define_metadata_filepath
-from disdrodb.configs import get_base_dir
+from disdrodb.api.search import available_stations
 
 
 def get_list_metadata(
     data_sources=None,
     campaign_names=None,
     station_names=None,
-    with_stations_data=True,
-    base_dir=None,
+    product=None,
+    available_data=False,
+    raise_error_if_empty=False,
+    invalid_fields_policy="raise",
+    data_archive_dir=None,
+    metadata_archive_dir=None,
+    **product_kwargs,
 ):
     """
-    Get the list of metadata filepaths in the DISDRODB raw archive.
+    Get station metadata filepaths.
+
+    By default, it returns the metadata file paths of stations present in the DISDRODB Metadata Archive matching
+    the filtering criteria.
+
+    If the DISDRODB product is specified, it lists only metadata file paths of stations with the specified product
+    present in the local DISDRODB Data Archive.
 
     Parameters
     ----------
-    data_sources : str or list of str
-        Name of data source(s) of interest.
-        The name(s) must be UPPER CASE.
-        The default is None
-    campaign_names : str or list of str
-        Name of the campaign(s) of interest.
-        The name(s) must be UPPER CASE.
-        The default is None
-    station_names : str or list of str
-        Station names of interest.
-        The default is None
-    with_stations_data : bool
-        If True, only return metadata filepaths that have corresponding data in the local DISDRODB raw archive.
-        The default is True
-    base_dir : str (optional)
-        Base directory of DISDRODB. Format: <...>/DISDRODB
-        If None (the default), the disdrodb config variable 'dir' is used.
+    product : str or None, optional
+        Name of the product to filter on (e.g., "RAW", "L0A", "L1").
 
-    Returns
-    -------
-    metadata_filepaths: list
-        List of metadata YAML file paths
+        If the DISDRODB product is not specified (default),
+        it returns the metadata file paths of stations present in the DISDRODB Metadata Archive matching
+        the filtering criteria.
 
-    """
-    base_dir = get_base_dir(base_dir)
-    if with_stations_data:
-        list_metadata = _get_list_metadata_with_data(
-            base_dir=base_dir,
-            data_sources=data_sources,
-            campaign_names=campaign_names,
-            station_names=station_names,
-        )
-    else:
-        list_metadata = _get_list_all_metadata(
-            base_dir=base_dir,
-            data_sources=data_sources,
-            campaign_names=campaign_names,
-            station_names=station_names,
-        )
-    return list_metadata
+        If the DISDRODB product is specified, it lists only metadata file paths of stations with the specified product
+        present in the local DISDRODB Data Archive.
 
+    available_data : bool, optional
 
-def _get_list_all_metadata(base_dir, data_sources=None, campaign_names=None, station_names=None):
-    """
-    Get the list of metadata filepaths in the DISDRODB raw archive.
+        If ``product`` is not specified:
 
-    Parameters
-    ----------
-    base_dir : str
-        Base directory of DISDRODB
-        Format: <...>/DISDRODB
-    data_sources : str or list of str
-        Name of data source(s) of interest.
-        The name(s) must be UPPER CASE.
-        The default is None
-    campaign_names : str or list of str
-        Name of the campaign(s) of interest.
-        The name(s) must be UPPER CASE.
-        The default is None
-    station_names : str or list of str
-        Station names of interest.
-        The default is None
-    with_stations_data : bool
-        If True, only return metadata filepaths that have corresponding data in the local DISDRODB raw archive.
-        The default is True
+            - if available_data is False, return metadata filepaths of stations present in the DISDRODB Metadata Archive
+            - if available_data is True, return metadata filepaths of stations with data available on the
+            online DISDRODB Decentralized Data Archive (i.e., stations with the disdrodb_data_url in the metadata).
 
-    Returns
-    -------
-    metadata_filepaths: list
-        List of metadata YAML file paths
+        If ``product`` is specified:
 
-    """
-    # Get all config files from the metadata directories
-    list_of_base_path = []
-    if data_sources:
-        if isinstance(data_sources, str):
-            data_sources = [data_sources]
-    else:
-        data_sources = ["**"]
-    if campaign_names:
-        if isinstance(campaign_names, str):
-            campaign_names = [campaign_names]
-    else:
-        campaign_names = ["**"]
-
-    for data_source in data_sources:
-        for campaign_name in campaign_names:
-            base_path = os.path.join(base_dir, "Raw", data_source, campaign_name)
-            list_of_base_path.append(base_path)
-
-    metadata_filepaths = []
-    for base_path in list_of_base_path:
-        if station_names:
-            if isinstance(station_names, str):
-                station_names = [station_names]
-            for station_name in station_names:
-                metadata_path = os.path.join(base_path, "**", "metadata", f"{station_name}.yml")
-                metadata_filepaths += glob.glob(metadata_path, recursive=True)
-        else:
-            metadata_path = os.path.join(base_path, "**", "metadata", "*.yml")
-            metadata_filepaths += glob.glob(metadata_path, recursive=True)
-
-    return list(set(metadata_filepaths))
-
-
-def _get_list_metadata_with_data(base_dir, data_sources=None, campaign_names=None, station_names=None):
-    """
-    Get the list of metadata filepaths that have corresponding data in the DISDRODB raw archive.
+            - if available_data is False, return metadata filepaths of stations where
+            the product directory exists in the in the local DISDRODB Data Archive
+            - if available_data is True, return metadata filepaths of stations where product data exists in the
+              in the local DISDRODB Data Archive.
+        The default is is False.
 
-    Parameters
-    ----------
-    base_dir : str
-        Base directory of DISDRODB
-        Format: <...>/DISDRODB
-    data_sources : str or list of str
-        Name of data source(s) of interest.
+    data_sources : str or sequence of str, optional
+        One or more data source identifiers to filter stations by.
         The name(s) must be UPPER CASE.
-        The default is None
-    campaign_names : str or list of str
-        Name of the campaign(s) of interest.
+        If None, no filtering on data source is applied. The default is is ``None``.
+    campaign_names : str or sequence of str, optional
+        One or more campaign names to filter stations by.
         The name(s) must be UPPER CASE.
-        The default is None
-    station_names : str or list of str
-        Station names of interest.
-        The default is None
+        If None, no filtering on campaign is applied. The default is is ``None``.
+    station_names : str or sequence of str, optional
+        One or more station names to include.
+        If None, all stations matching other filters are considered. The default is is ``None``.
+    raise_error_if_empty : bool, optional
+        If True and no stations satisfy the criteria, raise a ``ValueError``.
+        If False, return an empty list/tuple. The default is False.
+    invalid_fields_policy : {'raise', 'warn', 'ignore'}, optional
+        How to handle invalid filter values for ``data_sources``, ``campaign_names``,
+        or ``station_names`` that are not present in the metadata archive:
+
+          - 'raise' : raise a ``ValueError`` (default)
+          - 'warn'  : emit a warning, then ignore invalid entries
+          - 'ignore': silently drop invalid entries
+    data_archive_dir : str or Path-like, optional
+        Path to the root of the local DISDRODB Data Archive. Format: ``<...>/DISDRODB``
+        Required only if ``product``is specified.
+        If None, the``data_archive_dir`` path specified in the DISDRODB active configuration file is used.
+        The default is None.
+    metadata_archive_dir : str or Path-like, optional
+        Path to the root of the DISDRODB Metadata Archive. Format: ``<...>/DISDRODB``
+        If None, the``metadata_archive_dir`` path specified in the DISDRODB active configuratio. The default is None.
+    **product_kwargs : dict, optional
+        Additional arguments required for some products.
+        For example, for the "L2E" product, you need to specify ``rolling`` and
+        ``sample_interval``. For the "L2M" product, you need to specify also
+        the ``model_name``.
 
     Returns
     -------
@@ -169,39 +113,28 @@ def _get_list_metadata_with_data(base_dir, data_sources=None, campaign_names=Non
         List of metadata YAML file paths
 
     """
-    from disdrodb.api.io import available_stations
-
-    # Check inputs
-    if isinstance(data_sources, str):
-        data_sources = [data_sources]
-    if isinstance(campaign_names, str):
-        campaign_names = [campaign_names]
-
-    # Retrieve information of requested stations
-    # --> (data_source, campaign_name, station_name)
-
     list_info = available_stations(
-        base_dir=base_dir,
-        product="RAW",
         data_sources=data_sources,
         campaign_names=campaign_names,
+        station_names=station_names,
+        product=product,
+        available_data=available_data,
+        raise_error_if_empty=raise_error_if_empty,
+        invalid_fields_policy=invalid_fields_policy,
+        metadata_archive_dir=metadata_archive_dir,
+        data_archive_dir=data_archive_dir,
+        return_tuple=True,
+        **product_kwargs,
     )
 
-    # If no stations available, raise an error
-    if len(list_info) == 0:
-        raise ValueError("No stations are available !")
-
-    # Get metadata filepaths
+    # Define metadata filepath
     metadata_filepaths = [
         define_metadata_filepath(
-            product="Raw",
+            metadata_archive_dir=metadata_archive_dir,
             data_source=data_source,
             campaign_name=campaign_name,
             station_name=station_name,
-            base_dir=base_dir,
-            check_exists=False,
         )
         for data_source, campaign_name, station_name in list_info
     ]
-
-    return metadata_filepaths
+    return sorted(set(metadata_filepaths))