disdrodb 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl

disdrodb/api/io.py

@@ -16,311 +16,347 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
-"""Routines tot extract information from the DISDRODB infrastructure."""
-
+"""Routines to list and open DISDRODB products."""
 import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional
+
+from disdrodb.api.path import (
+    define_campaign_dir,
+    define_data_dir,
+    define_metadata_dir,
+    define_station_dir,
+)
+from disdrodb.l0.l0_reader import define_readers_directory
+from disdrodb.utils.directories import list_files
+from disdrodb.utils.logger import (
+    log_info,
+)
+
+####----------------------------------------------------------------------------------
+#### DISDRODB Search Product Files
+
+
+def filter_filepaths(filepaths, debugging_mode):
+    """Filter out filepaths if ``debugging_mode=True``."""
+    if debugging_mode:
+        max_files = min(3, len(filepaths))
+        filepaths = filepaths[0:max_files]
+    return filepaths
+
+
+def find_files(
+    data_source,
+    campaign_name,
+    station_name,
+    product,
+    debugging_mode: bool = False,
+    data_archive_dir: Optional[str] = None,
+    glob_pattern="*",
+    **product_kwargs,
+):
+    """Retrieve DISDRODB product files for a give station.
+
+    Parameters
+    ----------
+    data_source : str
+        The name of the institution (for campaigns spanning multiple countries) or
+        the name of the country (for campaigns or sensor networks within a single country).
+        Must be provided in UPPER CASE.
+    campaign_name : str
+        The name of the campaign. Must be provided in UPPER CASE.
+    station_name : str
+        The name of the station.
+    product : str
+        The name DISDRODB product.
+    debugging_mode : bool, optional
+        If ``True``, it select maximum 3 files for debugging purposes.
+        The default value is ``False``.
+    data_archive_dir : str, optional
+        The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``.
+        If not specified, the path specified in the DISDRODB active configuration will be used.
+    glob_pattern: str, optional
+        Glob pattern to search for raw data files. The default is "*".
+        The argument is used only if product="RAW".
+
+    Other Parameters
+    ----------------
+    sample_interval : int, optional
+        The sampling interval in seconds of the product.
+        It must be specified only for product L2E and L2M !
+    rolling : bool, optional
+        Whether the dataset has been resampled by aggregating or rolling.
+        It must be specified only for product L2E and L2M !
+    model_name : str
+        The model name of the statistical distribution for the DSD.
+        It must be specified only for product L2M !
+
+    Returns
+    -------
+    filepaths : list
+        List of file paths.
 
-import numpy as np
-
-from disdrodb.api.checks import check_product
-from disdrodb.api.path import get_disdrodb_path
-from disdrodb.configs import get_base_dir
-from disdrodb.utils.directories import count_files, list_directories, list_files
-
-
-def _get_list_stations_dirs(product, campaign_dir):
-    # Get directory where data are stored
-    # - Raw: <campaign>/data/<...>
-    # - Processed: <campaign>/L0A/L0B>
-    if product.upper() == "RAW":
-        product_dir = os.path.join(campaign_dir, "data")
-    else:
-        product_dir = os.path.join(campaign_dir, product)
-    # Check if the data directory exists
-    # - For a fresh disdrodb-data cloned repo, no "data" directories
-    if not os.path.exists(product_dir):
-        return []
-    # Get list of directories (stations)
-    stations_names = os.listdir(product_dir)
-    list_stations_dir = [os.path.join(product_dir, station_name) for station_name in stations_names]
-    return list_stations_dir
-
-
-def _get_list_stations_with_data(product, campaign_dir):
-    """Get the list of stations with data inside."""
-    # Get stations directory
-    list_stations_dir = _get_list_stations_dirs(product=product, campaign_dir=campaign_dir)
-    # Count number of files within directory
-    list_nfiles_per_station = [count_files(station_dir, "*", recursive=True) for station_dir in list_stations_dir]
-    # Keep only stations with at least one file
-    stations_names = [os.path.basename(path) for n, path in zip(list_nfiles_per_station, list_stations_dir) if n >= 1]
-    return stations_names
-
-
-def _get_list_stations_with_metadata(campaign_dir):
-    # Get directory where metadata are stored
-    metadata_path = os.path.join(campaign_dir, "metadata")
-    # List metadata files
-    metadata_filepaths = list_files(metadata_path, glob_pattern="*.yml", recursive=False)
-    # Return stations with metadata
-    stations_names = [os.path.basename(filepath).replace(".yml", "") for filepath in metadata_filepaths]
-    return stations_names
-
-
-def _get_campaign_stations(base_dir, product, data_source, campaign_name):
-    # Get campaign directory
-    campaign_dir = get_disdrodb_path(
-        base_dir=base_dir,
-        product=product,
+    """
+    # Retrieve data directory
+    data_dir = define_data_dir(
+        data_archive_dir=data_archive_dir,
         data_source=data_source,
         campaign_name=campaign_name,
+        station_name=station_name,
+        product=product,
+        # Product options
+        **product_kwargs,
     )
 
-    # Get list of stations with data and metadata
-    list_stations_data = _get_list_stations_with_data(product=product, campaign_dir=campaign_dir)
-    list_stations_metadata = _get_list_stations_with_metadata(campaign_dir)
-    # Get list of stations with both data and metadata
-    stations_names = list(set(list_stations_data).intersection(list_stations_metadata))
+    # Define or check the specified glob pattern
+    if product != "RAW":
+        glob_pattern = "*.parquet" if product == "L0A" else "*.nc"
 
-    # Return all available stations for a give campaign
-    return stations_names
+    # Retrieve files
+    filepaths = list_files(data_dir, glob_pattern=glob_pattern, recursive=True)
 
+    # Filter out filepaths if debugging_mode=True
+    filepaths = filter_filepaths(filepaths, debugging_mode=debugging_mode)
 
-def _get_campaigns_stations(base_dir, product, data_source, campaign_names):
-    if isinstance(campaign_names, str):
-        campaign_names = [campaign_names]
-    list_available_stations = []
-    for campaign_name in campaign_names:
-        # Get list of available stations
-        stations_names = _get_campaign_stations(
-            base_dir=base_dir,
-            product=product,
-            data_source=data_source,
-            campaign_name=campaign_name,
-        )
-        for station_name in stations_names:
-            list_available_stations.append((data_source, campaign_name, station_name))
+    # If no file available, raise error
+    if len(filepaths) == 0:
+        msg = f"No {product} files are available in {data_dir}. Run {product} processing first."
+        raise ValueError(msg)
 
-    # Return all available stations for the asked campaigns (and specific data source)
-    return list_available_stations
+    # Sort filepaths
+    filepaths = sorted(filepaths)
+    return filepaths
 
 
-def _get_data_source_stations(base_dir, product, data_source):
-    """Return list of available stations for a specific data source.
+####----------------------------------------------------------------------------------
+#### DISDRODB Open Product Files
+def open_dataset(
+    data_source,
+    campaign_name,
+    station_name,
+    product,
+    product_kwargs=None,
+    debugging_mode: bool = False,
+    data_archive_dir: Optional[str] = None,
+    parallel=False,
+    **open_kwargs,
+):
+    """Retrieve DISDRODB product files for a give station.
+
+    Parameters
+    ----------
+    data_source : str
+        The name of the institution (for campaigns spanning multiple countries) or
+        the name of the country (for campaigns or sensor networks within a single country).
+        Must be provided in UPPER CASE.
+    campaign_name : str
+        The name of the campaign. Must be provided in UPPER CASE.
+    station_name : str
+        The name of the station.
+    product : str
+        The name DISDRODB product.
+    sample_interval : int, optional
+        The sampling interval in seconds of the product.
+        It must be specified only for product L2E and L2M !
+    rolling : bool, optional
+        Whether the dataset has been resampled by aggregating or rolling.
+        It must be specified only for product L2E and L2M !
+    model_name : str
+        The model name of the statistical distribution for the DSD.
+        It must be specified only for product L2M !
+    debugging_mode : bool, optional
+        If ``True``, it select maximum 3 files for debugging purposes.
+        The default value is ``False``.
+    data_archive_dir : str, optional
+        The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``.
+        If not specified, the path specified in the DISDRODB active configuration will be used.
+
+    Returns
+    -------
+    xarray.Dataset
 
-    Returns a tuple (<DATA_SOURCE>, <CAMPAIGN_NAME>, <station_name>)
     """
-    # Get data source directory
-    data_source_dir = get_disdrodb_path(
-        base_dir=base_dir,
-        product=product,
-        data_source=data_source,
-        campaign_name="",
-    )
-    # Get possible campaign list
-    campaign_names = os.listdir(data_source_dir)
+    import xarray as xr
 
-    # For each campaign, retrieve available stations
-    list_available_stations = _get_campaigns_stations(
-        base_dir=base_dir,
-        product=product,
+    from disdrodb.l0.l0a_processing import read_l0a_dataframe
+
+    # Check product validity
+    if product == "RAW":
+        raise ValueError("It's not possible to open the raw data with this function.")
+    product_kwargs = product_kwargs if product_kwargs else {}
+
+    # List product files
+    filepaths = find_files(
+        data_archive_dir=data_archive_dir,
         data_source=data_source,
-        campaign_names=campaign_names,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        product=product,
+        debugging_mode=debugging_mode,
+        **product_kwargs,
     )
 
-    # Return all available stations for a specific data source
-    return list_available_stations
-
+    # Open L0A Parquet files
+    if product == "L0A":
+        return read_l0a_dataframe(filepaths)
+
+    # Open DISDRODB netCDF files using xarray
+    # - TODO: parallel option and add closers !
+    # - decode_timedelta -- > sample_interval not decoded to timedelta !
+    # list_ds = [xr.open_dataset(fpath, decode_timedelta=False, **open_kwargs) for fpath in filepaths]
+    # ds = xr.concat(list_ds, dim="time")
+    ds = xr.open_mfdataset(
+        filepaths,
+        engine="netcdf4",
+        combine="nested",  # 'by_coords',
+        concat_dim="time",
+        decode_timedelta=False,
+        parallel=parallel,
+        **open_kwargs,
+    )
+    return ds
 
-def _get_data_sources_stations(base_dir, product, data_sources):
-    if isinstance(data_sources, str):
-        data_sources = [data_sources]
-    list_available_stations = []
-    for data_source in data_sources:
-        list_available = _get_data_source_stations(
-            base_dir=base_dir,
-            product=product,
-            data_source=data_source,
-        )
-        for station_name in list_available:
-            list_available_stations.append(station_name)
 
-    # Return all available stations
-    return list_available_stations
+####----------------------------------------------------------------------------------
+#### DISDRODB Remove Product Files
 
 
-def _get_stations(base_dir, product):
-    # Get raw or processed directory
-    level_dir = get_disdrodb_path(
-        base_dir=base_dir,
+def remove_product(
+    data_archive_dir,
+    product,
+    data_source,
+    campaign_name,
+    station_name,
+    logger=None,
+    verbose=True,
+    **product_kwargs,
+):
+    """Remove all product files of a specific station."""
+    if product.upper() == "RAW":
+        raise ValueError("Removal of 'RAW' files is not allowed.")
+    data_dir = define_data_dir(
+        data_archive_dir=data_archive_dir,
         product=product,
-        data_source="",
-        campaign_name="",
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        **product_kwargs,
     )
-    # Get possible data sources
-    data_sources = os.listdir(level_dir)
+    if logger is not None:
+        log_info(logger=logger, msg="Removal of {product} files started.", verbose=verbose)
+    shutil.rmtree(data_dir)
+    if logger is not None:
+        log_info(logger=logger, msg="Removal of {product} files ended.", verbose=verbose)
 
-    # For each data_source, retrieve available stations
-    list_available_stations = _get_data_sources_stations(
-        base_dir=base_dir,
-        product=product,
-        data_sources=data_sources,
-    )
-    # Return all available stations
-    return list_available_stations
 
+####--------------------------------------------------------------------------.
+#### Open directories
 
-####---------------------------------------------------------------------------.
-#### I/O CHECKS
 
+def open_file_explorer(path):
+    """Open the native file-browser showing 'path'."""
+    p = Path(path).resolve()
+    if not p.exists():
+        raise FileNotFoundError(f"{p} does not exist")
 
-def _check_data_sources(base_dir, product, data_sources):
-    """Check DISDRODB data source.
+    if sys.platform.startswith("win"):
+        # Windows
+        os.startfile(str(p))
+    elif sys.platform == "darwin":
+        # macOS
+        subprocess.run(["open", str(p)], check=False)
+    else:
+        # Linux (most desktop environments)
+        subprocess.run(["xdg-open", str(p)], check=False)
 
-    It checks only if the directory exist.
-    """
-    # If data_sources is None, return None
-    if isinstance(data_sources, type(None)):
-        return data_sources
-    # Ensure is a list
-    if isinstance(data_sources, str):
-        data_sources = [data_sources]
-    # Remove duplicates
-    data_sources = np.unique(np.array(data_sources))
-    # Get directory
-    dir_path = get_disdrodb_path(base_dir=base_dir, product=product)
-    # Get data sources directory
-    list_dir = os.listdir(dir_path)
-    # Check if there are invalid data_sources
-    idx_invalid = np.where(np.isin(data_sources, list_dir, invert=True))[0]
-    if len(idx_invalid) > 0:
-        invalid_data_sources = data_sources[idx_invalid].tolist()
-        raise ValueError(f"These data sources are invalid: {invalid_data_sources}.")
-    # Return data_sources list
-    data_sources = data_sources.tolist()
-    return data_sources
-
-
-def _check_campaign_names(base_dir, product, campaign_names):
-    """Check DISDRODB campaign_names are valid.
-
-    It checks only if the directory exist within the product.
-    """
-    # If campaign_names is None, return None
-    if isinstance(campaign_names, type(None)):
-        return campaign_names
-    # Ensure is a list
-    if isinstance(campaign_names, str):
-        campaign_names = [campaign_names]
-    # Remove duplicates
-    campaign_names = np.unique(np.array(campaign_names))
-    # Get product directory path
-    dir_path = get_disdrodb_path(base_dir=base_dir, product=product)
-    # Get campaigns directory path
-    list_campaigns_path = list_directories(dir_path, glob_pattern=os.path.join("*", "*"), recursive=False)
-    # Get campaigns names
-    list_campaign_names = [os.path.basename(path) for path in list_campaigns_path]
-    # Remove duplicates
-    list_campaign_names = np.unique(list_campaign_names)
-    # Check if there are invalid campaign_names
-    idx_invalid = np.where(np.isin(campaign_names, list_campaign_names, invert=True))[0]
-    if len(idx_invalid) > 0:
-        invalid_campaign_names = campaign_names[idx_invalid].tolist()
-        raise ValueError(f"These campaign names are invalid: {invalid_campaign_names}.")
-    # Return campaign_names list
-    campaign_names = campaign_names.tolist()
-    return campaign_names
-
-
-####---------------------------------------------------------------------------.
-#### DISDRODB I/O INTERFACE
-
-
-def available_data_sources(product, base_dir=None):
-    """Return data sources for which stations data are available."""
-    base_dir = get_base_dir(base_dir)
-    product = check_product(product)
-    # Get available stations
-    list_available_stations = _get_stations(base_dir=base_dir, product=product)
-    data_sources = [info[0] for info in list_available_stations]
-    data_sources = np.unique(data_sources).tolist()
-    return data_sources
-
-
-def available_campaigns(product, data_sources=None, return_tuple=True, base_dir=None):
-    """Return campaigns for which stations data are available."""
-    # Checks
-    base_dir = get_base_dir(base_dir)
-    product = check_product(product)
-    data_sources = _check_data_sources(
-        base_dir=base_dir,
-        product=product,
-        data_sources=data_sources,
+
+def open_logs_directory(
+    data_source,
+    campaign_name,
+    station_name=None,  # noqa
+    data_archive_dir=None,
+):
+    """Open the DISDRODB Data Archive logs directory of a station."""
+    from disdrodb.configs import get_data_archive_dir
+
+    data_archive_dir = get_data_archive_dir(data_archive_dir)
+    campaign_dir = define_campaign_dir(
+        archive_dir=data_archive_dir,
+        product="L0A",
+        data_source=data_source,
+        campaign_name=campaign_name,
+        check_exists=True,
     )
-    # Get available stations
-    if data_sources is None:
-        list_available_stations = _get_stations(base_dir=base_dir, product=product)
-    else:
-        list_available_stations = _get_data_sources_stations(
-            base_dir=base_dir,
-            product=product,
-            data_sources=data_sources,
-        )
-    if not return_tuple:
-        campaigns = [info[1] for info in list_available_stations]
-        campaigns = np.unique(campaigns).tolist()
-        return campaigns
-    else:
-        data_source_campaigns = [(info[0], info[1]) for info in list_available_stations]
-        data_source_campaigns = list(set(data_source_campaigns))
-        return data_source_campaigns
+    logs_dir = os.path.join(campaign_dir, "logs")
+    open_file_explorer(logs_dir)
 
 
-def available_stations(
+def open_product_directory(
     product,
-    data_sources=None,
-    campaign_names=None,
-    return_tuple=True,
-    base_dir=None,
+    data_source,
+    campaign_name,
+    station_name,
+    data_archive_dir=None,
 ):
-    """Return stations for which data are available."""
-    base_dir = get_base_dir(base_dir)
-    # Checks
-    product = check_product(product)
-    data_sources = _check_data_sources(
-        base_dir=base_dir,
+    """Open the DISDRODB Data Archive station product directory."""
+    from disdrodb.configs import get_data_archive_dir
+
+    data_archive_dir = get_data_archive_dir(data_archive_dir)
+    station_dir = define_station_dir(
+        data_archive_dir=data_archive_dir,
         product=product,
-        data_sources=data_sources,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+        check_exists=True,
     )
-    campaign_names = _check_campaign_names(
-        base_dir=base_dir,
-        product=product,
-        campaign_names=campaign_names,
+    open_file_explorer(station_dir)
+
+
+def open_metadata_directory(
+    data_source,
+    campaign_name,
+    station_name=None,  # noqa
+    metadata_archive_dir=None,
+):
+    """Open the DISDRODB Metadata Archive station(s) metadata directory."""
+    from disdrodb.configs import get_metadata_archive_dir
+
+    metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
+    metadata_dir = define_metadata_dir(
+        metadata_archive_dir=metadata_archive_dir,
+        data_source=data_source,
+        campaign_name=campaign_name,
+        check_exists=True,
     )
-    # Format arguments to list
-    if isinstance(data_sources, str):
-        data_sources = [data_sources]
-    if isinstance(campaign_names, str):
-        campaign_names = [campaign_names]
-
-    # If data_source is None, first retrieve all stations
-    if data_sources is None:
-        list_info = _get_stations(base_dir=base_dir, product=product)
-    # Otherwise retrieve all stations for the specified data sources
-    else:
-        list_info = _get_data_sources_stations(
-            base_dir=base_dir,
-            data_sources=data_sources,
-            product=product,
-        )
-
-    # Then, if campaign_name is not None, subset by campaign_name
-    if campaign_names is not None:
-        list_info = [info for info in list_info if info[1] in campaign_names]
-
-    if return_tuple:
-        return list_info
-    else:
-        # TODO: ENSURE THAT NO DUPLICATED STATION NAMES ?
-        list_stations = [info[2] for info in list_info]
-        return list_stations
+    open_file_explorer(metadata_dir)
+
+
+def open_readers_directory():
+    """Open the disdrodb software readers directory."""
+    readers_directory = define_readers_directory()
+
+    open_file_explorer(readers_directory)
+
+
+def open_metadata_archive(
+    metadata_archive_dir=None,
+):
+    """Open the DISDRODB Metadata Archive."""
+    from disdrodb.configs import get_metadata_archive_dir
+
+    metadata_archive_dir = get_metadata_archive_dir(metadata_archive_dir)
+    open_file_explorer(metadata_archive_dir)
+
+
+def open_data_archive(
+    data_archive_dir=None,
+):
+    """Open the DISDRODB Data Archive."""
+    from disdrodb.configs import get_data_archive_dir
+
+    data_archive_dir = get_data_archive_dir(data_archive_dir)
+    open_file_explorer(data_archive_dir)