disdrodb 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

disdrodb/api/io.py

@@ -17,6 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """Routines to list and open DISDRODB products."""
+import datetime
 import os
 import shutil
 import subprocess
@@ -24,6 +25,14 @@ import sys
 from pathlib import Path
 from typing import Optional
 
+import numpy as np
+
+from disdrodb.api.checks import (
+    check_filepaths,
+    check_start_end_time,
+    get_current_utc_time,
+)
+from disdrodb.api.info import get_start_end_time_from_filepaths
 from disdrodb.api.path import (
     define_campaign_dir,
     define_data_dir,
@@ -48,6 +57,75 @@ def filter_filepaths(filepaths, debugging_mode):
     return filepaths
 
 
+def is_within_time_period(l_start_time, l_end_time, start_time, end_time):
+    """Assess which files are within the start and end time."""
+    # - Case 1
+    #     s               e
+    #     |               |
+    #   ---------> (-------->)
+    idx_select1 = np.logical_and(l_start_time <= start_time, l_end_time > start_time)
+    # - Case 2
+    #     s               e
+    #     |               |
+    #          ---------(-.)
+    idx_select2 = np.logical_and(l_start_time >= start_time, l_end_time <= end_time)
+    # - Case 3
+    #     s               e
+    #     |               |
+    #                -------------
+    idx_select3 = np.logical_and(l_start_time < end_time, l_end_time > end_time)
+    # - Get idx where one of the cases occur
+    idx_select = np.logical_or.reduce([idx_select1, idx_select2, idx_select3])
+    return idx_select
+
+
+def filter_by_time(filepaths, start_time=None, end_time=None):
+    """Filter filepaths by start_time and end_time.
+
+    Parameters
+    ----------
+    filepaths : list
+        List of filepaths.
+    start_time : datetime.datetime
+        Start time.
+        If ``None``, will be set to 1997-01-01.
+    end_time : datetime.datetime
+        End time.
+        If ``None`` will be set to current UTC time.
+
+    Returns
+    -------
+    filepaths : list
+        List of valid filepaths.
+        If no valid filepaths, returns an empty list !
+
+    """
+    # -------------------------------------------------------------------------.
+    # Check filepaths
+    if isinstance(filepaths, type(None)):
+        return []
+    filepaths = check_filepaths(filepaths)
+    if len(filepaths) == 0:
+        return []
+
+    # -------------------------------------------------------------------------.
+    # Check start_time and end_time
+    if start_time is None:
+        start_time = datetime.datetime(1978, 1, 1, 0, 0, 0)  # Dummy start
+    if end_time is None:
+        end_time = get_current_utc_time()  # Current time
+    start_time, end_time = check_start_end_time(start_time, end_time)
+
+    # -------------------------------------------------------------------------.
+    # - Retrieve start_time and end_time of GPM granules
+    l_start_time, l_end_time = get_start_end_time_from_filepaths(filepaths)
+
+    # -------------------------------------------------------------------------.
+    # Select granules with data within the start and end time
+    idx_select = is_within_time_period(l_start_time, l_end_time, start_time=start_time, end_time=end_time)
+    return np.array(filepaths)[idx_select].tolist()
+
+
 def find_files(
     data_source,
     campaign_name,
@@ -55,7 +133,9 @@ def find_files(
     product,
     debugging_mode: bool = False,
     data_archive_dir: Optional[str] = None,
-    glob_pattern="*",
+    glob_pattern=None,
+    start_time=None,
+    end_time=None,
     **product_kwargs,
 ):
     """Retrieve DISDRODB product files for a give station.
@@ -100,6 +180,8 @@ def find_files(
         List of file paths.
 
     """
+    from disdrodb.metadata import read_station_metadata
+
     # Retrieve data directory
     data_dir = define_data_dir(
         data_archive_dir=data_archive_dir,
@@ -110,8 +192,16 @@ def find_files(
         # Product options
         **product_kwargs,
     )
-
-    # Define or check the specified glob pattern
+    # For the DISDRODB RAW product, retrieve glob_pattern from metadata if not specified
+    if product == "RAW" and glob_pattern is None:
+        metadata = read_station_metadata(
+            data_source=data_source,
+            campaign_name=campaign_name,
+            station_name=station_name,
+        )
+        glob_pattern = metadata.get("raw_data_glob_pattern", "")
+
+    # For the others DISDRODB products, define the correct glob pattern
     if product != "RAW":
         glob_pattern = "*.parquet" if product == "L0A" else "*.nc"
 
@@ -126,6 +216,13 @@ def find_files(
         msg = f"No {product} files are available in {data_dir}. Run {product} processing first."
         raise ValueError(msg)
 
+    # Filter files by start_time and end_time
+    if product != "RAW":
+        filepaths = filter_by_time(filepaths=filepaths, start_time=start_time, end_time=end_time)
+        if len(filepaths) == 0:
+            msg = f"No {product} files are available between {start_time} and {end_time}."
+            raise ValueError(msg)
+
     # Sort filepaths
     filepaths = sorted(filepaths)
     return filepaths
@@ -133,6 +230,117 @@ def find_files(
 
 ####----------------------------------------------------------------------------------
 #### DISDRODB Open Product Files
+
+
+def open_raw_files(filepaths, data_source, campaign_name, station_name):
+    """Open raw files to DISDRODB L0A or L0B format.
+
+    Raw text files are opened into a DISDRODB L0A pandas Dataframe.
+    Raw netCDF files are opened into a DISDRODB L0B xarray Dataset.
+    """
+    from disdrodb.issue import read_station_issue
+    from disdrodb.l0 import generate_l0a, generate_l0b_from_nc, get_station_reader
+    from disdrodb.metadata import read_station_metadata
+
+    # Read station metadata
+    metadata = read_station_metadata(
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+    )
+    sensor_name = metadata["sensor_name"]
+
+    # Read station issue YAML file
+    try:
+        issue_dict = read_station_issue(
+            data_source=data_source,
+            campaign_name=campaign_name,
+            station_name=station_name,
+        )
+    except Exception:
+        issue_dict = None
+
+    # Get reader
+    reader = get_station_reader(
+        data_source=data_source,
+        campaign_name=campaign_name,
+        station_name=station_name,
+    )
+    # Return DISDRODB L0A dataframe if raw text files
+    if metadata["raw_data_format"] == "txt":
+        df = generate_l0a(
+            filepaths=filepaths,
+            reader=reader,
+            sensor_name=sensor_name,
+            issue_dict=issue_dict,
+            verbose=False,
+        )
+        return df
+
+    # Return DISDRODB L0B dataframe if raw netCDF files
+    ds = generate_l0b_from_nc(
+        filepaths=filepaths,
+        reader=reader,
+        sensor_name=sensor_name,
+        metadata=metadata,
+        issue_dict=issue_dict,
+        verbose=False,
+    )
+    return ds
+
+
+def open_netcdf_files(
+    filepaths,
+    chunks=-1,
+    start_time=None,
+    end_time=None,
+    variables=None,
+    parallel=False,
+    compute=True,
+    **open_kwargs,
+):
+    """Open DISDRODB netCDF files using xarray."""
+    import xarray as xr
+
+    # Ensure variables is a list
+    if variables is not None and isinstance(variables, str):
+        variables = [variables]
+    # Define preprocessing function for parallel opening
+    preprocess = (lambda ds: ds[variables]) if parallel and variables is not None else None
+
+    # Open netcdf
+    ds = xr.open_mfdataset(
+        filepaths,
+        chunks=chunks,
+        combine="nested",
+        concat_dim="time",
+        engine="netcdf4",
+        parallel=parallel,
+        preprocess=preprocess,
+        compat="no_conflicts",
+        combine_attrs="override",
+        coords="different",  # maybe minimal?
+        decode_timedelta=False,
+        cache=False,
+        autoclose=True,
+        **open_kwargs,
+    )
+    # - Subset variables
+    if variables is not None and preprocess is None:
+        ds = ds[variables]
+    # - Subset time
+    ds = ds.sel(time=slice(start_time, end_time))
+    # - If compute=True, load in memory and close connections to files
+    if compute:
+        dataset = ds.compute()
+        ds.close()
+        dataset.close()
+        del ds
+    else:
+        dataset = ds
+    return dataset
+
+
 def open_dataset(
     data_source,
     campaign_name,
@@ -141,7 +349,12 @@ def open_dataset(
     product_kwargs=None,
     debugging_mode: bool = False,
     data_archive_dir: Optional[str] = None,
+    chunks=-1,
     parallel=False,
+    compute=False,
+    start_time=None,
+    end_time=None,
+    variables=None,
     **open_kwargs,
 ):
     """Retrieve DISDRODB product files for a give station.
@@ -179,13 +392,8 @@ def open_dataset(
     xarray.Dataset
 
     """
-    import xarray as xr
-
     from disdrodb.l0.l0a_processing import read_l0a_dataframe
 
-    # Check product validity
-    if product == "RAW":
-        raise ValueError("It's not possible to open the raw data with this function.")
     product_kwargs = product_kwargs if product_kwargs else {}
 
     # List product files
@@ -196,25 +404,36 @@ def open_dataset(
         station_name=station_name,
         product=product,
         debugging_mode=debugging_mode,
+        start_time=start_time,
+        end_time=end_time,
         **product_kwargs,
     )
 
+    # Open RAW files
+    # - For raw txt files return DISDRODB L0A dataframe
+    # - For raw netCDF files return DISDRODB L0B dataframe
+    if product == "RAW":
+        obj = open_raw_files(
+            filepaths=filepaths,
+            data_source=data_source,
+            campaign_name=campaign_name,
+            station_name=station_name,
+        )
+        return obj
+
     # Open L0A Parquet files
     if product == "L0A":
         return read_l0a_dataframe(filepaths)
 
     # Open DISDRODB netCDF files using xarray
-    # - TODO: parallel option and add closers !
-    # - decode_timedelta -- > sample_interval not decoded to timedelta !
-    # list_ds = [xr.open_dataset(fpath, decode_timedelta=False, **open_kwargs) for fpath in filepaths]
-    # ds = xr.concat(list_ds, dim="time")
-    ds = xr.open_mfdataset(
-        filepaths,
-        engine="netcdf4",
-        combine="nested",  # 'by_coords',
-        concat_dim="time",
-        decode_timedelta=False,
+    ds = open_netcdf_files(
+        filepaths=filepaths,
+        chunks=chunks,
+        start_time=start_time,
+        end_time=end_time,
+        variables=variables,
         parallel=parallel,
+        compute=compute,
         **open_kwargs,
     )
     return ds

disdrodb/api/path.py

@@ -20,11 +20,12 @@
 import os
 
 from disdrodb.configs import get_data_archive_dir, get_metadata_archive_dir
+from disdrodb.constants import ARCHIVE_VERSION
 from disdrodb.utils.directories import check_directory_exists
 from disdrodb.utils.time import (
     ensure_sample_interval_in_seconds,
     get_file_start_end_time,
-    seconds_to_acronym,
+    seconds_to_temporal_resolution,
 )
 
 ####--------------------------------------------------------------------------.
@@ -68,8 +69,6 @@ def define_disdrodb_path(
     dir_path : str
         Directory path
     """
-    from disdrodb import ARCHIVE_VERSION
-
     if len(campaign_name) > 0 and len(data_source) == 0:
         raise ValueError("If campaign_name is specified, data_source must be specified.")
 
@@ -349,6 +348,55 @@ def define_config_dir(product):
 #### Directory/Filepaths L0A and L0B products
 
 
+def define_partitioning_tree(time, folder_partitioning):
+    """Define the time directory tree given a timestep.
+
+    Parameters
+    ----------
+    time : datetime.datetime
+        Timestep.
+    folder_partitioning : str or None
+        Define the subdirectory structure where saving files.
+        Allowed values are:
+          - None: Files are saved directly in data_dir.
+          - "year": Files are saved under a subdirectory for the year.
+          - "year/month": Files are saved under subdirectories for year and month.
+          - "year/month/day": Files are saved under subdirectories for year, month and day
+          - "year/month_name": Files are stored under subdirectories by year and month name
+          - "year/quarter": Files are saved under subdirectories for year and quarter.
+
+    Returns
+    -------
+    str
+        A time partitioned directory tree.
+    """
+    if folder_partitioning == "":
+        return ""
+    if folder_partitioning == "year":
+        year = str(time.year)
+        return year
+    if folder_partitioning == "year/month":
+        year = str(time.year)
+        month = str(time.month).zfill(2)
+        return os.path.join(year, month)
+    if folder_partitioning == "year/month/day":
+        year = str(time.year)
+        month = str(time.month).zfill(2)
+        day = str(time.day).zfill(2)
+        return os.path.join(year, month, day)
+    if folder_partitioning == "year/month_name":
+        year = str(time.year)
+        month = str(time.month_name())
+        return os.path.join(year, month)
+    if folder_partitioning == "year/quarter":
+        year = str(time.year)
+        # Calculate quarter: months 1-3 => Q1, 4-6 => Q2, etc.
+        quarter = (time.month - 1) // 3 + 1
+        quarter_dir = f"Q{quarter}"
+        return os.path.join(year, quarter_dir)
+    raise NotImplementedError(f"Unrecognized '{folder_partitioning}' folder partitioning scheme.")
+
+
 def define_file_folder_path(obj, data_dir, folder_partitioning):
     """
     Define the folder path where saving a file based on the dataset's starting time.
@@ -382,32 +430,9 @@ def define_file_folder_path(obj, data_dir, folder_partitioning):
     # Retrieve the starting time from the dataset.
     starting_time, _ = get_file_start_end_time(obj)
 
-    # Build the folder path based on the chosen partition scheme.
-    if folder_partitioning == "":
-        return data_dir
-    if folder_partitioning == "year":
-        year = str(starting_time.year)
-        return os.path.join(data_dir, year)
-    if folder_partitioning == "year/month":
-        year = str(starting_time.year)
-        month = str(starting_time.month).zfill(2)
-        return os.path.join(data_dir, year, month)
-    if folder_partitioning == "year/month/day":
-        year = str(starting_time.year)
-        month = str(starting_time.month).zfill(2)
-        day = str(starting_time.day).zfill(2)
-        return os.path.join(data_dir, year, month, day)
-    if folder_partitioning == "year/month_name":
-        year = str(starting_time.year)
-        month = str(starting_time.month_name())
-        return os.path.join(data_dir, year, month)
-    if folder_partitioning == "year/quarter":
-        year = str(starting_time.year)
-        # Calculate quarter: months 1-3 => Q1, 4-6 => Q2, etc.
-        quarter = (starting_time.month - 1) // 3 + 1
-        quarter_dir = f"Q{quarter}"
-        return os.path.join(data_dir, year, quarter_dir)
-    raise NotImplementedError(f"Unrecognized '{folder_partitioning}' folder partitioning scheme.")
+    # Build the folder path based on the chosen partition scheme
+    partitioning_tree = define_partitioning_tree(time=starting_time, folder_partitioning=folder_partitioning)
+    return os.path.join(data_dir, partitioning_tree)
 
 
 def define_product_dir_tree(
@@ -448,16 +473,16 @@ def define_product_dir_tree(
         sample_interval = product_kwargs.get("sample_interval")
         check_rolling(rolling)
         check_sample_interval(sample_interval)
-        sample_interval_acronym = define_accumulation_acronym(seconds=sample_interval, rolling=rolling)
-        return os.path.join(sample_interval_acronym)
+        temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
+        return os.path.join(temporal_resolution)
     if product == "L2M":
         rolling = product_kwargs.get("rolling")
         sample_interval = product_kwargs.get("sample_interval")
         model_name = product_kwargs.get("model_name")
         check_rolling(rolling)
         check_sample_interval(sample_interval)
-        sample_interval_acronym = define_accumulation_acronym(seconds=sample_interval, rolling=rolling)
-        return os.path.join(model_name, sample_interval_acronym)
+        temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
+        return os.path.join(model_name, temporal_resolution)
     raise ValueError(f"The product {product} is not defined.")
 
 
@@ -629,15 +654,15 @@ def define_data_dir(
 #### Filenames for DISDRODB products
 
 
-def define_accumulation_acronym(seconds, rolling):
-    """Define the accumulation acronnym.
+def define_temporal_resolution(seconds, rolling):
+    """Define the DISDRODB product temporal resolution.
 
-    Prefix the accumulation interval acronym with ROLL if rolling=True.
+    Prefix the measurement interval with ROLL if rolling=True.
     """
-    accumulation_acronym = seconds_to_acronym(seconds)
+    temporal_resolution = seconds_to_temporal_resolution(seconds)
     if rolling:
-        accumulation_acronym = f"ROLL{accumulation_acronym}"
-    return accumulation_acronym
+        temporal_resolution = f"ROLL{temporal_resolution}"
+    return temporal_resolution
 
 
 ####--------------------------------------------------------------------------.
@@ -685,32 +710,31 @@ def define_filename(
     str
         L0B file name.
     """
-    from disdrodb import ARCHIVE_VERSION
     from disdrodb.api.checks import check_product, check_product_kwargs
 
     product = check_product(product)
     product_kwargs = check_product_kwargs(product, product_kwargs)
 
     # -----------------------------------------.
-    # TODO: Define sample_interval_acronym
-    # - ADD sample_interval_acronym also to L0A and L0B
-    # - Add sample_interval_acronym also to L0C and L1
+    # TODO: Define temporal_resolution
+    # - ADD temporal_resolution also to L0A and L0B
+    # - Add temporal_resolution also to L0C and L1
 
     # -----------------------------------------.
-    # Define product acronym
-    product_acronym = f"{product}"
+    # Define product name
+    product_name = f"{product}"
     if product in ["L2E", "L2M"]:
         rolling = product_kwargs.get("rolling")
         sample_interval = product_kwargs.get("sample_interval")
-        sample_interval_acronym = define_accumulation_acronym(seconds=sample_interval, rolling=rolling)
-        product_acronym = f"L2E.{sample_interval_acronym}"
+        temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
+        product_name = f"L2E.{temporal_resolution}"
     if product in ["L2M"]:
         model_name = product_kwargs.get("model_name")
-        product_acronym = f"L2M_{model_name}.{sample_interval_acronym}"
+        product_name = f"L2M_{model_name}.{temporal_resolution}"
 
     # -----------------------------------------.
     # Define base filename
-    filename = f"{product_acronym}.{campaign_name}.{station_name}"
+    filename = f"{product_name}.{campaign_name}.{station_name}"
 
     # -----------------------------------------.
     # Add prefix
@@ -759,8 +783,6 @@ def define_l0a_filename(df, campaign_name: str, station_name: str) -> str:
     str
         L0A file name.
     """
-    from disdrodb import ARCHIVE_VERSION
-
     starting_time, ending_time = get_file_start_end_time(df)
     starting_time = starting_time.strftime("%Y%m%d%H%M%S")
     ending_time = ending_time.strftime("%Y%m%d%H%M%S")
@@ -786,8 +808,6 @@ def define_l0b_filename(ds, campaign_name: str, station_name: str) -> str:
     str
         L0B file name.
     """
-    from disdrodb import ARCHIVE_VERSION
-
     starting_time, ending_time = get_file_start_end_time(ds)
     starting_time = starting_time.strftime("%Y%m%d%H%M%S")
     ending_time = ending_time.strftime("%Y%m%d%H%M%S")
@@ -813,18 +833,14 @@ def define_l0c_filename(ds, campaign_name: str, station_name: str) -> str:
     str
         L0B file name.
     """
-    from disdrodb import ARCHIVE_VERSION
-
     # TODO: add sample_interval as argument
     sample_interval = int(ensure_sample_interval_in_seconds(ds["sample_interval"]).data.item())
-    sample_interval_acronym = define_accumulation_acronym(sample_interval, rolling=False)
+    temporal_resolution = define_temporal_resolution(sample_interval, rolling=False)
     starting_time, ending_time = get_file_start_end_time(ds)
     starting_time = starting_time.strftime("%Y%m%d%H%M%S")
     ending_time = ending_time.strftime("%Y%m%d%H%M%S")
     version = ARCHIVE_VERSION
-    filename = (
-        f"L0C.{sample_interval_acronym}.{campaign_name}.{station_name}.s{starting_time}.e{ending_time}.{version}.nc"
-    )
+    filename = f"L0C.{temporal_resolution}.{campaign_name}.{station_name}.s{starting_time}.e{ending_time}.{version}.nc"
     return filename
 
 
@@ -845,18 +861,14 @@ def define_l1_filename(ds, campaign_name, station_name: str) -> str:
     str
         L1 file name.
     """
-    from disdrodb import ARCHIVE_VERSION
-
     # TODO: add sample_interval as argument
     sample_interval = int(ensure_sample_interval_in_seconds(ds["sample_interval"]).data.item())
-    sample_interval_acronym = define_accumulation_acronym(sample_interval, rolling=False)
+    temporal_resolution = define_temporal_resolution(sample_interval, rolling=False)
     starting_time, ending_time = get_file_start_end_time(ds)
     starting_time = starting_time.strftime("%Y%m%d%H%M%S")
     ending_time = ending_time.strftime("%Y%m%d%H%M%S")
     version = ARCHIVE_VERSION
-    filename = (
-        f"L1.{sample_interval_acronym}.{campaign_name}.{station_name}.s{starting_time}.e{ending_time}.{version}.nc"
-    )
+    filename = f"L1.{temporal_resolution}.{campaign_name}.{station_name}.s{starting_time}.e{ending_time}.{version}.nc"
     return filename
 
 
@@ -877,16 +889,12 @@ def define_l2e_filename(ds, campaign_name: str, station_name: str, sample_interv
     str
         L0B file name.
     """
-    from disdrodb import ARCHIVE_VERSION
-
-    sample_interval_acronym = define_accumulation_acronym(seconds=sample_interval, rolling=rolling)
+    temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
     starting_time, ending_time = get_file_start_end_time(ds)
     starting_time = starting_time.strftime("%Y%m%d%H%M%S")
     ending_time = ending_time.strftime("%Y%m%d%H%M%S")
     version = ARCHIVE_VERSION
-    filename = (
-        f"L2E.{sample_interval_acronym}.{campaign_name}.{station_name}.s{starting_time}.e{ending_time}.{version}.nc"
-    )
+    filename = f"L2E.{temporal_resolution}.{campaign_name}.{station_name}.s{starting_time}.e{ending_time}.{version}.nc"
     return filename
 
 
@@ -914,15 +922,13 @@ def define_l2m_filename(
     str
         L0B file name.
     """
-    from disdrodb import ARCHIVE_VERSION
-
-    sample_interval_acronym = define_accumulation_acronym(seconds=sample_interval, rolling=rolling)
+    temporal_resolution = define_temporal_resolution(seconds=sample_interval, rolling=rolling)
     starting_time, ending_time = get_file_start_end_time(ds)
     starting_time = starting_time.strftime("%Y%m%d%H%M%S")
     ending_time = ending_time.strftime("%Y%m%d%H%M%S")
     version = ARCHIVE_VERSION
     filename = (
-        f"L2M_{model_name}.{sample_interval_acronym}.{campaign_name}."
+        f"L2M_{model_name}.{temporal_resolution}.{campaign_name}."
         + f"{station_name}.s{starting_time}.e{ending_time}.{version}.nc"
     )
     return filename

disdrodb/api/search.py

@@ -16,14 +16,13 @@ from disdrodb.api.path import (
     define_station_dir,
 )
 from disdrodb.configs import get_data_archive_dir, get_metadata_archive_dir
-from disdrodb.utils.directories import contains_files, contains_netcdf_or_parquet_files
+from disdrodb.constants import PRODUCTS_REQUIREMENTS
+from disdrodb.utils.directories import contains_files, contains_netcdf_or_parquet_files, list_directories, list_files
 from disdrodb.utils.yaml import read_yaml
 
 
 def get_required_product(product):
     """Determine the required product for input product processing."""
-    from disdrodb import PRODUCTS_REQUIREMENTS
-
     # Check input
     check_product(product)
     # Determine required product
@@ -37,7 +36,8 @@ def get_required_product(product):
 
 def list_data_sources(metadata_archive_dir, data_sources=None, invalid_fields_policy="raise"):
     """List data sources names in the DISDRODB Metadata Archive."""
-    available_data_sources = os.listdir(os.path.join(metadata_archive_dir, "METADATA"))
+    path = os.path.join(metadata_archive_dir, "METADATA")
+    available_data_sources = sorted(list_directories(path, return_paths=False))
     # Filter by optionally specified data_sources
     if data_sources is not None:
         available_data_sources = check_valid_fields(
@@ -52,7 +52,7 @@ def list_data_sources(metadata_archive_dir, data_sources=None, invalid_fields_po
 
 def _list_campaign_names(metadata_archive_dir, data_source):
     data_source_dir = define_data_source_dir(metadata_archive_dir, product="METADATA", data_source=data_source)
-    campaign_names = os.listdir(data_source_dir)
+    campaign_names = sorted(list_directories(data_source_dir, return_paths=False))
     return campaign_names
 
 
@@ -109,7 +109,7 @@ def _list_station_names(metadata_archive_dir, data_source, campaign_name):
         data_source=data_source,
         campaign_name=campaign_name,
     )
-    metadata_filenames = os.listdir(metadata_dir)
+    metadata_filenames = sorted(list_files(metadata_dir, glob_pattern="*.yml", return_paths=False))
     station_names = [fname.replace(".yml", "").replace(".yaml", "") for fname in metadata_filenames]
     return station_names