disdrodb 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

disdrodb/api/checks.py

@@ -17,9 +17,12 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------.
 """DISDRODB Checks Functions."""
+import datetime
+import difflib
 import logging
 import os
 import re
+import sys
 import warnings
 
 import numpy as np
@@ -30,8 +33,10 @@ from disdrodb.api.path import (
     define_issue_filepath,
     define_metadata_filepath,
 )
+from disdrodb.constants import PRODUCTS, PRODUCTS_ARGUMENTS
 from disdrodb.utils.directories import (
     ensure_string_path,
+    list_directories,
     list_files,
 )
 
@@ -83,7 +88,7 @@ def check_path_is_a_directory(dir_path, path_name=""):
 
 def check_directories_inside(dir_path):
     """Check there are directories inside the specified ``dir_path``."""
-    dir_paths = os.listdir(dir_path)
+    dir_paths = list_directories(dir_path, recursive=False)
     if len(dir_paths) == 0:
         raise ValueError(f"There are not directories within {dir_path}")
 
@@ -106,6 +111,15 @@ def check_metadata_archive_dir(metadata_archive_dir: str):
     return metadata_archive_dir
 
 
+def check_scattering_table_dir(scattering_table_dir: str):
+    """Raise an error if the directory does not exist."""
+    scattering_table_dir = str(scattering_table_dir)  # convert Pathlib to string
+    scattering_table_dir = os.path.normpath(scattering_table_dir)
+    if not os.path.exists(scattering_table_dir):
+        raise ValueError(f"The DISDRODB T-Matrix scattering tables directory {scattering_table_dir} does not exists.")
+    return scattering_table_dir
+
+
 def check_measurement_interval(measurement_interval):
     """Check measurement interval validity."""
     if isinstance(measurement_interval, str) and measurement_interval == "":
@@ -130,14 +144,14 @@ def check_measurement_intervals(measurement_intervals):
 
 def check_sample_interval(sample_interval):
     """Check sample_interval argument validity."""
-    if not isinstance(sample_interval, int):
-        raise ValueError("'sample_interval' must be an integer.")
+    if not isinstance(sample_interval, int) or isinstance(sample_interval, bool):
+        raise TypeError("'sample_interval' must be an integer.")
 
 
 def check_rolling(rolling):
     """Check rolling argument validity."""
     if not isinstance(rolling, bool):
-        raise ValueError("'rolling' must be a boolean.")
+        raise TypeError("'rolling' must be a boolean.")
 
 
 def check_folder_partitioning(folder_partitioning):
@@ -149,12 +163,12 @@ def check_folder_partitioning(folder_partitioning):
     folder_partitioning : str or None
         Defines the subdirectory structure based on the dataset's start time.
         Allowed values are:
-          - "": No additional subdirectories, files are saved directly in data_dir.
-          - "year": Files are stored under a subdirectory for the year (<data_dir>/2025).
-          - "year/month": Files are stored under subdirectories by year and month (<data_dir>/2025/04).
-          - "year/month/day": Files are stored under subdirectories by year, month and day (<data_dir>/2025/04/01).
-          - "year/month_name": Files are stored under subdirectories by year and month name (<data_dir>/2025/April).
-          - "year/quarter": Files are stored under subdirectories by year and quarter (<data_dir>/2025/Q2).
+          - "" or None: No additional subdirectories, files are saved directly in dir.
+          - "year": Files are stored under a subdirectory for the year (<dir>/2025).
+          - "year/month": Files are stored under subdirectories by year and month (<dir>/2025/04).
+          - "year/month/day": Files are stored under subdirectories by year, month and day (<dir>/2025/04/01).
+          - "year/month_name": Files are stored under subdirectories by year and month name (<dir>/2025/April).
+          - "year/quarter": Files are stored under subdirectories by year and quarter (<dir>/2025/Q2).
 
     Returns
     -------
@@ -162,6 +176,8 @@ def check_folder_partitioning(folder_partitioning):
         The verified folder partitioning scheme.
     """
     valid_options = ["", "year", "year/month", "year/month/day", "year/month_name", "year/quarter"]
+    if folder_partitioning is None:
+        folder_partitioning = ""
     if folder_partitioning not in valid_options:
         raise ValueError(
             f"Invalid folder_partitioning scheme '{folder_partitioning}'. Valid options are: {valid_options}.",
@@ -216,8 +232,6 @@ def check_data_source(data_source):
 
 def check_product(product):
     """Check DISDRODB product."""
-    from disdrodb import PRODUCTS
-
     if not isinstance(product, str):
         raise TypeError("`product` must be a string.")
     valid_products = PRODUCTS
@@ -248,8 +262,6 @@ def check_product_kwargs(product, product_kwargs):
     ValueError
         If required arguments are missing or if there are unexpected extra arguments.
     """
-    from disdrodb import PRODUCTS_ARGUMENTS
-
     required = set(PRODUCTS_ARGUMENTS.get(product, []))
     provided = set(product_kwargs.keys())
     missing = required - provided
@@ -267,8 +279,6 @@ def check_product_kwargs(product, product_kwargs):
 
 def select_required_product_kwargs(product, product_kwargs):
     """Select the required product arguments."""
-    from disdrodb import PRODUCTS_ARGUMENTS
-
     required = set(PRODUCTS_ARGUMENTS.get(product, []))
     provided = set(product_kwargs.keys())
     missing = required - provided
@@ -323,16 +333,37 @@ def check_valid_fields(fields, available_fields, field_name, invalid_fields_poli
         fields = [fields]
     fields = np.unique(np.array(fields))
     invalid_fields_policy = check_invalid_fields_policy(invalid_fields_policy)
+
     # Check for invalid fields
     fields = np.array(fields)
     is_valid = np.isin(fields, available_fields)
     invalid_fields_values = fields[~is_valid].tolist()
     fields = fields[is_valid].tolist()
+
+    # If invalid fields, suggest corrections using difflib
+    if invalid_fields_values:
+
+        # Format invalid fields nicely (avoid single-element lists)
+        if len(invalid_fields_values) == 1:
+            invalid_fields_str = f"'{invalid_fields_values[0]}'"
+        else:
+            invalid_fields_str = f"{invalid_fields_values}"
+
+        # Prepare suggestion string
+        suggestions = []
+        for invalid in invalid_fields_values:
+            matches = difflib.get_close_matches(invalid, available_fields, n=1, cutoff=0.4)
+            if matches:
+                suggestions.append(f"Did you mean '{matches[0]}' instead of '{invalid}'?")
+        suggestion_msg = " " + " ".join(suggestions) if suggestions else ""
+
     # Error handling for invalid fields were found
     if invalid_fields_policy == "warn" and invalid_fields_values:
-        warnings.warn(f"Ignoring invalid {field_name}: {invalid_fields_values}", UserWarning, stacklevel=2)
+        msg = f"Ignoring invalid {field_name}: {invalid_fields_str}.{suggestion_msg}"
+        warnings.warn(msg, UserWarning, stacklevel=2)
     elif invalid_fields_policy == "raise" and invalid_fields_values:
-        raise ValueError(f"These {field_name} does not exist: {invalid_fields_values}.")
+        msg = f"These {field_name} do not exist: {invalid_fields_str}.{suggestion_msg}"
+        raise ValueError(msg)
     else:  # "ignore" silently drop invalid entries
         pass
     # If no valid fields left, raise error
@@ -341,6 +372,46 @@ def check_valid_fields(fields, available_fields, field_name, invalid_fields_poli
     return fields
 
 
+def check_station_inputs(
+    data_source,
+    campaign_name,
+    station_name,
+    metadata_archive_dir=None,
+):
+    """Check validity of stations inputs."""
+    import disdrodb
+
+    # Check data source
+    valid_data_sources = disdrodb.available_data_sources(metadata_archive_dir=metadata_archive_dir)
+    if data_source not in valid_data_sources:
+        matches = difflib.get_close_matches(data_source, valid_data_sources, n=1, cutoff=0.4)
+        suggestion = f"Did you mean '{matches[0]}'?" if matches else ""
+        raise ValueError(f"DISDRODB does not include a data source named {data_source}. {suggestion}")
+
+    # Check campaign name
+    valid_campaigns = disdrodb.available_campaigns(data_sources=data_source, metadata_archive_dir=metadata_archive_dir)
+    if campaign_name not in valid_campaigns:
+        matches = difflib.get_close_matches(campaign_name, valid_campaigns, n=1, cutoff=0.4)
+        suggestion = f"Did you mean campaign '{matches[0]}' ?" if matches else ""
+        raise ValueError(
+            f"The {data_source} data source does not include a campaign named {campaign_name}. {suggestion}",
+        )
+
+    # Check station name
+    valid_stations = disdrodb.available_stations(
+        data_sources=data_source,
+        campaign_names=campaign_name,
+        metadata_archive_dir=metadata_archive_dir,
+        return_tuple=False,
+    )
+    if station_name not in valid_stations:
+        matches = difflib.get_close_matches(station_name, valid_stations, n=1, cutoff=0.4)
+        suggestion = f"Did you mean station '{matches[0]}'?" if matches else ""
+        raise ValueError(
+            f"The {data_source} {campaign_name} campaign does not have a station named {station_name}. {suggestion}",
+        )
+
+
 def has_available_data(
     data_source,
     campaign_name,
@@ -368,7 +439,7 @@ def has_available_data(
         return False
 
     # If no files, return False
-    filepaths = list_files(data_dir, glob_pattern="*", recursive=True)
+    filepaths = list_files(data_dir, recursive=True)
     nfiles = len(filepaths)
     return nfiles >= 1
 
@@ -412,7 +483,6 @@ def check_metadata_file(metadata_archive_dir, data_source, campaign_name, statio
             f"The metadata YAML file of {data_source} {campaign_name} {station_name} does not exist at"
             f" {metadata_filepath}."
         )
-        logger.error(msg)
         raise ValueError(msg)
 
     # Check validity
@@ -434,10 +504,9 @@ def check_issue_dir(data_source, campaign_name, metadata_archive_dir=None):
         campaign_name=campaign_name,
         check_exists=False,
     )
-    if not os.path.exists(issue_dir) and os.path.isdir(issue_dir):
-        msg = "The issue directory does not exist at {issue_dir}."
+    if not os.path.exists(issue_dir) or not os.path.isdir(issue_dir):
+        msg = f"The issue directory does not exist at {issue_dir}."
         logger.error(msg)
-        raise ValueError(msg)
     return issue_dir
 
 
@@ -458,7 +527,7 @@ def check_issue_file(data_source, campaign_name, station_name, metadata_archive_
         station_name=station_name,
         check_exists=False,
     )
-    # Check existence
+    # Check existence. If not, create one !
     if not os.path.exists(issue_filepath):
         create_station_issue(
             metadata_archive_dir=metadata_archive_dir,
@@ -475,3 +544,87 @@ def check_issue_file(data_source, campaign_name, station_name, metadata_archive_
         station_name=station_name,
     )
     return issue_filepath
+
+
+def check_filepaths(filepaths):
+    """Ensure filepaths is a list of string."""
+    if isinstance(filepaths, str):
+        filepaths = [filepaths]
+    if not isinstance(filepaths, list):
+        raise TypeError("Expecting a list of filepaths.")
+    return filepaths
+
+
+def get_current_utc_time():
+    """Get current UTC time."""
+    if sys.version_info >= (3, 11):
+        return datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
+    return datetime.datetime.utcnow()
+
+
+def check_start_end_time(start_time, end_time):
+    """Check start_time and end_time value validity."""
+    start_time = check_time(start_time)
+    end_time = check_time(end_time)
+
+    # Check start_time and end_time are chronological
+    if start_time > end_time:
+        raise ValueError("Provide 'start_time' occurring before of 'end_time'.")
+    # Check start_time and end_time are in the past
+    if start_time > get_current_utc_time():
+        raise ValueError("Provide 'start_time' occurring in the past.")
+    if end_time > get_current_utc_time():
+        raise ValueError("Provide 'end_time' occurring in the past.")
+    return (start_time, end_time)
+
+
+def check_time(time):
+    """Check time validity.
+
+    It returns a :py:class:`datetime.datetime` object to seconds precision.
+
+    Parameters
+    ----------
+    time : datetime.datetime, datetime.date, numpy.datetime64 or str
+        Time object.
+        Accepted types: ``datetime.datetime``, ``datetime.date``, ``numpy.datetime64`` or ``str``.
+        If string type, it expects the isoformat ``YYYY-MM-DD hh:mm:ss``.
+
+    Returns
+    -------
+    time: datetime.datetime
+
+    """
+    if not isinstance(time, (datetime.datetime, datetime.date, np.datetime64, np.ndarray, str)):
+        raise TypeError(
+            "Specify time with datetime.datetime objects or a " "string of format 'YYYY-MM-DD hh:mm:ss'.",
+        )
+
+    # If numpy array with datetime64 (and size=1)
+    if isinstance(time, np.ndarray):
+        if np.issubdtype(time.dtype, np.datetime64):
+            if time.size == 1:
+                time = time[0].astype("datetime64[s]").tolist()
+            else:
+                raise ValueError("Expecting a single timestep!")
+        else:
+            raise ValueError("The numpy array does not have a numpy.datetime64 dtype!")
+
+    # If np.datetime64, convert to datetime.datetime
+    if isinstance(time, np.datetime64):
+        time = time.astype("datetime64[s]").tolist()
+    # If datetime.date, convert to datetime.datetime
+    if not isinstance(time, (datetime.datetime, str)):
+        time = datetime.datetime(time.year, time.month, time.day, 0, 0, 0)
+    if isinstance(time, str):
+        try:
+            time = datetime.datetime.fromisoformat(time)
+        except ValueError:
+            raise ValueError("The time string must have format 'YYYY-MM-DD hh:mm:ss'")
+    # If datetime object carries timezone that is not UTC, raise error
+    if time.tzinfo is not None:
+        if str(time.tzinfo) != "UTC":
+            raise ValueError("The datetime object must be in UTC timezone if timezone is given.")
+        # If UTC, strip timezone information
+        time = time.replace(tzinfo=None)
+    return time

disdrodb/api/configs.py

@@ -23,6 +23,7 @@ import os
 
 from disdrodb.api.checks import check_product, check_sensor_name
 from disdrodb.api.path import define_config_dir
+from disdrodb.utils.directories import list_directories
 from disdrodb.utils.yaml import read_yaml
 
 logger = logging.getLogger(__name__)
@@ -53,8 +54,6 @@ def get_sensor_configs_dir(sensor_name: str, product: str) -> str:
     config_dir = define_config_dir(product=product)
     config_sensor_dir = os.path.join(config_dir, sensor_name)
     if not os.path.exists(config_sensor_dir):
-        list_sensors = sorted(os.listdir(config_dir))
-        print(f"Available sensor_name are {list_sensors}")
         raise ValueError(f"The config directory {config_sensor_dir} does not exist.")
     return config_sensor_dir
 
@@ -102,4 +101,5 @@ def available_sensor_names() -> list:
         Sorted list of the available sensors
     """
     config_dir = define_config_dir(product="L0A")
-    return sorted(os.listdir(config_dir))
+    list_sensors = sorted(list_directories(config_dir, recursive=False, return_paths=False))
+    return list_sensors

disdrodb/api/info.py

@@ -25,7 +25,7 @@ from pathlib import Path
 import numpy as np
 from trollsift import Parser
 
-from disdrodb.utils.time import acronym_to_seconds
+from disdrodb.utils.time import temporal_resolution_to_seconds
 
 ####---------------------------------------------------------------------------
 ########################
@@ -35,13 +35,13 @@ DISDRODB_FNAME_L0_PATTERN = (
     "{product:s}.{campaign_name:s}.{station_name:s}.s{start_time:%Y%m%d%H%M%S}.e{end_time:%Y%m%d%H%M%S}"
     ".{version:s}.{data_format:s}"
 )
-DISDRODB_FNAME_L2E_PATTERN = (  # also L0C and L1 --> accumulation_acronym = sample_interval
-    "{product:s}.{accumulation_acronym}.{campaign_name:s}.{station_name:s}.s{start_time:%Y%m%d%H%M%S}.e{end_time:%Y%m%d%H%M%S}"
+DISDRODB_FNAME_L2E_PATTERN = (  # also L0C and L1
+    "{product:s}.{temporal_resolution}.{campaign_name:s}.{station_name:s}.s{start_time:%Y%m%d%H%M%S}.e{end_time:%Y%m%d%H%M%S}"
     ".{version:s}.{data_format:s}"
 )
 
 DISDRODB_FNAME_L2M_PATTERN = (
-    "{product:s}_{subproduct:s}.{accumulation_acronym}.{campaign_name:s}.{station_name:s}.s{start_time:%Y%m%d%H%M%S}.e{end_time:%Y%m%d%H%M%S}"
+    "{product:s}_{subproduct:s}.{temporal_resolution}.{campaign_name:s}.{station_name:s}.s{start_time:%Y%m%d%H%M%S}.e{end_time:%Y%m%d%H%M%S}"
     ".{version:s}.{data_format:s}"
 )
 
@@ -76,8 +76,8 @@ def _get_info_from_filename(filename):
         raise ValueError(f"{filename} can not be parsed. Report the issue.")
 
     # Add additional information to info dictionary
-    if "accumulation_acronym" in info_dict:
-        info_dict["sample_interval"] = acronym_to_seconds(info_dict["accumulation_acronym"])
+    if "temporal_resolution" in info_dict:
+        info_dict["sample_interval"] = temporal_resolution_to_seconds(info_dict["temporal_resolution"])
 
     # Return info dictionary
     return info_dict
@@ -162,8 +162,8 @@ def get_start_end_time_from_filepaths(filepaths):
 
 def get_sample_interval_from_filepaths(filepaths):
     """Return the sample interval of the specified files."""
-    list_accumulation_acronym = get_key_from_filepaths(filepaths, key="accumulation_acronym")
-    list_sample_interval = [acronym_to_seconds(s) for s in list_accumulation_acronym]
+    list_temporal_resolution = get_key_from_filepaths(filepaths, key="temporal_resolution")
+    list_sample_interval = [temporal_resolution_to_seconds(s) for s in list_temporal_resolution]
     return list_sample_interval
 
 
@@ -345,7 +345,7 @@ FILE_KEYS = [
     "start_time",
     "end_time",
     "data_format",
-    "accumulation_acronym",
+    "temporal_resolution",
     "sample_interval",
 ]
 
@@ -410,8 +410,8 @@ def get_time_component(time, component):
     return str(func_dict[component](time))
 
 
-def _get_groups_value(groups, filepath):
-    """Return the value associated to the groups keys.
+def get_groups_value(groups, filepath):
+    """Return a string associated to the groups keys.
 
     If multiple keys are specified, the value returned is a string of format: ``<group_value_1>/<group_value_2>/...``
 
@@ -444,7 +444,7 @@ def group_filepaths(filepaths, groups=None):
     groups: list or str
         The group keys by which to group the filepaths.
         Valid group keys are ``product``, ``subproduct``, ``campaign_name``, ``station_name``,
-        ``start_time``, ``end_time``,``accumulation_acronym``,``sample_interval``,
+        ``start_time``, ``end_time``,``temporal_resolution``,``sample_interval``,
         ``data_format``,
         ``year``, ``month``, ``day``,  ``doy``, ``dow``, ``hour``, ``minute``, ``second``,
         ``month_name``, ``quarter``, ``season``.
@@ -463,5 +463,5 @@ def group_filepaths(filepaths, groups=None):
         return filepaths
     groups = check_groups(groups)
     filepaths_dict = defaultdict(list)
-    _ = [filepaths_dict[_get_groups_value(groups, filepath)].append(filepath) for filepath in filepaths]
+    _ = [filepaths_dict[get_groups_value(groups, filepath)].append(filepath) for filepath in filepaths]
     return dict(filepaths_dict)