eo-tides 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

eo_tides/__init__.py

@@ -28,9 +28,9 @@ validation : Load observed tide gauge data to validate modelled tides
 
 # Import commonly used functions for convenience
 from .eo import pixel_tides, tag_tides
-from .model import list_models, model_phases, model_tides
+from .model import model_phases, model_tides
 from .stats import pixel_stats, tide_stats
-from .utils import idw
+from .utils import clip_models, idw, list_models
 from .validation import eval_metrics, load_gauge_gesla
 
 # Define what should be imported with "from eo_tides import *"
@@ -42,7 +42,8 @@ __all__ = [
     "pixel_tides",
     "tide_stats",
     "pixel_stats",
-    "idw",
     "eval_metrics",
     "load_gauge_gesla",
+    "clip_models",
+    "idw",
 ]

eo_tides/eo.py

@@ -8,17 +8,15 @@ from typing import TYPE_CHECKING
 
 import numpy as np
 import odc.geo.xr
-import pandas as pd
 import xarray as xr
 from odc.geo.geobox import GeoBox
 
 # Only import if running type checking
 if TYPE_CHECKING:
-    import datetime
-
     from odc.geo import Shape2d
 
-from .model import DatetimeLike, _standardise_time, model_tides
+from .model import model_tides
+from .utils import DatetimeLike, _standardise_time
 
 
 def _resample_chunks(

eo_tides/model.py

@@ -1,15 +1,14 @@
 # Used to postpone evaluation of type annotations
 from __future__ import annotations
 
-import datetime
 import os
-import pathlib
 import textwrap
-import warnings
 from concurrent.futures import ProcessPoolExecutor
 from concurrent.futures.process import BrokenProcessPool
 from functools import partial
-from typing import TYPE_CHECKING, List, Union
+from typing import TYPE_CHECKING
+
+import psutil
 
 # Only import if running type checking
 if TYPE_CHECKING:
@@ -20,309 +19,9 @@ import numpy as np
 import pandas as pd
 import pyproj
 import pyTMD
-from colorama import Style, init
-from pyTMD.io.model import load_database, model
 from tqdm import tqdm
 
-from .utils import idw
-
-# Type alias for all possible inputs to "time" params
-DatetimeLike = Union[np.ndarray, pd.DatetimeIndex, pd.Timestamp, datetime.datetime, str, List[str]]
-
-
-def _set_directory(
-    directory: str | os.PathLike | None = None,
-) -> os.PathLike:
-    """
-    Set tide modelling files directory. If no custom
-    path is provided, try global environmental variable
-    instead.
-    """
-    if directory is None:
-        if "EO_TIDES_TIDE_MODELS" in os.environ:
-            directory = os.environ["EO_TIDES_TIDE_MODELS"]
-        else:
-            raise Exception(
-                "No tide model directory provided via `directory`, and/or no "
-                "`EO_TIDES_TIDE_MODELS` environment variable found. "
-                "Please provide a valid path to your tide model directory."
-            )
-
-    # Verify path exists
-    directory = pathlib.Path(directory).expanduser()
-    if not directory.exists():
-        raise FileNotFoundError(f"No valid tide model directory found at path `{directory}`")
-    else:
-        return directory
-
-
-def _standardise_time(
-    time: DatetimeLike | None,
-) -> np.ndarray | None:
-    """
-    Accept any time format accepted by `pd.to_datetime`,
-    and return a datetime64 ndarray. Return None if None
-    passed.
-    """
-    # Return time as-is if None
-    if time is None:
-        return None
-
-    # Use pd.to_datetime for conversion, then convert to numpy array
-    time = pd.to_datetime(time).to_numpy().astype("datetime64[ns]")
-
-    # Ensure that data has at least one dimension
-    return np.atleast_1d(time)
-
-
-def list_models(
-    directory: str | os.PathLike | None = None,
-    show_available: bool = True,
-    show_supported: bool = True,
-    raise_error: bool = False,
-) -> tuple[list[str], list[str]]:
-    """
-    List all tide models available for tide modelling.
-
-    This function scans the specified tide model directory
-    and returns a list of models that are available in the
-    directory as well as the full list of all models supported
-    by `eo-tides` and `pyTMD`.
-
-    For instructions on setting up tide models, see:
-    <https://geoscienceaustralia.github.io/eo-tides/setup/>
-
-    Parameters
-    ----------
-    directory : str, optional
-        The directory containing tide model data files. If no path is
-        provided, this will default to the environment variable
-        `EO_TIDES_TIDE_MODELS` if set, or raise an error if not.
-        Tide modelling files should be stored in sub-folders for each
-        model that match the structure required by `pyTMD`
-        (<https://geoscienceaustralia.github.io/eo-tides/setup/>).
-    show_available : bool, optional
-        Whether to print a list of locally available models.
-    show_supported : bool, optional
-        Whether to print a list of all supported models, in
-        addition to models available locally.
-    raise_error : bool, optional
-        If True, raise an error if no available models are found.
-        If False, raise a warning.
-
-    Returns
-    -------
-    available_models : list of str
-        A list of all tide models available within `directory`.
-    supported_models : list of str
-        A list of all tide models supported by `eo-tides`.
-    """
-    init()  # Initialize colorama
-
-    # Set tide modelling files directory. If no custom path is
-    # provided, try global environment variable.
-    directory = _set_directory(directory)
-
-    # Get full list of supported models from pyTMD database
-    model_database = load_database()["elevation"]
-    supported_models = list(model_database.keys())
-
-    # Extract expected model paths
-    expected_paths = {}
-    for m in supported_models:
-        model_file = model_database[m]["model_file"]
-        model_file = model_file[0] if isinstance(model_file, list) else model_file
-        expected_paths[m] = str(directory / pathlib.Path(model_file).expanduser().parent)
-
-    # Define column widths
-    status_width = 4  # Width for emoji
-    name_width = max(len(name) for name in supported_models)
-    path_width = max(len(path) for path in expected_paths.values())
-
-    # Print list of supported models, marking available and
-    # unavailable models and appending available to list
-    if show_available or show_supported:
-        total_width = min(status_width + name_width + path_width + 6, 80)
-        print("─" * total_width)
-        print(f"{'󠀠🌊':^{status_width}} | {'Model':<{name_width}} | {'Expected path':<{path_width}}")
-        print("─" * total_width)
-
-    available_models = []
-    for m in supported_models:
-        try:
-            model_file = model(directory=directory).elevation(m=m)
-            available_models.append(m)
-
-            if show_available:
-                # Mark available models with a green tick
-                status = "✅"
-                print(f"{status:^{status_width}}│ {m:<{name_width}} │ {expected_paths[m]:<{path_width}}")
-        except FileNotFoundError:
-            if show_supported:
-                # Mark unavailable models with a red cross
-                status = "❌"
-                print(
-                    f"{status:^{status_width}}│ {Style.DIM}{m:<{name_width}} │ {expected_paths[m]:<{path_width}}{Style.RESET_ALL}"
-                )
-
-    if show_available or show_supported:
-        print("─" * total_width)
-
-        # Print summary
-        print(f"\n{Style.BRIGHT}Summary:{Style.RESET_ALL}")
-        print(f"Available models: {len(available_models)}/{len(supported_models)}")
-
-    # Raise error or warning if no models are available
-    if not available_models:
-        warning_msg = textwrap.dedent(
-            f"""
-            No valid tide models are available in `{directory}`.
-            Are you sure you have provided the correct `directory` path, or set the
-            `EO_TIDES_TIDE_MODELS` environment variable to point to the location of your
-            tide model directory?
-            """
-        ).strip()
-
-        if raise_error:
-            raise Exception(warning_msg)
-        else:
-            warnings.warn(warning_msg, UserWarning)
-
-    # Return list of available and supported models
-    return available_models, supported_models
-
-
-def _model_tides(
-    model,
-    x,
-    y,
-    time,
-    directory,
-    crs,
-    crop,
-    method,
-    extrapolate,
-    cutoff,
-    output_units,
-    mode,
-):
-    """Worker function applied in parallel by `model_tides`. Handles the
-    extraction of tide modelling constituents and tide modelling using
-    `pyTMD`.
-    """
-    # Obtain model details
-    pytmd_model = pyTMD.io.model(directory).elevation(model)
-
-    # Reproject x, y to latitude/longitude
-    transformer = pyproj.Transformer.from_crs(crs, "EPSG:4326", always_xy=True)
-    lon, lat = transformer.transform(x.flatten(), y.flatten())
-
-    # Convert datetime
-    timescale = pyTMD.time.timescale().from_datetime(time.flatten())
-
-    try:
-        # Read tidal constants and interpolate to grid points
-        amp, ph, c = pytmd_model.extract_constants(
-            lon,
-            lat,
-            type=pytmd_model.type,
-            crop=crop,
-            bounds=None,
-            method=method,
-            extrapolate=extrapolate,
-            cutoff=cutoff,
-            append_node=False,
-            # append_node=True,
-        )
-
-        # TODO: Return constituents
-        # print(amp.shape, ph.shape, c)
-        # print(pd.DataFrame({"amplitude": amp}))
-
-    # Raise error if constituent files no not cover analysis extent
-    except IndexError as e:
-        error_msg = f"""
-        The {model} tide model constituent files do not cover the requested analysis extent.
-        This can occur if you are using clipped model files to improve run times.
-        Consider using model files that cover your entire analysis area, or set `crop=False`
-        to reduce the extent of tide model constituent files that is loaded.
-        """
-        raise Exception(textwrap.dedent(error_msg).strip()) from None
-
-    # Calculate complex phase in radians for Euler's
-    cph = -1j * ph * np.pi / 180.0
-
-    # Calculate constituent oscillation
-    hc = amp * np.exp(cph)
-
-    # Compute deltat based on model
-    if pytmd_model.corrections in ("OTIS", "ATLAS", "TMD3", "netcdf"):
-        # Use delta time at 2000.0 to match TMD outputs
-        deltat = np.zeros_like(timescale.tt_ut1)
-    else:
-        # Use interpolated delta times
-        deltat = timescale.tt_ut1
-
-    # Determine the number of points and times to process. If in
-    # "one-to-many" mode, these counts are used to repeat our extracted
-    # constituents and timesteps so we can extract tides for all
-    # combinations of our input times and tide modelling points.
-    # If in "one-to-many" mode, repeat constituents to length of time
-    # and number of input coords before passing to `predict_tide_drift`
-    # If in "one-to-one" mode, we avoid this step by setting counts to 1
-    # (e.g. "repeat 1 times")
-    points_repeat = len(x) if mode == "one-to-many" else 1
-    time_repeat = len(time) if mode == "one-to-many" else 1
-    t, hc, deltat = (
-        np.tile(timescale.tide, points_repeat),
-        hc.repeat(time_repeat, axis=0),
-        np.tile(deltat, points_repeat),
-    )
-
-    # Create arrays to hold outputs
-    tide = np.ma.zeros((len(t)), fill_value=np.nan)
-    tide.mask = np.any(hc.mask, axis=1)
-
-    # Predict tidal elevations at time and infer minor corrections
-    tide.data[:] = pyTMD.predict.drift(
-        t,
-        hc,
-        c,
-        deltat=deltat,
-        corrections=pytmd_model.corrections,
-    )
-    minor = pyTMD.predict.infer_minor(
-        t,
-        hc,
-        c,
-        deltat=deltat,
-        corrections=pytmd_model.corrections,
-        minor=pytmd_model.minor,
-    )
-    tide.data[:] += minor.data[:]
-
-    # Replace invalid values with fill value
-    tide.data[tide.mask] = tide.fill_value
-
-    # Convert data to pandas.DataFrame, and set index to our input
-    # time/x/y values
-    tide_df = pd.DataFrame({
-        "time": np.tile(time, points_repeat),
-        "x": np.repeat(x, time_repeat),
-        "y": np.repeat(y, time_repeat),
-        "tide_model": model,
-        "tide_height": tide,
-    }).set_index(["time", "x", "y"])
-
-    # Optionally convert outputs to integer units (can save memory)
-    if output_units == "m":
-        tide_df["tide_height"] = tide_df.tide_height.astype(np.float32)
-    elif output_units == "cm":
-        tide_df["tide_height"] = (tide_df.tide_height * 100).astype(np.int16)
-    elif output_units == "mm":
-        tide_df["tide_height"] = (tide_df.tide_height * 1000).astype(np.int16)
-
-    return tide_df
+from .utils import DatetimeLike, _set_directory, _standardise_time, idw, list_models
 
 
 def _ensemble_model(
@@ -490,6 +189,180 @@ def _ensemble_model(
     return pd.concat(ensemble_list)
 
 
+def _parallel_splits(
+    total_points: int,
+    model_count: int,
+    parallel_max: int | None = None,
+    min_points_per_split: int = 1000,
+) -> int:
+    """
+    Calculates the optimal number of parallel splits for data
+    processing based on system resources and processing constraints.
+
+    Parameters:
+    -----------
+    total_points : int
+        Total number of data points to process
+    model_count : int
+        Number of models that will be run in parallel
+    parallel_max : int, optional
+        Maximum number of parallel processes to use. If None, uses CPU core count
+    min_points_per_split : int, default=1000
+        Minimum number of points that should be processed in each split
+    """
+    # Get available CPUs. First see if `CPU_GUARANTEE` exists in
+    # environment (if running in JupyterHub); if not use psutil
+    # followed by standard CPU count
+    if parallel_max is None:
+        # Take the first valid output
+        raw_value = os.environ.get("CPU_GUARANTEE") or psutil.cpu_count(logical=False) or os.cpu_count() or 1
+
+        # Convert to integer
+        if isinstance(raw_value, str):
+            parallel_max = int(float(raw_value))
+        else:
+            parallel_max = int(raw_value)
+
+    # Calculate optimal number of splits based on constraints
+    splits_by_size = total_points / min_points_per_split
+    splits_by_cpu = parallel_max / model_count
+    optimal_splits = min(splits_by_size, splits_by_cpu)
+
+    # Convert to integer and ensure at least 1 split
+    final_split_count = int(max(1, optimal_splits))
+    return final_split_count
+
+
+def _model_tides(
+    model,
+    x,
+    y,
+    time,
+    directory,
+    crs,
+    crop,
+    method,
+    extrapolate,
+    cutoff,
+    output_units,
+    mode,
+):
+    """Worker function applied in parallel by `model_tides`. Handles the
+    extraction of tide modelling constituents and tide modelling using
+    `pyTMD`.
+    """
+    # Obtain model details
+    pytmd_model = pyTMD.io.model(directory).elevation(model)
+
+    # Reproject x, y to latitude/longitude
+    transformer = pyproj.Transformer.from_crs(crs, "EPSG:4326", always_xy=True)
+    lon, lat = transformer.transform(x.flatten(), y.flatten())
+
+    # Convert datetime
+    timescale = pyTMD.time.timescale().from_datetime(time.flatten())
+
+    try:
+        # Read tidal constants and interpolate to grid points
+        amp, ph, c = pytmd_model.extract_constants(
+            lon,
+            lat,
+            type=pytmd_model.type,
+            crop=crop,
+            method=method,
+            extrapolate=extrapolate,
+            cutoff=cutoff,
+            append_node=False,
+            # append_node=True,
+        )
+
+        # TODO: Return constituents
+        # print(amp.shape, ph.shape, c)
+        # print(pd.DataFrame({"amplitude": amp}))
+
+    # Raise error if constituent files no not cover analysis extent
+    except IndexError:
+        error_msg = f"""
+        The {model} tide model constituent files do not cover the analysis extent
+        ({min(lon):.2f}, {max(lon):.2f}, {min(lat):.2f}, {max(lat):.2f}).
+        This can occur if you are using clipped model files to improve run times.
+        Consider using model files that cover your entire analysis area, or set `crop=False`
+        to reduce the extent of tide model constituent files that is loaded.
+        """
+        raise Exception(textwrap.dedent(error_msg).strip()) from None
+
+    # Calculate complex phase in radians for Euler's
+    cph = -1j * ph * np.pi / 180.0
+
+    # Calculate constituent oscillation
+    hc = amp * np.exp(cph)
+
+    # Compute delta times based on model
+    if pytmd_model.corrections in ("OTIS", "ATLAS", "TMD3", "netcdf"):
+        # Use delta time at 2000.0 to match TMD outputs
+        deltat = np.zeros_like(timescale.tt_ut1)
+    else:
+        # Use interpolated delta times
+        deltat = timescale.tt_ut1
+
+    # In "one-to-many" mode, extracted tidal constituents and timesteps
+    # are repeated/multiplied out to match the number of input points and
+    # timesteps, enabling the modeling of tides across all combinations
+    # of input times and points. In "one-to-one" mode, no repetition is
+    # needed, so each repeat count is set to 1.
+    points_repeat = len(x) if mode == "one-to-many" else 1
+    time_repeat = len(time) if mode == "one-to-many" else 1
+    t, hc, deltat = (
+        np.tile(timescale.tide, points_repeat),
+        hc.repeat(time_repeat, axis=0),
+        np.tile(deltat, points_repeat),
+    )
+
+    # Create arrays to hold outputs
+    tide = np.ma.zeros((len(t)), fill_value=np.nan)
+    tide.mask = np.any(hc.mask, axis=1)
+
+    # Predict tidal elevations at time and infer minor corrections
+    tide.data[:] = pyTMD.predict.drift(
+        t,
+        hc,
+        c,
+        deltat=deltat,
+        corrections=pytmd_model.corrections,
+    )
+    minor = pyTMD.predict.infer_minor(
+        t,
+        hc,
+        c,
+        deltat=deltat,
+        corrections=pytmd_model.corrections,
+        minor=pytmd_model.minor,
+    )
+    tide.data[:] += minor.data[:]
+
+    # Replace invalid values with fill value
+    tide.data[tide.mask] = tide.fill_value
+
+    # Convert data to pandas.DataFrame, and set index to our input
+    # time/x/y values
+    tide_df = pd.DataFrame({
+        "time": np.tile(time, points_repeat),
+        "x": np.repeat(x, time_repeat),
+        "y": np.repeat(y, time_repeat),
+        "tide_model": model,
+        "tide_height": tide,
+    }).set_index(["time", "x", "y"])
+
+    # Optionally convert outputs to integer units (can save memory)
+    if output_units == "m":
+        tide_df["tide_height"] = tide_df.tide_height.astype(np.float32)
+    elif output_units == "cm":
+        tide_df["tide_height"] = (tide_df.tide_height * 100).astype(np.int16)
+    elif output_units == "mm":
+        tide_df["tide_height"] = (tide_df.tide_height * 1000).astype(np.int16)
+
+    return tide_df
+
+
 def model_tides(
     x: float | list[float] | xr.DataArray,
     y: float | list[float] | xr.DataArray,
@@ -498,12 +371,13 @@ def model_tides(
     directory: str | os.PathLike | None = None,
     crs: str = "EPSG:4326",
     crop: bool = True,
-    method: str = "spline",
+    method: str = "linear",
     extrapolate: bool = True,
     cutoff: float | None = None,
     mode: str = "one-to-many",
     parallel: bool = True,
-    parallel_splits: int = 5,
+    parallel_splits: int | str = "auto",
+    parallel_max: int | None = None,
     output_units: str = "m",
     output_format: str = "long",
     ensemble_models: list[str] | None = None,
@@ -564,11 +438,11 @@ def model_tides(
         1 degree buffer around all input points. Defaults to True.
     method : str, optional
         Method used to interpolate tidal constituents
-        from model files. Options include:
+        from model files. Defaults to "linear"; options include:
 
-        - "spline": scipy bivariate spline interpolation (default)
-        - "bilinear": quick bilinear interpolation
         - "linear", "nearest": scipy regular grid interpolations
+        - "spline": scipy bivariate spline interpolation
+        - "bilinear": quick bilinear interpolation
     extrapolate : bool, optional
         Whether to extrapolate tides for x and y coordinates outside of
         the valid tide modelling domain using nearest-neighbor.
@@ -594,12 +468,16 @@ def model_tides(
         parallel. Optionally, tide modelling can also be run in parallel
         across input x and y coordinates (see "parallel_splits" below).
         Default is True.
-    parallel_splits : int, optional
+    parallel_splits : str or int, optional
         Whether to split the input x and y coordinates into smaller,
         evenly-sized chunks that are processed in parallel. This can
         provide a large performance boost when processing large numbers
-        of coordinates. The default is 5 chunks, which will split
-        coordinates into 5 parallelised chunks.
+        of coordinates. The default is "auto", which will automatically
+        attempt to determine optimal splits based on available CPUs,
+        the number of input points, and the number of models.
+    parallel_max : int, optional
+        Maximum number of processes to run in parallel. The default of
+        None will automatically determine this from your available CPUs.
     output_units : str, optional
         Whether to return modelled tides in floating point metre units,
         or integer centimetre units (i.e. scaled by 100) or integer
@@ -729,13 +607,28 @@ def model_tides(
         mode=mode,
     )
 
-    # Ensure requested parallel splits is not smaller than number of points
-    parallel_splits = min(parallel_splits, len(x))
+    # If automatic parallel splits, calculate optimal value
+    # based on available parallelisation, number of points
+    # and number of models
+    if parallel_splits == "auto":
+        parallel_splits = _parallel_splits(
+            total_points=len(x),
+            model_count=len(models_to_process),
+            parallel_max=parallel_max,
+        )
+
+    # Verify that parallel splits are not larger than number of points
+    assert isinstance(parallel_splits, int)
+    if parallel_splits > len(x):
+        raise ValueError(f"Parallel splits ({parallel_splits}) cannot be larger than the number of points ({len(x)}).")
 
     # Parallelise if either multiple models or multiple splits requested
+
     if parallel & ((len(models_to_process) > 1) | (parallel_splits > 1)):
-        with ProcessPoolExecutor() as executor:
-            print(f"Modelling tides using {', '.join(models_to_process)} in parallel")
+        with ProcessPoolExecutor(max_workers=parallel_max) as executor:
+            print(
+                f"Modelling tides with {', '.join(models_to_process)} in parallel (models: {len(models_to_process)}, splits: {parallel_splits})"
+            )
 
             # Optionally split lon/lat points into `splits_n` chunks
             # that will be applied in parallel
@@ -783,7 +676,7 @@ def model_tides(
         model_outputs = []
 
         for model_i in models_to_process:
-            print(f"Modelling tides using {model_i}")
+            print(f"Modelling tides with {model_i}")
             tide_df = iter_func(model_i, x, y, time)
             model_outputs.append(tide_df)
 

eo_tides/stats.py

@@ -6,20 +6,18 @@ from typing import TYPE_CHECKING
 
 import matplotlib.pyplot as plt
 import numpy as np
-import odc.geo.xr
 import pandas as pd
 import xarray as xr
 from scipy import stats
 
 # Only import if running type checking
 if TYPE_CHECKING:
-    import datetime
-
     import xarray as xr
     from odc.geo.geobox import GeoBox
 
 from .eo import _standardise_inputs, pixel_tides, tag_tides
-from .model import DatetimeLike, model_tides
+from .model import model_tides
+from .utils import DatetimeLike
 
 
 def _plot_biases(

eo_tides/utils.py

@@ -1,5 +1,457 @@
+# Used to postpone evaluation of type annotations
+from __future__ import annotations
+
+import datetime
+import os
+import pathlib
+import textwrap
+import warnings
+from typing import List, Union
+
 import numpy as np
+import odc.geo
+import pandas as pd
+import xarray as xr
+from colorama import Style, init
+from odc.geo.geom import BoundingBox
+from pyTMD.io.model import load_database
+from pyTMD.io.model import model as pytmd_model
 from scipy.spatial import cKDTree as KDTree
+from tqdm import tqdm
+
+# Type alias for all possible inputs to "time" params
+DatetimeLike = Union[np.ndarray, pd.DatetimeIndex, pd.Timestamp, datetime.datetime, str, List[str]]
+
+
+def _set_directory(
+    directory: str | os.PathLike | None = None,
+) -> os.PathLike:
+    """
+    Set tide modelling files directory. If no custom
+    path is provided, try global `EO_TIDES_TIDE_MODELS`
+    environmental variable instead.
+    """
+    if directory is None:
+        if "EO_TIDES_TIDE_MODELS" in os.environ:
+            directory = os.environ["EO_TIDES_TIDE_MODELS"]
+        else:
+            raise Exception(
+                "No tide model directory provided via `directory`, and/or no "
+                "`EO_TIDES_TIDE_MODELS` environment variable found. "
+                "Please provide a valid path to your tide model directory."
+            )
+
+    # Verify path exists
+    directory = pathlib.Path(directory).expanduser()
+    if not directory.exists():
+        raise FileNotFoundError(f"No valid tide model directory found at path `{directory}`")
+    else:
+        return directory
+
+
+def _standardise_time(
+    time: DatetimeLike | None,
+) -> np.ndarray | None:
+    """
+    Accept any time format accepted by `pd.to_datetime`,
+    and return a datetime64 ndarray. Return None if None
+    passed.
+    """
+    # Return time as-is if None
+    if time is None:
+        return None
+
+    # Use pd.to_datetime for conversion, then convert to numpy array
+    time = pd.to_datetime(time).to_numpy().astype("datetime64[ns]")
+
+    # Ensure that data has at least one dimension
+    return np.atleast_1d(time)
+
+
+def _clip_model_file(
+    nc: xr.Dataset,
+    bbox: BoundingBox,
+    ydim: str,
+    xdim: str,
+    ycoord: str,
+    xcoord: str,
+) -> xr.Dataset:
+    """
+    Clips tide model netCDF datasets to a bounding box.
+
+    If the bounding box crosses 0 degrees longitude (e.g. Greenwich),
+    the function will clip the dataset into two parts and concatenate
+    them along the x-dimension to create a continuous result.
+
+    Parameters
+    ----------
+    nc : xr.Dataset
+        Input tide model xarray dataset.
+    bbox : odc.geo.geom.BoundingBox
+        A BoundingBox object for clipping the dataset in EPSG:4326
+        degrees coordinates. For example:
+        `BoundingBox(left=108, bottom=-48, right=158, top=-6, crs='EPSG:4326')`
+    ydim : str
+        The name of the xarray dimension representing the y-axis.
+        Depending on the tide model, this may or may not contain
+        actual latitude values.
+    xdim : str
+        The name of the xarray dimension representing the x-axis.
+        Depending on the tide model, this may or may not contain
+        actual longitude values.
+    ycoord : str
+        The name of the coordinate, variable or dimension containing
+        actual latitude values used for clipping the data.
+    xcoord : str
+        The name of the coordinate, variable or dimension containing
+        actual longitude values used for clipping the data.
+
+    Returns
+    -------
+    xr.Dataset
+        A dataset clipped to the specified bounding box, with
+        appropriate adjustments if the bounding box crosses 0
+        degrees longitude.
+
+    Examples
+    --------
+    >>> nc = xr.open_dataset("GOT5.5/ocean_tides/2n2.nc")
+    >>> bbox = BoundingBox(left=108, bottom=-48, right=158, top=-6, crs='EPSG:4326')
+    >>> clipped_nc = _clip_model_file(nc, bbox,  xdim="lon", ydim="lat", ycoord="latitude", xcoord="longitude")
+    """
+
+    # Extract x and y coords from xarray and load into memory
+    xcoords = nc[xcoord].compute()
+    ycoords = nc[ycoord].compute()
+
+    # If data falls within 0-360 degree bounds, then clip directly
+    if (bbox.left >= 0) & (bbox.right <= 360):
+        nc_clipped = nc.sel({
+            ydim: (ycoords >= bbox.bottom) & (ycoords <= bbox.top),
+            xdim: (xcoords >= bbox.left) & (xcoords <= bbox.right),
+        })
+
+    # If bbox crosses zero longitude, extract left and right
+    # separately and then combine into one concatenated dataset
+    elif (bbox.left < 0) & (bbox.right > 0):
+        # Convert longitudes to 0-360 range
+        left = bbox.left % 360
+        right = bbox.right % 360
+
+        # Extract data from left of 0 longitude, and convert lon
+        # coords to -180 to 0 range to enable continuous interpolation
+        # across 0 boundary
+        nc_left = nc.sel({
+            ydim: (ycoords >= bbox.bottom) & (ycoords <= bbox.top),
+            xdim: (xcoords >= left) & (xcoords <= 360),
+        }).assign({xcoord: lambda x: x[xcoord] - 360})
+
+        # Convert additional lon variables for TXPO
+        if "lon_v" in nc_left:
+            nc_left = nc_left.assign({
+                "lon_v": lambda x: x["lon_v"] - 360,
+                "lon_u": lambda x: x["lon_u"] - 360,
+            })
+
+        # Extract data to right of 0 longitude
+        nc_right = nc.sel({
+            ydim: (ycoords >= bbox.bottom) & (ycoords <= bbox.top),
+            xdim: (xcoords > 0) & (xcoords <= right),
+        })
+
+        # Combine left and right data along x dimension
+        nc_clipped = xr.concat([nc_left, nc_right], dim=xdim)
+
+        # Hack fix to remove expanded x dim on lat variables issue
+        # for TPXO data; remove x dim by selecting the first obs
+        for i in ["lat_z", "lat_v", "lat_u", "con"]:
+            try:
+                nc_clipped[i] = nc_clipped[i].isel(nx=0)
+            except:
+                pass
+
+    return nc_clipped
+
+
+def clip_models(
+    input_directory: str | os.PathLike,
+    output_directory: str | os.PathLike,
+    bbox: tuple[float, float, float, float],
+    model: list | None = None,
+    buffer: float = 1,
+    overwrite: bool = False,
+):
+    """
+    Clip NetCDF-format ocean tide models to a bounding box.
+
+    This function identifies all NetCDF-format tide models in a
+    given input directory, including "ATLAS-netcdf" (e.g. TPXO9-atlas-nc),
+    "FES-netcdf" (e.g. FES2022, EOT20), and "GOT-netcdf" (e.g. GOT5.5)
+    format files. Files for each model are then clipped to the extent of
+    the provided bounding box, handling model-specific file structures.
+    After each model is clipped, the result is exported to the output
+    directory and verified with `pyTMD` to ensure the clipped data is
+    suitable for tide modelling.
+
+    For instructions on accessing and downloading tide models, see:
+    <https://geoscienceaustralia.github.io/eo-tides/setup/>
+
+    Parameters
+    ----------
+    input_directory : str or os.PathLike
+        Path to directory containing input NetCDF-format tide model files.
+    output_directory : str or os.PathLike
+        Path to directory where clipped NetCDF files will be exported.
+    bbox : tuple of float
+        Bounding box for clipping the tide models in EPSG:4326 degrees
+        coordinates, specified as `(left, bottom, right, top)`.
+    model : str or list of str, optional
+        The tide model (or models) to clip. Defaults to None, which
+        will automatically identify and clip all NetCDF-format models
+        in the input directly.
+    buffer : float, optional
+        Buffer distance (in degrees) added to the bounding box to provide
+        sufficient data on edges of study area. Defaults to 1 degree.
+    overwrite : bool, optional
+        If True, overwrite existing files in the output directory.
+        Defaults to False.
+
+    Examples
+    --------
+    >>> clip_models(
+    ...     input_directory="tide_models/",
+    ...     output_directory="tide_models_clipped/",
+    ...     bbox=(-8.968392, 50.070574, 2.447160, 59.367122),
+    ... )
+    """
+
+    # Get input and output paths
+    input_directory = _set_directory(input_directory)
+    output_directory = pathlib.Path(output_directory)
+
+    # Prepare bounding box
+    bbox = odc.geo.geom.BoundingBox(*bbox, crs="EPSG:4326").buffered(buffer)
+
+    # Identify NetCDF models
+    model_database = load_database()["elevation"]
+    netcdf_formats = ["ATLAS-netcdf", "FES-netcdf", "GOT-netcdf"]
+    netcdf_models = {k for k, v in model_database.items() if v["format"] in netcdf_formats}
+
+    # Identify subset of available and requested NetCDF models
+    available_models, _ = list_models(directory=input_directory, show_available=False, show_supported=False)
+    requested_models = list(np.atleast_1d(model)) if model is not None else available_models
+    available_netcdf_models = list(set(available_models) & set(requested_models) & set(netcdf_models))
+
+    # Raise error if no valid models found
+    if len(available_netcdf_models) == 0:
+        raise ValueError(f"No valid NetCDF models found in {input_directory}.")
+
+    # If model list is provided,
+    print(f"Preparing to clip suitable NetCDF models: {available_netcdf_models}\n")
+
+    # Loop through suitable models and export
+    for m in available_netcdf_models:
+        # Get model file and grid file list if they exist
+        model_files = model_database[m].get("model_file", [])
+        grid_file = model_database[m].get("grid_file", [])
+
+        # Convert to list if strings and combine
+        model_files = model_files if isinstance(model_files, list) else [model_files]
+        grid_file = grid_file if isinstance(grid_file, list) else [grid_file]
+        all_files = model_files + grid_file
+
+        # Loop through each model file and clip
+        for file in tqdm(all_files, desc=f"Clipping {m}"):
+            # Skip if it exists in output directory
+            if (output_directory / file).exists() and not overwrite:
+                continue
+
+            # Load model file
+            nc = xr.open_mfdataset(input_directory / file)
+
+            # Open file and clip according to model
+            if m in (
+                "GOT5.5",
+                "GOT5.5_load",
+                "GOT5.5_extrapolated",
+                "GOT5.5D",
+                "GOT5.5D_extrapolated",
+                "GOT5.6",
+                "GOT5.6_extrapolated",
+            ):
+                nc_clipped = _clip_model_file(
+                    nc,
+                    bbox,
+                    xdim="lon",
+                    ydim="lat",
+                    ycoord="latitude",
+                    xcoord="longitude",
+                )
+
+            elif m in ("HAMTIDE11",):
+                nc_clipped = _clip_model_file(nc, bbox, xdim="LON", ydim="LAT", ycoord="LAT", xcoord="LON")
+
+            elif m in (
+                "EOT20",
+                "EOT20_load",
+                "FES2012",
+                "FES2014",
+                "FES2014_extrapolated",
+                "FES2014_load",
+                "FES2022",
+                "FES2022_extrapolated",
+                "FES2022_load",
+            ):
+                nc_clipped = _clip_model_file(nc, bbox, xdim="lon", ydim="lat", ycoord="lat", xcoord="lon")
+
+            elif m in (
+                "TPXO8-atlas-nc",
+                "TPXO9-atlas-nc",
+                "TPXO9-atlas-v2-nc",
+                "TPXO9-atlas-v3-nc",
+                "TPXO9-atlas-v4-nc",
+                "TPXO9-atlas-v5-nc",
+                "TPXO10-atlas-v2-nc",
+            ):
+                nc_clipped = _clip_model_file(
+                    nc,
+                    bbox,
+                    xdim="nx",
+                    ydim="ny",
+                    ycoord="lat_z",
+                    xcoord="lon_z",
+                )
+
+            else:
+                raise Exception(f"Model {m} not supported")
+
+            # Create directory and export
+            (output_directory / file).parent.mkdir(parents=True, exist_ok=True)
+            nc_clipped.to_netcdf(output_directory / file, mode="w")
+
+        # Verify that models are ready
+        pytmd_model(directory=output_directory).elevation(m=m).verify
+        print(" ✅ Clipped model exported and verified")
+
+    print(f"\nOutputs exported to {output_directory}")
+    list_models(directory=output_directory, show_available=True, show_supported=False)
+
+
+def list_models(
+    directory: str | os.PathLike | None = None,
+    show_available: bool = True,
+    show_supported: bool = True,
+    raise_error: bool = False,
+) -> tuple[list[str], list[str]]:
+    """
+    List all tide models available for tide modelling.
+
+    This function scans the specified tide model directory
+    and returns a list of models that are available in the
+    directory as well as the full list of all models supported
+    by `eo-tides` and `pyTMD`.
+
+    For instructions on setting up tide models, see:
+    <https://geoscienceaustralia.github.io/eo-tides/setup/>
+
+    Parameters
+    ----------
+    directory : str, optional
+        The directory containing tide model data files. If no path is
+        provided, this will default to the environment variable
+        `EO_TIDES_TIDE_MODELS` if set, or raise an error if not.
+        Tide modelling files should be stored in sub-folders for each
+        model that match the structure required by `pyTMD`
+        (<https://geoscienceaustralia.github.io/eo-tides/setup/>).
+    show_available : bool, optional
+        Whether to print a list of locally available models.
+    show_supported : bool, optional
+        Whether to print a list of all supported models, in
+        addition to models available locally.
+    raise_error : bool, optional
+        If True, raise an error if no available models are found.
+        If False, raise a warning.
+
+    Returns
+    -------
+    available_models : list of str
+        A list of all tide models available within `directory`.
+    supported_models : list of str
+        A list of all tide models supported by `eo-tides`.
+    """
+    init()  # Initialize colorama
+
+    # Set tide modelling files directory. If no custom path is
+    # provided, try global environment variable.
+    directory = _set_directory(directory)
+
+    # Get full list of supported models from pyTMD database
+    model_database = load_database()["elevation"]
+    supported_models = list(model_database.keys())
+
+    # Extract expected model paths
+    expected_paths = {}
+    for m in supported_models:
+        model_file = model_database[m]["model_file"]
+        model_file = model_file[0] if isinstance(model_file, list) else model_file
+        expected_paths[m] = str(directory / pathlib.Path(model_file).expanduser().parent)
+
+    # Define column widths
+    status_width = 4  # Width for emoji
+    name_width = max(len(name) for name in supported_models)
+    path_width = max(len(path) for path in expected_paths.values())
+
+    # Print list of supported models, marking available and
+    # unavailable models and appending available to list
+    if show_available or show_supported:
+        total_width = min(status_width + name_width + path_width + 6, 80)
+        print("─" * total_width)
+        print(f"{'󠀠🌊':^{status_width}} | {'Model':<{name_width}} | {'Expected path':<{path_width}}")
+        print("─" * total_width)
+
+    available_models = []
+    for m in supported_models:
+        try:
+            model_file = pytmd_model(directory=directory).elevation(m=m)
+            available_models.append(m)
+
+            if show_available:
+                # Mark available models with a green tick
+                status = "✅"
+                print(f"{status:^{status_width}}│ {m:<{name_width}} │ {expected_paths[m]:<{path_width}}")
+        except FileNotFoundError:
+            if show_supported:
+                # Mark unavailable models with a red cross
+                status = "❌"
+                print(
+                    f"{status:^{status_width}}│ {Style.DIM}{m:<{name_width}} │ {expected_paths[m]:<{path_width}}{Style.RESET_ALL}"
+                )
+
+    if show_available or show_supported:
+        print("─" * total_width)
+
+        # Print summary
+        print(f"\n{Style.BRIGHT}Summary:{Style.RESET_ALL}")
+        print(f"Available models: {len(available_models)}/{len(supported_models)}")
+
+    # Raise error or warning if no models are available
+    if not available_models:
+        warning_msg = textwrap.dedent(
+            f"""
+            No valid tide models are available in `{directory}`.
+            Are you sure you have provided the correct `directory` path, or set the
+            `EO_TIDES_TIDE_MODELS` environment variable to point to the location of your
+            tide model directory?
+            """
+        ).strip()
+
+        if raise_error:
+            raise Exception(warning_msg)
+        else:
+            warnings.warn(warning_msg, UserWarning)
+
+    # Return list of available and supported models
+    return available_models, supported_models
 
 
 def idw(
@@ -22,7 +474,7 @@ def idw(
     inverse distance to each neighbor, with weights descreasing with
     increasing distance.
 
-    Code inspired by: https://github.com/DahnJ/REM-xarray
+    Code inspired by: <https://github.com/DahnJ/REM-xarray>
 
     Parameters
     ----------

eo_tides/validation.py

@@ -1,5 +1,4 @@
 import datetime
-import glob
 import warnings
 from math import sqrt
 from numbers import Number
@@ -193,11 +192,12 @@ def load_gauge_gesla(
     metadata_path="/gdata1/data/sea_level/GESLA3_ALL 2.csv",
 ):
     """
-    Load and process all available Global Extreme Sea Level Analysis
-    (GESLA) tide gauge data with an `x, y, time` spatiotemporal query,
-    or from a list of specific tide gauges.
+    Load Global Extreme Sea Level Analysis (GESLA) tide gauge data.
 
-    Can optionally filter by gauge quality and append detailed gauge metadata.
+    Load and process all available GESLA measured sea-level data
+    with an `x, y, time` spatio-temporal query, or from a list of
+    specific tide gauges. Can optionally filter by gauge quality
+    and append detailed gauge metadata.
 
     Modified from original code in <https://github.com/philiprt/GeslaDataset>.
 

{eo_tides-0.2.0.dist-info → eo_tides-0.3.0.dist-info}/METADATA

@@ -1,7 +1,8 @@
 Metadata-Version: 2.1
 Name: eo-tides
-Version: 0.2.0
+Version: 0.3.0
 Summary: Tide modelling tools for large-scale satellite earth observation analysis
+Author: Stephen Sagar, Claire Phillips, Vanessa Newey
 Author-email: Robbi Bishop-Taylor <Robbi.BishopTaylor@ga.gov.au>
 Project-URL: Homepage, https://GeoscienceAustralia.github.io/eo-tides/
 Project-URL: Repository, https://github.com/GeoscienceAustralia/eo-tides
@@ -19,6 +20,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Python: <4.0,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -28,8 +30,9 @@ Requires-Dist: matplotlib >=3.8.0
 Requires-Dist: numpy >=1.26.0
 Requires-Dist: odc-geo >=0.4.7
 Requires-Dist: pandas >=2.2.0
+Requires-Dist: psutil >=5.8.0
 Requires-Dist: pyproj >=3.6.1
-Requires-Dist: pyTMD ==2.1.7
+Requires-Dist: pyTMD ==2.1.8
 Requires-Dist: scikit-learn >=1.4.0
 Requires-Dist: scipy >=1.11.2
 Requires-Dist: shapely >=2.0.6
@@ -44,16 +47,17 @@ Requires-Dist: planetary-computer >=1.0.0 ; extra == 'notebooks'
 
 # `eo-tides`: Tide modelling tools for large-scale satellite earth observation analysis
 
-<img align="right" width="200" src="docs/assets/eo-tides-logo.gif" alt="eo-tides logo" style="margin-right: 40px;">
+<img align="right" width="200" src="https://github.com/GeoscienceAustralia/eo-tides/blob/main/docs/assets/eo-tides-logo.gif?raw=true" alt="eo-tides logo" style="margin-right: 40px;">
 
 [![Release](https://img.shields.io/github/v/release/GeoscienceAustralia/eo-tides)](https://pypi.org/project/eo-tides/)
 [![Build status](https://img.shields.io/github/actions/workflow/status/GeoscienceAustralia/eo-tides/main.yml?branch=main)](https://github.com/GeoscienceAustralia/eo-tides/actions/workflows/main.yml?query=branch%3Amain)
-![Python Version from PEP 621 TOML](https://img.shields.io/pypi/pyversions/eo-tides)
+[![Python Version from PEP 621 TOML](https://img.shields.io/pypi/pyversions/eo-tides)](https://github.com/GeoscienceAustralia/eo-tides/blob/main/pyproject.toml)
 [![codecov](https://codecov.io/gh/GeoscienceAustralia/eo-tides/branch/main/graph/badge.svg)](https://codecov.io/gh/GeoscienceAustralia/eo-tides)
 [![License](https://img.shields.io/github/license/GeoscienceAustralia/eo-tides)](https://img.shields.io/github/license/GeoscienceAustralia/eo-tides)
 
-- **Github repository**: <https://github.com/GeoscienceAustralia/eo-tides/>
-- **Documentation** <https://GeoscienceAustralia.github.io/eo-tides/>
+- ⚙️ **Github repository**: <https://github.com/GeoscienceAustralia/eo-tides/>
+- 📘 **Documentation**: <https://GeoscienceAustralia.github.io/eo-tides/>
+- 🐍 **PyPI**: <https://pypi.org/project/eo-tides/>
 
 > [!CAUTION]
 > This package is a work in progress, and not currently ready for operational use.
@@ -64,12 +68,12 @@ Requires-Dist: planetary-computer >=1.0.0 ; extra == 'notebooks'
 
 These tools can be applied to petabytes of freely available satellite data (e.g. from [Digital Earth Australia](https://knowledge.dea.ga.gov.au/) or [Microsoft Planetary Computer](https://planetarycomputer.microsoft.com/)) loaded via Open Data Cube's [`odc-stac`](https://odc-stac.readthedocs.io/en/latest/) or [`datacube`](https://opendatacube.readthedocs.io/en/latest/) packages, supporting coastal and ocean earth observation analysis for any time period or location globally.
 
-![eo-tides abstract showing satellite data, tide data array and tide animation](docs/assets/eo-tides-abstract.gif)
+![eo-tides abstract showing satellite data, tide data array and tide animation](https://github.com/GeoscienceAustralia/eo-tides/blob/main/docs/assets/eo-tides-abstract.gif?raw=true)
 
 ## Highlights
 
 - 🌊 Model tide heights and phases (e.g. high, low, ebb, flow) from multiple global ocean tide models in parallel, and return a `pandas.DataFrame` for further analysis
-- 🛰️ "Tag" satellite data with tide height and stage based on the exact moment of image acquisition
+- 🛰️ "Tag" satellite data with tide heights based on the exact moment of image acquisition
 - 🌐 Model tides for every individual satellite pixel through time, producing three-dimensional "tide height" `xarray`-format datacubes that can be integrated with satellite data
 - 📈 Calculate statistics describing local tide dynamics, as well as biases caused by interactions between tidal processes and satellite orbits
 - 🛠️ Validate modelled tides using measured sea levels from coastal tide gauges (e.g. [GESLA Global Extreme Sea Level Analysis](https://gesla.org/))

eo_tides-0.3.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+eo_tides/__init__.py,sha256=rn6slQP0bAhqM9cL2W8omiEM8f0b7libt6WsQ5DvYTE,1655
+eo_tides/eo.py,sha256=Vc_AHqT0_IDqdwbOpNcONjHhiCtbCe8Osk2gvzUeNmU,22377
+eo_tides/model.py,sha256=T0IFvSiXjiTvlY-Vi7-jneHPfaCLt95pmcRd8CZxOjE,34478
+eo_tides/stats.py,sha256=lchWWJ5gBDuZWvaD8TF-12Xlo2qCWiNI2IcgAaKWy2U,22668
+eo_tides/utils.py,sha256=tqXAkLk_Dm8s3yuXPxBBkHW3PZ42ayV8VCQjNw3wDZw,22433
+eo_tides/validation.py,sha256=UREsc0yWRO4x0PJXvyoIx8gYiBZiRSim4z6TmAz_VDM,11857
+eo_tides-0.3.0.dist-info/LICENSE,sha256=owxWsXViCL2J6Ks3XYhot7t4Y93nstmXAT95Zf030Cc,11350
+eo_tides-0.3.0.dist-info/METADATA,sha256=lajVK9XoAA2fJqA9R3HHYJ7iUJFMtmxh3U07uGTs4_4,8040
+eo_tides-0.3.0.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+eo_tides-0.3.0.dist-info/top_level.txt,sha256=lXZDUUM1DlLdKWHRn8zdmtW8Rx-eQOIWVvt0b8VGiyQ,9
+eo_tides-0.3.0.dist-info/RECORD,,

eo_tides-0.2.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-eo_tides/__init__.py,sha256=MKL_HjRECVHHQVnMVg-TSz3J-vjxPZgbnyWu0RAXZ0U,1623
-eo_tides/eo.py,sha256=98llXpF2lSDfsVQBao-dpr8Z4zH5q0C2Rfu4X-qmPiE,22400
-eo_tides/model.py,sha256=kaYXU_jmv91ONam_CpoVN137aJsmqHvKHEPWe-DRJnI,38280
-eo_tides/stats.py,sha256=sOwXEh8RPb8muh2o9-z1c0GDnV5FJa_TgsiGb4rMkiU,22689
-eo_tides/utils.py,sha256=l9VXJawQzaRBYaFMsP8VBeaN5VA3rFDdzcvF7Rk04Vc,5620
-eo_tides/validation.py,sha256=JjTUqDfbR189m_6W1bpaSolQIHNTLicTHN7z9O_nr3s,11828
-eo_tides-0.2.0.dist-info/LICENSE,sha256=owxWsXViCL2J6Ks3XYhot7t4Y93nstmXAT95Zf030Cc,11350
-eo_tides-0.2.0.dist-info/METADATA,sha256=msiUYdlCm5pTix7LXA7RzM8Hg-9r4JHpWTSUyxdPpg4,7637
-eo_tides-0.2.0.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-eo_tides-0.2.0.dist-info/top_level.txt,sha256=lXZDUUM1DlLdKWHRn8zdmtW8Rx-eQOIWVvt0b8VGiyQ,9
-eo_tides-0.2.0.dist-info/RECORD,,