roms-tools 1.5.0__py3-none-any.whl → 1.6.1__py3-none-any.whl

roms_tools/setup/surface_forcing.py

@@ -7,8 +7,7 @@ from roms_tools.setup.grid import Grid
 from datetime import datetime
 import numpy as np
 from typing import Dict, Union, List
-from roms_tools.setup.fill import _lateral_fill, LateralFill
-from roms_tools.setup.regrid import _lateral_regrid, LateralRegrid
+from roms_tools.setup.regrid import LateralRegrid
 from roms_tools.setup.datasets import (
     ERA5Dataset,
     ERA5Correction,
@@ -99,51 +98,40 @@ class SurfaceForcing:
 
         data = self._get_data()
         data.choose_subdomain(
-            latitude_range=[
-                target_coords["lat"].min().values,
-                target_coords["lat"].max().values,
-            ],
-            longitude_range=[
-                target_coords["lon"].min().values,
-                target_coords["lon"].max().values,
-            ],
-            margin=2,
-            straddle=target_coords["straddle"],
+            target_coords,
+            buffer_points=20,  # lateral fill needs some buffer from data margin
         )
 
-        variable_info = self._set_variable_info(data)
+        data.apply_lateral_fill()
 
-        data_vars = {}
-        for var_name in data.var_names:
-            if var_name != "mask":
-                data_vars[var_name] = data.ds[data.var_names[var_name]]
-
-        data_vars = _lateral_fill(data_vars, data)
+        variable_info = self._set_variable_info(data)
+        var_names = variable_info.keys()
 
+        processed_fields = {}
         # lateral regridding
-        var_names = variable_info.keys()
-        data_vars = _lateral_regrid(
-            data, target_coords["lon"], target_coords["lat"], data_vars, var_names
-        )
+        lateral_regrid = LateralRegrid(target_coords, data.dim_names)
+        for var_name in var_names:
+            if var_name in data.var_names.keys():
+                processed_fields[var_name] = lateral_regrid.apply(
+                    data.ds[data.var_names[var_name]]
+                )
 
         # rotation of velocities and interpolation to u/v points
         if "uwnd" in variable_info and "vwnd" in variable_info:
-            data_vars["uwnd"], data_vars["vwnd"] = rotate_velocities(
-                data_vars["uwnd"],
-                data_vars["vwnd"],
+            processed_fields["uwnd"], processed_fields["vwnd"] = rotate_velocities(
+                processed_fields["uwnd"],
+                processed_fields["vwnd"],
                 target_coords["angle"],
                 interpolate=False,
             )
 
         # correct radiation
         if self.type == "physics" and self.correct_radiation:
-            data_vars = self._apply_correction(data_vars, data)
-
-        object.__setattr__(data, "data_vars", data_vars)
+            processed_fields = self._apply_correction(processed_fields, data)
 
         d_meta = get_variable_metadata()
 
-        ds = self._write_into_dataset(data, d_meta)
+        ds = self._write_into_dataset(processed_fields, data, d_meta)
 
         if self.use_coarse_grid:
             mask = self.grid.ds["mask_coarse"].rename(
@@ -152,13 +140,11 @@ class SurfaceForcing:
         else:
             mask = self.grid.ds["mask_rho"]
 
-        # NaN values at wet points indicate that the raw data did not cover the domain, and the following will raise a ValueError
-        for var in ds.data_vars:
-            nan_check(ds[var].isel(time=0), mask)
+        self._validate(ds, mask)
 
         # substitute NaNs over land by a fill value to avoid blow-up of ROMS
-        for var in ds.data_vars:
-            ds[var] = substitute_nans_by_fillvalue(ds[var])
+        for var_name in ds.data_vars:
+            ds[var_name] = substitute_nans_by_fillvalue(ds[var_name])
 
         object.__setattr__(self, "ds", ds)
 
@@ -272,7 +258,7 @@ class SurfaceForcing:
 
         return variable_info
 
-    def _apply_correction(self, data_vars, data):
+    def _apply_correction(self, processed_fields, data):
 
         correction_data = self._get_correction_data()
         # choose same subdomain as forcing data so that we can use same mask
@@ -285,44 +271,34 @@ class SurfaceForcing:
         correction_data.choose_subdomain(
             coords_correction, straddle=self.target_coords["straddle"]
         )
-
+        correction_data.ds["mask"] = data.ds["mask"]  # use mask from ERA5 data
+        correction_data.apply_lateral_fill()
         # regrid
-        lateral_fill = LateralFill(
-            data.ds["mask"],  # use mask from ERA5 data
-            [
-                correction_data.dim_names["latitude"],
-                correction_data.dim_names["longitude"],
-            ],
-        )
-        lateral_regrid = LateralRegrid(
-            correction_data, self.target_coords["lon"], self.target_coords["lat"]
-        )
-
-        filled = lateral_fill.apply(
+        lateral_regrid = LateralRegrid(self.target_coords, correction_data.dim_names)
+        corr_factor = lateral_regrid.apply(
             correction_data.ds[correction_data.var_names["swr_corr"]]
         )
-        corr_factor = lateral_regrid.apply(filled)
 
         # temporal interpolation
         corr_factor = interpolate_from_climatology(
             corr_factor,
             correction_data.dim_names["time"],
-            time=data_vars["swrad"].time,
+            time=processed_fields["swrad"].time,
         )
 
-        data_vars["swrad"] = data_vars["swrad"] * corr_factor
+        processed_fields["swrad"] = processed_fields["swrad"] * corr_factor
 
-        return data_vars
+        return processed_fields
 
-    def _write_into_dataset(self, data, d_meta):
+    def _write_into_dataset(self, processed_fields, data, d_meta):
 
         # save in new dataset
         ds = xr.Dataset()
 
-        for var in data.data_vars.keys():
-            ds[var] = data.data_vars[var].astype(np.float32)
-            ds[var].attrs["long_name"] = d_meta[var]["long_name"]
-            ds[var].attrs["units"] = d_meta[var]["units"]
+        for var_name in processed_fields.keys():
+            ds[var_name] = processed_fields[var_name].astype(np.float32)
+            ds[var_name].attrs["long_name"] = d_meta[var_name]["long_name"]
+            ds[var_name].attrs["units"] = d_meta[var_name]["units"]
 
         if self.use_coarse_grid:
             ds = ds.rename({"eta_coarse": "eta_rho", "xi_coarse": "xi_rho"})
@@ -388,11 +364,36 @@ class SurfaceForcing:
             ds = ds.drop_vars(["time"])
 
         variables_to_drop = ["lat_rho", "lon_rho", "lat_coarse", "lon_coarse"]
-        existing_vars = [var for var in variables_to_drop if var in ds]
+        existing_vars = [var_name for var_name in variables_to_drop if var_name in ds]
         ds = ds.drop_vars(existing_vars)
 
         return ds
 
+    def _validate(self, ds, mask):
+        """Validates the dataset by checking for NaN values at wet points, which would
+        indicate missing raw data coverage over the target domain.
+
+        Parameters
+        ----------
+        ds : xarray.Dataset
+            The dataset to validate.
+        mask : xarray.DataArray
+            Land mask (1=ocean, 0=land) to determine wet points in the domain.
+
+        Raises
+        ------
+        ValueError
+            If NaN values are found in any of the specified variables at wet points,
+            indicating incomplete data coverage.
+
+        Notes
+        -----
+        This check is applied to the first time step (`time=0`) of each variable in the provided dataset.
+        """
+
+        for var_name in ds.data_vars:
+            nan_check(ds[var_name].isel(time=0), mask)
+
     def _add_global_metadata(self, ds=None):
 
         if ds is None:
@@ -416,12 +417,12 @@ class SurfaceForcing:
 
         return ds
 
-    def plot(self, varname, time=0) -> None:
+    def plot(self, var_name, time=0) -> None:
         """Plot the specified surface forcing field for a given time slice.
 
         Parameters
         ----------
-        varname : str
+        var_name : str
             The name of the surface forcing field to plot. Options include:
 
             - "uwnd": 10 meter wind in x-direction.
@@ -450,7 +451,7 @@ class SurfaceForcing:
         Raises
         ------
         ValueError
-            If the specified varname is not found in dataset.
+            If the specified var_name is not found in dataset.
 
 
         Examples
@@ -458,10 +459,16 @@ class SurfaceForcing:
         >>> atm_forcing.plot("uwnd", time=0)
         """
 
-        if varname not in self.ds:
-            raise ValueError(f"Variable '{varname}' is not found in dataset.")
+        if var_name not in self.ds:
+            raise ValueError(f"Variable '{var_name}' is not found in dataset.")
+
+        field = self.ds[var_name].isel(time=time)
+        if self.use_dask:
+            from dask.diagnostics import ProgressBar
+
+            with ProgressBar():
+                field = field.load()
 
-        field = self.ds[varname].isel(time=time).load()
         title = field.long_name
 
         # assign lat / lon
@@ -476,14 +483,14 @@ class SurfaceForcing:
         )
 
         # choose colorbar
-        if varname in ["uwnd", "vwnd"]:
+        if var_name in ["uwnd", "vwnd"]:
             vmax = max(field.max().values, -field.min().values)
             vmin = -vmax
             cmap = plt.colormaps.get_cmap("RdBu_r")
         else:
             vmax = field.max().values
             vmin = field.min().values
-            if varname in ["swrad", "lwrad", "Tair", "qair"]:
+            if var_name in ["swrad", "lwrad", "Tair", "qair"]:
                 cmap = plt.colormaps.get_cmap("YlOrRd")
             else:
                 cmap = plt.colormaps.get_cmap("YlGnBu")
@@ -501,24 +508,26 @@ class SurfaceForcing:
         )
 
     def save(
-        self, filepath: Union[str, Path], np_eta: int = None, np_xi: int = None
+        self,
+        filepath: Union[str, Path],
+        np_eta: int = None,
+        np_xi: int = None,
+        group: bool = False,
     ) -> None:
-        """Save the surface forcing fields to netCDF4 files.
-
-        This method saves the dataset by grouping it into subsets based on the data frequency. The subsets are then written
-        to one or more netCDF4 files. The filenames of the output files reflect the temporal coverage of the data.
-
-        There are two modes of saving the dataset:
+        """Save the surface forcing fields to one or more netCDF4 files.
 
-          1. **Single File Mode (default)**:
+        This method saves the dataset either as a single file or as multiple files depending on the partitioning and grouping options.
+        The dataset can be saved in two modes:
 
-            If both `np_eta` and `np_xi` are `None`, the entire dataset, divided by temporal subsets, is saved as a single netCDF4 file
-            with the base filename specified by `filepath.nc`.
+        1. **Single File Mode (default)**:
+            - If both `np_eta` and `np_xi` are `None`, the entire dataset is saved as a single netCDF4 file.
+            - The file is named based on the `filepath`, with `.nc` automatically appended.
 
-          2. **Partitioned Mode**:
+        2. **Partitioned Mode**:
+            - If either `np_eta` or `np_xi` is specified, the dataset is partitioned into spatial tiles along the `eta` and `xi` axes.
+            - Each tile is saved as a separate netCDF4 file, and filenames are modified with an index (e.g., `"filepath_YYYYMM.0.nc"`, `"filepath_YYYYMM.1.nc"`).
 
-            - If either `np_eta` or `np_xi` is specified, the dataset is divided into spatial tiles along the eta-axis and xi-axis.
-            - Each spatial tile is saved as a separate netCDF4 file.
+        Additionally, if `group` is set to `True`, the dataset is first grouped into temporal subsets, resulting in multiple grouped files before partitioning and saving.
 
         Parameters
         ----------
@@ -530,6 +539,8 @@ class SurfaceForcing:
             The number of partitions along the `eta` direction. If `None`, no spatial partitioning is performed.
         np_xi : int, optional
             The number of partitions along the `xi` direction. If `None`, no spatial partitioning is performed.
+        group: bool, optional
+            If `True`, groups the dataset into multiple files based on temporal data frequency. Defaults to `False`.
 
         Returns
         -------
@@ -544,7 +555,18 @@ class SurfaceForcing:
         if filepath.suffix == ".nc":
             filepath = filepath.with_suffix("")
 
-        dataset_list, output_filenames = group_dataset(self.ds.load(), str(filepath))
+        if self.use_dask:
+            from dask.diagnostics import ProgressBar
+
+            with ProgressBar():
+                self.ds.load()
+
+        if group:
+            dataset_list, output_filenames = group_dataset(self.ds, str(filepath))
+        else:
+            dataset_list = [self.ds]
+            output_filenames = [str(filepath)]
+
         saved_filenames = save_datasets(
             dataset_list, output_filenames, np_eta=np_eta, np_xi=np_xi
         )
@@ -602,7 +624,7 @@ class SurfaceForcing:
             # Write header
             file.write(header)
             # Write YAML data
-            yaml.dump(yaml_data, file, default_flow_style=False)
+            yaml.dump(yaml_data, file, default_flow_style=False, sort_keys=False)
 
     @classmethod
     def from_yaml(

roms_tools/setup/tides.py

@@ -20,8 +20,7 @@ from roms_tools.setup.utils import (
     rotate_velocities,
     get_vector_pairs,
 )
-from roms_tools.setup.fill import _lateral_fill
-from roms_tools.setup.regrid import _lateral_regrid
+from roms_tools.setup.regrid import LateralRegrid
 import matplotlib.pyplot as plt
 from pathlib import Path
 
@@ -77,71 +76,64 @@ class TidalForcing:
         data = self._get_data()
         data.check_number_constituents(self.ntides)
         data.choose_subdomain(
-            latitude_range=[
-                target_coords["lat"].min().values,
-                target_coords["lat"].max().values,
-            ],
-            longitude_range=[
-                target_coords["lon"].min().values,
-                target_coords["lon"].max().values,
-            ],
-            margin=2,
-            straddle=target_coords["straddle"],
+            target_coords,
+            buffer_points=20,
         )
         # select desired number of constituents
         object.__setattr__(data, "ds", data.ds.isel(ntides=slice(None, self.ntides)))
         self._correct_tides(data)
 
-        variable_info = self._set_variable_info()
-
-        data_vars = {}
-        for var_name in data.var_names:
-            data_vars[var_name] = data.ds[data.var_names[var_name]]
+        data.apply_lateral_fill()
 
-        data_vars = _lateral_fill(data_vars, data)
+        variable_info = self._set_variable_info()
+        var_names = variable_info.keys()
 
+        processed_fields = {}
         # lateral regridding
-        var_names = variable_info.keys()
-        data_vars = _lateral_regrid(
-            data, target_coords["lon"], target_coords["lat"], data_vars, var_names
-        )
+        lateral_regrid = LateralRegrid(target_coords, data.dim_names)
+        for var_name in var_names:
+            if var_name in data.var_names.keys():
+                processed_fields[var_name] = lateral_regrid.apply(
+                    data.ds[data.var_names[var_name]]
+                )
 
         # rotation of velocities and interpolation to u/v points
         vector_pairs = get_vector_pairs(variable_info)
         for pair in vector_pairs:
             u_component = pair[0]
             v_component = pair[1]
-            if u_component in data_vars and v_component in data_vars:
-                (data_vars[u_component], data_vars[v_component],) = rotate_velocities(
-                    data_vars[u_component],
-                    data_vars[v_component],
+            if u_component in processed_fields and v_component in processed_fields:
+                (
+                    processed_fields[u_component],
+                    processed_fields[v_component],
+                ) = rotate_velocities(
+                    processed_fields[u_component],
+                    processed_fields[v_component],
                     target_coords["angle"],
                     interpolate=False,
                 )
 
         # convert to barotropic velocity
-        for varname in ["u_Re", "v_Re", "u_Im", "v_Im"]:
-            data_vars[varname] = data_vars[varname] / self.grid.ds.h
+        for var_name in ["u_Re", "v_Re", "u_Im", "v_Im"]:
+            processed_fields[var_name] = processed_fields[var_name] / self.grid.ds.h
 
         # interpolate from rho- to velocity points
         for uname in ["u_Re", "u_Im"]:
-            data_vars[uname] = interpolate_from_rho_to_u(data_vars[uname])
+            processed_fields[uname] = interpolate_from_rho_to_u(processed_fields[uname])
         for vname in ["v_Re", "v_Im"]:
-            data_vars[vname] = interpolate_from_rho_to_v(data_vars[vname])
+            processed_fields[vname] = interpolate_from_rho_to_v(processed_fields[vname])
 
         d_meta = get_variable_metadata()
-        ds = self._write_into_dataset(data_vars, d_meta)
+        ds = self._write_into_dataset(processed_fields, d_meta)
         ds["omega"] = data.ds["omega"]
 
         ds = self._add_global_metadata(ds)
 
-        # NaN values at wet points indicate that the raw data did not cover the domain, and the following will raise a ValueError
-        for var in ["ssh_Re", "u_Re", "v_Im"]:
-            nan_check(ds[var].isel(ntides=0), self.grid.ds.mask_rho)
+        self._validate(ds)
 
         # substitute NaNs over land by a fill value to avoid blow-up of ROMS
-        for var in ds.data_vars:
-            ds[var] = substitute_nans_by_fillvalue(ds[var])
+        for var_name in ds.data_vars:
+            ds[var_name] = substitute_nans_by_fillvalue(ds[var_name])
 
         object.__setattr__(self, "ds", ds)
 
@@ -216,15 +208,15 @@ class TidalForcing:
 
         return variable_info
 
-    def _write_into_dataset(self, data_vars, d_meta):
+    def _write_into_dataset(self, processed_fields, d_meta):
 
         # save in new dataset
         ds = xr.Dataset()
 
-        for var in data_vars.keys():
-            ds[var] = data_vars[var].astype(np.float32)
-            ds[var].attrs["long_name"] = d_meta[var]["long_name"]
-            ds[var].attrs["units"] = d_meta[var]["units"]
+        for var_name in processed_fields.keys():
+            ds[var_name] = processed_fields[var_name].astype(np.float32)
+            ds[var_name].attrs["long_name"] = d_meta[var_name]["long_name"]
+            ds[var_name].attrs["units"] = d_meta[var_name]["units"]
 
         ds = ds.drop_vars(["lat_rho", "lon_rho"])
 
@@ -247,12 +239,35 @@ class TidalForcing:
 
         return ds
 
-    def plot(self, varname, ntides=0) -> None:
+    def _validate(self, ds):
+        """Validates the dataset by checking for NaN values at wet points, which would
+        indicate missing raw data coverage over the target domain.
+
+        Parameters
+        ----------
+        ds : xarray.Dataset
+            The dataset to validate, containing tidal variables and a mask for wet points.
+
+        Raises
+        ------
+        ValueError
+            If NaN values are found in any of the specified variables at wet points,
+            indicating incomplete data coverage.
+
+        Notes
+        -----
+        This check is applied to the first constituent (`ntides=0`) of each variable in the dataset.
+        The method utilizes `self.grid.ds.mask_rho` to determine the wet points in the domain.
+        """
+        for var_name in ds.data_vars:
+            nan_check(ds[var_name].isel(ntides=0), self.grid.ds.mask_rho)
+
+    def plot(self, var_name, ntides=0) -> None:
         """Plot the specified tidal forcing variable for a given tidal constituent.
 
         Parameters
         ----------
-        varname : str
+        var_name : str
             The tidal forcing variable to plot. Options include:
 
             - "ssh_Re": Real part of tidal elevation.
@@ -285,7 +300,14 @@ class TidalForcing:
         >>> tidal_forcing.plot("ssh_Re", nc=0)
         """
 
-        field = self.ds[varname].isel(ntides=ntides).compute()
+        field = self.ds[var_name].isel(ntides=ntides)
+
+        if self.use_dask:
+            from dask.diagnostics import ProgressBar
+
+            with ProgressBar():
+                field = field.load()
+
         if all(dim in field.dims for dim in ["eta_rho", "xi_rho"]):
             field = field.where(self.grid.ds.mask_rho)
             field = field.assign_coords(
@@ -306,7 +328,7 @@ class TidalForcing:
         else:
             ValueError("provided field does not have two horizontal dimension")
 
-        title = "%s, ntides = %i" % (field.long_name, self.ds[varname].ntides[ntides])
+        title = "%s, ntides = %i" % (field.long_name, self.ds[var_name].ntides[ntides])
 
         vmax = max(field.max(), -field.min())
         vmin = -vmax
@@ -363,7 +385,13 @@ class TidalForcing:
         if filepath.suffix == ".nc":
             filepath = filepath.with_suffix("")
 
-        dataset_list = [self.ds.load()]
+        if self.use_dask:
+            from dask.diagnostics import ProgressBar
+
+            with ProgressBar():
+                self.ds.load()
+
+        dataset_list = [self.ds]
         output_filenames = [str(filepath)]
 
         saved_filenames = save_datasets(
@@ -404,8 +432,8 @@ class TidalForcing:
             "TidalForcing": {
                 "source": self.source,
                 "ntides": self.ntides,
-                "model_reference_date": self.model_reference_date.isoformat(),
                 "allan_factor": self.allan_factor,
+                "model_reference_date": self.model_reference_date.isoformat(),
             }
         }
 
@@ -416,7 +444,7 @@ class TidalForcing:
             # Write header
             file.write(header)
             # Write YAML data
-            yaml.dump(yaml_data, file, default_flow_style=False)
+            yaml.dump(yaml_data, file, default_flow_style=False, sort_keys=False)
 
     @classmethod
     def from_yaml(