tfv-get-tools 0.2.0__py3-none-any.whl

tfv_get_tools/providers/wave/cawcr.py

@@ -0,0 +1,166 @@
+from pathlib import Path
+from typing import List, Tuple
+
+import numpy as np
+import pandas as pd
+import xarray as xr
+from pandas.tseries.offsets import MonthEnd
+from tqdm import tqdm
+
+from tfv_get_tools.providers._downloader import BaseDownloader
+from tfv_get_tools.providers._merger import BaseMerger
+from tfv_get_tools.providers._utilities import todstr
+
+
+class DownloadCAWCR(BaseDownloader):
+    """CAWCR Wave downloader"""
+    
+    def _init_specific(self, **kwargs):
+        """Set source and mode - matches original interface"""
+        if self.model == 'default':
+            self.log("Default model has been selected == 'glob_24m'")
+            self.model = 'glob_24m'
+        
+        self.source = "CAWCR"
+        self.mode = "WAVE"
+        
+        # Validate model selection
+        MODELS = {"pac_4m", "pac_10m", "glob_24m", "aus_4m", "aus_10m"}
+        if self.model not in MODELS:
+            raise ValueError(f"Model must be one of {MODELS}")
+        
+        self._load_config()
+        
+        # CAWCR-specific: track whether we've checked for valid data
+        self.checked_data = False
+    
+    def _get_output_filename(self, ts: pd.Timestamp, te: pd.Timestamp) -> Path:
+        """CAWCR filename pattern includes model name"""
+        return self.outdir / f"{self.prefix}_{self.model}_{todstr(ts)}_{todstr(te)}.nc"
+    
+    def _construct_opendap_url(self, date: pd.Timestamp) -> str:
+        """Construct the OPeNDAP URL for the given date"""
+        date_str = date.strftime("%Y%m")
+        url = f"{self.base_url}/ww3.{self.model}.{date_str}.nc"
+        return url
+    
+    def _check_data_validity(self, ds: xr.Dataset) -> None:
+        """Check if the dataset contains valid data (CAWCR-specific)"""
+        if not self.checked_data:
+            # Check first variable for valid data
+            v = next(iter(self.variables))
+            nonnan = np.nansum(ds[v][0].values)
+            if nonnan == 0.0:
+                raise ValueError(
+                    "No valid data detected in netcdf - please check requested extents against the data source"
+                )
+            else:
+                self.checked_data = True
+    
+    def _download_single_file(self, fname: Path, url: str) -> bool:
+        """Download single file from CAWCR OPeNDAP server"""
+        try:
+            ds = xr.open_dataset(url)
+
+            # Handle longitude selection (CAWCR-specific logic)
+            if self.xlims[0] < self.xlims[1]:
+                lon_idx = np.hstack(
+                    np.where(
+                        (self.xlims[0] <= ds["longitude"])
+                        & (ds["longitude"] <= self.xlims[1])
+                    )
+                )
+            else:
+                lon_idx = np.hstack(
+                    (
+                        np.where(ds["longitude"] <= self.xlims[1])[0],
+                        np.where(ds["longitude"] >= self.xlims[0])[0],
+                    )
+                )
+            
+            if lon_idx.size <= 1:
+                raise ValueError("No longitude points selected! You may need to increase your grid extents")
+
+            # Select latitude indices
+            lat_idx = np.hstack(
+                np.where(
+                    (self.ylims[0] <= ds["latitude"]) & (ds["latitude"] <= self.ylims[1])
+                )
+            )
+            
+            if lat_idx.size <= 1:
+                raise ValueError("No latitude points selected! You may need to increase your grid extents")
+
+            # Subset dataset by latitude and longitude
+            ds = ds.isel(longitude=lon_idx, latitude=lat_idx)
+
+            # Subset dataset to requested variables
+            ds = ds[self.variables]
+
+            # CAWCR-specific: Check for valid data on first download
+            self._check_data_validity(ds)
+
+            # Save to file
+            ds.to_netcdf(fname)
+            return True
+            
+        except Exception as e:
+            if self.verbose:
+                print(f"Failed to download {url}: {e}")
+            return False
+    
+    def download(self):
+        """CAWCR-specific download loop - yields tasks for new base class"""
+        for ts in self.times:
+            te = ts + MonthEnd() + pd.Timedelta("23.9h")
+            
+            output_file = self._get_output_filename(ts, te)
+            url = self._construct_opendap_url(ts)
+            
+            yield {
+                'file_path': output_file,
+                'url': url,
+                'timestamp': ts,
+                'variable': f"{len(self.variables)}_vars",
+                'download_func': lambda f=output_file, u=url: self._download_single_file(f, u)
+            }
+
+class MergeCAWCR(BaseMerger):
+    def _init_specific(self) -> None:
+        self.source = "CAWCR"
+        self.mode = "WAVE"
+        
+        if self.model == 'default':
+            self.model = 'glob_24m'
+        
+        self._load_config()
+
+    def merge_files(self, file_list: List[Path]) -> Tuple[xr.Dataset, List[Path]]:
+        """Merge CAWCR wave files using time concatenation."""
+        if not file_list:
+            raise ValueError("No files provided for merging")
+            
+        datasets = []
+        skipped_files = []
+        
+        for file_path in tqdm(file_list, disable=not self.verbose):
+            ds = self._open_subset_netcdf(file_path)
+            if ds is not None:
+                datasets.append(ds)
+            else:
+                skipped_files.append(file_path)
+                
+        if not datasets:
+            raise ValueError("No valid datasets could be loaded")
+            
+        # Concatenate and clean up
+        merged = xr.concat(datasets, dim="time", combine_attrs="override", 
+                          data_vars="minimal", coords="minimal", compat="override")
+        
+        # Remove duplicates and sort
+        merged = merged.sortby("time")
+        _, unique_idx = np.unique(merged["time"], return_index=True)
+        merged = merged.isel(time=np.sort(unique_idx))
+        
+        return merged, skipped_files
+        

tfv_get_tools/providers/wave/cfgs/cawcr_aus_10m.yaml

@@ -0,0 +1,39 @@
+_BASE_URL: "https://data-cbr.csiro.au/thredds/dodsC/catch_all/CMAR_CAWCR-Wave_archive/CAWCR_Wave_Hindcast_aggregate/gridded"
+_INFO_URL: https://data.csiro.au/collection/csiro:39819
+
+_SOURCE_XLIMS: [100., 175.]
+_SOURCE_YLIMS: [-50., 3.]
+_SOURCE_TIMELIMS: ["1979-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+_VARIABLES: ["hs", "fp", "dir", "t02", "spr"]
+_DATASETS: { 1979-01-01: { ww3.aus_10m.%y%m: default } }
+
+# KEY: The standard NC Variable for merged data (and per `_standard_attrs.py`)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+hs:
+  source_var: "hs"
+  tfv_var: "WVHT"
+  bc_scale: 1
+  bc_offset: 0
+
+tp:
+  source_var: "fp"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+  pre_process: freq_to_period(fp)
+
+tm02:
+  source_var: "t02"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+
+mwd:
+  source_var: "dir"
+  tfv_var: "WVDIR"
+  bc_scale: 1
+  bc_offset: 0

tfv_get_tools/providers/wave/cfgs/cawcr_aus_4m.yaml

@@ -0,0 +1,39 @@
+_BASE_URL: "https://data-cbr.csiro.au/thredds/dodsC/catch_all/CMAR_CAWCR-Wave_archive/CAWCR_Wave_Hindcast_aggregate/gridded"
+_INFO_URL: https://data.csiro.au/collection/csiro:39819
+
+_SOURCE_XLIMS: [100., 175.]
+_SOURCE_YLIMS: [-50., 3.]
+_SOURCE_TIMELIMS: ["1979-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+_VARIABLES: ["hs", "fp", "dir", "t02", "spr"]
+_DATASETS: { 1979-01-01: { ww3.aus_4m.%y%m: default } }
+
+# KEY: The standard NC Variable for merged data (and per `_standard_attrs.py`)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+hs:
+  source_var: "hs"
+  tfv_var: "WVHT"
+  bc_scale: 1
+  bc_offset: 0
+
+tp:
+  source_var: "fp"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+  pre_process: freq_to_period(fp)
+
+tm02:
+  source_var: "t02"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+
+mwd:
+  source_var: "dir"
+  tfv_var: "WVDIR"
+  bc_scale: 1
+  bc_offset: 0

tfv_get_tools/providers/wave/cfgs/cawcr_glob_24m.yaml

@@ -0,0 +1,39 @@
+_BASE_URL: "https://data-cbr.csiro.au/thredds/dodsC/catch_all/CMAR_CAWCR-Wave_archive/CAWCR_Wave_Hindcast_aggregate/gridded"
+_INFO_URL: https://data.csiro.au/collection/csiro:39819
+
+_SOURCE_XLIMS: [-180, 360]
+_SOURCE_YLIMS: [-78, 78]
+_SOURCE_TIMELIMS: ["1979-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+_VARIABLES: ["hs", "fp", "dir", "t02", "spr"]
+_DATASETS: { 1979-01-01: { ww3.glob_24.%y%m: default } }
+
+# KEY: The standard NC Variable for merged data (and per `_standard_attrs.py`)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+hs:
+  source_var: "hs"
+  tfv_var: "WVHT"
+  bc_scale: 1
+  bc_offset: 0
+
+tp:
+  source_var: "fp"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+  pre_process: freq_to_period(fp)
+
+tm02:
+  source_var: "t02"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+
+mwd:
+  source_var: "dir"
+  tfv_var: "WVDIR"
+  bc_scale: 1
+  bc_offset: 0

tfv_get_tools/providers/wave/cfgs/cawcr_pac_10m.yaml

@@ -0,0 +1,39 @@
+_BASE_URL: "https://data-cbr.csiro.au/thredds/dodsC/catch_all/CMAR_CAWCR-Wave_archive/CAWCR_Wave_Hindcast_aggregate/gridded"
+_INFO_URL: https://data.csiro.au/collection/csiro:39819
+
+_SOURCE_XLIMS: [125., 210.]
+_SOURCE_YLIMS: [-30., 20.]
+_SOURCE_TIMELIMS: ["1979-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+_VARIABLES: ["hs", "fp", "dir", "t02", "spr"]
+_DATASETS: { 1979-01-01: { ww3.pac_10m.%y%m: default } }
+
+# KEY: The standard NC Variable for merged data (and per `_standard_attrs.py`)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+hs:
+  source_var: "hs"
+  tfv_var: "WVHT"
+  bc_scale: 1
+  bc_offset: 0
+
+tp:
+  source_var: "fp"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+  pre_process: freq_to_period(fp)
+
+tm02:
+  source_var: "t02"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+
+mwd:
+  source_var: "dir"
+  tfv_var: "WVDIR"
+  bc_scale: 1
+  bc_offset: 0

tfv_get_tools/providers/wave/cfgs/cawcr_pac_4m.yaml

@@ -0,0 +1,39 @@
+_BASE_URL: "https://data-cbr.csiro.au/thredds/dodsC/catch_all/CMAR_CAWCR-Wave_archive/CAWCR_Wave_Hindcast_aggregate/gridded"
+_INFO_URL: https://data.csiro.au/collection/csiro:39819
+
+_SOURCE_XLIMS: [125., 210.]
+_SOURCE_YLIMS: [-30., 20.]
+_SOURCE_TIMELIMS: ["1979-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+_VARIABLES: ["hs", "fp", "dir", "t02", "spr"]
+_DATASETS: { 1979-01-01: { ww3.pac_4m.%y%m: default } }
+
+# KEY: The standard NC Variable for merged data (and per `_standard_attrs.py`)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+hs:
+  source_var: "hs"
+  tfv_var: "WVHT"
+  bc_scale: 1
+  bc_offset: 0
+
+tp:
+  source_var: "fp"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+  pre_process: freq_to_period(fp)
+
+tm02:
+  source_var: "t02"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+
+mwd:
+  source_var: "dir"
+  tfv_var: "WVDIR"
+  bc_scale: 1
+  bc_offset: 0

tfv_get_tools/providers/wave/cfgs/copernicus_glo.yaml

@@ -0,0 +1,56 @@
+_BASE_URL: "N/A"
+
+_SOURCE_XLIMS: [-180, 360]
+_SOURCE_YLIMS: [-90, 90]
+_SOURCE_TIMELIMS: ["1993-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+
+# Default variables to extract if no `variables` arg is provided.
+_VARIABLES: ["VHM0", "VTPK", "VMDR", "VTM02"]
+
+# This is the map of datasets and what variables to snatch from each.
+# default == take the _VARIABLES above.
+_DATASETS:
+  {
+    1993-01-01:
+      { cmems_mod_glo_wav_my_0.2deg_PT3H-i: ["VHM0", "VTPK", "VMDR", "VTM02"] },
+    2023-05-01:
+      {
+        cmems_mod_glo_phy_myint_0.083deg_P1D-m:
+          ["VHM0", "VTPK", "VMDR", "VTM02"],
+      },
+    forecast:
+      {
+        cmems_mod_glo_wav_anfc_0.083deg_PT3H-i:
+          ["VHM0", "VTPK", "VMDR", "VTM02"],
+      },
+  }
+
+# KEY: The standard NC Variable for merged data (and per `_standard_attrs.py`)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+hs:
+  source_var: "VHM0"
+  tfv_var: "WVHT"
+  bc_scale: 1
+  bc_offset: 0
+
+tp:
+  source_var: "VTPK"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+
+tm02:
+  source_var: "VTM02"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+
+mwd:
+  source_var: "VMDR"
+  tfv_var: "WVDIR"
+  bc_scale: 1
+  bc_offset: 0

tfv_get_tools/providers/wave/cfgs/copernicus_nws.yaml

@@ -0,0 +1,51 @@
+_BASE_URL: "N/A"
+
+_SOURCE_XLIMS: [-16.0, 13.0]
+_SOURCE_YLIMS: [46.0, 62.74]
+_SOURCE_ZLIMS: [null, null]
+_SOURCE_TIMELIMS: ["1980-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+
+# Default variables to extract if no `variables` arg is provided.
+_VARIABLES: ["VHM0", "VTPK", "VMDR", "VTM02"]
+
+# This is the map of datasets and what variables to snatch from each.
+# default == take the _VARIABLES above.
+_DATASETS:
+  {
+    1993-01-01: { MetO-NWS-WAV-RAN: ["VHM0", "VTPK", "VMDR", "VTM02"] },
+    forecast:
+      {
+        cmems_mod_nws_wav_anfc_0.05deg_PT1H-i:
+          ["VHM0", "VTPK", "VMDR", "VTM02"],
+      },
+  }
+
+# KEY: The standard NC Variable for merged data (and per `_standard_attrs.py`)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+hs:
+  source_var: "VHM0"
+  tfv_var: "WVHT"
+  bc_scale: 1
+  bc_offset: 0
+
+tp:
+  source_var: "VTPK"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+
+tm02:
+  source_var: "VTM02"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+
+mwd:
+  source_var: "VMDR"
+  tfv_var: "WVDIR"
+  bc_scale: 1
+  bc_offset: 0

tfv_get_tools/providers/wave/cfgs/era5.yaml

@@ -0,0 +1,48 @@
+_BASE_URL: "N/A"
+_INFO_URL: https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels
+
+_SOURCE_XLIMS: [-180, 360]
+_SOURCE_YLIMS: [-90, 90]
+_SOURCE_TIMELIMS: ["1940-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+
+# Default variables to extract if no `variables` arg is provided.
+_VARIABLES:
+  [
+    "significant_height_of_combined_wind_waves_and_swell",
+    "peak_wave_period",
+    "mean_wave_direction",
+    "wave_spectral_directional_width",
+  ]
+
+# ERA5 only has the single single-levels dataset.
+_DATASETS: { 1940-01-01: { reanalysis-era5-single-levels: default } }
+
+# KEY: The standard NC Variable for merged data (and per `_standard_attrs.py`)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+hs:
+  source_var: "swh"
+  tfv_var: "WVHT"
+  bc_scale: 1
+  bc_offset: 0
+
+tp:
+  source_var: "pp1d"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+
+mwd:
+  source_var: "mwd"
+  tfv_var: "WVDIR"
+  bc_scale: 1
+  bc_offset: 0
+
+spr:
+  source_var: "wdw"
+  tfv_var: null
+  bc_scale: 1
+  bc_offset: 0

tfv_get_tools/providers/wave/cfgs/era5_gcp.yaml

@@ -0,0 +1,48 @@
+_BASE_URL: "gs://gcp-public-data-arco-era5/ar/full_37-1h-0p25deg-chunk-1.zarr-v3"
+_INFO_URL: https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels
+
+_SOURCE_XLIMS: [-180, 360]
+_SOURCE_YLIMS: [-90, 90]
+_SOURCE_TIMELIMS: ["1940-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+
+# Default variables to extract if no `variables` arg is provided.
+_VARIABLES:
+  [
+    "significant_height_of_combined_wind_waves_and_swell",
+    "peak_wave_period",
+    "mean_wave_direction",
+    "wave_spectral_directional_width",
+  ]
+
+# ERA5 only has the single single-levels dataset.
+_DATASETS: { 1940-01-01: { reanalysis-era5-single-levels: default } }
+
+# KEY: The standard NC Variable for merged data (and per `_standard_attrs.py`)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+hs:
+  source_var: "swh"
+  tfv_var: "WVHT"
+  bc_scale: 1
+  bc_offset: 0
+
+tp:
+  source_var: "pp1d"
+  tfv_var: "WVPER"
+  bc_scale: 1
+  bc_offset: 0
+
+mwd:
+  source_var: "mwd"
+  tfv_var: "WVDIR"
+  bc_scale: 1
+  bc_offset: 0
+
+spr:
+  source_var: "wdw"
+  tfv_var: null
+  bc_scale: 1
+  bc_offset: 0

tfv_get_tools/providers/wave/copernicus_wave.py

@@ -0,0 +1,38 @@
+"""
+This copernicus wave model is branched from the Copernicus ocean downloader.
+"""
+
+from tfv_get_tools.providers.ocean.copernicus_ocean import (
+    DownloadCopernicusOcean,
+    MergeCopernicusOcean,
+)
+import logging
+
+
+class DownloadCopernicusWave(DownloadCopernicusOcean):
+    def _init_specific(self):
+        if self.model == "default":
+            self.log("Default model has been selected == 'GLO'")
+            self.model = "GLO"
+
+        self.source = "COPERNICUS"
+        self.mode = "WAVE"
+        self._load_config()
+
+        # User login check not yet performed
+        self._logged_in = False
+
+        # Cache for temporal extents
+        self._temporal_extents_cache = {}
+
+        if not self.verbose:
+            logging.getLogger("copernicusmarine").setLevel(logging.WARNING)
+
+
+class MergeCopernicusWave(MergeCopernicusOcean):
+    def _init_specific(self):
+        self.source = "COPERNICUS"
+        self.mode = "WAVE"
+        if self.model == "default":
+            self.model = "GLO"
+        self._load_config()