tfv-get-tools 0.2.0__py3-none-any.whl

tfv_get_tools/providers/atmos/cfgs/cfsr.yaml

@@ -0,0 +1,68 @@
+_BASE_URL: "https://thredds.rda.ucar.edu/thredds/dodsC/files/g"
+_INFO_URL: https://climatedataguide.ucar.edu/climate-data/climate-forecast-system-reanalysis-cfsr
+
+_SOURCE_XLIMS: [-180.0, 359.5]
+_SOURCE_YLIMS: [-90.0, 90.0]
+_SOURCE_TIMELIMS: ["1979-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+
+# Default variables to extract if no `variables` arg is provided.
+_VARIABLES: ["wnd10m", "prmsl", "dlwsfc", "dswsfc", "tmp2m", "rh2m", "prate"]
+
+# This CFSR dataset mapping links the dataset id and system name to the valid date.
+# The full opendap url takes the form <base_url>/<dset_id>/<var>.<sys>.<datestr>.grb2
+_DATASETS: {
+    1979-01-01: { d093001-gdas: default },
+    2011-01-01: { d094001-gdas: default },
+    2011-04-01: { d094001-cdas1: default },
+  } # CFSR Selected Hourly # CFSv2 Selected Hourly on GDAS # CFSv2 Selected Hourly on CDAS1
+
+# For CFSR, the merged grid will be based on the u10 variable
+u10:
+  source_var: "u-component_of_wind_height_above_ground"
+  tfv_var: "W10_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+v10:
+  source_var: "v-component_of_wind_height_above_ground"
+  tfv_var: "W10_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+mslp:
+  source_var: "Pressure_reduced_to_MSL_msl"
+  tfv_var: "MSLP_GRID"
+  bc_scale: 0.01
+  bc_offset: 0
+
+dlwrf:
+  source_var: "Downward_Long-Wave_Radp_Flux_surface_Mixed_intervals_Average"
+  tfv_var: "LW_RAD_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+dswrf:
+  source_var: "Downward_Short-Wave_Radiation_Flux_surface_Mixed_intervals_Average"
+  tfv_var: "SW_RAD_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+t2m:
+  source_var: "Temperature_height_above_ground"
+  tfv_var: "AIR_TEMP_GRID"
+  bc_scale: 1
+  bc_offset: -273.15
+
+relhum:
+  source_var: "Relative_humidity_height_above_ground"
+  tfv_var: "REL_HUM_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+prate:
+  source_var: "Precipitation_rate_surface_Mixed_intervals_Average"
+  tfv_var: PRECIP_GRID
+  bc_scale: 86.4 # Convert from kg/m2/s to mm/day
+  bc_offset: 0

tfv_get_tools/providers/atmos/cfgs/era5.yaml

@@ -0,0 +1,77 @@
+_BASE_URL: "N/A"
+_INFO_URL: https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels
+
+_SOURCE_XLIMS: [-180, 360]
+_SOURCE_YLIMS: [-90, 90]
+_SOURCE_TIMELIMS: ["1940-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+
+# Default variables to extract if no `variables` arg is provided.
+_VARIABLES:
+  [
+    "10m_u_component_of_wind",
+    "10m_v_component_of_wind",
+    "mean_sea_level_pressure",
+    "mean_surface_downward_long_wave_radiation_flux",
+    "mean_surface_downward_short_wave_radiation_flux",
+    "2m_temperature",
+    "mean_total_precipitation_rate",
+    "2m_dewpoint_temperature",
+  ]
+
+# ERA5 only has the single single-levels dataset.
+_DATASETS: { 1940-01-01: { reanalysis-era5-single-levels: default } }
+
+# KEY: The standard NC Variable for merged data (and per _standard_attrs.py)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+u10:
+  source_var: "u10"
+  tfv_var: "W10_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+v10:
+  source_var: "v10"
+  tfv_var: "W10_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+mslp:
+  source_var: "msl"
+  tfv_var: "MSLP_GRID"
+  bc_scale: 0.01
+  bc_offset: 0
+
+dlwrf:
+  source_var: "msdwlwrf"
+  tfv_var: LW_RAD_GRID
+  bc_scale: 1
+  bc_offset: 0
+
+dswrf:
+  source_var: "msdwswrf"
+  tfv_var: SW_RAD_GRID
+  bc_scale: 1
+  bc_offset: 0
+
+t2m:
+  source_var: "t2m"
+  tfv_var: AIR_TEMP_GRID
+  bc_scale: 1
+  bc_offset: 0
+
+prate:
+  source_var: "mtpr"
+  tfv_var: PRECIP_GRID
+  bc_scale: 86.4 # Convert from kg/m2/s to mm/day
+  bc_offset: 0
+
+relhum:
+  source_var: d2m
+  tfv_var: "relative_humidity"
+  bc_scale: 1
+  bc_offset: 0
+  pre_process: dewpt_to_relhum(d2m, t2m)

tfv_get_tools/providers/atmos/cfgs/era5_gcp.yaml

@@ -0,0 +1,77 @@
+_BASE_URL: "gs://gcp-public-data-arco-era5/ar/full_37-1h-0p25deg-chunk-1.zarr-v3"
+_INFO_URL: https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels
+
+_SOURCE_XLIMS: [-180, 360]
+_SOURCE_YLIMS: [-90, 90]
+_SOURCE_TIMELIMS: ["1940-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+
+# Default variables to extract if no `variables` arg is provided.
+_VARIABLES:
+  [
+    "10m_u_component_of_wind",
+    "10m_v_component_of_wind",
+    "mean_sea_level_pressure",
+    "mean_surface_downward_long_wave_radiation_flux",
+    "mean_surface_downward_short_wave_radiation_flux",
+    "2m_temperature",
+    "mean_total_precipitation_rate",
+    "2m_dewpoint_temperature",
+  ]
+
+# ERA5 only has the single single-levels dataset.
+_DATASETS: { 1940-01-01: { reanalysis-era5-single-levels: default } }
+
+# KEY: The standard NC Variable for merged data (and per _standard_attrs.py)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+u10:
+  source_var: "u10"
+  tfv_var: "W10_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+v10:
+  source_var: "v10"
+  tfv_var: "W10_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+mslp:
+  source_var: "msl"
+  tfv_var: "MSLP_GRID"
+  bc_scale: 0.01
+  bc_offset: 0
+
+dlwrf:
+  source_var: "msdwlwrf"
+  tfv_var: LW_RAD_GRID
+  bc_scale: 1
+  bc_offset: 0
+
+dswrf:
+  source_var: "msdwswrf"
+  tfv_var: SW_RAD_GRID
+  bc_scale: 1
+  bc_offset: 0
+
+t2m:
+  source_var: "t2m"
+  tfv_var: AIR_TEMP_GRID
+  bc_scale: 1
+  bc_offset: 0
+
+prate:
+  source_var: "mtpr"
+  tfv_var: PRECIP_GRID
+  bc_scale: 86.4 # Convert from kg/m2/s to mm/day
+  bc_offset: 0
+
+relhum:
+  source_var: d2m
+  tfv_var: "relative_humidity"
+  bc_scale: 1
+  bc_offset: 0
+  pre_process: dewpt_to_relhum(d2m, t2m)

tfv_get_tools/providers/atmos/cfsr.py

@@ -0,0 +1,207 @@
+"""
+CFSR/CFSv2
+"""
+
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+import numpy as np
+import pandas as pd
+import xarray as xr
+from pandas.tseries.offsets import MonthEnd
+from tqdm import tqdm
+
+from tfv_get_tools.providers._downloader import BaseDownloader
+from tfv_get_tools.providers._merger import BaseMerger
+from tfv_get_tools.providers._utilities import todstr
+
+
+class DownloadCFSRAtmos(BaseDownloader):
+    """CFSR downloader - only the source-specific parts"""
+    
+    def _init_specific(self, **kwargs):
+        """Set source and mode - that's it"""
+        self.source = "CFSR"
+        self.mode = "ATMOS"
+        self._load_config()
+    
+    def _get_output_filename(self, ts: pd.Timestamp, te: pd.Timestamp, var: str) -> Path:
+        """CFSR-specific filename pattern"""
+        return self.outdir / f"{self.prefix}_{var}_{todstr(ts)}_{todstr(te)}.nc"
+    
+    def _construct_opendap_url(self, date: pd.Timestamp, var: str) -> str:
+        """CFSR-specific URL construction"""
+        dataset_dates = list(self.dsmap.keys())
+        idx = np.where([date.date() >= x for x in dataset_dates])[0][-1]
+        dataset_time = dataset_dates[idx]
+        
+        ds_entry = list(self.dsmap[dataset_time].keys())[0]
+        dataset_id, sys = ds_entry.split("-")
+
+        datestr = date.strftime("%Y%m")
+        year = date.year
+
+        fname = f"{var}.{sys}.{datestr}.grb2"
+        url = f"{self.base_url}/{dataset_id}/{year}/{fname}"
+        return url
+    
+    def _download_single_file(self, fname: Path, url: str) -> bool:
+        """CFSR-specific download and processing"""
+        try:
+            ds = xr.open_dataset(url)
+
+            # Handle longitude selection (CFSR-specific logic)
+            if self.xlims[0] < self.xlims[1]:
+                lon_idx = np.hstack(
+                    np.where(
+                        (self.xlims[0] <= ds["lon"])
+                        & (ds["lon"] <= self.xlims[1])
+                    )
+                )
+            else:
+                lon_idx = np.hstack(
+                    (
+                        np.where(ds["lon"] <= self.xlims[1])[0],
+                        np.where(ds["lon"] >= self.xlims[0])[0],
+                    )
+                )
+            
+            assert lon_idx.size > 1, "No longitude points selected!"
+
+            # Select latitude indices
+            lat_idx = np.hstack(
+                np.where(
+                    (self.ylims[0] <= ds["lat"]) 
+                    & (ds["lat"] <= self.ylims[1])
+                )
+            )
+            assert lat_idx.size > 1, "No latitude points selected!"
+
+            # Subset and save
+            ds = ds.isel(lon=lon_idx, lat=lat_idx)
+            ds.to_netcdf(fname)
+            return True
+
+        except Exception as e:
+            if self.verbose:
+                print(f"Failed to download {url}: {e}")
+            return False
+    
+    def download(self):
+        """CFSR-specific download loop - only the parts that differ from base"""
+        # The base class handles result tracking, progress, etc.
+        # We just need to yield the download tasks
+        
+        for ts in self.times:
+            te = ts + MonthEnd() + pd.Timedelta("23.9h")
+            
+            for var in self.variables:
+                out_file = self._get_output_filename(ts, te, var)
+                url = self._construct_opendap_url(ts, var)
+                
+                # Let base class handle the file existence check, timing, etc.
+                yield {
+                    'file_path': out_file,
+                    'url': url,
+                    'timestamp': ts,
+                    'variable': var,
+                    'download_func': lambda f=out_file, u=url: self._download_single_file(f, u)
+                }
+
+class MergeCFSRAtmos(BaseMerger):
+    def _init_specific(self) -> None:
+        self.source = "CFSR"
+        self.mode = "ATMOS"
+        self._load_config()
+
+    def _extract_target_coordinates(self, datasets: List[xr.Dataset]) -> Tuple[np.ndarray, np.ndarray]:
+        """Extract appropriate wind coordinates, preferring CFSv2 over CFSR."""
+        cfsv2_coords = None
+        cfsr_coords = None
+        
+        for ds in datasets:
+            if ds is None or 'u-component_of_wind_height_above_ground' not in ds:
+                continue
+                
+            time_values = pd.to_datetime(ds['time'].values)
+            first_time = time_values[0] if len(time_values) > 0 else pd.Timestamp.min
+            
+            if first_time >= pd.Timestamp(2011, 1, 1):
+                cfsv2_coords = (ds['lon'].values, ds['lat'].values)
+            else:
+                cfsr_coords = (ds['lon'].values, ds['lat'].values)
+        
+        # Prefer CFSv2, fallback to CFSR, then any available
+        return cfsv2_coords or cfsr_coords or (datasets[0]['lon'].values, datasets[0]['lat'].values)
+
+    def _process_file(self, file_path: Path) -> Optional[xr.Dataset]:
+        """Load file and filter 30-minute data."""
+        ds = self._open_subset_netcdf(file_path, chunks=dict(time=24))
+        if ds is not None:
+            # Filter out 30-minute data points
+            time_values = pd.to_datetime(ds['time'].values)
+            valid_indices = [i for i, t in enumerate(time_values) if t.minute != 30]
+            ds = ds.isel(time=valid_indices)
+        return ds
+
+    def merge_files(self, file_list: List[Path]) -> Tuple[xr.Dataset, List[Path]]:
+        """Merge CFSR files: group by startdate, merge variables, then concat time."""
+        if not file_list:
+            raise ValueError("No files provided for merging")
+            
+        # Group files by start date
+        startdates = [f.stem.split('_')[-2] for f in file_list]
+        grouped_files = {date: [] for date in np.unique(startdates)}
+        
+        # Load files and group them
+        all_datasets = []
+        skipped_files = []
+        
+        for i, file_path in enumerate(tqdm(file_list, disable=not self.verbose)):
+            ds = self._process_file(file_path)
+            if ds is not None:
+                grouped_files[startdates[i]].append(ds)
+                all_datasets.append(ds)
+            else:
+                skipped_files.append(file_path)
+        
+        if not all_datasets:
+            raise ValueError("No valid datasets could be loaded")
+        
+        # Extract target coordinates for interpolation
+        target_lon, target_lat = self._extract_target_coordinates(all_datasets)
+        
+        if self.verbose:
+            print("Concatenating and interpolating xarray dataset")
+        
+        # Merge variables for each start date group
+        merged_by_date = []
+        for date_group in grouped_files.values():
+            if date_group:
+                # Interpolate all datasets to common grid
+                interpolated = [
+                    ds.interp(lon=target_lon, lat=target_lat, method='linear', 
+                             kwargs=dict(fill_value='extrapolate'))
+                    for ds in date_group
+                ]
+                merged_by_date.append(xr.merge(interpolated, compat='override'))
+        
+        # Concatenate along time dimension
+        merged = xr.concat(merged_by_date, dim="time", combine_attrs="override",
+                          data_vars="minimal", coords="minimal", compat="override")
+        
+        # Final cleanup
+        merged = merged.mean(dim='height_above_ground', skipna=True)
+        merged = merged.drop_vars(['reftime', 'time_bounds', 'GaussLatLon_Projection', 
+                                  'LatLon_Projection', 'height_above_ground'], errors='ignore')
+        
+        # Sort and remove duplicates
+        merged = merged.sortby('time')
+        _, unique_idx = np.unique(merged['time'], return_index=True)
+        merged = merged.isel(time=unique_idx)
+        
+        # Fill gaps and standardise coordinates
+        merged = merged.bfill('time', limit=3).ffill('time', limit=3)
+        merged = merged.rename({'lon': 'longitude', 'lat': 'latitude'})
+        
+        return merged, skipped_files

tfv_get_tools/providers/atmos/era5.py

@@ -0,0 +1,20 @@
+"""
+This ERA5 ATMOS model has been branched from the ERA5 Wave Downloader
+"""
+
+from tfv_get_tools.providers.wave.era5 import DownloadERA5Wave, MergeERA5Wave
+
+class DownloadERA5Atmos(DownloadERA5Wave):
+    def _init_specific(self):
+        self.source = 'ERA5'
+        self.mode = 'ATMOS'
+        self._load_config()
+        
+        # User login check not yet performed
+        self._logged_in = False        
+
+class MergeERA5Atmos(MergeERA5Wave):
+    def _init_specific(self):
+        self.source = 'ERA5'
+        self.mode = 'ATMOS'
+        self._load_config()

tfv_get_tools/providers/atmos/era5_gcp.py

@@ -0,0 +1,20 @@
+"""
+This ERA5 ATMOS model has been branched from the ERA5 Wave Downloader
+"""
+
+from tfv_get_tools.providers.wave.era5 import DownloadERA5Wave, MergeERA5Wave
+
+class DownloadERA5AtmosGCP(DownloadERA5Wave):
+    def _init_specific(self):
+        self.source = 'ERA5_GCP'
+        self.mode = 'ATMOS'
+        self._load_config()
+        
+        # User login check not yet performed
+        self._logged_in = False        
+
+class MergeERA5Atmos(MergeERA5Wave):
+    def _init_specific(self):
+        self.source = 'ERA5'
+        self.mode = 'ATMOS'
+        self._load_config()

tfv_get_tools/providers/ocean/cfgs/copernicus_blk.yaml

@@ -0,0 +1,64 @@
+_BASE_URL: "N/A"
+
+_SOURCE_XLIMS: [27.37, 41.96]
+_SOURCE_YLIMS: [40.86, 46.8]
+_SOURCE_ZLIMS: [0, 2300] # Approx 2248m is max in BLKSEA Forecast.
+_SOURCE_TIMELIMS: ["1993-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+
+# Default copernicus ocean variables for global
+_VARIABLES: ["zos", "uo", "vo", "so", "thetao"]
+
+# Variable mapping for individual datasets
+_DATASETS:
+  {
+    1993-01-01:
+      {
+        cmems_mod_blk_phy-cur_my_2.5km_P1D-m: ["uo", "vo"],
+        cmems_mod_blk_phy-sal_my_2.5km_P1D-m: ["so"],
+        cmems_mod_blk_phy-tem_my_2.5km_P1D-m: ["thetao"],
+        cmems_mod_blk_phy-ssh_my_2.5km_P1D-m: ["zos"],
+      },
+    forecast:
+      {
+        cmems_mod_blk_phy-cur_anfc_2.5km_P1D-m: ["uo", "vo"],
+        cmems_mod_blk_phy-sal_anfc_2.5km_P1D-m: ["so"],
+        cmems_mod_blk_phy-tem_anfc_2.5km_P1D-m: ["thetao"],
+        cmems_mod_blk_phy-ssh_anfc_2.5km_P1D-m: ["zos"],
+      },
+  }
+
+# KEY: The standard NC Variable for merged data (and per _standard_attrs.py)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+surf_el:
+  source_var: "zos"
+  tfv_var: "H"
+  bc_scale: 1
+  bc_offset: 0
+
+water_u:
+  source_var: "uo"
+  tfv_var: "V_x"
+  bc_scale: 1
+  bc_offset: 0
+
+water_v:
+  source_var: "vo"
+  tfv_var: "V_y"
+  bc_scale: 1
+  bc_offset: 0
+
+salinity:
+  source_var: "so"
+  tfv_var: "SAL"
+  bc_scale: 1
+  bc_offset: 0
+
+water_temp:
+  source_var: "thetao"
+  tfv_var: "TEMP"
+  bc_scale: 1
+  bc_offset: 0

tfv_get_tools/providers/ocean/cfgs/copernicus_glo.yaml

@@ -0,0 +1,67 @@
+_BASE_URL: "N/A"
+
+_SOURCE_XLIMS: [-180, 360]
+_SOURCE_YLIMS: [-90, 90]
+_SOURCE_ZLIMS: [null, null]
+_SOURCE_TIMELIMS: ["1993-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+
+# Default copernicus ocean variables for global
+_VARIABLES: ["zos", "uo", "vo", "so", "thetao"]
+
+# Variable mapping for individual datasets
+_DATASETS:
+  {
+    1993-01-01:
+      {
+        cmems_mod_glo_phy_my_0.083deg_P1D-m:
+          ["zos", "uo", "vo", "so", "thetao"],
+      },
+    2021-07-01:
+      {
+        cmems_mod_glo_phy_myint_0.083deg_P1D-m:
+          ["zos", "uo", "vo", "so", "thetao"],
+      },
+    forecast:
+      {
+        cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i: ["uo", "vo"],
+        cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i: ["so"],
+        cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i: ["thetao"],
+        cmems_mod_glo_phy_anfc_0.083deg_PT1H-m: ["zos"],
+      },
+  }
+
+# KEY: The standard NC Variable for merged data (and per _standard_attrs.py)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+surf_el:
+  source_var: "zos"
+  tfv_var: "H"
+  bc_scale: 1
+  bc_offset: 0
+
+water_u:
+  source_var: "uo"
+  tfv_var: "V_x"
+  bc_scale: 1
+  bc_offset: 0
+
+water_v:
+  source_var: "vo"
+  tfv_var: "V_y"
+  bc_scale: 1
+  bc_offset: 0
+
+salinity:
+  source_var: "so"
+  tfv_var: "SAL"
+  bc_scale: 1
+  bc_offset: 0
+
+water_temp:
+  source_var: "thetao"
+  tfv_var: "TEMP"
+  bc_scale: 1
+  bc_offset: 0

tfv_get_tools/providers/ocean/cfgs/copernicus_nws.yaml

@@ -0,0 +1,62 @@
+_BASE_URL: "N/A"
+
+_SOURCE_XLIMS: [-19.89, 13.0]
+_SOURCE_YLIMS: [40.07, 65.0]
+_SOURCE_ZLIMS: [0, 5000]
+_SOURCE_TIMELIMS: ["1993-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+
+# Default copernicus ocean variables for global
+_VARIABLES: ["zos", "uo", "vo", "so", "thetao"]
+
+# Variable mapping for individual datasets
+_DATASETS:
+  {
+    1993-01-01:
+      {
+        cmems_mod_nws_phy-uv_my_7km-3D_P1D-m: ["uo", "vo"],
+        cmems_mod_nws_phy-s_my_7km-3D_P1D-m: ["so"],
+        cmems_mod_nws_phy-t_my_7km-3D_P1D-m: ["thetao"],
+        cmems_mod_nws_phy-ssh_my_7km-2D_PT1H-i: ["zos"],
+      },
+    forecast:
+      {
+        cmems_mod_nws_phy_anfc_0.027deg-2D_PT1H-m: ["zos"],
+        cmems_mod_nws_phy_anfc_0.027deg-3D_PT1H-m: ["uo", "vo", "so", "thetao"],
+      },
+  }
+
+# KEY: The standard NC Variable for merged data (and per _standard_attrs.py)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+surf_el:
+  source_var: "zos"
+  tfv_var: "H"
+  bc_scale: 1
+  bc_offset: 0
+
+water_u:
+  source_var: "uo"
+  tfv_var: "V_x"
+  bc_scale: 1
+  bc_offset: 0
+
+water_v:
+  source_var: "vo"
+  tfv_var: "V_y"
+  bc_scale: 1
+  bc_offset: 0
+
+salinity:
+  source_var: "so"
+  tfv_var: "SAL"
+  bc_scale: 1
+  bc_offset: 0
+
+water_temp:
+  source_var: "thetao"
+  tfv_var: "TEMP"
+  bc_scale: 1
+  bc_offset: 0