tfv-get-tools 0.2.0__py3-none-any.whl

tfv_get_tools/providers/wave/era5.py

@@ -0,0 +1,232 @@
+"""
+ERA5 Wave downloader
+"""
+
+import shutil
+import zipfile
+from pathlib import Path
+from typing import List, Tuple
+
+import cdsapi
+import numpy as np
+import pandas as pd
+import xarray as xr
+from pandas.tseries.offsets import MonthEnd
+from tqdm import tqdm
+
+from tfv_get_tools.providers._downloader import BaseDownloader
+from tfv_get_tools.providers._merger import BaseMerger
+from tfv_get_tools.providers._utilities import todstr
+
+class DownloadERA5Wave(BaseDownloader):
+    """ERA5 Wave downloader via CDS API"""
+    
+    def _init_specific(self, **kwargs):
+        """Set source and mode - matches original interface"""
+        self.source = "ERA5"
+        self.mode = "WAVE"
+        self._load_config()
+    
+    def _get_output_filename(self, ts: pd.Timestamp, te: pd.Timestamp) -> Path:
+        """ERA5 Wave filename pattern (no variable in filename)"""
+        fname = f"{self.prefix}_{todstr(ts)}_{todstr(te)}.nc"
+        return self.outdir / fname
+    
+    def _construct_cds_request(self, date: pd.Timestamp) -> dict:
+        """Construct CDS API request parameters"""
+        limstr = self._to_limstr(self.xlims, self.ylims)
+        
+        year = date.year
+        month = date.month
+        times = [f"{x:02}:00" for x in range(0, 24, 1)]  # Hourly
+        days = [str(x) for x in range(1, 32)]
+        
+        return {
+            "product_type": "reanalysis",
+            "variable": self.variables,
+            "year": [year],
+            "month": [month],
+            "day": days,
+            "time": times,
+            "area": limstr,
+            "data_format": "netcdf",
+            "download_format": "unarchived",
+        }
+    
+    @staticmethod
+    def _to_limstr(x, y):
+        """Convert coordinate bounds to ERA5 area string format"""
+        return f"{y[1]}/{x[0]}/{y[0]}/{x[1]}"
+    
+    def _download_single_file(self, temp_file: Path, final_file: Path, cds_request: dict) -> bool:
+        """Download single file via CDS API"""
+        try:
+            c = cdsapi.Client()
+            
+            # Download to temporary file first
+            c.retrieve("reanalysis-era5-single-levels", cds_request, temp_file)
+            
+            if temp_file.exists():
+                # ERA5 sometimes returns zip files, handle both cases
+                self._convert_split_netcdf_data(temp_file, final_file)
+                return True
+            else:
+                return False
+                    
+        except Exception as e:
+            error_msg = str(e).lower()
+            
+            # Check for specific error types and provide helpful messages
+            if "missing/incomplete configuration file" in error_msg or ".cdsapirc" in error_msg:
+                print("\n" + "="*60)
+                print("CDS API CONFIGURATION ERROR")
+                print("="*60)
+                print("The CDS API configuration file is missing or incomplete.")
+                print("To fix this issue:")
+                print("1. Register at https://cds.climate.copernicus.eu/")
+                print("2. Go to your profile page and copy your API key")
+                print("3. Create a file called '.cdsapirc' in your home directory with:")
+                print("   url: https://cds.climate.copernicus.eu/api")
+                print("   key: YOUR_API_KEY_HERE")
+                print("="*60)
+                
+            elif "authentication" in error_msg or "invalid key" in error_msg:
+                print("\n" + "="*60)
+                print("CDS API AUTHENTICATION ERROR")
+                print("="*60)
+                print("Your CDS API key appears to be invalid.")
+                print("Please check your .cdsapirc file and ensure your API key is correct.")
+                print("You can find your key at: https://cds.climate.copernicus.eu/user/")
+                print("="*60)
+                
+            else:
+                # For any other errors, show the original message if verbose
+                if self.verbose:
+                    print(f"Failed to download via CDS API: {e}")
+                else:
+                    # Always show some info for unhandled errors
+                    print(f"\nCDS API Error: {e}")
+                    
+            return False
+    
+    @staticmethod
+    def _convert_split_netcdf_data(file_handle_temp: Path, file_handle: Path) -> bool:
+        """
+        Handle ERA5 zip files or direct NetCDF files
+        """
+        file_path = Path(file_handle_temp)
+        file_path_out = Path(file_handle)
+        
+        # Check if file is a zip file
+        if zipfile.is_zipfile(file_path):
+            datasets = []
+            with zipfile.ZipFile(file_path, 'r') as zip_ref:
+                # Extract all files to a temporary directory
+                temp_dir = file_path.parent / 'temp_netcdf'
+                temp_dir.mkdir(exist_ok=True)
+                zip_ref.extractall(temp_dir)
+                
+                # Load all NetCDF files
+                for extracted_file in temp_dir.glob('*.nc'):
+                    with xr.open_dataset(extracted_file) as dsx:
+                        dsx.load()
+                    datasets.append(dsx)
+                
+                # Clean up extracted files
+                for file in temp_dir.iterdir():
+                    file.unlink()
+                temp_dir.rmdir()
+            
+            if not datasets:
+                raise ValueError("No NetCDF files found in zip archive")
+            
+            # Combine all datasets
+            ds = xr.merge(datasets)
+            ds.to_netcdf(file_path_out)
+            
+            # Delete the zip file
+            file_path.unlink()
+            
+        else:
+            # Move the NetCDF file to the output path
+            shutil.move(file_path, file_path_out)
+        
+        return True
+    
+    def download(self):
+        """ERA5 Wave-specific download loop - yields tasks for new base class"""
+        for ts in self.times:
+            te = ts + MonthEnd() + pd.Timedelta("23.9h")
+            
+            final_file = self._get_output_filename(ts, te)
+            temp_file = self.outdir / '_temp_era5_file'
+            cds_request = self._construct_cds_request(ts)
+            
+            yield {
+                'file_path': final_file,
+                'url': 'CDS_API',  # Not a URL but API call
+                'timestamp': ts,
+                'variable': 'all_variables',  # ERA5 downloads all vars together
+                'download_func': lambda tf=temp_file, ff=final_file, req=cds_request: 
+                    self._download_single_file(tf, ff, req)
+            }
+
+
+class MergeERA5Wave(BaseMerger):
+    def _init_specific(self) -> None:
+        self.source = "ERA5"
+        self.mode = "WAVE"
+        self._load_config()
+
+    def _process_era5_dataset(self, ds: xr.Dataset) -> xr.Dataset:
+        """Apply ERA5-specific coordinate and dimension processing."""
+        # Rename time coordinate
+        if "valid_time" in ds.coords:
+            ds = ds.rename({"valid_time": "time"})
+            
+        # Handle experiment version dimension  
+        if "expver" in ds.dims:
+            ds = ds.mean(dim="expver", keep_attrs=True)
+        ds = ds.drop_vars("expver", errors='ignore')
+        
+        # Standardise coordinate names
+        coord_mappings = {'lon': 'longitude', 'lng': 'longitude', 'lat': 'latitude'}
+        rename_dict = {old: new for old, new in coord_mappings.items() if old in ds.coords}
+        if rename_dict:
+            ds = ds.rename(rename_dict)
+            
+        return ds
+
+    def merge_files(self, file_list: List[Path]) -> Tuple[xr.Dataset, List[Path]]:
+        """Merge ERA5 wave files with time concatenation."""
+        if not file_list:
+            raise ValueError("No files provided for merging")
+            
+        datasets = []
+        skipped_files = []
+        
+        for file_path in tqdm(file_list, disable=not self.verbose):
+            ds = self._open_subset_netcdf(file_path, time=("time", "valid_time"))
+            if ds is not None:
+                ds = self._process_era5_dataset(ds)
+                datasets.append(ds)
+            else:
+                skipped_files.append(file_path)
+                
+        if not datasets:
+            raise ValueError("No valid datasets could be loaded")
+            
+        # Concatenate and clean up
+        merged = xr.concat(datasets, dim="time", combine_attrs="override", 
+                          data_vars="minimal", coords="minimal", compat="override")
+        
+        # Remove duplicates and sort
+        merged = merged.sortby("time")
+        _, unique_idx = np.unique(merged["time"], return_index=True)
+        merged = merged.isel(time=np.sort(unique_idx))
+        
+        # Sort latitudes south to north
+        if 'latitude' in merged.coords:
+            merged = merged.sortby("latitude")
+            
+        return merged, skipped_files

tfv_get_tools/providers/wave/era5_gcp.py

@@ -0,0 +1,169 @@
+"""
+AEW - This is an alternative source ERA5 that we used for awhile while CDSAPI wasn't playing ball.
+It's marginally faster, requires no registration, but runs several months behind CDSAPI which
+got called out a few times, so we should stick with CDSAPI. Also generally believe it's good form to
+have users go through CDSAPI as it is their data. 
+"""
+
+from pathlib import Path
+
+import numpy as np
+import xarray as xr
+import pandas as pd
+from tqdm import tqdm
+
+from pandas.tseries.offsets import MonthEnd
+
+from tfv_get_tools.providers._utilities import todstr
+from tfv_get_tools.providers._downloader import BaseDownloader
+from tfv_get_tools.providers._merger import BaseMerger
+
+
+class DownloadERA5WaveGCP(BaseDownloader):
+    """Updated version that makes use of the public GCP Zarr ERA5 release"""
+
+    def _init_specific(self, **kwargs):
+        """Set source and mode - matches original interface"""
+        self.source = "ERA5_GCP"
+        self.mode = "WAVE"
+        self._load_config()
+        
+        # Initialize the Zarr dataset connection
+        self.era5_ds = None
+        self.last_valid = None
+    
+    def _initialize_zarr_connection(self):
+        """Initialize connection to ERA5 Zarr dataset"""
+        if self.era5_ds is None:
+            if self.verbose:
+                print('Opening connection to ERA5 data - please wait')
+            
+            self.era5_ds = xr.open_zarr(
+                self.base_url,
+                chunks=dict(time=744),  # 31 day chunk
+                storage_options=dict(token='anon'),
+            )
+
+            self.last_valid = pd.Timestamp(self.era5_ds.attrs['valid_time_stop']).floor('1d')
+
+            self.era5_ds = self.era5_ds.sel(time=slice(
+                pd.Timestamp(1940, 1, 1), self.last_valid
+            ))
+    
+    def _get_output_filename(self, ts: pd.Timestamp, te: pd.Timestamp) -> Path:
+        """ERA5 GCP filename pattern (no variable in filename - downloads all together)"""
+        fname = f"{self.prefix}_{todstr(ts)}_{todstr(te)}.nc"
+        return self.outdir / fname
+    
+    def _download_single_time_period(self, ts: pd.Timestamp, te: pd.Timestamp, output_file: Path) -> bool:
+        """Download single time period from Zarr dataset"""
+        try:
+            # Check if we're past the valid time range
+            if ts > self.last_valid:
+                if self.verbose:
+                    print(f'The final valid ERA5 time on this database is {self.last_valid.strftime("%Y-%m-%d")}.')
+                    print('Skipping this time period')
+                return False
+            
+            # Apply slices and variable filters
+            dsx = self.era5_ds.sel(
+                time=slice(ts, te),
+                latitude=slice(*self.ylims[::-1]), 
+                longitude=slice(*self.xlims),
+            )[self.variables]
+            
+            if self.verbose:
+                print(f"... Downloading {output_file.name}")
+            
+            # Wrap the cut-down piece back to -180 to 180
+            dsx = dsx.assign_coords({'longitude': (dsx['longitude'] + 180) % 360 - 180})
+            dsx = dsx.sortby('longitude')
+
+            dsx.to_netcdf(output_file)
+            return True
+            
+        except Exception as e:
+            if self.verbose:
+                print(f"Failed to download {output_file.name}: {e}")
+            return False
+    
+    def download(self):
+        """ERA5 GCP-specific download loop - yields tasks for new base class"""
+        # Initialize the Zarr connection once
+        self._initialize_zarr_connection()
+        
+        if self.verbose:
+            print('Starting downloading loop')
+        
+        for ts in self.times:
+            te = ts + MonthEnd() + pd.Timedelta("23.9h")
+            
+            # Check if we're past the valid time range
+            if ts > self.last_valid:
+                if self.verbose:
+                    print(f'The final valid ERA5 time on this database is {self.last_valid.strftime("%Y-%m-%d")}.')
+                    print('Exiting download loop early')
+                break
+            
+            output_file = self._get_output_filename(ts, te)
+            
+            yield {
+                'file_path': output_file,
+                'url': f'zarr://{self.base_url}',  # Pseudo-URL for logging
+                'timestamp': ts,
+                'variable': 'all_variables',  # ERA5 downloads all vars together
+                'download_func': lambda start=ts, end=te, out=output_file: 
+                    self._download_single_time_period(start, end, out)
+            }
+
+class MergeERA5WaveGCP(BaseMerger):
+    def _init_specific(self):
+        self.source = "ERA5"
+        self.mode = "WAVE"
+        self._load_config()
+
+    def merge_files(self, file_list):
+        """
+        ERA5 merging logic.
+
+        ERA5 names the time variable "valid_time" which we rename after opening.
+
+        Args:
+            file_list (list): list of path objects to open and concat.
+
+        Returns:
+            xr.Dataset: merged xarray dataset
+            list: files unable to be merged
+        """
+        dsset = []
+        skipped_list = []
+        for f in tqdm(file_list):
+            dsx = self._open_subset_netcdf(f, time=("time", "valid_time"))
+            if dsx is not None:
+                if "valid_time" in dsx:
+                    dsx = dsx.rename(valid_time="time")
+                if "expver" in dsx.dims:
+                    dsx = dsx.mean(dim="expver", keep_attrs=True)
+                dsx = dsx.drop("expver", errors='ignore')
+                dsset.append(dsx)
+            else:
+                skipped_list.append(f)
+
+        print("Concatenating xarray dataset")
+        ds = xr.concat(
+            dsset,
+            dim="time",
+            combine_attrs="override",
+            data_vars="minimal",
+            coords="minimal",
+            compat="override",
+        )
+
+        # Sort by time and drop duplicates (from overlaps)
+        ds = ds.sortby("time")
+        _, idx = np.unique(ds["time"], return_index=True)
+        ds = ds.isel(time=idx)
+
+        ds = ds.sortby("latitude")  # Latitudes should go south to north
+
+        return ds, skipped_list

tfv_get_tools/tide/__init__.py

@@ -0,0 +1,2 @@
+from ._tidal_base import get_constituents, predict_waterlevel_timeseries, ExtractTide
+from ._nodestring import load_nodestring_shapefile, process_nodestring_gdf

tfv_get_tools/tide/_nodestring.py

@@ -0,0 +1,214 @@
+from pathlib import Path
+from typing import Union, List, Any
+
+import numpy as np
+import pandas as pd
+import geopandas as gpd
+from pyproj import CRS
+from geopandas import GeoDataFrame
+from shapely.geometry import LineString, Point
+
+
+def _convert_id(x: Any):
+    """Simple ID entry conversion
+
+    Try to make each ID an integer, but fall back on a string.
+
+    Args:
+        x (Any): ID Field (e.g., 2, 'NS1')
+
+    Returns:
+        Union[int, str]: ID as either an int or str, with preference on integer.
+    """
+    try:
+        x = int(x)
+    except:
+        x = str(x)
+    return x
+
+
+def load_nodestring_shapefile(
+    filename: Union[Path, str], crs: int = None, process_ids: Union[tuple, list] = None
+) -> GeoDataFrame:
+    """Load a TUFLOW FV nodestring shapefile as a GeoDataFrame.
+
+    The CRS will be read from the .prj file if present. Otherwise, `crs=X` can be passed as an EPSG integer code.
+
+    By default, all the features in the nodestring will be loaded. Use `process_ids` arg to filter to only certain features.
+
+    Args:
+        filename (Union[Path, str]): Path to the nodestring .shp file
+        crs (int, optional): Coordinate reference system EPSG code. Defaults to None.
+        process_ids (Union[tuple, list], optional): List of ID's to process. Defaults to None.
+
+    Returns:
+        GeoDataFrame: Frame containing the geometry and ID features ready for processing.
+    """
+
+    gdf = gpd.read_file(filename, columns=["ID"])
+
+    # Set the CRS of the geodataframe. Assume 4326 as backup.
+    if crs is None:
+        if not gdf.crs:
+            print(
+                "No CRS could be read from the shapefile. Assuming nodestring is in latitude/longitude (EPSG 4326)"
+            )
+            gdf.set_crs(4326)
+    else:
+        try:
+            crs = CRS.from_epsg(crs)
+            gdf = gdf.set_crs(crs)
+        except:
+            raise ValueError(
+                f"Supplied CRS `{crs}` is not valid. Please provide an EPSG code as an integer, e.g., 7856"
+            )
+
+    # Parse the process ID's
+    assert "ID" in gdf.columns, "There must be an `ID` column present in the shapefile"
+    shp_ids = gdf["ID"].apply(_convert_id).tolist()
+
+    if process_ids is None:
+        # If no id's are supplied, then we take everything except obvious nan's.
+        process_ids = [x for x in shp_ids if x != "nan"]
+    else:
+        # If they are supplied, then we first convert to INT or STR
+        process_ids = [_convert_id(x) for x in process_ids]
+        for x in process_ids:
+            assert (
+                x in shp_ids
+            ), f"Nodestring feature ID `{x}` was not found in the shapefile"
+
+    # Do a check on geometry before moving on
+    checked_ids = []
+    for x in process_ids:
+        idx = shp_ids.index(x)
+        geo = gdf.loc[idx, "geometry"]
+
+        if geo is None:
+            print(
+                f"Warning - No geometry detected for Nodestring ID {x}. Skipping this feature..."
+            )
+        elif not isinstance(geo, LineString):
+            print(
+                f"Warning - Invalid geometry detected for Nodestring ID {x}. Must be a Linestring type. Skipping this feature..."
+            )
+        else:
+            checked_ids.append(x)
+
+    msk = [shp_ids.index(x) for x in checked_ids]
+    gdf = gdf.loc[msk]
+
+    return gdf
+
+
+def process_nodestring_gdf(gdf: GeoDataFrame, spacing=2500.0) -> dict:
+    """Generates a dictionary containing a Nx2 array of lat/lon coordinates from a nodestring geodataframe.
+
+    The geodataframe must have a `crs` set. It does not need to be lat/lon.
+
+    All features in the geodataframe will be processed - it is assumed that filtering has already happened.
+
+    Args:
+        gdf (GeoDataFrame): nodestring geodataframe.
+        spacing (float, optional): Spacing in meters. Defaults to 2500.0
+
+    Returns:
+        dict: nodestring dictionary where key is the `ID` and values are a Nx2 np.ndarray.
+    """
+
+    if gdf.crs is None:
+        raise ValueError("The nodestring geodataframe must have a CRS defined.")
+
+    coords = sample_coordinates_along_linestring(gdf, spacing)
+
+    process_ids = [_convert_id(x) for x in gdf["ID"]]
+
+    ns_dat = {}
+    for i, k in enumerate(process_ids):
+        coord_array = np.squeeze(np.asarray([x.xy for x in coords.iloc[i]]))
+        ns_dat[k] = coord_array
+
+    return ns_dat
+
+
+def sample_coordinates_along_linestring(
+    gdf: gpd.GeoDataFrame, spacing: float
+) -> gpd.GeoSeries:
+    """
+    Sample coordinates along LineString geometries in a GeoDataFrame.
+
+    Args:
+        gdf (gpd.GeoDataFrame): The input GeoDataFrame containing LineString geometries.
+        spacing (float): The desired spacing between sampled points in meters.
+
+    Returns:
+        gpd.GeoSeries: A GeoSeries where each element is a list of sampled Points in WGS84 coordinates.
+    """
+    return gdf.apply(lambda row: process_row(row, spacing, gdf.crs), axis=1)
+
+
+def sample_linestring(line: LineString, spacing_meters: float) -> List[Point]:
+    """
+    Sample points along a LineString at a specified spacing.
+
+    Args:
+        line (LineString): The LineString to sample points from.
+        spacing_meters (float): The spacing between points in meters.
+
+    Returns:
+        List[Point]: A list of sampled points along the LineString.
+
+    Raises:
+        ValueError: If the line is empty or the spacing is not positive.
+    """
+    if line.is_empty:
+        raise ValueError("The LineString is empty.")
+    if spacing_meters <= 0:
+        raise ValueError("Spacing must be a positive number.")
+
+    line_length = line.length
+    num_points = int(line_length / spacing_meters) + 1
+    distances = np.linspace(0, line_length, num_points)
+    return [line.interpolate(distance) for distance in distances]
+
+
+def process_row(row: pd.Series, spacing: float, crs: CRS) -> List[Point]:
+    """
+    Process a single row of a GeoDataFrame, sampling points along its LineString geometry.
+
+    This function handles both geographic (lat/lon) and projected coordinate systems.
+    It always returns the sampled points in WGS84 (EPSG:4326) lat/lon coordinates.
+
+    Args:
+        row (pd.Series): A row from a GeoDataFrame containing a 'geometry' column
+                         with a LineString object.
+        spacing (float): The desired spacing between sampled points in meters.
+        crs (CRS): The coordinate reference system of the input geometry.
+
+    Returns:
+        List[Point]: A list of sampled points along the LineString, in lat/lon coordinates.
+
+    Raises:
+        ValueError: If the input geometry is not a LineString.
+    """
+    if not isinstance(row.geometry, LineString):
+        raise ValueError("The geometry must be a LineString.")
+
+    if crs.is_geographic:
+        # Convert to UTM for accurate distance calculations
+        centroid = row.geometry.centroid
+        utm_zone = int((centroid.x + 180) // 6 + 1)
+        utm_crs = CRS.from_proj4(
+            f"+proj=utm +zone={utm_zone} +datum=WGS84 +units=m +no_defs"
+        )
+        utm_geometry = gpd.GeoSeries([row.geometry], crs=crs).to_crs(utm_crs)[0]
+        sampled_points = sample_linestring(utm_geometry, spacing)
+        points_gdf = gpd.GeoDataFrame(geometry=sampled_points, crs=utm_crs)
+    else:
+        # If already projected, sample directly
+        sampled_points = sample_linestring(row.geometry, spacing)
+        points_gdf = gpd.GeoDataFrame(geometry=sampled_points, crs=crs)
+
+    # Convert sampled points to lat/lon (WGS84)
+    points_gdf = points_gdf.to_crs(epsg=4326)
+    return points_gdf.geometry.tolist()