tfv-get-tools 0.2.0__py3-none-any.whl

tfv_get_tools/__init__.py

@@ -0,0 +1,4 @@
+from tfv_get_tools.ocean import DownloadOcean, MergeOcean
+from tfv_get_tools.wave import DownloadWave, MergeWave
+from tfv_get_tools.atmos import DownloadAtmos, MergeAtmos
+from tfv_get_tools.tide._tidal_base import ExtractTide

tfv_get_tools/_standard_attrs.py

@@ -0,0 +1,107 @@
+"""
+Set of standard variable attributes for the merged dataset
+"""
+
+STDVARS = {
+    "u10": {
+        "long_name": "10m_eastward_wind",
+        "standard_name": "eastward_wind",
+        "units": "m s-1"
+    },
+    "v10": {
+        "long_name": "10m_northward_wind",
+        "standard_name": "northward_wind",
+        "units": "m s-1"
+    },
+    "mslp": {
+        "long_name": "mean_sea_level_pressure",
+        "standard_name": "air_pressure_at_sea_level",
+        "units": "Pa"
+    },
+    "dlwrf": {
+        "long_name": "surface_downward_longwave_flux",
+        "standard_name": "surface_downwelling_longwave_flux_in_air",
+        "units": "W m-2"
+    },
+    "dswrf": {
+        "long_name": "surface_downward_shortwave_flux",
+        "standard_name": "surface_downwelling_shortwave_flux_in_air",
+        "units": "W m-2"
+    },
+    "t2m": {
+        "long_name": "2m_air_temperature",
+        "standard_name": "air_temperature",
+        "units": "K"
+    },
+    "prate": {
+        "long_name": "precipitation_rate",
+        "standard_name": "precipitation_flux",
+        "units": "kg m-2 s-1"
+    },
+    "relhum": {
+        "long_name": "relative_humidity",
+        "standard_name": "relative_humidity",
+        "units": "1"
+    },
+    "water_u": {
+        "long_name": "eastward_sea_water_velocity",
+        "standard_name": "eastward_sea_water_velocity",
+        "units": "m s-1"
+    },
+    "water_v": {
+        "long_name": "northward_sea_water_velocity",
+        "standard_name": "northward_sea_water_velocity",
+        "units": "m s-1"
+    },
+    "surf_el": {
+        "long_name": "sea_surface_height_above_geoid",
+        "standard_name": "sea_surface_height_above_geoid",
+        "units": "m"
+    },
+    "water_temp": {
+        "long_name": "sea_water_temperature",
+        "standard_name": "sea_water_temperature",
+        "units": "K"
+    },
+    "salinity": {
+        "long_name": "sea_water_salinity",
+        "standard_name": "sea_water_practical_salinity",
+        "units": "1e-3"
+    },
+    # Wave parameters
+    "hs": {
+        "long_name": "significant_wave_height",
+        "standard_name": "sea_surface_wave_significant_height",
+        "units": "m"
+    },
+    "tp": {
+        "long_name": "peak_wave_period",
+        "standard_name": "sea_surface_wave_period_at_variance_spectral_density_maximum",
+        "units": "s"
+    },
+    "tm02": {
+        "long_name": "mean_wave_period_tm02",
+        "standard_name": "sea_surface_wave_mean_period_from_variance_spectral_density_second_frequency_moment",
+        "units": "s"
+    },
+    "tm10": {
+        "long_name": "mean_wave_period_tm10",
+        "standard_name": "sea_surface_wave_mean_period_from_variance_spectral_density_inverse_frequency_moment",
+        "units": "s"
+    },
+    "mwd": {
+        "long_name": "mean_wave_direction",
+        "standard_name": "sea_surface_wave_from_direction",
+        "units": "degree"
+    },
+    "pwd": {
+        "long_name": "peak_wave_direction",
+        "standard_name": "sea_surface_wave_from_direction_at_variance_spectral_density_maximum",
+        "units": "degree"
+    },
+    "spr": {
+        "long_name": "mean_wave_directional_spreading",
+        "standard_name": "sea_surface_wave_directional_spread",
+        "units": "degree"
+    }
+}

tfv_get_tools/atmos.py

@@ -0,0 +1,167 @@
+from datetime import datetime
+from pathlib import Path
+from typing import Union, Tuple, Optional, List
+import pandas as pd
+from tfv_get_tools.providers._downloader import BatchDownloadResult
+
+
+def DownloadAtmos(
+    start_date: Union[str, datetime, pd.Timestamp],
+    end_date: Union[str, datetime, pd.Timestamp],
+    xlims: Tuple[float, float],
+    ylims: Tuple[float, float],
+    out_path: Union[str, Path] = Path("./raw"),
+    source: str = "ERA5",
+    model: str = "default",
+    prefix: Optional[str] = None,
+    verbose: bool = False,
+    variables: Optional[List[str]] = None,
+    skip_check: bool = False,
+    **kwargs,
+) -> BatchDownloadResult:
+    """Download Atmospheric Data - the proper user-facing API
+    
+    Users should call this function, not the individual downloader classes directly.
+    
+    Args:
+        start_date: Start date
+        end_date: End date  
+        xlims: Longitude bounds
+        ylims: Latitude bounds
+        out_path: Output directory
+        source: Data source ('ERA5', 'CFSR', 'BARRA2')
+        model: Model variant
+        prefix: File prefix
+        verbose: Verbose output
+        variables: Custom variables to download
+        skip_check: Skip user confirmation
+        **kwargs: Additional arguments
+        
+    Returns:
+        BatchDownloadResult: Results of the download operation
+    """
+    
+    if source.lower() == "cfsr":
+        from tfv_get_tools.providers.atmos.cfsr import DownloadCFSRAtmos
+        
+        downloader = DownloadCFSRAtmos(
+            start_date=start_date,
+            end_date=end_date,
+            xlims=xlims,
+            ylims=ylims,
+            out_path=out_path,
+            model=model,
+            prefix=prefix,
+            verbose=verbose,
+            variables=variables,
+            skip_check=skip_check,
+            **kwargs
+        )
+        return downloader.execute_download()
+    
+    elif source.lower() == "era5":
+        from tfv_get_tools.providers.atmos.era5 import DownloadERA5Atmos
+        
+        downloader = DownloadERA5Atmos(
+            start_date=start_date,
+            end_date=end_date,
+            xlims=xlims,
+            ylims=ylims,
+            out_path=out_path,
+            model=model,
+            prefix=prefix,
+            verbose=verbose,
+            variables=variables,
+            skip_check=skip_check,
+            **kwargs
+        )
+        return downloader.execute_download()
+        
+    elif source.lower() == "barra2":
+        from tfv_get_tools.providers.atmos.barra2 import DownloadBARRA2
+        
+        downloader = DownloadBARRA2(
+            start_date=start_date,
+            end_date=end_date,
+            xlims=xlims,
+            ylims=ylims,
+            out_path=out_path,
+            model=model,
+            prefix=prefix,
+            verbose=verbose,
+            variables=variables,
+            skip_check=skip_check,
+            **kwargs
+        )
+        return downloader.execute_download()
+        
+    else:
+        raise ValueError(f'Unrecognised source {source}. Must be one of: CFSR, ERA5, BARRA2')
+
+
+def MergeAtmos(
+    in_path: Path = Path("./raw"),
+    out_path: Path = Path("."),
+    fname: str = None,
+    source: str = 'ERA5',
+    model: str = 'default',
+    time_start: str = None,
+    time_end: str = None,
+    reproject: int = None,
+    local_tz: Tuple[float, str] = None,
+    pad_dry: bool = False,
+    wrapto360: bool = False,
+    write: bool = True,
+):
+    """
+    Merge raw downloaded atmos datafiles into a single netcdf file. 
+    
+    **Use the same `source` and `model` that was supplied to the Downloader function**
+
+    Args:
+        in_path (Path, optional): Directory of the raw ocean data-files. Defaults to Path(".").
+        out_path (Path, optional): Output directory for the merged ocean netcdf and (opt) the fvc. Defaults to Path(".").
+        fname (str, optional): Merged ocean netcdf filename. Defaults to None.
+        source (str, optional): Source to be merged, defaults to "ERA5". 
+        model (str, optional): Model for source to be merged. Defaults are listed in the wiki documentation.
+        time_start (str, optional): Start time limit of the merged dataset (str: "YYYY-mm-dd HH:MM"). Defaults to None.
+        time_end (str, optional): End time limit of the merged dataset (str: "YYYY-mm-dd HH:MM"). Defaults to None.
+        reproject (int, optional): Optionally reproject based, based on EPSG code. Defaults to None.
+        local_tz: (Tuple(float, str): optional): Add local timezone format is a tuple with Offset[float] and Label[str]
+        pad_dry: (bool, optional): Optionally pad landwards (i.e., fill nans horizontally). Defaults to False.
+        wrapto360: (bool, optional): Optionally wrap longitude to (0, 360) rather than (-180, 180). Defaults to False.
+        write (bool): Write the dataset. If False, the virtual merged dataset will be returned.
+    """
+    
+    args = tuple()
+
+    kwargs = dict(
+        in_path=in_path,
+        out_path=out_path,
+        fname=fname,
+        source=source,
+        model=model,
+        time_start=time_start,
+        time_end=time_end,
+        reproject=reproject,
+        local_tz=local_tz,
+        pad_dry=pad_dry,
+        wrapto360=wrapto360,
+        write=write,
+    )
+    
+    if source.lower() == "era5":
+        from tfv_get_tools.providers.atmos.era5 import MergeERA5Atmos
+        mrg = MergeERA5Atmos(*args, **kwargs)
+    
+    elif source.lower() == "cfsr":
+        from tfv_get_tools.providers.atmos.cfsr import MergeCFSRAtmos
+        mrg = MergeCFSRAtmos(*args, **kwargs)
+
+    elif source.lower() == "barra2":
+        from tfv_get_tools.providers.atmos.barra2 import MergeBARRA2
+        mrg = MergeBARRA2(*args, **kwargs)
+
+    # If the user requested no-write, return the dataset object.
+    if not write:
+        return mrg.ds

tfv_get_tools/cli/_cli_base.py

@@ -0,0 +1,173 @@
+"""Base CLI Class
+
+This is where all the default arguments are setup for the CLI tools (for Downloader "A" or Merger "B")
+
+For the mode specific arguments and messaging, go to X_cli.py program.
+"""
+
+from abc import ABC, abstractmethod
+import argparse
+from pathlib import Path
+import re
+import sys
+       
+
+class CLIBase(ABC):
+    def __init__(self, prog_name, description):
+        self.parser = argparse.ArgumentParser(
+            prog=prog_name,
+            description=description,
+            epilog="See '<command> --help' to read about a specific sub-command.",
+        )
+        self.subparsers = self.parser.add_subparsers(
+            dest="command", help="Sub-commands"
+        )
+
+    def dir_path(self, path):
+        if Path(path).is_dir():
+            return Path(path)
+        else:
+            raise argparse.ArgumentTypeError(
+                f"--path:{path} is not a valid path - check that it exists"
+            )
+
+    @abstractmethod
+    def add_source_arguments(self, parser):
+        pass
+
+    def add_download_parser(self, name, help_text):
+        parser = self.subparsers.add_parser(name, help=help_text)
+        parser.add_argument(
+            "time_start", type=str, help='Start time in format "yyyy-mm-dd"'
+        )
+        parser.add_argument("time_end", type=str, help='End time in format "yyyy-mm-dd"')
+        parser.add_argument(
+            "bbox",
+            nargs=4,
+            type=float,
+            help='Bounding box lon/lat extents as a list "xmin xmax ymin ymax"',
+        )
+        parser.add_argument(
+            "-p",
+            "--path",
+            default=".",
+            type=self.dir_path,
+            help="Output directory, needs to exist first",
+        )
+        self.add_source_arguments(parser)
+        parser.set_defaults(func=self.run_download)
+
+        return parser
+
+    def add_merge_parser(self, name, help_text):
+        parser = self.subparsers.add_parser(name, help=help_text)
+        parser.add_argument(
+            "-f",
+            "--file_name",
+            help='Merged netcdf filename. Default: "<Type>_<time_start>_<time_end>.nc"',
+        )
+        parser.add_argument(
+            "--time_start", help='Start time in format "yyyy-mm-dd"'
+        )
+        parser.add_argument("--time_end", help='End time in format "yyyy-mm-dd"')
+        parser.add_argument(
+            "-i",
+            "--in_path",
+            default=".",
+            type=self.dir_path,
+            help="Path to the directory holding the raw data files",
+        )
+        parser.add_argument(
+            "-o",
+            "--out_path",
+            default="./raw",
+            type=self.dir_path,
+            help="Output directory for the merged dataset, should exist first. Defaults to `./raw`",
+        )
+        parser.add_argument(
+            "-fvc",
+            "--write_fvc",
+            action="store_true",
+            help="Write a TUFLOW FV '.fvc' file to accompany merged dataset",
+        )
+        parser.add_argument(
+            "-rp",
+            "--reproject",
+            type=int,
+            default=None,
+            help="Reproject coordinates to a new coord system, input as EPSG code (e.g., 4326)",
+        )
+        parser.add_argument(
+            "-tz",
+            "--timezone_offset",
+            type=float,
+            default=None,
+            help='Fixed offset hours for local timezone, e.g. "-tz 10"',
+        )
+        parser.add_argument(
+            "-ltz",
+            "--timezone_label",
+            type=str,
+            default=None,
+            help='Custom timezone label, e.g. "-ltz AEST"',
+        )
+        parser.add_argument(
+            "--wrapto360",
+            action="store_true",
+            help="Format longitude as (0, 360) rather than (-180, 180)",
+        )
+        parser.add_argument(
+            "--pad_dry",
+            action="store_true",
+            help="Flag to pad values out over land or dry cells by applied nearest neighbour extrapolation",
+        )
+        self.add_source_arguments(parser)
+        parser.set_defaults(func=self.run_merge)
+
+        return parser
+
+    @abstractmethod
+    def run_download(self, args):
+        pass
+
+    @abstractmethod
+    def run_merge(self, args):
+        pass
+
+
+def check_bbox(args):
+    """
+    Unfortunate edge case hackfix for programmer style floats like "-28." isntead of "-28.0".
+    Fixes negative numbers with trailing dots in the final 4 arguments (bbox).
+    Only modifies if all 4 final args are numeric-like!!
+    Otherwise spits fire
+    """
+    # Early exit if only 1 or 2 args are provided - or --help  / -h
+    if len(args) == 1 or len(args) == 2 or (len(args) >= 2 and (('-h' in args) or ('--help' in args))):
+        return args
+        
+    if (len(args) < 4):
+        raise ValueError("Must supply at minimum 6 arguments - time_start time_end xmin xmax ymin ymax")
+
+    # Get the last 4 arguments ( Should be bbox )
+    bbox_args = args[-4:]
+    other_args = args[:-4]
+
+    # Check if all 4 look like numbers (including problematic trailing dots)
+    numeric_pattern = r'^-?\d*\.?\d*$'
+    if not all(re.match(numeric_pattern, arg) and arg not in ['', '.', '-.', '-'] for arg in bbox_args):
+        raise ValueError("The final 4 arguments don't appear to be valid bbox coordinates. "
+                        "Expected format: xmin xmax ymin ymax (numeric values)")
+
+    # Fix trailing dots by adding '0'
+    trailing_dot_pattern = r'^-?\d+\.$'
+    fixed_bbox = [arg + '0' if re.match(trailing_dot_pattern, arg) else arg for arg in bbox_args]
+
+    # Verify they're actually parseable as floats
+    try:
+        [float(arg) for arg in fixed_bbox]
+    except ValueError:
+        raise ValueError("The final 4 arguments cannot be parsed as bbox coordinates."
+                        "Expected format: xmin xmax ymin ymax (numeric values)")
+
+    return other_args + fixed_bbox

tfv_get_tools/cli/atmos_cli.py

@@ -0,0 +1,192 @@
+"""ATMOS CLI
+
+This tool provides two sub-programs:
+- Downloading raw atmospheric data  
+- Merging raw atmospheric data and assisting with TUFLOW FV fvc template setup
+"""
+
+from argparse import ArgumentParser
+import sys
+import textwrap
+
+from tfv_get_tools import DownloadAtmos, MergeAtmos
+from tfv_get_tools.cli._cli_base import CLIBase, check_bbox
+
+def print_atmos_info():
+    """Returns detailed help text for GetAtmos"""
+    return textwrap.dedent("""
+        GetAtmos
+        =======================================
+        
+        This tool is designed to support a user with downloading  atmospheric datasets
+        from common online publicly available sources, as well as subsequent 
+        pre-processing and collation for TUFLOW FV modelling. 
+        The ideology of the tool is to provide the datasets in a true to original
+        raw format (program "A"), and then in a processed "simplified"
+        merged format ready for TUFLOW FV modelling (program "B").
+
+        GetAtmos works with atmospheric reanalysis and analysis data, including 10m wind 
+        velocity components, mean sea-level pressure, temperature, relative humidity
+        and short/long wave downward radiation. 
+
+        There are several data sources and sub-models available currently. These are listed 
+        below. Some of these may require registration. 
+        
+        Available data sources and sub-models:
+        -------------
+        1. (Default) ECMWF's "ERA5"  - registration required
+            - "default" - A global reanalysis 
+        
+        2. NOAA's "CFSR"
+            - "default" - A global reanalysis 
+
+        3. BoM's "BARRA2"
+           - "R2" - An 11-km grid reanalysis covering Australia
+           - "C2" - An 4-km grid reanalysis covering Australia, with only wind and pressure fields downloaded. 
+           - "RE2" - (Testing Only) An experimental ensembles 22-km grid covering Australia.
+        
+        Example Usage:
+        ---------
+        Download ERA5 Reanalysis Data - all defaults
+            `GetAtmos A 2011-01-01 2012-01-01 145 150 -30 -25`
+        
+        Merge ERA5 Reanalysis Data - all defaults, merge all data in the raw folder
+            `GetAtmos B`
+
+        Download BARRA2 C2 Dataset
+            `GetAtmos A -s BARRA2 -m R2 2011-01-01 2012-01-01 150 153 -30 -25`
+
+        Merge BARRA2 C2 Dataset with reprojection and local time
+            `GetAtmos B -s BARRA2 -m R2 -tz 10 -ltz AEST -rp 7856` 
+    
+        For more specific help, please use:
+        `GetAtmos A -h` or `GetAtmos B -h`
+        """)
+
+
+def entry():
+    """This is the entrypoint to the CLI, linked in the pyproject.toml"""
+    cli = GetAtmos()
+    sys.argv = check_bbox(sys.argv)
+    cli.run_cli()
+
+
+class GetAtmos(CLIBase):
+    def __init__(self, download_func=None, merge_func=None):
+        super().__init__("GetAtmos", "Tool for downloading and merging Atmos data")
+        
+        # Allow injection of functions for testing
+        self.download_func = download_func or DownloadAtmos
+        self.merge_func = merge_func or MergeAtmos
+        
+        self.add_download_parser(
+            "A",
+            "Download raw Atmos files for a set time period and bounding box extents",
+        )
+        self.add_merge_parser(
+            "B",
+            "Merge raw Atmos files into a single netcdf, and optionally write a TUFLOW FV FVC file",
+        )
+
+        self.add_info_parser()
+
+    def run_cli(self):
+        """Parse arguments and execute the appropriate function."""
+        args = self.parser.parse_args()
+        if args.command is not None:
+            args.func(args)
+        else:
+            self.parser.print_help()
+
+    def add_info_parser(self):
+        parser = self.subparsers.add_parser('info', help="Print detailed information about this program and the options")
+        parser.set_defaults(func=self.print_detailed_info)
+    
+    def print_detailed_info(self, args):
+        text = print_atmos_info()
+        print(text)
+
+    def add_source_arguments(self, parser: ArgumentParser):
+        """Add source arguments
+        
+        Args:
+            parser: The parser to add arguments to.
+        """
+        super().add_source_arguments(parser)
+
+        parser.add_argument(
+            '--info',
+            action="store_true",
+            help="Display the full program help"
+        )
+
+        parser.add_argument(
+            "-s",
+            "--source",
+            type=str,
+            default="ERA5",
+            help='Atmos data source. Default = "ERA5". Optionally others see wiki.',
+        )
+        parser.add_argument(
+            "-m",
+            "--model",
+            type=str,
+            default="default",
+            help='Model from source. Default is "default". Optionally others see wiki.',
+        )
+        parser.add_argument(
+            "--test",
+            action="store_true",
+            help="Run in test mode - no actual downloads performed",
+        )
+
+    def run_download(self, args):
+        """Call the DownloadAtmos function
+
+        Args:
+            args: CLI argument parser
+        """
+        xlims = tuple([float(x) for x in args.bbox[:2]])
+        ylims = tuple([float(x) for x in args.bbox[2:]])
+
+        self.download_func(
+            args.time_start,
+            args.time_end,
+            xlims,
+            ylims,
+            source=args.source,
+            out_path=args.path,
+            TEST_MODE=args.test,
+        )
+
+    def run_merge(self, args):
+        """Call the MergeAtmos function
+
+        Args:
+            args: CLI argument parser
+        """
+        # Sort out the timezone arguments
+        if (args.timezone_offset is not None) and (args.timezone_label is not None):
+            tz = (args.timezone_offset, args.timezone_label)
+        elif args.timezone_offset is not None:
+            sign = "+" if args.timezone_offset > 0 else "-"
+            tz = (args.timezone_offset, f"UTC{sign}{abs(args.timezone_offset):0.1f}")
+        elif (args.timezone_offset is None) and (args.timezone_label is not None):
+            raise ValueError("Need to supply a timezone_offset!")
+        else:
+            tz = None
+
+        self.merge_func(
+            in_path=args.in_path,
+            out_path=args.out_path,
+            fname=args.file_name,
+            source=args.source,
+            model=args.model,
+            time_start=args.time_start,
+            time_end=args.time_end,
+            # write_fvc=args.write_fvc,
+            reproject=args.reproject,
+            local_tz=tz,
+            wrapto360=args.wrapto360,
+            pad_dry=args.pad_dry,
+        )