tfv-get-tools 0.2.0__py3-none-any.whl

tfv_get_tools/providers/_utilities.py

@@ -0,0 +1,255 @@
+import sys
+from pathlib import Path
+
+import pandas as pd
+import xarray as xr
+import yaml
+
+root = Path(__file__).parent
+
+def _get_config(mode:str, source: str, model=None):
+    """Load a data source config file
+
+    Args:
+        mode (str): Ocean data type {'ocean', 'wave', 'atmos'}
+        source (str): Source e.g {'hycom', 'copernicus'}
+
+    Returns:
+        dict: cfg dictionary
+        str: Base data source URL (e.g., for THREDDS)
+    """
+    # Replace the 'default' tag with a None for this check
+    model = None if model == 'default' else model
+    
+    # If there is a model specified, append to filename
+    if model is None:
+        cfgname = f'{source}.yaml'
+    else:
+        cfgname = f'{source}_{model}.yaml'
+    
+    path = root / f"{mode}/cfgs/{cfgname}".lower()
+    
+    if not path.exists():
+        raise ValueError(f'Config file mode {mode} and source/model {cfgname} does not exist! Please review your source / model request')
+    
+    with open(path) as f:
+        cfg = yaml.safe_load(f)
+        BASE_URL = cfg.pop("_BASE_URL", None)
+        assert BASE_URL, "Please check config file, missing BASE_URL"
+        
+    return cfg, BASE_URL
+
+
+def todstr(datetime):
+    """Return a YYYYmmdd formatted datestr
+    For use in all downloaders and mergers
+
+    Args:
+        datetime (pd.Timestamp): Timestamp for conversion
+
+    Returns:
+        str: YYYYmmdd format string
+    """
+    return datetime.strftime("%Y%m%d")
+
+def _check_time_interval(x):
+    assert int(x) == float(x), "Timestep unit should be an integer representing hours"
+    x = int(x)
+    assert (
+        x >= 3
+    ), "The highest resolution timestep available is 3-hourly output - please check"
+    assert (
+        x % 3 == 0
+    ), "Timestep should be a multiple of 3 (highest available timestep resolution is 3hrly) - please check"
+    return x
+
+
+def _conv_date(date):
+    # Convert date-types, if necessary
+    if isinstance(date, str):
+        if len(date) == 10:
+            fmt = "%Y-%m-%d"
+        elif len(date) == 13:
+            fmt = "%Y-%m-%d %H"
+        elif len(date) == 16:
+            fmt = "%Y-%m-%d %H:%M"
+        elif len(date) == 19:
+            fmt = "%Y-%m-%d %H:%M:%S"
+        elif len(date) == 8:
+            fmt = "%Y%m%d"
+        elif len(date) == 15:
+            fmt = "%Y%m%d.%H%M%S"
+        date = datetime.strptime(date, fmt)
+    elif isinstance(date, pd.Timestamp):
+        date = date.to_pydatetime()
+    return date
+
+
+def validate_request(
+    timelims, xlims, ylims, src_timelims, src_xlims, src_ylims, source_name
+):
+    """Validate the time range of the source dataset.
+
+    Args:
+        timelims (tuple): start and end time of data request.
+        xlims (tuple): x limits of the data request
+        ylims (tuple): y limits of the data request
+        src_timelims (tuple): tuple of start and end timeframes for the dataset.
+        src_xlims (tuple): tuple of start and end longitudes for the dataset.
+        src_ylims (tuple): tuple of start and end latitudes for the dataset.
+        source_name (_type_): data source name
+    """
+    # Capitalise src name for consistency
+    source_name = source_name.upper()
+
+    # Validate time limits
+    src_start, src_end = src_timelims
+
+    time_start, time_end = timelims
+    if src_start is not None:
+        assert time_start >= pd.Timestamp(
+            src_start
+        ), f"Start time is outside of {source_name} data temporal extents ({src_start} to {src_end})"
+
+
+    if src_end is not None:
+        assert time_end <= pd.Timestamp(
+            src_end
+        ), f"End time is outside of {source_name} data temporal extents ({src_start} to {src_end})"
+    else:
+        assert time_end >= pd.Timestamp(
+            src_start
+        ), f"End time is outside of {source_name} data temporal extents ({src_start} to {src_end})"
+
+    # Validate x limits
+    x_start, x_end = xlims
+    src_x_start, src_x_end = src_xlims
+
+    if src_x_start is not None:
+        assert (
+            x_start >= src_x_start
+        ), f"x start is outside of {source_name} data spatial extents ({src_x_start} to {src_x_end})"
+
+    if src_x_end is not None:
+        assert (
+            x_end <= src_x_end
+        ), f"x end is outside of {source_name} data spatial extents ({src_x_start} to {src_x_end})"
+
+    # Validate y limits
+    y_start, y_end = ylims
+    src_y_start, src_y_end = src_ylims
+
+    if src_y_start is not None:
+        assert (
+            y_start >= src_y_start
+        ), f"y start is outside of {source_name} data spatial extents ({src_y_start} to {src_y_end})"
+
+    if src_y_end is not None:
+        assert (
+            y_end <= src_y_end
+        ), f"y end is outside of {source_name} data spatial extents ({src_y_start} to {src_y_end})"
+
+
+def _open_netcdf_file(file: Path) -> xr.Dataset:
+    """Open a subset netcdf file and assert validity
+
+    Args:
+        file (Path): path to the netcdf file
+
+    Returns:
+        xr.Dataset: Subset netcdf dataset
+    """
+    try:
+        # Attempt to open the file
+        ds = xr.open_dataset(file)
+
+        # Check if 'time' is properly formatted
+        if pd.api.types.is_datetime64_any_dtype(ds["time"]):
+            return ds
+        else:
+            print(f"Skipping file {file.name} - time error")
+            return None
+    except Exception as e:
+        print(f"Skipping file {file.name}: {str(e)}")
+        return None
+
+
+def wrap_longitude(ds, wrapto360=False, xname="longitude"):
+    """Function to wrap a dataset longitude around 360 or 180.
+    Defaults to -180 to 180.
+
+    :param ds: dataset to be wrapped
+    :param wrapto360: boolean to wrap to 360 (Defaults to False).
+    :param xname: X-var name (Defaults to 'longitude')
+    :return: ds (xr.Dataset): The wrapped dataset
+    """
+    attrs = ds[xname].attrs
+    if wrapto360 is True:
+        x = ds[xname].values
+        x[x < 0] = x[x < 0] + 360
+    else:
+        x = ds[xname].values
+        x[x > 180] = x[x > 180] - 360
+
+    ds = ds.assign_coords({xname: x})
+    ds = ds.sortby(xname)
+    
+    # Ensure attributes get copied in correctly
+    ds[xname].attrs = attrs
+    
+    return ds
+
+
+def check_path(path):
+    # Convert str path to Pathlib; assert existence.
+    if isinstance(path, str):
+        path = Path(path)
+
+    if path.is_dir():
+        return path
+    else:
+        assert False, f"{path.as_posix()} is not a valid path - check that it exists"
+
+
+def query_yes_no(question, default="yes"):
+    """Ask a yes/no question via raw_input() and return their answer.
+
+    "question" is a string that is presented to the user.
+    "default" is the presumed answer if the user just hits <Enter>.
+            It must be "yes" (the default), "no" or None (meaning
+            an answer is required of the user).
+
+    The "answer" return value is True for "yes" or False for "no".
+    """
+    valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
+    if default is None:
+        prompt = " [y/n] "
+    elif default == "yes":
+        prompt = " [Y/n] "
+    elif default == "no":
+        prompt = " [y/N] "
+    else:
+        raise ValueError("invalid default answer: '%s'" % default)
+
+    while True:
+        sys.stdout.write(question + prompt)
+        choice = input().lower()
+        if default is not None and choice == "":
+            return valid[default]
+        elif choice in valid:
+            return valid[choice]
+        else:
+            sys.stdout.write("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n")
+
+
+def is_notebook() -> bool:
+    try:
+        shell = get_ipython().__class__.__name__
+        if shell == "ZMQInteractiveShell":
+            return True  # Jupyter notebook or qtconsole
+        elif shell == "TerminalInteractiveShell":
+            return False  # Terminal running IPython
+        else:
+            return False  # Other type (?)
+    except NameError:
+        return False  # Probably standard Python interpreter

tfv_get_tools/providers/atmos/barra2.py

@@ -0,0 +1,209 @@
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import xarray as xr
+from pandas.tseries.offsets import MonthEnd
+from tqdm import tqdm
+
+from tfv_get_tools.providers._downloader import BaseDownloader
+from tfv_get_tools.providers._merger import BaseMerger
+from tfv_get_tools.providers._utilities import todstr
+
+
+class DownloadBARRA2(BaseDownloader):
+    """BARRA2 Downloader
+    
+    This is a THREDDS Dataset with each variable saved under a sub-url.
+    Compared to BARRA1, the new BARRA2 appears to be more self-consistent so we'll use a 
+    standard THREDDS server downloader (like CAWCR)    
+    
+    """
+    def _init_specific(self):
+        if self.model == 'default':
+            self.log("Default model has been selected == 'R2'")
+            self.model = 'r2'
+        
+        self.source = "BARRA2"
+        self.mode = "ATMOS"
+        
+        MODELS = {"c2", "r2", "re2"}
+        if self.model.lower() not in MODELS:
+            raise ValueError(f"Model must be one of {MODELS}")
+        
+        self._load_config()
+
+    def _get_output_filename(self, ts: pd.Timestamp, te: pd.Timestamp, var: str) -> Path:
+        """BARAR2 filename pattern"""
+        fname = f"{self.prefix}_{self.model}_{var}_{todstr(ts)}_{todstr(te)}.nc"
+        return self.outdir / fname
+
+        
+    def download(self):
+        """Begin download of files.
+
+        Approach:
+            - Loop through each time in times vector
+            - Try again if failures.
+        """
+        for ts in self.times:
+            te = ts + MonthEnd() + pd.Timedelta("23.9h")
+            
+            for var in self.variables:
+                out_file = self._get_output_filename(ts, te, var)
+                url = self._construct_opendap_url(ts, var)
+                
+                # Let base class handle the file existence check, timing, etc.
+                yield {
+                    'file_path': out_file,
+                    'url': url,
+                    'timestamp': ts,
+                    'variable': var,
+                    'download_func': lambda f=out_file, u=url: self._download_single_file(f, u)
+                }
+
+    def _construct_opendap_url(self, date: pd.Timestamp, var: str) -> str:
+        """
+        Construct the OPeNDAP URL for the given date for BARRA2
+
+        Args:
+            date (pd.Timestamp): The date for which to construct the URL.
+            var (str): Variable name to download (BARRA2 var)
+
+        Returns:
+            str: The constructed URL.
+        """
+        date_str = date.strftime("%Y%m")
+        
+        # Crap way of getting the dset template name from nexted dict.
+        name_tmp = list(self.dsmap[list(self.dsmap.keys())[0]].keys())[0]      
+        
+        # Apply replacements
+        name = name_tmp.replace('<VAR>', var).replace('<DATE>', date_str)
+        
+        url = f"{self.base_url}/{var}/latest/{name}"
+        
+        return url
+    
+
+    def _download_single_file(self, fname: Path, url: str) -> bool:
+        """
+        Download a single file from the specified URL and save it to the specified filename.
+
+        Args:
+            fname (Path): The output filename.
+            url (str): The URL to download the data from.
+
+        Returns:
+            bool: True if the download was successful, False otherwise.
+        """
+        try:
+            ds = xr.open_dataset(url)
+
+            # Handle longitude selection (CFSR-specific logic)
+            if self.xlims[0] < self.xlims[1]:
+                lon_idx = np.hstack(
+                    np.where(
+                        (self.xlims[0] <= ds["lon"])
+                        & (ds["lon"] <= self.xlims[1])
+                    )
+                )
+            else:
+                lon_idx = np.hstack(
+                    (
+                        np.where(ds["lon"] <= self.xlims[1])[0],
+                        np.where(ds["lon"] >= self.xlims[0])[0],
+                    )
+                )
+            
+            assert lon_idx.size > 1, "No longitude points selected!"
+
+            # Select latitude indices
+            lat_idx = np.hstack(
+                np.where(
+                    (self.ylims[0] <= ds["lat"]) 
+                    & (ds["lat"] <= self.ylims[1])
+                )
+            )
+            assert lat_idx.size > 1, "No latitude points selected!"
+
+            # Subset and save
+            ds = ds.isel(lon=lon_idx, lat=lat_idx)
+            ds.to_netcdf(fname)
+            return True
+
+        except Exception as e:
+            if self.verbose:
+                print(f"Failed to download {url}: {e}")
+            return False
+
+class MergeBARRA2(BaseMerger):
+    def _init_specific(self):
+        self.source = "BARRA2"
+        self.mode = "ATMOS"
+        
+        if self.model == 'default':
+            self.model = 'R2'
+        
+        self._load_config()
+
+    
+    def merge_files(self, file_list):
+        """Specific merging logic
+        
+        # BARRA2 requires us to first merge on varabies before concatenating over time.
+        
+        Args:
+            file_list (list): list of path objects to open and concat.
+
+        Returns:
+            xr.Dataset: merged xarray dataset
+            list: files unable to be merged 
+        """
+        skipped_list = []
+        
+        startdates = [x.stem.split('_')[-2] for x in file_list]
+        unq_startdates = np.unique(startdates)
+        
+        dsset = {k: [] for k in unq_startdates}
+        
+        for i, f in enumerate(tqdm(file_list)):
+            dsx = self._open_subset_netcdf(f, chunks=dict(time=24))
+            if dsx is not None:
+                dsx['time'] = pd.to_datetime(dsx['time'].values).floor('1h')
+                
+                # Drop all the extra variables
+                dsx = dsx.drop_vars(['height', 'level_height', 'model_level_number', 'sigma', 'crs'], errors='ignore')
+
+                dsset[startdates[i]].append(dsx)
+            else:
+                skipped_list.append(f)
+                
+        print("Concatenating xarray dataset")
+        
+        # Merge the common start_dates first, then concatenate by time afterwards
+        dssetm = []
+        for v in dsset.values():
+            dssetm.append(xr.merge(v))
+        
+        ds = xr.concat(
+            dssetm,
+            dim="time",
+            combine_attrs="override",
+            data_vars="minimal",
+            coords="minimal",
+            compat="override",
+        )
+        
+        # Drop the redundant time_bnds var
+        ds = ds.drop_vars(['time_bnds', 'bnds'], errors='ignore')
+
+        # Sort and drop nans
+        ds = ds.sortby('time')
+        _, idx = np.unique(ds['time'], return_index=True)
+        ds = ds.isel(time=idx)
+        
+        # Rename the original coords
+        ds = ds.rename({'lon': 'longitude', 'lat': 'latitude'})
+        
+        return ds, skipped_list

tfv_get_tools/providers/atmos/cfgs/barra2_c2.yaml

@@ -0,0 +1,52 @@
+_BASE_URL: "https://thredds.nci.org.au/thredds/dodsC/ob53/output/reanalysis/AUST-04/BOM/ERA5/historical/hres/BARRA-C2/v1/1hr"
+_INFO_URL: http://www.bom.gov.au/research/publications/researchreports/BRR-067.pdf
+
+# BARRA2 is a THREDDS server dataset. Each variable is stored in suburls from the main dataset url above.
+# The data covers 1979 to near present (~5 month lag?)
+# No short or long wave radiation components are provided - leaving it as a wind and pressure dataset for now.
+
+# Dataset FNAME is read from the _DATASETS param below. <VAR> and <DATE> should be replaced:
+# - VAR = variable name, e.g., uasmean, vasmean, psl
+# - DATE = Date string in %Y%m format. It should be the same (e.g., 197901-197901)
+
+_SOURCE_XLIMS: [108.02, 159.9]
+_SOURCE_YLIMS: [-45.69, -5.01]
+_SOURCE_TIMELIMS: ["1979-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+_VARIABLES: ["uasmean", "vasmean", "psl", "pr"] # u10, v10, pressure. {temp: 'tasmean', relhum: 'hurs'}
+_DATASETS:
+  {
+    1979-01-01:
+      {
+        <VAR>_AUST-04_ERA5_historical_hres_BOM_BARRA-C2_v1_1hr_<DATE>-<DATE>.nc: default,
+      },
+  }
+
+# KEY: The standard NC Variable for merged data (and per _standard_attrs.py)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+u10:
+  source_var: "uasmean"
+  tfv_var: "W10_GRID"
+  bc_scale: 1
+  bc_offset: 0.
+
+v10:
+  source_var: "vasmean"
+  tfv_var: "W10_GRID"
+  bc_scale: 1
+  bc_offset: 0.
+
+mslp:
+  source_var: "psl"
+  tfv_var: "MSLP_GRID"
+  bc_scale: 0.01
+  bc_offset: 0.
+
+prate:
+  source_var: "pr"
+  tfv_var: "PRECIP_GRID"
+  bc_scale: 86.4 # Convert from kg/m2/s to mm/day
+  bc_offset: 0.

tfv_get_tools/providers/atmos/cfgs/barra2_r2.yaml

@@ -0,0 +1,85 @@
+_BASE_URL: "https://thredds.nci.org.au/thredds/dodsC/ob53/output/reanalysis/AUS-11/BOM/ERA5/historical/hres/BARRA-R2/v1/1hr"
+_INFO_URL: http://www.bom.gov.au/research/publications/researchreports/BRR-067.pdf
+
+# BARRA2 is a THREDDS server dataset. Each variable is stored in suburls from the main dataset url above.
+# The data covers 1979 to near present (~5 month lag?)
+# No short or long wave radiation components are provided - leaving it as a wind and pressure dataset for now.
+
+# Dataset FNAME is read from the _DATASETS param below. <VAR> and <DATE> should be replaced:
+# - VAR = variable name, e.g., uasmean, vasmean, psl
+# - DATE = Date string in %Y%m format. It should be the same (e.g., 197901-197901)
+
+_SOURCE_XLIMS: [88.48, 207.39]
+_SOURCE_YLIMS: [-57.97, 12.98]
+_SOURCE_TIMELIMS: ["1979-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+_VARIABLES: [
+    "uasmean",
+    "vasmean",
+    "psl",
+    "tasmean",
+    "hurs",
+    "rsds",
+    "rlds",
+    "pr",
+  ] # u10, v10, mslp, temp, relhum, swr, lwr
+_DATASETS:
+  {
+    1979-01-01:
+      {
+        <VAR>_AUS-11_ERA5_historical_hres_BOM_BARRA-R2_v1_1hr_<DATE>-<DATE>.nc: default,
+      },
+  }
+
+# KEY: The standard NC Variable for merged data (and per _standard_attrs.py)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+u10:
+  source_var: "uasmean"
+  tfv_var: "W10_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+v10:
+  source_var: "vasmean"
+  tfv_var: "W10_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+mslp:
+  source_var: "psl"
+  tfv_var: "MSLP_GRID"
+  bc_scale: 0.01
+  bc_offset: 0
+
+dlwrf:
+  source_var: "rlds"
+  tfv_var: "LW_RAD_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+dswrf:
+  source_var: "rsds"
+  tfv_var: "SW_RAD_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+t2m:
+  source_var: "tasmean"
+  tfv_var: "AIR_TEMP_GRID"
+  bc_scale: 1
+  bc_offset: -273.15
+
+relhum:
+  source_var: "hurs"
+  tfv_var: "REL_HUM_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+prate:
+  source_var: "pr"
+  tfv_var: "PRECIP_GRID"
+  bc_scale: 86.4 # Convert from kg/m2/s to mm/day
+  bc_offset: 0.

tfv_get_tools/providers/atmos/cfgs/barra2_re2.yaml

@@ -0,0 +1,70 @@
+_BASE_URL: "https://thredds.nci.org.au/thredds/dodsC/ob53/output/reanalysis/AUS-22/BOM/ERA5/historical/eda/BARRA-RE2/v1/1hr"
+_INFO_URL: http://www.bom.gov.au/research/publications/researchreports/BRR-067.pdf
+
+# BARRA2 is a THREDDS server dataset. Each variable is stored in suburls from the main dataset url above.
+# The data covers 1979 to near present (~5 month lag?)
+# No short or long wave radiation components are provided - leaving it as a wind and pressure dataset for now.
+
+# Dataset FNAME is read from the _DATASETS param below. <VAR> and <DATE> should be replaced:
+# - VAR = variable name, e.g., uasmean, vasmean, psl
+# - DATE = Date string in %Y%m format. It should be the same (e.g., 197901-197901)
+
+_SOURCE_XLIMS: [89.53, 206.13]
+_SOURCE_YLIMS: [-56.49, 11.71]
+_SOURCE_TIMELIMS: ["1979-01-01 00:00:00", null]
+
+_DOWNLOAD_INTERVAL: monthly
+_VARIABLES: ["uasmean", "vasmean", "psl", "tasmean", "hurs", "rsds", "rlds"] # u10, v10, mslp, temp, relhum, swr, lwr
+_DATASETS:
+  {
+    1979-01-01:
+      {
+        <VAR>_AUS-22_ERA5_historical_eda_BOM_BARRA-RE2_v1_1hr_<DATE>-<DATE>.nc: default,
+      },
+  }
+
+# KEY: The standard NC Variable for merged data (and per _standard_attrs.py)
+# source var == var as in the original dataset
+# tfv_var == standard tuflow fv var name (for FVC writer template)
+
+u10:
+  source_var: "uasmean"
+  tfv_var: "W10_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+v10:
+  source_var: "vasmean"
+  tfv_var: "W10_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+mslp:
+  source_var: "psl"
+  tfv_var: "MSLP_GRID"
+  bc_scale: 0.01
+  bc_offset: 0
+
+dlwrf:
+  source_var: "rlds"
+  tfv_var: "LW_RAD_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+dswrf:
+  source_var: "rsds"
+  tfv_var: "SW_RAD_GRID"
+  bc_scale: 1
+  bc_offset: 0
+
+t2m:
+  source_var: "tasmean"
+  tfv_var: "AIR_TEMP_GRID"
+  bc_scale: 1
+  bc_offset: -273.15
+
+relhum:
+  source_var: "hurs"
+  tfv_var: "REL_HUM_GRID"
+  bc_scale: 1
+  bc_offset: 0