tfv-get-tools 0.2.1__tar.gz → 0.2.3__tar.gz

{tfv_get_tools-0.2.1/src/tfv_get_tools.egg-info → tfv_get_tools-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tfv_get_tools
-Version: 0.2.1
+Version: 0.2.3
 Summary: Tool for downloading and processing data for TUFLOW FV modelling
 Author-email: Alex Waterhouse <alex.waterhouse@apac.bmt.org>, Mitchell Smith <mitchell.smith@apac.bmt.org>, TUFLOW Support <support@tuflow.com>
 License: MIT
@@ -60,22 +60,22 @@ TFV Get Tools is a Python package that simplifies the process of downloading and
 ### Supported Data Sources
 
 **Atmospheric Data:**
-- [ECMWF ERA5](https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5) (Default - registration required, see [CDS API](https://cds.climate.copernicus.eu/api-how-to))
+- [ECMWF ERA5](https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5) (*Atmos Default* - registration required, see [CDS API](https://cds.climate.copernicus.eu/api-how-to))
 - [NOAA CFSR](https://www.ncei.noaa.gov/data/climate-forecast-system/) (Climate Forecast System Reanalysis)
 - [BARRA2](http://www.bom.gov.au/research/projects/reanalysis/) (Australian Bureau of Meteorology)
 
 **Ocean Data:**
-- [HYCOM](https://www.hycom.org/) (Naval Research Laboratory - Global Ocean Forecast System)
+- [HYCOM](https://www.hycom.org/) (*Ocean Default* Naval Research Laboratory - Global Ocean Forecast System)
 - [Copernicus Marine](https://marine.copernicus.eu/) Global and NWS (registration required, see [Copernicus Marine Service](https://marine.copernicus.eu/))
 
 **Wave Data:**
-- [CSIRO CAWCR](https://data.csiro.au/collection/csiro:39819) (glob_24m, aus_10m, aus_4m, pac_10m, pac_4m)
+- [CSIRO CAWCR](https://data.csiro.au/collection/csiro:39819) (*Wave Default* glob_24m, aus_10m, aus_4m, pac_10m, pac_4m)
 - [Copernicus Marine](https://marine.copernicus.eu/) Global and NWS
 - [ECMWF ERA5](https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5) (registration required, see [CDS API](https://cds.climate.copernicus.eu/api-how-to))
 
 **Tidal Data:**
 - [FES2014](https://www.aviso.altimetry.fr/en/data/products/auxiliary-products/global-tide-fes.html) (AVISO+ Finite Element Solution 2014)
-- [FES2022](https://www.aviso.altimetry.fr/en/data/products/auxiliary-products/global-tide-fes.html) (AVISO+ Finite Element Solution 2022)
+- [FES2022](https://www.aviso.altimetry.fr/en/data/products/auxiliary-products/global-tide-fes.html) (*Tide Default* AVISO+ Finite Element Solution 2022)
 
 ## Installation
 
@@ -276,7 +276,7 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 ## Authors
 
-Developed by [BMT](https://www.tuflow.com/), 2025
+Developed by [TUFLOW](https://www.tuflow.com/), 2025
 
 ## Project Status
 

{tfv_get_tools-0.2.1 → tfv_get_tools-0.2.3}/README.md

@@ -14,22 +14,22 @@ TFV Get Tools is a Python package that simplifies the process of downloading and
 ### Supported Data Sources
 
 **Atmospheric Data:**
-- [ECMWF ERA5](https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5) (Default - registration required, see [CDS API](https://cds.climate.copernicus.eu/api-how-to))
+- [ECMWF ERA5](https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5) (*Atmos Default* - registration required, see [CDS API](https://cds.climate.copernicus.eu/api-how-to))
 - [NOAA CFSR](https://www.ncei.noaa.gov/data/climate-forecast-system/) (Climate Forecast System Reanalysis)
 - [BARRA2](http://www.bom.gov.au/research/projects/reanalysis/) (Australian Bureau of Meteorology)
 
 **Ocean Data:**
-- [HYCOM](https://www.hycom.org/) (Naval Research Laboratory - Global Ocean Forecast System)
+- [HYCOM](https://www.hycom.org/) (*Ocean Default* Naval Research Laboratory - Global Ocean Forecast System)
 - [Copernicus Marine](https://marine.copernicus.eu/) Global and NWS (registration required, see [Copernicus Marine Service](https://marine.copernicus.eu/))
 
 **Wave Data:**
-- [CSIRO CAWCR](https://data.csiro.au/collection/csiro:39819) (glob_24m, aus_10m, aus_4m, pac_10m, pac_4m)
+- [CSIRO CAWCR](https://data.csiro.au/collection/csiro:39819) (*Wave Default* glob_24m, aus_10m, aus_4m, pac_10m, pac_4m)
 - [Copernicus Marine](https://marine.copernicus.eu/) Global and NWS
 - [ECMWF ERA5](https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5) (registration required, see [CDS API](https://cds.climate.copernicus.eu/api-how-to))
 
 **Tidal Data:**
 - [FES2014](https://www.aviso.altimetry.fr/en/data/products/auxiliary-products/global-tide-fes.html) (AVISO+ Finite Element Solution 2014)
-- [FES2022](https://www.aviso.altimetry.fr/en/data/products/auxiliary-products/global-tide-fes.html) (AVISO+ Finite Element Solution 2022)
+- [FES2022](https://www.aviso.altimetry.fr/en/data/products/auxiliary-products/global-tide-fes.html) (*Tide Default* AVISO+ Finite Element Solution 2022)
 
 ## Installation
 
@@ -230,7 +230,7 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 ## Authors
 
-Developed by [BMT](https://www.tuflow.com/), 2025
+Developed by [TUFLOW](https://www.tuflow.com/), 2025
 
 ## Project Status
 

{tfv_get_tools-0.2.1 → tfv_get_tools-0.2.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tfv_get_tools"
-version = "0.2.1"
+version = "0.2.3"
 description = "Tool for downloading and processing data for TUFLOW FV modelling"
 authors = [
     { name = "Alex Waterhouse", email = "alex.waterhouse@apac.bmt.org" },

{tfv_get_tools-0.2.1 → tfv_get_tools-0.2.3}/src/tfv_get_tools/providers/_merger.py

@@ -208,30 +208,23 @@ class BaseMerger(ABC):
         download_interval = self.cfg.get("_DOWNLOAD_INTERVAL", "monthly")
 
         if download_interval == "monthly":
-            start_time_strings = [x.stem.split("_")[-2] for x in file_list]
-            end_time_strings = [x.stem.split("_")[-1] for x in file_list]
+            time_stings = [re.search(r'_(\d{8})_(\d{8})', x.stem) for x in file_list]
+            start_time_strings = [x.group(1) for x in time_stings]
+            end_time_strings = [x.group(2) for x in time_stings]
+            
             start_times = pd.DatetimeIndex(
-                [pd.Timestamp(x, unit="h") for x in start_time_strings]
+                [pd.to_datetime(x, format="%Y%m%d") for x in start_time_strings]
             )
             end_times = pd.DatetimeIndex(
-                [pd.Timestamp(x, unit="h") for x in end_time_strings]
+                [pd.to_datetime(x, format="%Y%m%d") for x in end_time_strings]
             )
 
         elif download_interval == "daily":
-            start_time_strings = [x.stem.split("_")[-1] for x in file_list]
-
-            # Handle legacy HYCOM files
-            if start_time_strings and start_time_strings[0] == "0000":
-                if self.verbose:
-                    print("WARNING: Detected legacy HYCOM files with time suffixes")
-                start_time_strings = [
-                    x.name.split(".")[0].split("_")[-2] for x in file_list
-                ]
-
+            start_time_strings = [re.search(r'_(\d{8})_', x.stem).group(1) for x in file_list]
             start_times = pd.DatetimeIndex(
-                [pd.Timestamp(x, unit="h") for x in start_time_strings]
+                [pd.to_datetime(x, format="%Y%m%d") for x in start_time_strings]
             )
-            end_times = start_times + pd.Timedelta("23.9h")
+            end_times = start_times + pd.Timedelta("23.99h")
         else:
             raise ValueError(f"Unknown download interval: {download_interval}")
 

{tfv_get_tools-0.2.1 → tfv_get_tools-0.2.3}/src/tfv_get_tools/providers/ocean/hycom.py

@@ -516,7 +516,7 @@ class MergeHYCOM(BaseMerger):
         
         if time_vars:
             # Create single rolling object and apply to all variables
-            rolling_ds = ds[time_vars].rolling(time=25, center=True).reduce(np.nanmean)
+            rolling_ds = ds[time_vars].rolling(time=25, center=True, min_periods=1).reduce(np.nanmean)
             
             for var_name in time_vars:
                 # Only replace post-cutoff values
@@ -572,10 +572,10 @@ class MergeHYCOM(BaseMerger):
                 if all_datasets and self._check_sub_daily_data(all_datasets[0]):
                     apply_tidal_filtering = True
         
-        if self.verbose:
-            print("Concatenating and interpolating xarray dataset")
-            if has_post_cutoff_data and apply_tidal_filtering:
-                print('... Dataset contains sub-daily data post-2024-08-10 (HYCOM ESPC-D-V02), applying tidal filtering.')
+        print("Concatenating and interpolating xarray dataset")
+        if has_post_cutoff_data and apply_tidal_filtering:
+            print('... Dataset contains sub-daily data post-2024-08-10 (HYCOM ESPC-D-V02), applying tidal filtering using a simple 25h rolling mean.')
+            print('... Warning: Your dataset should be padded at least 1 full day either side before using in TUFLOW FV.')
         
         # Merge variables for each start date group
         merged_by_date = []
@@ -600,7 +600,13 @@ class MergeHYCOM(BaseMerger):
         
         # Apply tidal filtering to the merged dataset if needed
         if apply_tidal_filtering:
+            # Copy the original surface elevation data to a raw variable for later
+            raw_surf_el = merged['surf_el'].copy()
             merged = self._apply_tidal_filtering(merged)
+
+            # Copy original surface elevation
+            merged['raw_surf_el'] = raw_surf_el
+            merged['raw_surf_el'].attrs['note'] = 'Original HYCOM water-level containing tides after 2024-08-10'
         
         # Final cleanup
         merged = merged.rename({'lon': 'longitude', 'lat': 'latitude'})

{tfv_get_tools-0.2.1 → tfv_get_tools-0.2.3}/src/tfv_get_tools/providers/wave/era5.py

@@ -99,13 +99,19 @@ class DownloadERA5Wave(BaseDownloader):
                 print("You can find your key at: https://cds.climate.copernicus.eu/user/")
                 print("="*60)
                 
+            elif "cds-beta.climate.copernicus.eu" in error_msg:
+                print("\n" + "="*60)
+                print("CDS API AUTHENTICATION ERROR")
+                print("="*60)
+                print("Your CDS API key appears to be invalid.")
+                print("This is likely due to an update by the Copernicus Climate Data Store.")
+                print("Please check your .cdsapirc file and ensure your API key is correct.")
+                print("You can find your key at: https://cds.climate.copernicus.eu/user/")
+                print("="*60)
+
             else:
-                # For any other errors, show the original message if verbose
-                if self.verbose:
-                    print(f"Failed to download via CDS API: {e}")
-                else:
-                    # Always show some info for unhandled errors
-                    print(f"\nCDS API Error: {e}")
+                print(f"Failed to download via CDS API: {e}")
+
                     
             return False
     
@@ -141,7 +147,7 @@ class DownloadERA5Wave(BaseDownloader):
                 raise ValueError("No NetCDF files found in zip archive")
             
             # Combine all datasets
-            ds = xr.merge(datasets)
+            ds = xr.merge(datasets, compat='override')
             ds.to_netcdf(file_path_out)
             
             # Delete the zip file
@@ -229,4 +235,4 @@ class MergeERA5Wave(BaseMerger):
         if 'latitude' in merged.coords:
             merged = merged.sortby("latitude")
             
-        return merged, skipped_files
+        return merged, skipped_files

{tfv_get_tools-0.2.1 → tfv_get_tools-0.2.3/src/tfv_get_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tfv_get_tools
-Version: 0.2.1
+Version: 0.2.3
 Summary: Tool for downloading and processing data for TUFLOW FV modelling
 Author-email: Alex Waterhouse <alex.waterhouse@apac.bmt.org>, Mitchell Smith <mitchell.smith@apac.bmt.org>, TUFLOW Support <support@tuflow.com>
 License: MIT
@@ -60,22 +60,22 @@ TFV Get Tools is a Python package that simplifies the process of downloading and
 ### Supported Data Sources
 
 **Atmospheric Data:**
-- [ECMWF ERA5](https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5) (Default - registration required, see [CDS API](https://cds.climate.copernicus.eu/api-how-to))
+- [ECMWF ERA5](https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5) (*Atmos Default* - registration required, see [CDS API](https://cds.climate.copernicus.eu/api-how-to))
 - [NOAA CFSR](https://www.ncei.noaa.gov/data/climate-forecast-system/) (Climate Forecast System Reanalysis)
 - [BARRA2](http://www.bom.gov.au/research/projects/reanalysis/) (Australian Bureau of Meteorology)
 
 **Ocean Data:**
-- [HYCOM](https://www.hycom.org/) (Naval Research Laboratory - Global Ocean Forecast System)
+- [HYCOM](https://www.hycom.org/) (*Ocean Default* Naval Research Laboratory - Global Ocean Forecast System)
 - [Copernicus Marine](https://marine.copernicus.eu/) Global and NWS (registration required, see [Copernicus Marine Service](https://marine.copernicus.eu/))
 
 **Wave Data:**
-- [CSIRO CAWCR](https://data.csiro.au/collection/csiro:39819) (glob_24m, aus_10m, aus_4m, pac_10m, pac_4m)
+- [CSIRO CAWCR](https://data.csiro.au/collection/csiro:39819) (*Wave Default* glob_24m, aus_10m, aus_4m, pac_10m, pac_4m)
 - [Copernicus Marine](https://marine.copernicus.eu/) Global and NWS
 - [ECMWF ERA5](https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5) (registration required, see [CDS API](https://cds.climate.copernicus.eu/api-how-to))
 
 **Tidal Data:**
 - [FES2014](https://www.aviso.altimetry.fr/en/data/products/auxiliary-products/global-tide-fes.html) (AVISO+ Finite Element Solution 2014)
-- [FES2022](https://www.aviso.altimetry.fr/en/data/products/auxiliary-products/global-tide-fes.html) (AVISO+ Finite Element Solution 2022)
+- [FES2022](https://www.aviso.altimetry.fr/en/data/products/auxiliary-products/global-tide-fes.html) (*Tide Default* AVISO+ Finite Element Solution 2022)
 
 ## Installation
 
@@ -276,7 +276,7 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 ## Authors
 
-Developed by [BMT](https://www.tuflow.com/), 2025
+Developed by [TUFLOW](https://www.tuflow.com/), 2025
 
 ## Project Status
 

{tfv_get_tools-0.2.1 → tfv_get_tools-0.2.3}/tests/test_all_downloaders.py

@@ -9,11 +9,16 @@ from tfv_get_tools import DownloadAtmos, DownloadOcean, DownloadWave
 
 
 # Loopy test :) Add new sources here
+# Turned off Copernicus because of login checks
+# We could mock this properly but I think it 
+# is low value... 
+# There is a test for login failure low which will only trigger
+# if it asks for stdin user/password, which means it is working 
 @pytest.mark.parametrize(
     "source,downloader_class,model",
     [
-        ("COPERNICUS", DownloadOcean, "GLO"),
-        ("COPERNICUS", DownloadWave, "GLO"),
+        # ("COPERNICUS", DownloadOcean, "GLO"),
+        # ("COPERNICUS", DownloadWave, "GLO"),
         ("HYCOM", DownloadOcean, None),
         ("CAWCR", DownloadWave, "glob_24m"),
         ("CAWCR", DownloadWave, "aus_10m"),
@@ -59,36 +64,44 @@ class TestDownloaderIntegration:
     skip_check = True
 
     def test_copernicus_ocean_glo(self):
-        """Test COPERNICUS ocean downloader with GLO model."""
+        """Test COPERNICUS ocean downloader with GLO model.
+        These could be better - I am testing that copernicus kicks off ok
+        and that it triggers a manual user login via terminal"""
         with tempfile.TemporaryDirectory() as temp_dir:
-            result = DownloadOcean(
-                start_date=self.start_date,
-                end_date=self.end_date,
-                xlims=self.xlims,
-                ylims=self.ylims,
-                source="COPERNICUS",
-                model="GLO",
-                out_path=temp_dir,
-                TEST_MODE=self.test_mode,
-                skip_check=self.skip_check,
-            )
-            assert result is not None
+            
+            with pytest.raises(OSError,
+                               match="reading from stdin"):
+                result = DownloadOcean(
+                    start_date=self.start_date,
+                    end_date=self.end_date,
+                    xlims=self.xlims,
+                    ylims=self.ylims,
+                    source="COPERNICUS",
+                    model="GLO",
+                    out_path=temp_dir,
+                    TEST_MODE=self.test_mode,
+                    skip_check=self.skip_check,
+                )
+            
 
     def test_copernicus_wave_glo(self):
-        """Test COPERNICUS wave downloader with GLO model."""
+        """Test COPERNICUS wave downloader with GLO model.
+        These could be better - I am testing that copernicus kicks off ok
+        and that it triggers a manual user login via terminal"""
         with tempfile.TemporaryDirectory() as temp_dir:
-            result = DownloadWave(
-                start_date=self.start_date,
-                end_date=self.end_date,
-                xlims=self.xlims,
-                ylims=self.ylims,
-                source="COPERNICUS",
-                model="GLO",
-                out_path=temp_dir,
-                TEST_MODE=self.test_mode,
-                skip_check=self.skip_check,
-            )
-            assert result is not None
+            with pytest.raises(OSError,
+                    match="reading from stdin"):
+                result = DownloadWave(
+                    start_date=self.start_date,
+                    end_date=self.end_date,
+                    xlims=self.xlims,
+                    ylims=self.ylims,
+                    source="COPERNICUS",
+                    model="GLO",
+                    out_path=temp_dir,
+                    TEST_MODE=self.test_mode,
+                    skip_check=self.skip_check,
+                )
 
     def test_hycom_ocean(self):
         """Test HYCOM ocean downloader."""

{tfv_get_tools-0.2.1 → tfv_get_tools-0.2.3}/tests/test_mergers.py

@@ -378,7 +378,7 @@ class TestFileHandling:
         assert "RANDOM_FILE.nc" not in era5_files
         assert "ERA5_WAVE_20220101_20220131.nc" not in era5_files
 
-    def test_time_filtering(self):
+    def test_time_filtering_simple_ERA5(self):
         """Test filtering files by time range."""
         # Create files with different dates
         test_files = [
@@ -408,6 +408,41 @@ class TestFileHandling:
         assert "ERA5_ATMOS_20220201_20220228.nc" in filtered_files
         # Should exclude March file
         assert "ERA5_ATMOS_20220301_20220331.nc" not in filtered_files
+        
+    def test_time_filtering_painful_HYCOM(self):
+        """Test filtering files by time range."""
+        # Create files with different dates
+        test_files = [
+            "HYCOM_OCEAN_20220101_03h.nc",
+            "HYCOM_OCEAN_20220102_03h_somedbase.nc",
+            "A_LONG_PREFIX_BOUND_TO_GET_YA_HYCOM_OCEAN_20220103_03h_somedbase.nc",
+            "KEEP_ME_HYCOM_OCEAN_20210101_03h.nc",
+            "EXCLUDE_ME_HYCOM_OCEAN_20230101_03h.nc",
+            "WHY_DID_I_EVER_ALLOW_SUCH_FLEXIBILITY_HYCOM_OCEAN_20220301_03h.nc",
+        ]
+        
+        for filename in test_files:
+            (self.in_path / filename).touch()
+        
+        # Test filtering with time constraints
+        merger = MergeHYCOM(
+            in_path=self.in_path,
+            out_path=self.out_path,
+            time_start="2021-01-01", # Keep em all except the EXCLUDE_ME one
+            time_end="2022-09-01",
+            execute=False,
+            verbose=False
+        )
+        
+        files = merger.get_file_list()
+        filtered_files = [f.name for f in files]
+        
+        # Should include Jan and Feb files
+        assert "KEEP_ME_HYCOM_OCEAN_20210101_03h.nc" in filtered_files
+        
+        # Should exclude March file
+        assert "EXCLUDE_ME_HYCOM_OCEAN_20230101_03h.nc" not in filtered_files
+        
 
     def test_no_files_found_error(self):
         """Test error handling when no files match pattern."""