OceanDataStore 0.3.0__py3-none-any.whl

OceanDataStore/data/NSIDC/process_NSIDC_SSI_Antarctic_data.py

@@ -0,0 +1,130 @@
+"""
+process_NSIDC_SSI_Antarctic_data.py
+
+Description: Python script to post-process NSIDC Sea Ice Index
+data for the Antarctic 1978-2025, including sea ice concentration,
+extent and total area.
+
+Created by: Ollie Tooth (oliver.tooth@noc.ac.uk)
+Created on: 2025-02-21
+"""
+# -- Import Python packages -- #
+import logging
+from datetime import datetime
+from glob import glob
+
+import numpy as np
+import xarray as xr
+
+from OceanDataStore.cli import initialise_logging
+
+
+# -- Define Utility Functions -- #
+def get_datetimes_from_filenames(file_list):
+    # Extract filenames from paths:
+    filenames = [file.split('/')[-1] for file in file_list]
+
+    # Convert filenames date str to datetime:
+    datetimes = np.array([datetime(year=int(file[2:6]), month=int(file[6:8]), day=15) for file in filenames])
+
+    return datetimes
+
+
+def main():
+    # ========== Initialize Logging and Print Banner ========== #
+    initialise_logging()
+
+    logging.info('In Progress: Post-Processing NSIDC Sea Ice Index Antarctic Observations...')
+
+    # ========== Load NSIDC Ancillary Data ========== #
+    # Define filepath to ancillary data:
+    anc_fpath = "/dssgfs01/scratch/otooth/npd_data/observations/NSIDC/ancillary/NSIDC0771_LatLon_PS_S25km_v1.0.nc"
+    # Open NSIDC ancillary data as dataset:
+    ds_si = xr.open_dataset(anc_fpath)
+
+    # Define filepath to NSIDC ancillary file - grid cell area:
+    area_fpath = "/dssgfs01/scratch/otooth/npd_data/observations/NSIDC/ancillary/NSIDC0771_CellArea_PS_S25km_v1.0.nc"
+    # Open NSIDC grid cell area:
+    ds_area = xr.open_dataset(area_fpath)
+    logging.info("-> Completed: Loaded NSIDC ancillary data and grid cell area.")
+
+    # ========== Load NSIDC Monthly Data ========== #
+    # Define directory path:
+    dir_path = "/dssgfs01/scratch/otooth/npd_data/observations/NSIDC/antarctic/"
+
+    # Get the list of files in the directory:
+    file_paths = glob(f"{dir_path}*.tif")
+    file_paths.sort()
+
+    # Retrieve sea ice mask & concentration files:
+    version_str = "v4.0"  # Options: "v3.0" or "v4.0"
+    mask_files = [f for f in file_paths if f"extent_{version_str}.tif" in f]
+    conc_files = [f for f in file_paths if f"concentration_{version_str}.tif" in f]
+
+    # ========== Post-Process Sea Ice Mask Data ========== #
+    # Define the time dimension:
+    time_simask = xr.DataArray(data=get_datetimes_from_filenames(file_list=mask_files), dims='time', name='time')
+    # Load and concatenate all sea ice mask GeoTIFFs:
+    simask = xr.concat([xr.open_dataset(i) for i in mask_files], dim=time_simask)
+    logging.info("-> Completed: Loaded NSIDC sea ice mask and concentration GeoTIFF files.")
+
+    # Sea Ice Mask is defined by [1: sea ice, 0: ocean]:
+    # Values greater than 1 (missing or land) are set to NaN:
+    ds_si['simask'] = xr.where(simask['band_data'] > 1, np.nan, simask['band_data']).squeeze(drop=True)
+    ds_si["simask"].attrs = {'units': '1', "long_name": "Sea Ice Mask", "standard_name": "sea_ice_mask", "comment": "1 = sea ice, 0 = ocean"}
+
+    # ========== Post-Process Sea Ice Concentration Data ========== #
+    # Define the time dimension:
+    time_siconc = xr.DataArray(data=get_datetimes_from_filenames(file_list=conc_files), dims='time', name='time')
+    # Load and concatenate all sea ice extent GeoTIFFs:
+    siconc = xr.concat([xr.open_dataset(i) for i in conc_files], dim=time_siconc)
+    logging.info("-> Completed: Loaded NSIDC sea ice concentration GeoTIFF files.")
+
+    # Sea Ice Area Fraction:
+    # Note concentration percentage is scaled by 10 -> requires division by 1000.
+    # Values greater than 1 (missing or land) are set to NaN:
+    ds_si['siconc'] = xr.where(siconc['band_data'] > 1000, np.nan, siconc['band_data']).squeeze(drop=True) / 1000
+    ds_si['siconc'].attrs = {'units': '1', 'long_name': 'Sea Ice Area Fraction', 'standard_name': 'sea_ice_area_fraction', "comment": "0 = ocean, 0.01-0.15 = statistically insignificant, > 0.15 = sea ice"}
+
+    # ========== Calculate sea ice area (m2) ========== #
+    ds_si['cell_area'] = ds_area['cell_area']
+    ds_si['cell_area'].attrs = {'units': 'm2', 'long_name': 'Grid-Cell Area for Sea Ice Variables', "standard_name": "cell_area"}
+
+    ds_si['siextent'] = (ds_si['cell_area']*ds_si['simask']).sum(dim=['x', 'y'])
+    ds_si['siextent'].attrs = {'units': 'm2', 'long_name': 'Total Area Where Sea Ice Area Fraction Exceeds 15%', 'standard_name': 'sea_ice_extent'}
+
+    # ========== Update Coordinates ========== #
+    ds_si.coords['lon'] = ds_si['longitude']
+    ds_si.coords['lat'] = ds_si['latitude']
+    # Drop auxiliary variables:
+    ds_si = ds_si.drop_vars(["spatial_ref", "crs", "longitude", "latitude"])
+    # Rename coordinates:
+    ds_si = ds_si.rename({'lon': 'longitude', 'lat': 'latitude'})
+
+    # ========== Update attributes to ensure CF-compliance: ========== #
+    ds_si['x'].attrs = {'standard_name': 'projection_x_coordinate', 'long_name': 'x coordinate of projection', 'units': 'meters'}
+    ds_si['y'].attrs = {'standard_name': 'projection_y_coordinate', 'long_name': 'y coordinate of projection', 'units': 'meters'}
+    ds_si['longitude'].attrs = {'standard_name': 'longitude', 'long_name': 'Longitude', 'units': 'degrees_east'}
+    ds_si['latitude'].attrs = {'standard_name': 'latitude', 'long_name': 'Latitude', 'units': 'degrees_north'}
+    ds_si['simask'].attrs.pop("valid_range", None)
+
+    # ========== Save NSIDC Sea Ice Index Dataset ========== #
+    # Update variable encodings:
+    for var in ds_si.variables:
+        if ds_si[var].dtype == 'float64':
+            ds_si[var].encoding['missing_value'] = None
+            ds_si[var].encoding['_FillValue'] = None
+
+    # Define output filepath:
+    out_fpath = "/dssgfs01/scratch/otooth/npd_data/observations/NSIDC/NSIDC_Sea_Ice_Index_v4_Antarctic_combined_1978_2025.nc"
+    # Save dataset to netCDF file:
+    logging.info(f'In Progress: Saving NSIDC Sea Ice Index Antarctic Observations to netCDF file: {out_fpath}...')
+    ds_si.to_netcdf(out_fpath, unlimited_dims='time')
+    # Close files associated with datasets:
+    ds_si.close()
+    ds_area.close()
+
+    logging.info(f'Completed: Saved NSIDC Sea Ice Index Antarctic Observations to netCDF file: {out_fpath}.')
+
+if __name__ == "__main__":
+    main()

OceanDataStore/data/NSIDC/process_NSIDC_SSI_Arctic_data.py

@@ -0,0 +1,129 @@
+"""
+process_NSIDC_SSI_Arctic_data.py
+
+Description: Python script to post-process NSIDC Sea Ice Index
+data for the Arctic 1978-2025, including sea ice concentration,
+extent and total area.
+
+Created by: Ollie Tooth (oliver.tooth@noc.ac.uk)
+Created on: 2025-02-21
+"""
+# -- Import Python packages -- #
+import logging
+from datetime import datetime
+from glob import glob
+
+import numpy as np
+import xarray as xr
+
+from OceanDataStore.cli import initialise_logging
+
+
+# -- Define Utility Functions -- #
+def get_datetimes_from_filenames(file_list):
+    # Extract filenames from paths:
+    filenames = [file.split('/')[-1] for file in file_list]
+
+    # Convert filenames date str to datetime:
+    datetimes = np.array([datetime(year=int(file[2:6]), month=int(file[6:8]), day=15) for file in filenames])
+
+    return datetimes
+
+def main():
+    # ========== Initialize Logging and Print Banner ========== #
+    initialise_logging()
+
+    logging.info('In Progress: Post-Processing NSIDC Sea Ice Index Arctic Observations...')
+
+    # ========== Load NSIDC Ancillary Data ========== #
+    # Define filepath to ancillary data:
+    anc_fpath = "/dssgfs01/scratch/otooth/npd_data/observations/NSIDC/ancillary/NSIDC0771_LatLon_PS_N25km_v1.0.nc"
+    # Open NSIDC ancillary data as dataset:
+    ds_si = xr.open_dataset(anc_fpath)
+
+    # Define filepath to NSIDC ancillary file - grid cell area:
+    area_fpath = "/dssgfs01/scratch/otooth/npd_data/observations/NSIDC/ancillary/NSIDC0771_CellArea_PS_N25km_v1.0.nc"
+    # Open NSIDC grid cell area:
+    ds_area = xr.open_dataset(area_fpath)
+    logging.info("-> Completed: Loaded NSIDC ancillary data and grid cell area.")
+
+    # ========== Load NSIDC Monthly Data ========== #
+    # Define directory path:
+    dir_path = "/dssgfs01/scratch/otooth/npd_data/observations/NSIDC/arctic/"
+
+    # Get the list of files in the directory:
+    file_paths = glob(f"{dir_path}*.tif")
+    file_paths.sort()
+
+    # Retrieve sea ice mask & concentration files:
+    version_str = "v4.0"  # Options: "v3.0" or "v4.0"
+    mask_files = [f for f in file_paths if f"extent_{version_str}.tif" in f]
+    conc_files = [f for f in file_paths if f"concentration_{version_str}.tif" in f]
+
+    # ========== Post-Process Sea Ice Mask Data ========== #
+    # Define the time dimension:
+    time_simask = xr.DataArray(data=get_datetimes_from_filenames(file_list=mask_files), dims='time', name='time')
+    # Load and concatenate all sea ice mask GeoTIFFs:
+    simask = xr.concat([xr.open_dataset(i) for i in mask_files], dim=time_simask)
+    logging.info("-> Completed: Loaded NSIDC sea ice mask and concentration GeoTIFF files.")
+
+    # Sea Ice Mask is defined by [1: sea ice, 0: ocean]:
+    # Values greater than 1 (missing or land) are set to NaN:
+    ds_si['simask'] = xr.where(simask['band_data'] > 1, np.nan, simask['band_data']).squeeze(drop=True)
+    ds_si["simask"].attrs = {'units': '1', "long_name": "Sea Ice Mask", "standard_name": "sea_ice_mask", "comment": "1 = sea ice, 0 = ocean"}
+
+    # ========== Post-Process Sea Ice Concentration Data ========== #
+    # Define the time dimension:
+    time_siconc = xr.DataArray(data=get_datetimes_from_filenames(file_list=conc_files), dims='time', name='time')
+    # Load and concatenate all sea ice extent GeoTIFFs:
+    siconc = xr.concat([xr.open_dataset(i) for i in conc_files], dim=time_siconc)
+    logging.info("-> Completed: Loaded NSIDC sea ice concentration GeoTIFF files.")
+
+    # Sea Ice Area Fraction:
+    # Note concentration percentage is scaled by 10 -> requires division by 1000.
+    # Values greater than 1 (missing or land) are set to NaN:
+    ds_si['siconc'] = xr.where(siconc['band_data'] > 1000, np.nan, siconc['band_data']).squeeze(drop=True) / 1000
+    ds_si['siconc'].attrs = {'units': '1', 'long_name': 'Sea Ice Area Fraction', 'standard_name': 'sea_ice_area_fraction', "comment": "0 = ocean, 0.01-0.15 = statistically insignificant, > 0.15 = sea ice"}
+
+    # ========== Calculate sea ice area (m2) ========== #
+    ds_si['cell_area'] = ds_area['cell_area']
+    ds_si['cell_area'].attrs = {'units': 'm2', 'long_name': 'Grid-Cell Area for Sea Ice Variables', "standard_name": "cell_area"}
+
+    ds_si['siextent'] = (ds_si['cell_area']*ds_si['simask']).sum(dim=['x', 'y'])
+    ds_si['siextent'].attrs = {'units': 'm2', 'long_name': 'Total Area Where Sea Ice Area Fraction Exceeds 15%', 'standard_name': 'sea_ice_extent'}
+
+    # ========== Update Coordinates ========== #
+    ds_si.coords['lon'] = ds_si['longitude']
+    ds_si.coords['lat'] = ds_si['latitude']
+    # Drop auxiliary variables:
+    ds_si = ds_si.drop_vars(["spatial_ref", "crs", "longitude", "latitude"])
+    # Rename coordinates:
+    ds_si = ds_si.rename({'lon': 'longitude', 'lat': 'latitude'})
+
+    # ========== Update attributes to ensure CF-compliance: ========== #
+    ds_si['x'].attrs = {'standard_name': 'projection_x_coordinate', 'long_name': 'x coordinate of projection', 'units': 'meters'}
+    ds_si['y'].attrs = {'standard_name': 'projection_y_coordinate', 'long_name': 'y coordinate of projection', 'units': 'meters'}
+    ds_si['longitude'].attrs = {'standard_name': 'longitude', 'long_name': 'Longitude', 'units': 'degrees_east'}
+    ds_si['latitude'].attrs = {'standard_name': 'latitude', 'long_name': 'Latitude', 'units': 'degrees_north'}
+    ds_si['simask'].attrs.pop("valid_range", None)
+
+    # ========== Save NSIDC Sea Ice Index Dataset ========== #
+    # Update variable encodings:
+    for var in ds_si.variables:
+        if ds_si[var].dtype == 'float64':
+            ds_si[var].encoding['missing_value'] = None
+            ds_si[var].encoding['_FillValue'] = None
+
+    # Define output filepath:
+    out_fpath = "/dssgfs01/scratch/otooth/npd_data/observations/NSIDC/NSIDC_Sea_Ice_Index_v4_Arctic_combined_1978_2025.nc"
+    # Save dataset to netCDF file:
+    logging.info(f"-> Saving NSIDC Sea Ice Index Arctic Observations to netCDF file: {out_fpath}...")
+    ds_si.to_netcdf(out_fpath, unlimited_dims='time')
+    # Close files associated with datasets:
+    ds_si.close()
+    ds_area.close()
+
+    logging.info(f'Completed: Saved NSIDC Sea Ice Index Arctic Observations to netCDF file: {out_fpath}.')
+
+if __name__ == "__main__":
+    main()

OceanDataStore/data/NSIDC/run_send_NSIDC_v4.0_to_OS.slurm

@@ -0,0 +1,32 @@
+#!/bin/bash
+#SBATCH --job-name=xfer_NSIDC_sea_ice_index_v4.0
+#SBATCH --partition=test
+#SBATCH --time=00:20:00
+#SBATCH --ntasks-per-core=1
+#SBATCH --ntasks-per-node=64
+#SBATCH --ntasks-per-socket=32
+#SBATCH --nodes=1
+
+# ================================================================
+# run_send_NSIDC_v4.0_to_OS.slurm
+#
+# Description: SLURM script to send the NSIDC sea ice observations
+# data to the JASMIN Object Store.
+#
+# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
+# Created On: 2026-06-05
+#
+# ================================================================
+set -euo pipefail
+
+# -- Python Environment -- #
+# Activate miniconda environment:
+source .../miniforge3/bin/activate
+conda activate env_ods
+
+# -- Send NSIDC Sea Ice Index v4.0 to JASMIN OS -- #
+echo "In Progress: Sending NSIDC Sea Ice Index v4.0 to Icechunk..."
+
+python3 send_NSIDC_SII_v4.0_to_os.py
+
+echo "Completed: Sent NSIDC Sea Ice Index v4.0 to Icechunk."

OceanDataStore/data/NSIDC/send_NSIDC_SII_v4.0_to_os.py

@@ -0,0 +1,140 @@
+# =========================================================
+# send_NSIDC_SII_v4.0_to_os.py
+#
+# Script to write NSIDC Sea Ice Index version 4.0 to
+# Icechunk repositories in JASMIN cloud object storage.
+#
+# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
+# =========================================================
+import logging
+
+import xarray as xr
+import zarr
+
+from OceanDataStore.cli import send_to_icechunk, initialise_logging
+
+logger = logging.getLogger(__name__)
+
+
+def main():
+    # ========== Initialise OceanDataStore Logging ========== #
+    initialise_logging()
+
+    # ========== Prepare Data ========== #
+    # Open NSIDC Sea Ice Index v4.0 datasets:
+    ds_si_arctic = xr.open_dataset("/dssgfs01/scratch/otooth/npd_data/observations/NSIDC/NSIDC_Sea_Ice_Index_v4_Arctic_combined_1978_2025.nc")
+
+    ds_si_antarctic = xr.open_dataset("/dssgfs01/scratch/otooth/npd_data/observations/NSIDC/NSIDC_Sea_Ice_Index_v4_Antarctic_combined_1978_2025.nc")
+
+    # Optimise chunk sizes for spatial analysis:
+    ds_si_arctic = ds_si_arctic.chunk({'time': 12, 'y': 448, 'x': 304})
+    ds_si_antarctic = ds_si_antarctic.chunk({'time': 12, 'y': 332, 'x': 316})
+
+    # Update variable encodings:
+    blosccodec = zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.shuffle)
+    for var in list(ds_si_arctic.data_vars) + list(ds_si_arctic.coords):
+        ds_si_arctic[var].encoding['compressors'] = [blosccodec]
+    for var in list(ds_si_antarctic.data_vars) + list(ds_si_antarctic.coords):
+        ds_si_antarctic[var].encoding['compressors'] = [blosccodec]
+
+    # Update global CF-metadata attributes:
+    ds_si_arctic.attrs.clear()
+    ds_si_arctic = ds_si_arctic.assign_attrs({
+        "Conventions": "CF-1.6",
+        "title": "NSIDC Sea Ice Index, Version 4 - Arctic",
+        "description": "NSIDC Sea Ice Index version 4.0 - Arctic sea ice area fraction, sea ice extent and total sea ice area timeseries.",
+        "source": "Satellite observations: Sea Ice Concentrations from Nimbus-7 SMMR and DMSP SSM/I-SSMIS Passive Microwave Data (GSFC). AMSR2 Daily Polar Gridded Sea Ice Concentrations (AMSR2).",
+        "dataset_type": "observation",
+        "product_type": "timeseries",
+        "product_version" : "1.0",
+        "institution": "National Snow and Ice Data Center; Cooperative Institute for Research in Environmental Sciences; University of Colorado at Boulder; Boulder, CO",
+        "citation": "Fetterer, F., Knowles, K., Meier, W. N., Savoie, M., Windnagel, A. K. & Stafford, T. (2025). Sea Ice Index. (G02135, Version 4). [Data Set]. Boulder, Colorado USA. National Snow and Ice Data Center. https://doi.org/10.7265/a98x-0f50. Date Accessed 05-29-2026.",
+        "references": "Windnagel, A., Stafford, T., Fetterer, F., Meier, W. (2025). Sea Ice Index Version 4 Analysis. NSIDC Special Report 28. Boulder CO, USA: National Snow and Ice Data Center.",
+        "acknowledgement": "These data are produced and supported by the NASA National Snow and Ice Data Center Distributed Active Archive Center.",
+        "license": "NSIDC Sea Ice Index, Version 4 data were obtained from https://nsidc.org/data/g02135/versions/4 and are provided under a U.S. Government Works License https://www.usa.gov/government-works",
+        "doi": "10.7265/a98x-0f50",
+        "platform": "gn",
+        "horizontal_grid_type": "curvilinear",
+        "horizontal_grid_resolution": "25 km",
+        "aggregation": "mean",
+        "aggregation_frequency": "monthly",
+        "status": "ongoing",
+        "update_frequency": "quarterly",
+        "bbox": "[-180.0, 180.0, 30.98, 90.0]",
+    })
+
+    ds_si_antarctic.attrs.clear()
+    ds_si_antarctic = ds_si_antarctic.assign_attrs({
+        "Conventions": "CF-1.6",
+        "title": "NSIDC Sea Ice Index, Version 4 - Antarctic",
+        "description": "NSIDC Sea Ice Index, Version 4 - Antarctic sea ice area fraction, sea ice extent and total sea ice area timeseries.",
+        "source": "Satellite observations: Sea Ice Concentrations from Nimbus-7 SMMR and DMSP SSM/I-SSMIS Passive Microwave Data (GSFC). AMSR2 Daily Polar Gridded Sea Ice Concentrations (AMSR2).",
+        "dataset_type": "observation",
+        "product_type": "timeseries",
+        "product_version" : "1.0",
+        "institution": "National Snow and Ice Data Center; Cooperative Institute for Research in Environmental Sciences; University of Colorado at Boulder; Boulder, CO",
+        "citation": "Fetterer, F., Knowles, K., Meier, W. N., Savoie, M., Windnagel, A. K. & Stafford, T. (2025). Sea Ice Index. (G02135, Version 4). [Data Set]. Boulder, Colorado USA. National Snow and Ice Data Center. https://doi.org/10.7265/a98x-0f50. Date Accessed 05-29-2026.",
+        "references": "Windnagel, A., Stafford, T., Fetterer, F., Meier, W. (2025). Sea Ice Index Version 4 Analysis. NSIDC Special Report 28. Boulder CO, USA: National Snow and Ice Data Center.",
+        "acknowledgement": "These data are produced and supported by the NASA National Snow and Ice Data Center Distributed Active Archive Center.",
+        "license": "NSIDC Sea Ice Index, Version 4 data were obtained from https://nsidc.org/data/g02135/versions/4 and are provided under a U.S. Government Works License https://www.usa.gov/government-works",
+        "doi": "10.7265/a98x-0f50",
+        "platform": "gn",
+        "horizontal_grid_type": "curvilinear",
+        "horizontal_grid_resolution": "25 km",
+        "aggregation": "mean",
+        "aggregation_frequency": "monthly",
+        "status": "ongoing",
+        "update_frequency": "quarterly",
+        "bbox": "[-180.0, 180.0, -90.0, -39.23089]",
+    })
+
+    # ========== Send to Icechunk Repository ========== #
+    bucket = "nsidc"
+    exists = False
+    store_credentials_json = ".../credentials/jasmin_os_credentials.json"
+    branch = "main"
+    variable_commits = True
+    config_kwargs = {
+            "temporary_directory":".../OceanDataStore/OceanDataStore/data/NSIDC/",
+            "local_directory":".../OceanDataStore/OceanDataStore/data/NSIDC/"
+        }
+    cluster_kwargs = {
+            "n_workers" : 10,
+            "threads_per_worker" : 1,
+            "memory_limit":"3GB"
+        }
+
+    # -- Sea Ice Index v4.0 - Arctic -- #
+    send_to_icechunk(
+        file=ds_si_arctic,
+        bucket=bucket,
+        object_prefix="nsidc_sea_ice_index_v4_arctic_monthly",
+        store_credentials_json=store_credentials_json,
+        exists=exists,
+        append_dim='time',
+        attrs=ds_si_arctic.attrs,
+        branch=branch,
+        commit_message="Added NSIDC Sea Ice Index version 4 - Arctic (1978-01-2025-12).",
+        variable_commits=variable_commits,
+        dask_config_kwargs=config_kwargs,
+        dask_cluster_kwargs=cluster_kwargs,
+        )
+    
+    # -- Sea Ice Index v4.0 - Antarctic -- #
+    send_to_icechunk(
+        file=ds_si_antarctic,
+        bucket=bucket,
+        object_prefix="nsidc_sea_ice_index_v4_antarctic_monthly",
+        store_credentials_json=store_credentials_json,
+        exists=exists,
+        append_dim='time',
+        attrs=ds_si_antarctic.attrs,
+        branch=branch,
+        commit_message="Added NSIDC Sea Ice Index version 4 - Antarctic (1978-01-2025-12).",
+        variable_commits=variable_commits,
+        dask_config_kwargs=config_kwargs,
+        dask_cluster_kwargs=cluster_kwargs,
+        )
+
+if __name__ == "__main__":
+    main()

OceanDataStore/data/OISST/create_OISSTv2_daily_climatology.py

@@ -0,0 +1,83 @@
+import argparse
+import glob
+
+import dask
+import numpy as np
+import xarray as xr
+from dask.distributed import Client, LocalCluster
+
+
+def main(
+    year_start=1991,
+    year_end=2020,
+    data_path="/dssgfs01/scratch/otooth/npd_data/observations/OISST/daily/",
+    output="./sst.daily.climatology.nc",
+    dask_cluster_kwargs={
+        "n_workers" : 8,
+        "threads_per_worker" : 1,
+        "memory_limit":"10GB"
+        },
+    dask_config_kwargs={
+            "temporary_directory":"/dssgfs01/working/otooth/Software/OceanDataStore/OceanDataStore/data/OISST/",
+            "local_directory":"/dssgfs01/working/otooth/Software/OceanDataStore/OceanDataStore/data/OISST/"
+        }
+    ):
+    xr.set_options(use_flox=True)
+
+    if dask_config_kwargs is not None:
+        dask.config.set(dask_config_kwargs)
+
+    with LocalCluster(**dask_cluster_kwargs) as cluster, Client(cluster) as client:
+        print(f"Created LocalCluster with {dask_cluster_kwargs['n_workers']} workers @ Client: {client.dashboard_link}")
+
+        # Find all SST files
+        files = sorted(glob.glob(f"{data_path}/sst.day.mean.????.nc"))
+        selected_files = [file for file in files if int(file[-7:-3]) >= year_start and int(file[-7:-3]) <= year_end]
+        print(f"Selected files for climatology computation: {selected_files}", flush=True)
+
+        # Open multiple files
+        ds = xr.open_mfdataset(selected_files, combine="by_coords",
+                            parallel=True, engine='h5netcdf',
+                            chunks={"time": 31, "latitude": 720, "longitude": 360},
+                            preprocess=lambda ds: ds['sst']
+                            )
+
+        # Compute daily climatology (day of year)
+        ds = ds.chunk({
+            "time": -1,
+            "lat": 100,
+            "lon": 100,
+        })
+        g_sst = ds["sst"].groupby("time.dayofyear")   # Group once for readability
+
+        mean = g_sst.mean("time")
+        mean = mean.persist()
+
+        p10 = g_sst.quantile(0.10, dim="time")
+        p10 = p10.persist()
+
+        p90 = g_sst.quantile(0.90, dim="time")
+        p90 = p90.persist()
+
+        # Build output dataset
+        clim = xr.Dataset()
+        clim["sst_mean"] = mean
+        clim["sst_p10"] = p10.astype(np.float32)
+        clim["sst_p90"] = p90.astype(np.float32)
+
+        # Save output
+        print(f"In Progress: Saving Climatology to {output}")
+        clim.to_netcdf(output, engine='h5netcdf', mode='w')
+        print(f"Completed: Climatology saved to {output}", flush=True)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Compute OISST v2.1 SST Daily Climatology")
+    parser.add_argument("--year_start", type=int, default=1996, help="Start year for climatology")
+    parser.add_argument("--year_end", type=int, default=2025, help="End year for climatology")
+    parser.add_argument("--data_path", default="/dssgfs01/scratch/otooth/npd_data/observations/OISST/daily", help="Directory containing SST files")
+    parser.add_argument("--output", default="./sst.daily.climatology.nc", help="Output file")
+
+    args = parser.parse_args()
+
+    main(args.year_start, args.year_end, args.data_path, args.output)

OceanDataStore/data/OISST/download_oisstv2_data.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# ----------------------------------------------------------------
+# download_OISSTv2_data.sh
+#
+# Description: Download the OISSTv2 dataset from the
+# NOAA website:
+# https://psl.noaa.gov/data/gridded/data.noaa.oisst.v2.highres.html
+#
+# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
+# Created On: 2026-06-24
+# ----------------------------------------------------------------
+set -euo pipefail
+
+# --- Inputs --- #
+output_dir="/dssgfs01/scratch/otooth/npd_data/observations/OISST/daily/" 
+
+# -- Defaults -- #
+base_url="https://downloads.psl.noaa.gov//Datasets/noaa.oisst.v2.highres"
+
+# --- Main Script --- #
+echo "==================================================="
+echo "          Downloading OISSTv2 Data"
+echo "                     v0.1.0"
+echo "               Oliver J. Tooth, NOC"
+echo "==================================================="
+echo "In Progress: Downloading OISSTv2 dataset..."
+# Iterate over years:
+for yr in {2012..2026}; do
+    # Construct URL for current year:
+    url="$base_url/sst.day.mean.${yr}.nc"
+
+    # Download file if not in output directory:
+    filepath="$output_dir/$(basename $url)"
+    if [ ! -f "$filepath" ]; then
+        wget -P $output_dir $url
+        echo "-> Completed: Downloaded $filepath."
+    else
+        echo "-> Skipping Download: NetCDF file for ${yr} already exists in $output_dir."
+    fi
+done
+
+echo "==================================================="

OceanDataStore/data/OISST/run_create_OISSTv2_daily_climatology.slurm

@@ -0,0 +1,44 @@
+#!/bin/bash
+#SBATCH --job-name=oisstv2_daily_climatology
+#SBATCH --partition=compute
+#SBATCH --time=05:00:00
+#SBATCH --ntasks-per-core=1
+#SBATCH --ntasks-per-node=64
+#SBATCH --ntasks-per-socket=32
+#SBATCH --nodes=1
+
+# ==============================================================
+# run_create_OISSTv2_daily_climatology.slurm
+#
+# Description: SLURM script to create the OISSTv2.1 daily 
+# climatology datasets.
+#
+# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
+# Created On: 2026-06-25
+#
+# ==============================================================
+set -euo pipefail
+TIME1=`date +%s`
+
+# -- Python Environment -- #
+# Activate miniconda environment:
+source /dssgfs01/working/otooth/miniforge3/bin/activate
+conda activate env_ods
+
+# -- Create OISSTv2.1 daily climatology datasets -- #
+dpath="/dssgfs01/scratch/otooth/npd_data/observations/OISST/daily/"
+opath="/dssgfs01/scratch/otooth/npd_data/observations/OISST/climatology"
+year_start=1996
+year_end=2025
+
+echo "Start year  : " $year_start
+echo "End year    : " $year_end
+echo "Output file : " $opath/OISST_v2.1_sst_climatology_${year_start}-${year_end}.nc
+
+python3 create_OISSTv2_daily_climatology.py --year_start $year_start --year_end $year_end --data_path $dpath --output $opath/oisst_climatology_${year_start}-${year_end}.nc
+
+# -- Report Job Timing -- #
+TIME2=`date +%s`
+DIFFSEC=`expr ${TIME2} - ${TIME1}`
+echo Job Completed in: ${DIFFSEC} seconds.
+echo Job Took: `date +%H:%M:%S -ud @${DIFFSEC}`

OceanDataStore/data/OISST/run_send_OISSTv2_daily_climatology_to_os.slurm

@@ -0,0 +1,32 @@
+#!/bin/bash
+#SBATCH --job-name=oisstv2_daily_climatology
+#SBATCH --partition=test
+#SBATCH --time=00:20:00
+#SBATCH --ntasks-per-core=1
+#SBATCH --ntasks-per-node=64
+#SBATCH --ntasks-per-socket=32
+#SBATCH --nodes=1
+
+# ==============================================================
+# run_send_OISSTv2_daily_climatology_to_os.slurm
+#
+# Description: SLURM script to send the OISSTv2.1 daily 
+# climatology datasets to Icechunk repository.
+#
+# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
+# Created On: 2026-06-09
+#
+# ==============================================================
+set -euo pipefail
+
+# -- Python Environment -- #
+# Activate miniconda environment:
+source /dssgfs01/working/otooth/miniforge3/bin/activate
+conda activate env_ods
+
+# -- Send OISSTv2.1 daily climatology datasets to JASMIN OS -- #
+echo "In Progress: Sending OISSTv2.1 daily climatology to Icechunk..."
+
+python3 send_OISSTv2_daily_climatology_to_os.py
+
+echo "Completed: Sent OISSTv2.1 daily climatology to Icechunk."

OceanDataStore/data/OISST/run_send_OISSTv2_daily_to_os.slurm

@@ -0,0 +1,32 @@
+#!/bin/bash
+#SBATCH --job-name=send_oisstv2_daily
+#SBATCH --partition=compute
+#SBATCH --time=03:00:00
+#SBATCH --ntasks-per-core=1
+#SBATCH --ntasks-per-node=64
+#SBATCH --ntasks-per-socket=32
+#SBATCH --nodes=1
+
+# ==============================================================
+# run_send_OISSTv2_daily_to_os.slurm
+#
+# Description: SLURM script to send the OISSTv2.1 daily 
+# time-series dataset to Icechunk repository.
+#
+# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
+# Created On: 2026-06-09
+#
+# ==============================================================
+set -euo pipefail
+
+# -- Python Environment -- #
+# Activate miniconda environment:
+source /dssgfs01/working/otooth/miniforge3/bin/activate
+conda activate env_ods
+
+# -- Send OISSTv2.1 daily time-series datasets to JASMIN OS -- #
+echo "In Progress: Sending OISSTv2.1 Daily time-series to Icechunk..."
+
+python3 send_OISSTv2_daily_to_os.py
+
+echo "Completed: Sent OISSTv2.1 Daily time-series to Icechunk."