OceanDataStore 0.3.0__py3-none-any.whl

OceanDataStore/data/OISST/send_OISSTv2_monthly_climatology_to_os.py

@@ -0,0 +1,150 @@
+# =========================================================
+# send_OISSTv2_monthly_climatology_to_os.py
+#
+# Script to write OISST v2.1 long-term monthly climatologies
+# to Icechunk repositories in JASMIN cloud object storage.
+#
+# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
+# =========================================================
+import logging
+
+import numpy as np
+import xarray as xr
+import zarr
+
+from OceanDataStore.cli import initialise_logging, send_to_icechunk
+from OceanDataStore.data.utils import (
+    compute_cell_area,
+    compute_dx,
+    compute_dy,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def main():
+    # ========== Initialise OceanDataStore Logging ========== #
+    initialise_logging()
+
+    # ========== Send to Icechunk Repository ========== #
+    bucket = "oisst"
+    exists = False
+    store_credentials_json = ".../credentials/jasmin_os_credentials.json"
+    branch = "main"
+    variable_commits = True
+
+    # Define climatology period:
+    start_yr = 1991
+    end_yr = 2020
+    
+    logging.info(f"In Progress: Sending OISSTv2.1 monthly climatology for {start_yr}-{end_yr} to Icechunk...")
+    # Open OISSTv2 dataset:
+    filepaths = [f"/dssgfs01/scratch/otooth/npd_data/observations/OISST/icec.mon.ltm.{start_yr}-{end_yr}.nc",
+                 f"/dssgfs01/scratch/otooth/npd_data/observations/OISST/sst.mon.ltm.{start_yr}-{end_yr}.nc"
+                ]
+    ds = xr.merge([xr.open_dataset(filepath, decode_times=False).drop_vars("valid_yr_count") for filepath in filepaths], compat="no_conflicts")
+    # Open OISSTv2 land-sea mask dataset:
+    ds_mask = xr.open_dataset("http://psl.noaa.gov/thredds/dodsC/Datasets/noaa.oisst.v2.highres/lsmask.oisst.nc", decode_times=False)
+    ds_mask = ds_mask.squeeze(drop=True).rename({"lon": "longitude", "lat": "latitude", "lsmask": "mask"})
+    ds_mask = ds_mask.assign_coords(
+        longitude=((ds_mask["longitude"] + 180) % 360) - 180
+    )
+
+    # Standardise coordinate dimension names:
+    ds = ds.rename({"lon": "longitude", "lat": "latitude", "time": "month"})
+
+    # Update longitude coordinates to be in the range [-180, 180]:
+    ds = ds.assign_coords(
+        longitude=((ds["longitude"] + 180) % 360) - 180
+    )
+    ds = ds.sortby("longitude")
+
+    # Add month of year coordinate (1-12):
+    ds = ds.assign_coords(
+        month=np.arange(1, 13)
+    )
+
+    # Rename variables to standard names:
+    ds = ds.rename({"sst": "tos",
+                    "icec": "siconc",
+                    "climatology_bounds": "time_bnds",
+                    })
+
+    # Add standard names and units:
+    ds["tos"].attrs["standard_name"] = "sea_surface_temperature"
+    ds["siconc"].attrs["standard_name"] = "sea_ice_area_fraction"
+    ds["siconc"].attrs["units"] = "1"
+
+    # Add OISSTv2 land mask:
+    ds["mask"] = ds_mask["mask"]
+    ds["mask"].attrs.clear()
+    ds["mask"] = ds["mask"].assign_attrs({'long_name': "Land-Sea Binary Mask",
+                                          "standard_name": "sea_binary_mask",
+                                          "comment": "1 = sea, 0 = land"
+                                          })
+
+    # Add horizontal grid cell area:
+    ds['dx'] = compute_dx(ds)
+    ds['dy'] = compute_dy(ds)
+    ds['cell_area'] = compute_cell_area(ds)
+
+    # Update time bounds to reflect climatological period:
+    ds['time_bnds'] = ds['time_bnds'].astype('datetime64[ns]')
+    ds['time_bnds'].data[:, 0] = (np.datetime64(f'{start_yr}-01', 'M') + (np.timedelta64(1, 'M') * np.arange(ds['month'].size))).astype('datetime64[ns]')
+    ds['time_bnds'].data[:, 1] = (np.datetime64(f'{end_yr}-01', 'M') + (np.timedelta64(1, 'M') * np.arange(ds['month'].size))).astype('datetime64[ns]')
+    ds.time_bnds.attrs.clear()
+
+    # Update global attributes:
+    ds.attrs.clear()
+    ds = ds.assign_attrs({
+        "Conventions": "CF-1.5",
+        "title": f"NOAA OISSTv2.1 Monthly Climatology ({start_yr}-{end_yr})",
+        "description": f"NOAA 1/4° Monthly Optimum Interpolation Sea Surface Temperature (OISST) version 2.1 monthly sea surface temperature and sea ice fraction climatology ({start_yr}-{end_yr}).",
+        "source": "Numerical models: Optimal Interpolation. In-situ observations: ICOADS-D R3.0.2, Argo GDAC. Satellite observations: Advanced Very High Resolution Radiometer (AVHRR).",
+        "dataset_type": "observation",
+        "product_type": "climatology",
+        "product_version": "2.1",
+        "institution": "NOAA National Centers for Environmental Information (NCEI)",
+        "citation": "Huang, B., C. Liu, V. Banzon, E. Freeman, G. Graham, B. Hankins, T. Smith, and H.-M. Zhang, 2021: Improvements of the Daily Optimum Interpolation Sea Surface Temperature (DOISST) Version 2.1, Journal of Climate, 34, 2923-2939. doi: 10.1175/JCLI-D-20-0166.1",
+        "references": "Huang, B., C. Liu, V. Banzon, E. Freeman, G. Graham, B. Hankins, T. Smith, and H.-M. Zhang, 2020: Improvements of the Daily Optimum Interpolation Sea Surface Temperature (DOISST) Version 2.1, Journal of Climate, 34, 2923-2939. doi: 10.1175/JCLI-D-20-0166.1. Banzon, V., Smith, T. M., Chin, T. M., Liu, C., and Hankins, W., 2016: A long-term record of blended satellite and in situ sea-surface temperature for climate monitoring, modeling and environmental studies. Earth Syst. Sci. Data, 8, 165-176, doi:10.5194/essd-8-165-2016. Reynolds, R. W., T. M. Smith, C. Liu, D. B. Chelton, K. S. Casey, and M. G. Schlax, 2007: Daily high-resolution-blended analyses for sea surface temperature. Journal of Climate, 20, 5473-5496, doi:10.1175/JCLI-D-14-00293.1",
+        "acknowledgement": "NOAA OI SST V2 High Resolution Dataset data provided by the NOAA PSL, Boulder, Colorado, USA, from their website at https://psl.noaa.gov.",
+        "license": "OISST v2.1 data were obtained from https://psl.noaa.gov/data/gridded/data.noaa.oisst.v2.highres.html and are provided under a Creative Commons CC0 1.0 Universal License https://creativecommons.org/publicdomain/zero/1.0/",
+        "doi": "10.1175/JCLI-D-20-0166.1",
+        "platform": "gr",
+        "horizontal_grid_type": "regular rectilinear",
+        "horizontal_grid_resolution": "0.25 degree",
+        "aggregation": "mean",
+        "aggregation_frequency": "monthly",
+        "status": "completed",
+        "update_frequency": "None",
+        "bbox": "[-180.0, 180.0, -90.0, 90.0]",
+    })
+
+    # Optimise chunk sizes for spatial analysis:
+    ds = ds.chunk({'month': 1, 'latitude': 720, 'longitude': 1440})
+
+    # Update variable encodings:
+    blosccodec = zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.shuffle)
+    for var in list(ds.data_vars) + list(ds.coords):
+        ds[var].encoding['compressors'] = [blosccodec]
+
+    # Define prefix and commit message based on climatology period:
+    prefix = f"oisst_v2.1_{start_yr}_{end_yr}_monthly_climatology"
+    commit_message = f"Added OISSTv2.1 Sea Surface Temperature Climatology ({start_yr}-{end_yr})."
+
+    send_to_icechunk(
+        file=ds,
+        bucket=bucket,
+        object_prefix=prefix,
+        store_credentials_json=store_credentials_json,
+        exists=exists,
+        append_dim='month',
+        branch=branch,
+        commit_message=commit_message,
+        variable_commits=variable_commits,
+        dask_config_kwargs=None,
+        dask_cluster_kwargs=None,
+        )
+
+if __name__ == "__main__":
+    main()

OceanDataStore/data/OISST/send_OISSTv2_monthly_to_os.py

@@ -0,0 +1,145 @@
+# =========================================================
+# send_OISSTv2_monthly_climatology_to_os.py
+#
+# Script to write OISST v2.1 long-term monthly climatologies
+# to Icechunk repositories in JASMIN cloud object storage.
+#
+# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
+# =========================================================
+import logging
+
+import numpy as np
+import xarray as xr
+import zarr
+
+from OceanDataStore.cli import initialise_logging, send_to_icechunk
+from OceanDataStore.data.utils import (
+    compute_cell_area,
+    compute_dx,
+    compute_dy,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def main():
+    # ========== Initialise OceanDataStore Logging ========== #
+    initialise_logging()
+
+    # ========== Send to Icechunk Repository ========== #
+    bucket = "oisst"
+    exists = False
+    store_credentials_json = ".../credentials/jasmin_os_credentials.json"
+    branch = "main"
+    variable_commits = True
+    
+    logging.info("In Progress: Sending OISSTv2.1 monthly mean time series to Icechunk...")
+    # Open OISSTv2 dataset:
+    filepaths = ["/dssgfs01/scratch/otooth/npd_data/observations/OISST/icec.mon.mean.nc",
+                 "/dssgfs01/scratch/otooth/npd_data/observations/OISST/sst.mon.mean.nc"
+                ]
+    ds = xr.merge([xr.open_dataset(filepath, decode_times=False) for filepath in filepaths], compat="no_conflicts")
+    # Open OISSTv2 land-sea mask dataset:
+    ds_mask = xr.open_dataset("http://psl.noaa.gov/thredds/dodsC/Datasets/noaa.oisst.v2.highres/lsmask.oisst.nc", decode_times=False)
+    ds_mask = ds_mask.squeeze(drop=True).rename({"lon": "longitude", "lat": "latitude", "lsmask": "mask"})
+    ds_mask = ds_mask.assign_coords(
+        longitude=((ds_mask["longitude"] + 180) % 360) - 180
+    )
+
+    # Standardise coordinate dimension names:
+    ds = ds.rename({"lon": "longitude", "lat": "latitude"})
+
+    # Update longitude coordinates to be in the range [-180, 180]:
+    ds = ds.assign_coords(
+        longitude=((ds["longitude"] + 180) % 360) - 180
+    )
+    ds = ds.sortby("longitude")
+
+    # Update time coordinate to be datetime objects:
+    ds = ds.assign_coords(time=(np.datetime64('1800-01-01', 'D') + (np.timedelta64(1, 'D') * ds['time'])).astype('datetime64[ns]'))
+    ds['time'].attrs.pop('units', None)
+
+    # Rename variables to standard names:
+    ds = ds.rename({"sst": "tos",
+                    "icec": "siconc",
+                    })
+
+    # Add standard names and units:
+    ds["tos"].attrs["standard_name"] = "sea_surface_temperature"
+    ds["siconc"].attrs["standard_name"] = "sea_ice_area_fraction"
+    ds["siconc"].attrs["units"] = "1"
+
+    # Add OISSTv2 land mask:
+    ds["mask"] = ds_mask["mask"]
+    ds["mask"].attrs.clear()
+    ds["mask"] = ds["mask"].assign_attrs({'long_name': "Land-Sea Binary Mask",
+                                          "standard_name": "sea_binary_mask",
+                                          "comment": "1 = sea, 0 = land"
+                                          })
+
+    # Add horizontal grid cell area:
+    ds["dx"] = compute_dx(ds)
+    ds["dy"] = compute_dy(ds)
+    ds['cell_area'] = compute_cell_area(ds)
+
+    # Add Northern and Southern Hemisphere sea ice area timeseries:
+    ds['siarea_NH'] = (ds['siconc'].where(ds['latitude'] > 0) * ds['cell_area']).sum(dim=['latitude', 'longitude'])
+    ds['siarea_NH'].attrs = {'long_name': 'Total Northern Hemisphere Sea Ice Area', 'standard_name': 'sea_ice_area', 'units': 'm2'}
+
+    ds['siarea_SH'] = (ds['siconc'].where(ds['latitude'] < 0) * ds['cell_area']).sum(dim=['latitude', 'longitude'])
+    ds['siarea_SH'].attrs = {'long_name': 'Total Southern Hemisphere Sea Ice Area', 'standard_name': 'sea_ice_area', 'units': 'm2'}
+
+    # Update global attributes:
+    ds.attrs.clear()
+    ds = ds.assign_attrs({
+        "Conventions": "CF-1.5",
+        "title": "NOAA OISSTv2.1 Monthly Timeseries",
+        "description": "NOAA 1/4° Monthly Optimum Interpolation Sea Surface Temperature (OISST) version 2.1 monthly sea surface temperature and sea ice fraction timeseries.",
+        "source": "Numerical models: Optimal Interpolation. In-situ observations: ICOADS-D R3.0.2, Argo GDAC. Satellite observations: Advanced Very High Resolution Radiometer (AVHRR).",
+        "dataset_type": "observation",
+        "product_type": "timeseries",
+        "product_version": "2.1",
+        "institution": "NOAA National Centers for Environmental Information (NCEI)",
+        "citation": "Huang, B., C. Liu, V. Banzon, E. Freeman, G. Graham, B. Hankins, T. Smith, and H.-M. Zhang, 2021: Improvements of the Daily Optimum Interpolation Sea Surface Temperature (DOISST) Version 2.1, Journal of Climate, 34, 2923-2939. doi: 10.1175/JCLI-D-20-0166.1",
+        "references": "Huang, B., C. Liu, V. Banzon, E. Freeman, G. Graham, B. Hankins, T. Smith, and H.-M. Zhang, 2020: Improvements of the Daily Optimum Interpolation Sea Surface Temperature (DOISST) Version 2.1, Journal of Climate, 34, 2923-2939. doi: 10.1175/JCLI-D-20-0166.1. Banzon, V., Smith, T. M., Chin, T. M., Liu, C., and Hankins, W., 2016: A long-term record of blended satellite and in situ sea-surface temperature for climate monitoring, modeling and environmental studies. Earth Syst. Sci. Data, 8, 165-176, doi:10.5194/essd-8-165-2016. Reynolds, R. W., T. M. Smith, C. Liu, D. B. Chelton, K. S. Casey, and M. G. Schlax, 2007: Daily high-resolution-blended analyses for sea surface temperature. Journal of Climate, 20, 5473-5496, doi:10.1175/JCLI-D-14-00293.1",
+        "acknowledgement": "NOAA OI SST V2 High Resolution Dataset data provided by the NOAA PSL, Boulder, Colorado, USA, from their website at https://psl.noaa.gov.",
+        "license": "OISST v2.1 data were obtained from https://psl.noaa.gov/data/gridded/data.noaa.oisst.v2.highres.html and are provided under a Creative Commons CC0 1.0 Universal License https://creativecommons.org/publicdomain/zero/1.0/",
+        "doi": "10.1175/JCLI-D-20-0166.1",
+        "platform": "gr",
+        "horizontal_grid_type": "regular rectilinear",
+        "horizontal_grid_resolution": "0.25 degree",
+        "aggregation": "mean",
+        "aggregation_frequency": "monthly",
+        "status": "ongoing",
+        "update_frequency": "quarterly",
+        "bbox": "[-180.0, 180.0, -90.0, 90.0]",
+    })
+
+    # Optimise chunk sizes for spatial analysis:
+    ds = ds.chunk({'time': 1, 'latitude': 720, 'longitude': 1440})
+
+    # Update variable encodings:
+    blosccodec = zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.shuffle)
+    for var in list(ds.data_vars) + list(ds.coords):
+        ds[var].encoding['compressors'] = [blosccodec]
+
+    # Define prefix and commit message based on climatology period:
+    prefix = "oisst_v2.1_monthly"
+    commit_message = "Added OISSTv2.1 Sea Surface Temperature & Sea Ice Fraction Monthly Timeseries (1981-2026)."
+
+    send_to_icechunk(
+        file=ds,
+        bucket=bucket,
+        object_prefix=prefix,
+        store_credentials_json=store_credentials_json,
+        exists=exists,
+        append_dim='time',
+        branch=branch,
+        commit_message=commit_message,
+        variable_commits=variable_commits,
+        dask_config_kwargs=None,
+        dask_cluster_kwargs=None,
+        )
+
+if __name__ == "__main__":
+    main()

OceanDataStore/data/OISST/update_OISSTv2_daily_to_os.py

@@ -0,0 +1,142 @@
+# =========================================================
+# update_OISSTv2_daily_to_os.py
+#
+# Script to write OISST v2.1 daily mean timeseries
+# to Icechunk repositories in JASMIN cloud object storage.
+#
+# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
+# =========================================================
+import logging
+from pathlib import Path
+
+import xarray as xr
+import zarr
+
+from OceanDataStore.cli import initialise_logging, update_icechunk
+from OceanDataStore.data.utils import (
+    compute_cell_area,
+    compute_dx,
+    compute_dy,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def main():
+    # ========== Initialise OceanDataStore Logging ========== #
+    initialise_logging()
+
+    # ========== Update Icechunk Repository ========== #
+    bucket = "oisst"
+    store_credentials_json = ".../credentials/jasmin_os_credentials.json"
+    branch = "main"
+    config_kwargs = {
+            "temporary_directory":"/dssgfs01/working/otooth/Software/OceanDataStore/OceanDataStore/data/OISST/",
+            "local_directory":"/dssgfs01/working/otooth/Software/OceanDataStore/OceanDataStore/data/OISST/"
+        }
+    cluster_kwargs = {
+            "n_workers" : 15,
+            "threads_per_worker" : 1,
+            "memory_limit":"6GB"
+        }
+    
+    logging.info("In Progress: Updating OISSTv2.1 daily mean time series in Icechunk...")
+    # Open OISSTv2 dataset:
+    filepaths = []
+    base = Path("/dssgfs01/scratch/otooth/npd_data/observations/OISST/daily/")
+    for year in range(2026, 2027):
+        filepaths.extend(sorted(base.glob(f"sst.day.mean.{year}.nc")))   
+    ds = xr.open_mfdataset(filepaths,
+                           combine="by_coords",
+                           data_vars="all",
+                           engine="h5netcdf",
+                           )
+
+    # Open OISSTv2 land-sea mask dataset:
+    ds_mask = xr.open_dataset("http://psl.noaa.gov/thredds/dodsC/Datasets/noaa.oisst.v2.highres/lsmask.oisst.nc", decode_times=False)
+    ds_mask = ds_mask.squeeze(drop=True).rename({"lon": "longitude", "lat": "latitude", "lsmask": "mask"})
+    ds_mask = ds_mask.assign_coords(
+        longitude=((ds_mask["longitude"] + 180) % 360) - 180
+    )
+
+    # Standardise coordinate dimension names:
+    ds = ds.rename({"lon": "longitude", "lat": "latitude"})
+
+    # Update longitude coordinates to be in the range [-180, 180]:
+    ds = ds.assign_coords(
+        longitude=((ds["longitude"] + 180) % 360) - 180
+    )
+    ds = ds.sortby("longitude")
+
+    # Rename variables to standard names:
+    ds = ds.rename({"sst": "tos"})
+
+    # Add standard names and units:
+    ds["tos"].attrs["standard_name"] = "sea_surface_temperature"
+
+    # Add OISSTv2 land mask:
+    ds["mask"] = ds_mask["mask"]
+    ds["mask"].attrs.clear()
+    ds["mask"] = ds["mask"].assign_attrs({"long_name": "Land-Sea Binary Mask",
+                                          "standard_name": "sea_binary_mask",
+                                          "comment": "1 = sea, 0 = land"
+                                          })
+
+    # Add horizontal grid cell area:
+    ds["dx"] = compute_dx(ds)
+    ds["dy"] = compute_dy(ds)
+    ds['cell_area'] = compute_cell_area(ds)
+
+    # Update global attributes:
+    ds.attrs.clear()
+    ds = ds.assign_attrs({
+        "Conventions": "CF-1.5",
+        "title": "NOAA OISSTv2.1 Daily Timeseries",
+        "description": "NOAA 1/4° Daily Optimum Interpolation Sea Surface Temperature (OISST) version 2.1 daily sea surface temperature timeseries.",
+        "source": "Numerical models: Optimal Interpolation. In-situ observations: ICOADS-D R3.0.2, Argo GDAC. Satellite observations: Advanced Very High Resolution Radiometer (AVHRR).",
+        "dataset_type": "observation",
+        "product_type": "timeseries",
+        "product_version": "2.1",
+        "institution": "NOAA National Centers for Environmental Information (NCEI)",
+        "citation": "Huang, B., C. Liu, V. Banzon, E. Freeman, G. Graham, B. Hankins, T. Smith, and H.-M. Zhang, 2021: Improvements of the Daily Optimum Interpolation Sea Surface Temperature (DOISST) Version 2.1, Journal of Climate, 34, 2923-2939. doi: 10.1175/JCLI-D-20-0166.1",
+        "references": "Huang, B., C. Liu, V. Banzon, E. Freeman, G. Graham, B. Hankins, T. Smith, and H.-M. Zhang, 2020: Improvements of the Daily Optimum Interpolation Sea Surface Temperature (DOISST) Version 2.1, Journal of Climate, 34, 2923-2939. doi: 10.1175/JCLI-D-20-0166.1. Banzon, V., Smith, T. M., Chin, T. M., Liu, C., and Hankins, W., 2016: A long-term record of blended satellite and in situ sea-surface temperature for climate monitoring, modeling and environmental studies. Earth Syst. Sci. Data, 8, 165-176, doi:10.5194/essd-8-165-2016. Reynolds, R. W., T. M. Smith, C. Liu, D. B. Chelton, K. S. Casey, and M. G. Schlax, 2007: Daily high-resolution-blended analyses for sea surface temperature. Journal of Climate, 20, 5473-5496, doi:10.1175/JCLI-D-14-00293.1",
+        "acknowledgement": "NOAA OI SST V2 High Resolution Dataset data provided by the NOAA PSL, Boulder, Colorado, USA, from their website at https://psl.noaa.gov.",
+        "license": "OISST v2.1 data were obtained from https://psl.noaa.gov/data/gridded/data.noaa.oisst.v2.highres.html and are provided under a Creative Commons CC0 1.0 Universal License https://creativecommons.org/publicdomain/zero/1.0/",
+        "doi": "10.1175/JCLI-D-20-0166.1",
+        "platform": "gr",
+        "horizontal_grid_type": "regular rectilinear",
+        "horizontal_grid_resolution": "0.25 degree",
+        "aggregation": "mean",
+        "aggregation_frequency": "daily",
+        "status": "ongoing",
+        "update_frequency": "quarterly",
+        "bbox": "[-180.0, 180.0, -90.0, 90.0]",
+    })
+
+    # Optimise chunk sizes for time-series analysis:
+    ds = ds.chunk({'time': ds['time'].size, 'latitude': 50, 'longitude': 50})
+
+    # Update variable encodings:
+    blosccodec = zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.shuffle)
+    for var in list(ds.data_vars) + list(ds.coords):
+        ds[var].encoding.clear()
+        ds[var].encoding['compressors'] = [blosccodec]
+
+    # Define prefix and commit message based on climatology period:
+    prefix = "oisst_v2.1_daily"
+    commit_message = "Added OISSTv2.1 Sea Surface Temperature Daily Timeseries (2026-01-2026-06)." 
+
+    update_icechunk(
+        file=ds,
+        bucket=bucket,
+        object_prefix=prefix,
+        store_credentials_json=store_credentials_json,
+        append_dim='time',
+        branch=branch,
+        commit_message=commit_message,
+        dask_config_kwargs=config_kwargs,
+        dask_cluster_kwargs=cluster_kwargs,
+        )
+
+if __name__ == "__main__":
+    main()

OceanDataStore/data/OSTIA/create_OSTIA_daily_climatology.py

@@ -0,0 +1,120 @@
+# =========================================================
+# create_OSTIA_daily_climatology.py
+#
+# Script to calculate daily mean and quantile climatology
+# for OSTIA sea surface temperature data.
+#
+# Created By: Oliver Tooth (oliver.tooth@noc.ac.uk)
+# =========================================================
+import argparse
+import logging
+
+import numpy as np
+import xarray as xr
+import zarr
+from dask.distributed import Client
+
+from OceanDataStore.cli import initialise_logging
+
+logger = logging.getLogger(__name__)
+
+def pooled_days(day: int, half_width: int=5, ndays: int=366):
+    """
+    Return cyclic day window centred on `day`.
+    """
+    offsets = np.arange(-half_width, half_width + 1)
+    return ((day - 1 + offsets) % ndays) + 1
+
+
+def main(start_year, end_year, data_path="/dssgfs01/working/otooth/data/observations/OSTIA/", output="sst_climatology.zarr"):
+
+    # ========== Initialise OceanDataStore Logging ========== #
+    initialise_logging()
+
+    # ========== Initialise Dask Client ========== #
+    client = Client(n_workers=20, threads_per_worker=1)
+    logging.info(f"Dask client initialized: {client}")
+
+    # ========== Compute Daily Climatology from OSTIA Dataset ========== #
+    logging.info(f"Computing OSTIA Daily Climatology from {start_year} to {end_year}")
+
+    # Open multiple files
+    ds_ostia_rep = xr.open_mfdataset([f"{data_path}ostia_global_sst_daily_NA_1990_1999.zarr",
+                                      f"{data_path}ostia_global_sst_daily_NA_2000_2025.zarr"
+                                      ], engine="zarr", parallel=True)
+
+    ds_ostia_nrt = xr.open_dataset(f"{data_path}ostia_global_sst_daily_NA_nrt_2025_2026.zarr",
+                                   engine="zarr",
+                                   )
+    ds_ostia_nrt = ds_ostia_nrt.sel(time=slice("2025-12-19", None))
+
+    ds = xr.concat([ds_ostia_rep, ds_ostia_nrt], dim="time")
+    ds = ds.sel(time=slice(f"{start_year}-01-01", f"{end_year}-12-31"))
+    logging.info(f"Completed: Opened OSTIA dataset for years: {start_year} to {end_year}")
+
+    ds = ds.chunk({
+        "time": -1,
+        "latitude": 100,
+        "longitude": 100
+    })
+
+    # Add Climatological Day of Year (clim_day) coordinate to the dataset
+    doy = ds['time'].dt.dayofyear
+    is_leap = ds['time'].dt.is_leap_year
+    after_feb28 = (~is_leap) & (doy >= 60)
+    clim_day = xr.where(after_feb28, doy + 1, doy)
+    ds = ds.assign_coords(clim_day=("time", clim_day.data))
+
+    # Rename variables and transform temperature from Kelvin to Celsius
+    ds = ds.rename({"analysed_sst": "tos",
+                    "analysis_error": "tos_error",
+                    "sea_ice_fraction": "siconc",
+                    })
+
+    ds['tos'] = ds['tos'] - 273.15  # Convert from Kelvin to Celsius
+    ds['tos_error'] = ds['tos_error'] - 273.15  # Convert from Kelvin to Celsius
+
+    # Compute daily climatology (day of year):
+    for day in range(1, 367):
+        logging.info(f"Calculating Mean, 10th and 90th percentiles for day {day}...")
+        pooled = ds['tos'][ds['clim_day'].isin(pooled_days(day=day))]
+
+        # Build output dataset
+        clim = xr.Dataset()
+        clim["tos_mean"] = pooled.mean(dim="time", skipna=True)
+        clim["tos_p10"] = pooled.quantile(q=0.1, dim="time", skipna=True).astype(np.float32)
+        clim["tos_p90"] = pooled.quantile(q=0.9, dim="time", skipna=True).astype(np.float32)
+        clim = clim.expand_dims(clim_day=[day])
+        logging.info("Completed: Created OSTIA Daily Climatology SST dataset.")
+
+        clim = clim.chunk({
+            "clim_day": 1,
+            "latitude": 721,
+            "longitude": 1440
+        })
+
+        # Update variable encodings:
+        blosccodec = zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.shuffle)
+        for var in list(clim.data_vars) + list(clim.coords):
+            clim[var].encoding.clear()
+            clim[var].encoding['compressors'] = [blosccodec]
+
+        # Save output
+        logging.info(f"In Progress: Saving climatology to {output}...")
+        if day == 1:
+            clim.to_zarr(store=output, mode="w", zarr_format=3)
+        else:
+            clim.to_zarr(store=output, append_dim='clim_day', zarr_format=3)
+        logging.info(f"Completed: Saved climatology to {output}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Compute SST Daily Climatology")
+    parser.add_argument("start_year", type=int, help="Start year (e.g. 2000)")
+    parser.add_argument("end_year", type=int, help="End year (e.g. 2010)")
+    parser.add_argument("--data_path", default=".", help="Directory containing SST files")
+    parser.add_argument("--output", default="sst_climatology.zarr", help="Output file")
+
+    args = parser.parse_args()
+
+    main(args.start_year, args.end_year, args.data_path, args.output)

OceanDataStore/data/OSTIA/download_OSTIA_NRT.py

@@ -0,0 +1,42 @@
+"""
+Download North Atlantic subdomain of OSTIA Global SST dataset from Copernicus Marine.
+This script downloads near-real time data from 2025 onwards.
+
+Subdomain is defined as: (-45 degE, 14 degE, 31 degN, 85 degN)
+
+Created By: Ollie Tooth
+Created On: 2026-06-27
+Contact: oliver.tooth@noc.ac.uk
+
+Virtual Environment: env_ods.
+"""
+
+# Import the Copernicus Marine API toolbox:
+import copernicusmarine
+
+# Define filepath to credentials:
+credentials_fpath = "~/.copernicusmarine/.copernicusmarine-credentials"
+# Define output directory:
+out_fdir = "/dssgfs01/working/otooth/data/observations/OSTIA/"
+
+# Define start and end years:
+year_start = 2025
+year_end = 2026
+
+# Download the OSTIA Global SST dataset for North Atlantic subdomain:
+print(f"In Progress: Downloading OSTIA Global SST dataset for: {year_start} to {year_end}...")
+copernicusmarine.subset(
+    dataset_id="METOFFICE-GLO-SST-L4-NRT-OBS-SST-V2",
+    variables=["analysed_sst", "analysis_error", "mask", "sea_ice_fraction"],
+    start_datetime=f"{year_start}-01-01T00:00:00",
+    end_datetime=f"{year_end}-12-31T23:59:59",
+    minimum_longitude=-45,
+    maximum_longitude=14,
+    minimum_latitude=31,
+    maximum_latitude=85,
+    credentials_file=credentials_fpath,
+    output_directory=out_fdir,
+    file_format="zarr",
+    output_filename=f"ostia_global_sst_daily_NA_nrt_{year_start}_{year_end}.zarr",
+)
+print(f"Completed: downloading OSTIA Global SST dataset for: {year_start} to {year_end}.")

OceanDataStore/data/OSTIA/download_OSTIA_REP_1981_2025.py

@@ -0,0 +1,42 @@
+"""
+Download North Atlantic subdomain of OSTIA Global SST dataset from Copernicus Marine.
+This script downloads the monthly reprocessed files from 1981 to 2025.
+
+Subdomain is defined as: (-45 degE, 14 degE, 31 degN, 85 degN)
+
+Created By: Ollie Tooth
+Created On: 2026-06-27
+Contact: oliver.tooth@noc.ac.uk
+
+Virtual Environment: env_ods.
+"""
+
+# Import the Copernicus Marine API toolbox:
+import copernicusmarine
+
+# Define filepath to credentials:
+credentials_fpath = "~/.copernicusmarine/.copernicusmarine-credentials"
+# Define output directory:
+out_fdir = "/dssgfs01/working/otooth/data/observations/OSTIA/"
+
+# Define start and end years:
+year_start = 2000
+year_end = 2025
+
+# Download the OSTIA Global SST dataset for North Atlantic subdomain:
+print(f"In Progress: Downloading OSTIA Global SST dataset for: {year_start} to {year_end}...")
+copernicusmarine.subset(
+    dataset_id="METOFFICE-GLO-SST-L4-REP-OBS-SST",
+    variables=["analysed_sst", "analysis_error", "mask", "sea_ice_fraction"],
+    start_datetime=f"{year_start}-01-01T00:00:00",
+    end_datetime=f"{year_end}-12-31T23:59:59",
+    minimum_longitude=-45,
+    maximum_longitude=14,
+    minimum_latitude=31,
+    maximum_latitude=85,
+    credentials_file=credentials_fpath,
+    output_directory=out_fdir,
+    file_format="zarr",
+    output_filename=f"ostia_global_sst_daily_NA_rep_{year_start}_{year_end}.zarr",
+)
+print(f"Completed: downloading OSTIA Global SST dataset for: {year_start} to {year_end}.")