OceanDataStore 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- OceanDataStore/__init__.py +21 -0
- OceanDataStore/catalog/__init__.py +12 -0
- OceanDataStore/catalog/oceandatacatalog.py +1242 -0
- OceanDataStore/catalog/stac/README.md +34 -0
- OceanDataStore/catalog/stac/__init__.py +30 -0
- OceanDataStore/catalog/stac/create_noc_stac.py +109 -0
- OceanDataStore/catalog/stac/npd_era5_collection.py +364 -0
- OceanDataStore/catalog/stac/npd_jra55_collection.py +196 -0
- OceanDataStore/catalog/stac/ods_obs_collection.py +534 -0
- OceanDataStore/catalog/stac/rapid_evo_collection.py +309 -0
- OceanDataStore/catalog/stac/template_collection.py +85 -0
- OceanDataStore/catalog/stac/utils.py +476 -0
- OceanDataStore/cli/__init__.py +34 -0
- OceanDataStore/cli/arg_parser.py +182 -0
- OceanDataStore/cli/cli.py +203 -0
- OceanDataStore/cli/exceptions.py +83 -0
- OceanDataStore/cli/icechunk.py +888 -0
- OceanDataStore/cli/logging.py +52 -0
- OceanDataStore/cli/object_store.py +293 -0
- OceanDataStore/cli/utils.py +275 -0
- OceanDataStore/cli/zarr.py +870 -0
- OceanDataStore/data/ARMOR3D/create_ARMOR3D_P1M-m_monthly_climatology.py +135 -0
- OceanDataStore/data/ARMOR3D/download_ARMOR3D_0.125def_P1M-m_1993_2024.py +33 -0
- OceanDataStore/data/ARMOR3D/run_create_ARMOR3D_P1M-m_monthly_climatology.slurm +32 -0
- OceanDataStore/data/ARMOR3D/run_send_ARMOR3D_P1M-m_climatology_to_os.slurm +32 -0
- OceanDataStore/data/ARMOR3D/run_send_ARMOR3D_P1M-m_monthly_to_os.slurm +32 -0
- OceanDataStore/data/ARMOR3D/run_update_ARMOR3D_P1m-m_monthly_to_os.slurm +32 -0
- OceanDataStore/data/ARMOR3D/send_ARMOR3D_P1m-m_monthly_climatology_to_os.py +99 -0
- OceanDataStore/data/ARMOR3D/send_ARMOR3D_P1m-m_monthly_to_os.py +147 -0
- OceanDataStore/data/ARMOR3D/update_ARMOR3D_P1m-m_monthly_to_os.py +143 -0
- OceanDataStore/data/EN.4.2.2/create_EN4.2.2_analysis_g10_climatology.py +162 -0
- OceanDataStore/data/EN.4.2.2/download_EN4.2.2_analysis_g10_data.sh +51 -0
- OceanDataStore/data/EN.4.2.2/run_send_EN4.2.2_analysis_g10_climatology_to_os.slurm +32 -0
- OceanDataStore/data/EN.4.2.2/run_send_EN4.2.2_analysis_g10_monthly_to_os.slurm +32 -0
- OceanDataStore/data/EN.4.2.2/run_update_EN4.2.2_analysis_g10_monthly_to_os.slurm +32 -0
- OceanDataStore/data/EN.4.2.2/send_EN4.2.2_analysis_g10_monthly_climatology_to_os.py +76 -0
- OceanDataStore/data/EN.4.2.2/send_EN4.2.2_analysis_g10_monthly_to_os.py +165 -0
- OceanDataStore/data/EN.4.2.2/update_EN4.2.2_analysis_g10_monthly_to_os.py +161 -0
- OceanDataStore/data/ERA5/create_ERA5_daily_climatology.py +110 -0
- OceanDataStore/data/ERA5/create_ERA5_daily_mean.py +69 -0
- OceanDataStore/data/ERA5/create_ERA5_monthly_mean.py +74 -0
- OceanDataStore/data/ERA5/run_create_ERA5_daily_climatology.slurm +54 -0
- OceanDataStore/data/ERA5/run_send_ERA5_daily_climatology_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/run_send_ERA5_daily_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/run_send_ERA5_monthly_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/run_update_ERA5_daily_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/run_update_ERA5_monthly_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/send_ERA5_daily_climatology_to_os.py +159 -0
- OceanDataStore/data/ERA5/send_ERA5_daily_to_os.py +141 -0
- OceanDataStore/data/ERA5/send_ERA5_monthly_to_os.py +173 -0
- OceanDataStore/data/ERA5/update_ERA5_daily_to_os.py +141 -0
- OceanDataStore/data/ERA5/update_ERA5_monthly_to_os.py +169 -0
- OceanDataStore/data/HadISST/download_HadISST1_data.sh +43 -0
- OceanDataStore/data/HadISST/run_send_HadISST1_monthly_to_os.slurm +32 -0
- OceanDataStore/data/HadISST/send_HadISST1_monthly_to_os.py +133 -0
- OceanDataStore/data/NSIDC/download_NSIDC_monthly_1979_2025_data.sh +54 -0
- OceanDataStore/data/NSIDC/process_NSIDC_SSI_Antarctic_data.py +130 -0
- OceanDataStore/data/NSIDC/process_NSIDC_SSI_Arctic_data.py +129 -0
- OceanDataStore/data/NSIDC/run_send_NSIDC_v4.0_to_OS.slurm +32 -0
- OceanDataStore/data/NSIDC/send_NSIDC_SII_v4.0_to_os.py +140 -0
- OceanDataStore/data/OISST/create_OISSTv2_daily_climatology.py +83 -0
- OceanDataStore/data/OISST/download_oisstv2_data.sh +43 -0
- OceanDataStore/data/OISST/run_create_OISSTv2_daily_climatology.slurm +44 -0
- OceanDataStore/data/OISST/run_send_OISSTv2_daily_climatology_to_os.slurm +32 -0
- OceanDataStore/data/OISST/run_send_OISSTv2_daily_to_os.slurm +32 -0
- OceanDataStore/data/OISST/run_send_OISSTv2_monthly_climatology_to_os.slurm +32 -0
- OceanDataStore/data/OISST/run_send_OISSTv2_monthly_to_os.slurm +32 -0
- OceanDataStore/data/OISST/run_update_OISSTv2_daily_to_os.slurm +32 -0
- OceanDataStore/data/OISST/send_OISSTv2_daily_climatology_to_os.py +154 -0
- OceanDataStore/data/OISST/send_OISSTv2_daily_ltm_climatology_to_os.py +151 -0
- OceanDataStore/data/OISST/send_OISSTv2_daily_to_os.py +142 -0
- OceanDataStore/data/OISST/send_OISSTv2_monthly_climatology_to_os.py +150 -0
- OceanDataStore/data/OISST/send_OISSTv2_monthly_to_os.py +145 -0
- OceanDataStore/data/OISST/update_OISSTv2_daily_to_os.py +142 -0
- OceanDataStore/data/OSTIA/create_OSTIA_daily_climatology.py +120 -0
- OceanDataStore/data/OSTIA/download_OSTIA_NRT.py +42 -0
- OceanDataStore/data/OSTIA/download_OSTIA_REP_1981_2025.py +42 -0
- OceanDataStore/data/OSTIA/run_create_OSTIA_daily_climatology.slurm +54 -0
- OceanDataStore/data/OSTIA/run_send_OSTIA_daily_climatology_to_os.slurm +32 -0
- OceanDataStore/data/OSTIA/run_send_OSTIA_nrt_daily_to_os.slurm +32 -0
- OceanDataStore/data/OSTIA/run_send_OSTIA_rep_daily_to_os.slurm +32 -0
- OceanDataStore/data/OSTIA/run_update_OSTIA_daily_to_os.slurm +33 -0
- OceanDataStore/data/OSTIA/send_OSTIA_daily_climatology_to_os.py +194 -0
- OceanDataStore/data/OSTIA/send_OSTIA_nrt_daily_to_os.py +141 -0
- OceanDataStore/data/OSTIA/send_OSTIA_rep_daily_to_os.py +145 -0
- OceanDataStore/data/OSTIA/update_OSTIA_copernicus_nrt_daily_to_os.py +144 -0
- OceanDataStore/data/OSTIA/update_OSTIA_nrt_daily_to_os.py +137 -0
- OceanDataStore/data/WOA23/download_WOA23_climatology.sh +41 -0
- OceanDataStore/data/WOA23/run_send_WOA23_annual_climatology_to_os.slurm +32 -0
- OceanDataStore/data/WOA23/run_send_WOA23_monthly_climatology_to_os.slurm +32 -0
- OceanDataStore/data/WOA23/send_WOA23_annual_climatology_to_os.py +263 -0
- OceanDataStore/data/WOA23/send_WOA23_monthly_climatology_to_os.py +292 -0
- OceanDataStore/data/update_icechunk_repo_attrs.py +76 -0
- OceanDataStore/data/update_noc_npd_era5v1_attrs.py +172 -0
- OceanDataStore/data/utils.py +506 -0
- OceanDataStore/zarr.py +993 -0
- oceandatastore-0.3.0.dist-info/METADATA +184 -0
- oceandatastore-0.3.0.dist-info/RECORD +104 -0
- oceandatastore-0.3.0.dist-info/WHEEL +5 -0
- oceandatastore-0.3.0.dist-info/entry_points.txt +2 -0
- oceandatastore-0.3.0.dist-info/licenses/LICENSE +201 -0
- oceandatastore-0.3.0.dist-info/scm_file_list.json +154 -0
- oceandatastore-0.3.0.dist-info/scm_version.json +8 -0
- oceandatastore-0.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# =========================================================
|
|
2
|
+
# update_OSTIA_daily_to_os.py
|
|
3
|
+
#
|
|
4
|
+
# Script to write OSTIA daily data to Icechunk repositories
|
|
5
|
+
# in JASMIN cloud object storage.
|
|
6
|
+
#
|
|
7
|
+
# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
|
|
8
|
+
# =========================================================
|
|
9
|
+
import logging
|
|
10
|
+
|
|
11
|
+
import copernicusmarine
|
|
12
|
+
import xarray as xr
|
|
13
|
+
import zarr
|
|
14
|
+
|
|
15
|
+
from OceanDataStore.cli import initialise_logging, update_icechunk
|
|
16
|
+
from OceanDataStore.data.utils import (
|
|
17
|
+
compute_cell_area,
|
|
18
|
+
compute_dx,
|
|
19
|
+
compute_dy,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def main():
|
|
26
|
+
# ========== Initialise OceanDataStore Logging ========== #
|
|
27
|
+
initialise_logging()
|
|
28
|
+
|
|
29
|
+
# ========== Send to Icechunk Repository ========== #
|
|
30
|
+
bucket = "ostia"
|
|
31
|
+
store_credentials_json = ".../credentials/jasmin_os_credentials.json"
|
|
32
|
+
branch = "main"
|
|
33
|
+
|
|
34
|
+
logging.info("In Progress: Sending OSTIA daily data to Icechunk...")
|
|
35
|
+
# Open OSTIA dataset:
|
|
36
|
+
ds = copernicusmarine.open_dataset(
|
|
37
|
+
dataset_id="METOFFICE-GLO-SST-L4-NRT-OBS-SST-V2",
|
|
38
|
+
variables=["analysed_sst", "analysis_error", "mask", "sea_ice_fraction"],
|
|
39
|
+
start_datetime="2026-06-27T00:00:00",
|
|
40
|
+
end_datetime="2026-06-29T23:59:59",
|
|
41
|
+
minimum_longitude=-45,
|
|
42
|
+
maximum_longitude=14,
|
|
43
|
+
minimum_latitude=31,
|
|
44
|
+
maximum_latitude=85,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
# Update variable units and attributes:
|
|
48
|
+
for var in ds.data_vars:
|
|
49
|
+
if "sst" in var:
|
|
50
|
+
# Transform units degK -> degC:
|
|
51
|
+
ds[var] = ds[var] - 273.15
|
|
52
|
+
ds[var].attrs["units"] = "degC"
|
|
53
|
+
|
|
54
|
+
# Rename variables:
|
|
55
|
+
ds = ds.rename({"analysed_sst": "tos",
|
|
56
|
+
"analysis_error": "tos_error",
|
|
57
|
+
"sea_ice_fraction": "siconc",
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
# Update variable long names:
|
|
61
|
+
ds["tos"].attrs["long_name"] = "Daily Mean Sea Surface Temperature"
|
|
62
|
+
ds["tos_error"].attrs["long_name"] = "Daily Standard Deviation Error Sea Surface Temperature"
|
|
63
|
+
ds["siconc"].attrs["long_name"] = "Daily Sea Ice Area Fraction"
|
|
64
|
+
|
|
65
|
+
# Add climatological day coordinate:
|
|
66
|
+
doy = ds['time'].dt.dayofyear
|
|
67
|
+
is_leap = ds['time'].dt.is_leap_year
|
|
68
|
+
after_feb28 = (~is_leap) & (doy >= 60)
|
|
69
|
+
clim_day = xr.where(after_feb28, doy + 1, doy)
|
|
70
|
+
ds = ds.assign_coords(clim_day=("time", clim_day.data))
|
|
71
|
+
ds["clim_day"].attrs["long_name"] = "Climatological Day of Year"
|
|
72
|
+
ds["clim_day"].attrs["description"] = "Climatological calendar day of year (1-366) with leap-year alignment for daily climatology calculations."
|
|
73
|
+
|
|
74
|
+
# Add ancillary variables:
|
|
75
|
+
ds['dx'] = compute_dx(ds)
|
|
76
|
+
ds['dy'] = compute_dy(ds)
|
|
77
|
+
ds['cell_area'] = compute_cell_area(ds)
|
|
78
|
+
|
|
79
|
+
# Update global attributes:
|
|
80
|
+
ds.attrs.clear()
|
|
81
|
+
ds = ds.assign_attrs({
|
|
82
|
+
"Conventions": "CF-1.4",
|
|
83
|
+
"title": "OSTIA Near-Real-Time North Atlantic Daily Timeseries",
|
|
84
|
+
"description": "The Operational Sea Surface Temperature and Ice Analysis (OSTIA) Near-Real-Time - North Atlantic Sea Surface Temperature & Sea Ice Area Fraction Daily Timeseries.",
|
|
85
|
+
"source": "Satellite observations: GCOM-W, AQUA, GOES<13,16>, MetoSat<08,09,10,11>, Sentinel-<3a,3b>, ERS<1,2>, Envisat, NOAA-<07,08,09,11,12,14,15,16,17,18,19,MTA>, GPM-Core, Suomi-NPP, NOAA-20.",
|
|
86
|
+
"dataset_type": "observation",
|
|
87
|
+
"product_type": "timeseries",
|
|
88
|
+
"product_version": "3.2",
|
|
89
|
+
"institution": "Met Office, UK",
|
|
90
|
+
"citation": "Ocean OSTIA Sea Surface Temperature and Sea Ice Analysis. E.U. Copernicus Marine Service Information (CMEMS). Marine Data Store (MDS). DOI: 10.48670/moi-00165 (Accessed on 27 06 2026).",
|
|
91
|
+
"references": "Good, S.; Fiedler, E.; Mao, C.; Martin, M.J.; Maycock, A.; Reid, R.; Roberts-Jones, J.; Searle, T.; Waters, J.; While, J.; Worsfold, M. The Current Configuration of the OSTIA System for Operational Production of Foundation Sea Surface Temperature and Ice Concentration Analyses. Remote Sens. 2020, 12, 720. doi: 10.3390/rs12040720. Donlon, C.J., Martin, M., Stark, J., Roberts-Jones, J., Fiedler, E., and Wimmer, W., 2012, The Operational Sea Surface Temperature and Sea Ice Analysis (OSTIA) system. Remote Sensing of the Environment. doi: 10.1016/j.rse.2010.10.017 2011. John D. Stark, Craig J. Donlon, Matthew J. Martin and Michael E. McCulloch, 2007, OSTIA : An operational, high resolution, real time, global sea surface temperature analysis system., Oceans 07 IEEE Aberdeen, conference proceedings. Marine challenges: coastline to deep sea. Aberdeen, Scotland.IEEE.",
|
|
92
|
+
"acknowledgement": "Generated using E.U. Copernicus Marine Service Information; https://doi.org/10.48670/moi-00165. These data were provided by GHRSST, Met Office and CMEMS.",
|
|
93
|
+
"license": "OSTIA data were obtained from https://doi.org/10.48670/moi-00165, and are provided under the Copernicus Marine Environment Monitoring Service Service Level Agreement (SLA) https://marine.copernicus.eu/user-corner/service-commitments-and-licence?pk_vid=42ac3e352be888641780994034c3bb6e",
|
|
94
|
+
"doi": "10.48670/moi-00165",
|
|
95
|
+
"platform": "gr",
|
|
96
|
+
"horizontal_grid_type": "regular rectilinear",
|
|
97
|
+
"horizontal_grid_resolution": "0.05 degree",
|
|
98
|
+
"aggregation": "mean",
|
|
99
|
+
"aggregation_frequency": "daily",
|
|
100
|
+
"status": "ongoing",
|
|
101
|
+
"update_frequency": "quarterly",
|
|
102
|
+
"bbox": "[-44.975, 13.975, 31.025, 84.975]",
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
# Optimise chunk sizes for time-series analysis:
|
|
106
|
+
# ds = ds.chunk({'time': ds['time'].size, 'latitude': 50, 'longitude': 50})
|
|
107
|
+
ds = ds.chunk({'time': 1, 'latitude': 1080, 'longitude': 1180})
|
|
108
|
+
|
|
109
|
+
# Update variable encodings:
|
|
110
|
+
blosccodec = zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.shuffle)
|
|
111
|
+
for var in list(ds.data_vars) + list(ds.coords):
|
|
112
|
+
ds[var].encoding.clear()
|
|
113
|
+
ds[var].encoding['compressors'] = [blosccodec]
|
|
114
|
+
|
|
115
|
+
# Define prefix and commit message based on timeseries period:
|
|
116
|
+
# prefix = "ostia_nrt_na_daily_timeseries"
|
|
117
|
+
prefix = "ostia_nrt_na_daily_spatial"
|
|
118
|
+
commit_message = "Added OSTIA Near-Real-Time North Atlantic Daily Timeseries (2026-01-01 - 2026-06-26)."
|
|
119
|
+
|
|
120
|
+
# Dask LocalCluster configuration:
|
|
121
|
+
config_kwargs = {
|
|
122
|
+
"temporary_directory":"/dssgfs01/working/otooth/Software/OceanDataStore/OceanDataStore/data/OSTIA/",
|
|
123
|
+
"local_directory":"/dssgfs01/working/otooth/Software/OceanDataStore/OceanDataStore/data/OSTIA/"
|
|
124
|
+
}
|
|
125
|
+
cluster_kwargs = {
|
|
126
|
+
"n_workers" : 20,
|
|
127
|
+
"threads_per_worker" : 1,
|
|
128
|
+
"memory_limit":"6GB"
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
update_icechunk(
|
|
132
|
+
file=ds,
|
|
133
|
+
bucket=bucket,
|
|
134
|
+
object_prefix=prefix,
|
|
135
|
+
store_credentials_json=store_credentials_json,
|
|
136
|
+
append_dim='time',
|
|
137
|
+
branch=branch,
|
|
138
|
+
commit_message=commit_message,
|
|
139
|
+
dask_config_kwargs=config_kwargs,
|
|
140
|
+
dask_cluster_kwargs=cluster_kwargs,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
if __name__ == "__main__":
|
|
144
|
+
main()
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# =========================================================
|
|
2
|
+
# update_OSTIA_daily_to_os.py
|
|
3
|
+
#
|
|
4
|
+
# Script to write OSTIA daily data to Icechunk repositories
|
|
5
|
+
# in JASMIN cloud object storage.
|
|
6
|
+
#
|
|
7
|
+
# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
|
|
8
|
+
# =========================================================
|
|
9
|
+
import logging
|
|
10
|
+
|
|
11
|
+
import xarray as xr
|
|
12
|
+
import zarr
|
|
13
|
+
|
|
14
|
+
from OceanDataStore.cli import initialise_logging, update_icechunk
|
|
15
|
+
from OceanDataStore.data.utils import (
|
|
16
|
+
compute_cell_area,
|
|
17
|
+
compute_dx,
|
|
18
|
+
compute_dy,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def main():
|
|
25
|
+
# ========== Initialise OceanDataStore Logging ========== #
|
|
26
|
+
initialise_logging()
|
|
27
|
+
|
|
28
|
+
# ========== Send to Icechunk Repository ========== #
|
|
29
|
+
bucket = "ostia"
|
|
30
|
+
store_credentials_json = ".../credentials/jasmin_os_credentials.json"
|
|
31
|
+
branch = "main"
|
|
32
|
+
|
|
33
|
+
logging.info("In Progress: Sending OSTIA daily data to Icechunk...")
|
|
34
|
+
# Open OSTIA dataset:
|
|
35
|
+
data_path = "/dssgfs01/working/otooth/data/observations/OSTIA"
|
|
36
|
+
ds = xr.open_dataset(f"{data_path}/ostia_global_sst_daily_NA_nrt_2025_2026.zarr",
|
|
37
|
+
engine="zarr",
|
|
38
|
+
).sel(time=slice("2026-01", "2026-06"))
|
|
39
|
+
|
|
40
|
+
# Update variable units and attributes:
|
|
41
|
+
for var in ds.data_vars:
|
|
42
|
+
if "sst" in var:
|
|
43
|
+
# Transform units degK -> degC:
|
|
44
|
+
ds[var] = ds[var] - 273.15
|
|
45
|
+
ds[var].attrs["units"] = "degC"
|
|
46
|
+
|
|
47
|
+
# Rename variables:
|
|
48
|
+
ds = ds.rename({"analysed_sst": "tos",
|
|
49
|
+
"analysis_error": "tos_error",
|
|
50
|
+
"sea_ice_fraction": "siconc",
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
# Update variable long names:
|
|
54
|
+
ds["tos"].attrs["long_name"] = "Daily Mean Sea Surface Temperature"
|
|
55
|
+
ds["tos_error"].attrs["long_name"] = "Daily Standard Deviation Error Sea Surface Temperature"
|
|
56
|
+
ds["siconc"].attrs["long_name"] = "Daily Sea Ice Area Fraction"
|
|
57
|
+
|
|
58
|
+
# Add climatological day coordinate:
|
|
59
|
+
doy = ds['time'].dt.dayofyear
|
|
60
|
+
is_leap = ds['time'].dt.is_leap_year
|
|
61
|
+
after_feb28 = (~is_leap) & (doy >= 60)
|
|
62
|
+
clim_day = xr.where(after_feb28, doy + 1, doy)
|
|
63
|
+
ds = ds.assign_coords(clim_day=("time", clim_day.data))
|
|
64
|
+
ds["clim_day"].attrs["long_name"] = "Climatological Day of Year"
|
|
65
|
+
ds["clim_day"].attrs["description"] = "Climatological calendar day of year (1-366) with leap-year alignment for daily climatology calculations."
|
|
66
|
+
|
|
67
|
+
# Add ancillary variables:
|
|
68
|
+
ds['dx'] = compute_dx(ds)
|
|
69
|
+
ds['dy'] = compute_dy(ds)
|
|
70
|
+
ds['cell_area'] = compute_cell_area(ds)
|
|
71
|
+
|
|
72
|
+
# Update global attributes:
|
|
73
|
+
ds.attrs.clear()
|
|
74
|
+
ds = ds.assign_attrs({
|
|
75
|
+
"Conventions": "CF-1.4",
|
|
76
|
+
"title": "OSTIA Near-Real-Time North Atlantic Daily Timeseries",
|
|
77
|
+
"description": "The Operational Sea Surface Temperature and Ice Analysis (OSTIA) Near-Real-Time - North Atlantic Sea Surface Temperature & Sea Ice Area Fraction Daily Timeseries.",
|
|
78
|
+
"source": "Satellite observations: GCOM-W, AQUA, GOES<13,16>, MetoSat<08,09,10,11>, Sentinel-<3a,3b>, ERS<1,2>, Envisat, NOAA-<07,08,09,11,12,14,15,16,17,18,19,MTA>, GPM-Core, Suomi-NPP, NOAA-20.",
|
|
79
|
+
"dataset_type": "observation",
|
|
80
|
+
"product_type": "timeseries",
|
|
81
|
+
"product_version": "3.2",
|
|
82
|
+
"institution": "Met Office, UK",
|
|
83
|
+
"citation": "Ocean OSTIA Sea Surface Temperature and Sea Ice Analysis. E.U. Copernicus Marine Service Information (CMEMS). Marine Data Store (MDS). DOI: 10.48670/moi-00165 (Accessed on 27 06 2026).",
|
|
84
|
+
"references": "Good, S.; Fiedler, E.; Mao, C.; Martin, M.J.; Maycock, A.; Reid, R.; Roberts-Jones, J.; Searle, T.; Waters, J.; While, J.; Worsfold, M. The Current Configuration of the OSTIA System for Operational Production of Foundation Sea Surface Temperature and Ice Concentration Analyses. Remote Sens. 2020, 12, 720. doi: 10.3390/rs12040720. Donlon, C.J., Martin, M., Stark, J., Roberts-Jones, J., Fiedler, E., and Wimmer, W., 2012, The Operational Sea Surface Temperature and Sea Ice Analysis (OSTIA) system. Remote Sensing of the Environment. doi: 10.1016/j.rse.2010.10.017 2011. John D. Stark, Craig J. Donlon, Matthew J. Martin and Michael E. McCulloch, 2007, OSTIA : An operational, high resolution, real time, global sea surface temperature analysis system., Oceans 07 IEEE Aberdeen, conference proceedings. Marine challenges: coastline to deep sea. Aberdeen, Scotland.IEEE.",
|
|
85
|
+
"acknowledgement": "Generated using E.U. Copernicus Marine Service Information; https://doi.org/10.48670/moi-00165. These data were provided by GHRSST, Met Office and CMEMS.",
|
|
86
|
+
"license": "OSTIA data were obtained from https://doi.org/10.48670/moi-00165, and are provided under the Copernicus Marine Environment Monitoring Service Service Level Agreement (SLA) https://marine.copernicus.eu/user-corner/service-commitments-and-licence?pk_vid=42ac3e352be888641780994034c3bb6e",
|
|
87
|
+
"doi": "10.48670/moi-00165",
|
|
88
|
+
"platform": "gr",
|
|
89
|
+
"horizontal_grid_type": "regular rectilinear",
|
|
90
|
+
"horizontal_grid_resolution": "0.05 degree",
|
|
91
|
+
"aggregation": "mean",
|
|
92
|
+
"aggregation_frequency": "daily",
|
|
93
|
+
"status": "ongoing",
|
|
94
|
+
"update_frequency": "quarterly",
|
|
95
|
+
"bbox": "[-44.975, 13.975, 31.025, 84.975]",
|
|
96
|
+
})
|
|
97
|
+
|
|
98
|
+
# Optimise chunk sizes for time-series analysis:
|
|
99
|
+
# ds = ds.chunk({'time': ds['time'].size, 'latitude': 50, 'longitude': 50})
|
|
100
|
+
ds = ds.chunk({'time': 1, 'latitude': 1080, 'longitude': 1180})
|
|
101
|
+
|
|
102
|
+
# Update variable encodings:
|
|
103
|
+
blosccodec = zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.shuffle)
|
|
104
|
+
for var in list(ds.data_vars) + list(ds.coords):
|
|
105
|
+
ds[var].encoding.clear()
|
|
106
|
+
ds[var].encoding['compressors'] = [blosccodec]
|
|
107
|
+
|
|
108
|
+
# Define prefix and commit message based on timeseries period:
|
|
109
|
+
# prefix = "ostia_nrt_na_daily_timeseries"
|
|
110
|
+
prefix = "ostia_nrt_na_daily_spatial"
|
|
111
|
+
commit_message = "Added OSTIA Near-Real-Time North Atlantic Daily Timeseries (2026-01-01 - 2026-06-26)."
|
|
112
|
+
|
|
113
|
+
# Dask LocalCluster configuration:
|
|
114
|
+
config_kwargs = {
|
|
115
|
+
"temporary_directory":"/dssgfs01/working/otooth/Software/OceanDataStore/OceanDataStore/data/OSTIA/",
|
|
116
|
+
"local_directory":"/dssgfs01/working/otooth/Software/OceanDataStore/OceanDataStore/data/OSTIA/"
|
|
117
|
+
}
|
|
118
|
+
cluster_kwargs = {
|
|
119
|
+
"n_workers" : 20,
|
|
120
|
+
"threads_per_worker" : 1,
|
|
121
|
+
"memory_limit":"6GB"
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
update_icechunk(
|
|
125
|
+
file=ds,
|
|
126
|
+
bucket=bucket,
|
|
127
|
+
object_prefix=prefix,
|
|
128
|
+
store_credentials_json=store_credentials_json,
|
|
129
|
+
append_dim='time',
|
|
130
|
+
branch=branch,
|
|
131
|
+
commit_message=commit_message,
|
|
132
|
+
dask_config_kwargs=config_kwargs,
|
|
133
|
+
dask_cluster_kwargs=cluster_kwargs,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
if __name__ == "__main__":
|
|
137
|
+
main()
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
# ----------------------------------------------------------------
|
|
4
|
+
# download_WOA23_climatology.sh
|
|
5
|
+
#
|
|
6
|
+
# Description: Download the World Ocean Atlas 2023 30-year climatologies
|
|
7
|
+
# using URLs provided by the National Centers for Environmental Information.
|
|
8
|
+
# WOA23 provides Climate Normals for temperature and salinity at 0.25 degree
|
|
9
|
+
# resolution as 30-year averages for validating models.
|
|
10
|
+
#
|
|
11
|
+
# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
|
|
12
|
+
# Created On: 2026-05-27
|
|
13
|
+
# ----------------------------------------------------------------
|
|
14
|
+
set -euo pipefail
|
|
15
|
+
|
|
16
|
+
echo "==================================================="
|
|
17
|
+
echo " Downloading WOA23 Climatologies"
|
|
18
|
+
echo " v0.1.0"
|
|
19
|
+
echo " Oliver J. Tooth, NOC"
|
|
20
|
+
echo "==================================================="
|
|
21
|
+
echo "In Progress: Downloading WOA23 Upper 1500m Climatologies..."
|
|
22
|
+
for i in {00..16}; do echo $i; wget https://www.ncei.noaa.gov/data/oceans/woa/WOA23/DATA/temperature/netcdf/decav71A0/0.25/woa23_decav71A0_t${i}_04.nc ; done
|
|
23
|
+
|
|
24
|
+
for i in {00..16}; do echo $i; wget https://www.ncei.noaa.gov/data/oceans/woa/WOA23/DATA/temperature/netcdf/decav81B0/0.25/woa23_decav81B0_t${i}_04.nc ; done
|
|
25
|
+
|
|
26
|
+
for i in {00..16}; do echo $i; wget https://www.ncei.noaa.gov/data/oceans/woa/WOA23/DATA/temperature/netcdf/decav91C0/0.25/woa23_decav91C0_t${i}_04.nc ; done
|
|
27
|
+
echo "Completed: Downloaded Upper 1500m WOA23 temperature climatologies"
|
|
28
|
+
|
|
29
|
+
echo "In Progress: Downloading WOA23 Upper 1500m Salinity Climatologies..."
|
|
30
|
+
for i in {00..16}; do echo $i; wget https://www.ncei.noaa.gov/data/oceans/woa/WOA23/DATA/salinity/netcdf/decav71A0/0.25/woa23_decav71A0_s${i}_04.nc ; done
|
|
31
|
+
|
|
32
|
+
for i in {00..16}; do echo $i; wget https://www.ncei.noaa.gov/data/oceans/woa/WOA23/DATA/salinity/netcdf/decav81B0/0.25/woa23_decav81B0_s${i}_04.nc ; done
|
|
33
|
+
|
|
34
|
+
for i in {00..16}; do echo $i; wget https://www.ncei.noaa.gov/data/oceans/woa/WOA23/DATA/salinity/netcdf/decav91C0/0.25/woa23_decav91C0_s${i}_04.nc ; done
|
|
35
|
+
echo "Completed: Downloaded Upper 1500m WOA23 salinity climatologies"
|
|
36
|
+
|
|
37
|
+
# Winter average of full time period, covering full ocean depth:
|
|
38
|
+
echo "In Progress: Downloading WOA23 Full Depth Climatologies..."
|
|
39
|
+
wget https://www.ncei.noaa.gov/data/oceans/woa/WOA23/DATA/temperature/netcdf/decav/0.25/woa23_decav_t13_04.nc
|
|
40
|
+
wget https://www.ncei.noaa.gov/data/oceans/woa/WOA23/DATA/salinity/netcdf/decav/0.25/woa23_decav_s13_04.nc
|
|
41
|
+
echo "Completed: Downloaded WOA23 Full Depth Climatologies"
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
#SBATCH --job-name=woa23_annual_climatology
|
|
3
|
+
#SBATCH --partition=compute
|
|
4
|
+
#SBATCH --time=01:00:00
|
|
5
|
+
#SBATCH --ntasks-per-core=1
|
|
6
|
+
#SBATCH --ntasks-per-node=64
|
|
7
|
+
#SBATCH --ntasks-per-socket=32
|
|
8
|
+
#SBATCH --nodes=1
|
|
9
|
+
|
|
10
|
+
# ==============================================================
|
|
11
|
+
# run_send_WOA23_annual_climatology_to_os.slurm
|
|
12
|
+
#
|
|
13
|
+
# Description: SLURM script to send the WOA23 Annual Climatology
|
|
14
|
+
# datasets to Icechunk repository.
|
|
15
|
+
#
|
|
16
|
+
# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
|
|
17
|
+
# Created On: 2026-06-05
|
|
18
|
+
#
|
|
19
|
+
# ==============================================================
|
|
20
|
+
set -euo pipefail
|
|
21
|
+
|
|
22
|
+
# -- Python Environment -- #
|
|
23
|
+
# Activate miniconda environment:
|
|
24
|
+
source .../miniforge3/bin/activate
|
|
25
|
+
conda activate env_ods
|
|
26
|
+
|
|
27
|
+
# -- Send WOA23 annual climatology to JASMIN OS -- #
|
|
28
|
+
echo "In Progress: Sending WOA23 Annual Climatologies to Icechunk..."
|
|
29
|
+
|
|
30
|
+
python3 send_WOA23_annual_climatology_to_os.py
|
|
31
|
+
|
|
32
|
+
echo "Completed: Sent WOA23 Annual Climatologies to Icechunk."
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
#SBATCH --job-name=woa23_monthly_climatology
|
|
3
|
+
#SBATCH --partition=compute
|
|
4
|
+
#SBATCH --time=01:00:00
|
|
5
|
+
#SBATCH --ntasks-per-core=1
|
|
6
|
+
#SBATCH --ntasks-per-node=64
|
|
7
|
+
#SBATCH --ntasks-per-socket=32
|
|
8
|
+
#SBATCH --nodes=1
|
|
9
|
+
|
|
10
|
+
# ==============================================================
|
|
11
|
+
# run_send_WOA23_monthly_climatology_to_os.slurm
|
|
12
|
+
#
|
|
13
|
+
# Description: SLURM script to send the WOA23 Monthly Climatology
|
|
14
|
+
# datasets to Icechunk repository.
|
|
15
|
+
#
|
|
16
|
+
# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
|
|
17
|
+
# Created On: 2026-06-06
|
|
18
|
+
#
|
|
19
|
+
# ==============================================================
|
|
20
|
+
set -euo pipefail
|
|
21
|
+
|
|
22
|
+
# -- Python Environment -- #
|
|
23
|
+
# Activate miniconda environment:
|
|
24
|
+
source .../miniforge3/bin/activate
|
|
25
|
+
conda activate env_ods
|
|
26
|
+
|
|
27
|
+
# -- Send WOA23 monthly climatology to JASMIN OS -- #
|
|
28
|
+
echo "In Progress: Sending WOA23 Monthly Climatologies to Icechunk..."
|
|
29
|
+
|
|
30
|
+
python3 send_WOA23_monthly_climatology_to_os.py
|
|
31
|
+
|
|
32
|
+
echo "Completed: Sent WOA23 Monthly Climatologies to Icechunk."
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
# =========================================================
|
|
2
|
+
# send_WOA23_annual_climatology_to_os.py
|
|
3
|
+
#
|
|
4
|
+
# Script to write WOA23 annual climatologies to Icechunk
|
|
5
|
+
# repository in JASMIN cloud object storage.
|
|
6
|
+
#
|
|
7
|
+
# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
|
|
8
|
+
# =========================================================
|
|
9
|
+
import logging
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
import pandas as pd
|
|
13
|
+
import xarray as xr
|
|
14
|
+
import zarr
|
|
15
|
+
|
|
16
|
+
from OceanDataStore.cli import initialise_logging, send_to_icechunk
|
|
17
|
+
from OceanDataStore.data.utils import (
|
|
18
|
+
compute_cell_area,
|
|
19
|
+
compute_dx,
|
|
20
|
+
compute_dy,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def main():
|
|
27
|
+
# ========== Initialise OceanDataStore Logging ========== #
|
|
28
|
+
initialise_logging()
|
|
29
|
+
logging.info("In Progress: Sending WOA23 Annual Climatologies to JASMIN Object Store.")
|
|
30
|
+
|
|
31
|
+
# ========== Prepare Data ========== #
|
|
32
|
+
filedir = "/dssgfs01/scratch/otooth/npd_data/observations/WOA2023"
|
|
33
|
+
# Define file paths for WOA23 annual climatologies:
|
|
34
|
+
salinity_paths = [f"{filedir}/woa23_decav71A0_s00_04.nc",
|
|
35
|
+
f"{filedir}/woa23_decav81B0_s00_04.nc",
|
|
36
|
+
f"{filedir}/woa23_decav91C0_s00_04.nc",
|
|
37
|
+
]
|
|
38
|
+
temperature_paths = [f"{filedir}/woa23_decav71A0_t00_04.nc",
|
|
39
|
+
f"{filedir}/woa23_decav81B0_t00_04.nc",
|
|
40
|
+
f"{filedir}/woa23_decav91C0_t00_04.nc"
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
# Define year bounds for each WOA23 annual climatology:
|
|
44
|
+
year_bound_start = [1971, 1981, 1991]
|
|
45
|
+
year_bound_end = [2000, 2010, 2020]
|
|
46
|
+
|
|
47
|
+
for temp_path, sal_path, start_year, end_year in zip(temperature_paths, salinity_paths, year_bound_start, year_bound_end):
|
|
48
|
+
logging.info(f"-> In Progress: Preparing WOA23 {start_year}-{end_year} annual climatology...")
|
|
49
|
+
# Open WOA23 annual climatologies dataset:
|
|
50
|
+
ds_s = xr.open_dataset(sal_path, decode_times=False)
|
|
51
|
+
ds_t = xr.open_dataset(temp_path, decode_times=False)
|
|
52
|
+
|
|
53
|
+
ds = xr.merge([ds_s, ds_t], compat='no_conflicts').squeeze(drop=True)
|
|
54
|
+
ds['climatology_bounds'].values = np.array([np.datetime64(f'{start_year}-01-01', 'ns'), np.datetime64(f'{end_year}-12-31', 'ns')])
|
|
55
|
+
|
|
56
|
+
ds = ds.rename({"lon": "longitude",
|
|
57
|
+
"lat": "latitude",
|
|
58
|
+
"climatology_bounds": "time_bnds"
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
# Use OceanDataStore standard variable names:
|
|
62
|
+
for var in ds.data_vars:
|
|
63
|
+
if var.startswith("t_"):
|
|
64
|
+
ds = ds.rename({var: var.replace("t_", "thetao_")})
|
|
65
|
+
elif var.startswith("s_"):
|
|
66
|
+
ds = ds.rename({var: var.replace("s_", "so_")})
|
|
67
|
+
|
|
68
|
+
# Add ancillary variables:
|
|
69
|
+
ds['dx'] = compute_dx(ds)
|
|
70
|
+
ds['dy'] = compute_dy(ds)
|
|
71
|
+
ds['cell_area'] = compute_cell_area(ds)
|
|
72
|
+
|
|
73
|
+
# Custom ancillary variables:
|
|
74
|
+
ds['cell_thickness'] = ds['depth_bnds'].isel(nbounds=1) - ds['depth_bnds'].isel(nbounds=0)
|
|
75
|
+
ds['cell_volume'] = ds['cell_thickness'] * ds['cell_area']
|
|
76
|
+
|
|
77
|
+
# Update attributes for custom ancillary variables:
|
|
78
|
+
ds['cell_thickness'].attrs.update({
|
|
79
|
+
'long_name': "Grid-Cell Thickness",
|
|
80
|
+
'standard_name': "cell_thickness",
|
|
81
|
+
'units': "m",
|
|
82
|
+
})
|
|
83
|
+
ds['cell_volume'].attrs.update({
|
|
84
|
+
'long_name': "Grid-Cell Volume",
|
|
85
|
+
'standard_name': "cell_volume",
|
|
86
|
+
'units': "m3",
|
|
87
|
+
})
|
|
88
|
+
logging.info(f"Completed: Prepared WOA23 {start_year}-{end_year} annual climatology dataset with ancillary variables.")
|
|
89
|
+
|
|
90
|
+
# ========== Prepare Ancillary Data ========== #
|
|
91
|
+
# Open WOA23 land sea mask:
|
|
92
|
+
df_mask = pd.read_table("/dssgfs01/working/otooth/Software/OceanDataStore/OceanDataStore/data/WOA23/data/landsea_04.msk",
|
|
93
|
+
delimiter=',',
|
|
94
|
+
header=0
|
|
95
|
+
)
|
|
96
|
+
# Define level of sea floor (i.e. bottom standard level) at each grid cell:
|
|
97
|
+
ds['bottom_level'] = xr.full_like(ds['cell_area'], fill_value=np.nan).squeeze(drop=True)
|
|
98
|
+
ds['bottom_level'].data = df_mask['Bottom_Standard_Level'].values.reshape(720, 1440)
|
|
99
|
+
ds['bottom_level'].name = "bottom_level"
|
|
100
|
+
ds['bottom_level'].attrs = {"standard_name": "model_level_number_at_sea_floor",
|
|
101
|
+
"long_name": "Model Level Number at Sea Floor",
|
|
102
|
+
"units": "1"}
|
|
103
|
+
# Define land sea mask (1 for ocean grid cells, 0 for land grid cells):
|
|
104
|
+
ds['mask'] = ds['bottom_level'] > 1
|
|
105
|
+
ds['mask'].name = "land_binary_mask"
|
|
106
|
+
ds["mask"] = ds["mask"].assign_attrs({'long_name': "Land-Sea Binary Mask",
|
|
107
|
+
"standard_name": "sea_binary_mask",
|
|
108
|
+
"comment": "1 = sea, 0 = land"
|
|
109
|
+
})
|
|
110
|
+
logging.info("Completed: Prepared land sea mask & bottom level variables.")
|
|
111
|
+
|
|
112
|
+
# Open WOA23 basin mask:
|
|
113
|
+
df_basin = pd.read_table("/dssgfs01/working/otooth/Software/OceanDataStore/OceanDataStore/data/WOA23/data/basinmask_04.msk",
|
|
114
|
+
delimiter=',',
|
|
115
|
+
header=0
|
|
116
|
+
)
|
|
117
|
+
# Define basin mask (integer values for each ocean basin, NaN for land grid cells):
|
|
118
|
+
ds['basin_mask'] = xr.full_like(ds['cell_area'], fill_value=np.nan).squeeze(drop=True)
|
|
119
|
+
for basin, longitude, latitude in df_basin[["Basin_0m", "Longitude", "Latitude"]].itertuples(index=False):
|
|
120
|
+
ds['basin_mask'].loc[dict(latitude=latitude, longitude=longitude)] = basin
|
|
121
|
+
|
|
122
|
+
ds['basin_mask'].attrs = {
|
|
123
|
+
"standard_name": "ocean_basin_mask",
|
|
124
|
+
"long_name": "Ocean Basin Mask",
|
|
125
|
+
"basin_name": {
|
|
126
|
+
1: "Atlantic Ocean",
|
|
127
|
+
2: "Pacific Ocean",
|
|
128
|
+
3: "Indian Ocean",
|
|
129
|
+
4: "Mediterranean Sea",
|
|
130
|
+
5: "Baltic Sea",
|
|
131
|
+
6: "Black Sea",
|
|
132
|
+
7: "Red Sea",
|
|
133
|
+
8: "Persian Gulf",
|
|
134
|
+
9: "Hudson Bay",
|
|
135
|
+
10: "Southern Ocean",
|
|
136
|
+
11: "Arctic Ocean",
|
|
137
|
+
12: "Sea of Japan",
|
|
138
|
+
13: "Kara Sea",
|
|
139
|
+
14: "Sulu Sea",
|
|
140
|
+
15: "Baffin Bay",
|
|
141
|
+
16: "East Mediterranean",
|
|
142
|
+
17: "West Mediterranean",
|
|
143
|
+
18: "Sea of Okhotsk",
|
|
144
|
+
19: "Banda Sea",
|
|
145
|
+
20: "Caribbean Sea",
|
|
146
|
+
21: "Andaman Basin",
|
|
147
|
+
22: "North Caribbean",
|
|
148
|
+
23: "Gulf of Mexico",
|
|
149
|
+
24: "Beaufort Sea",
|
|
150
|
+
25: "South China Sea",
|
|
151
|
+
26: "Barents Sea",
|
|
152
|
+
27: "Celebes Sea",
|
|
153
|
+
28: "Aleutian Basin",
|
|
154
|
+
29: "Fiji Basin",
|
|
155
|
+
30: "North American Basin",
|
|
156
|
+
31: "West European Basin",
|
|
157
|
+
32: "Southeast Indian Basin",
|
|
158
|
+
33: "Coral Sea",
|
|
159
|
+
34: "East Indian Basin",
|
|
160
|
+
35: "Central Indian Basin",
|
|
161
|
+
36: "Southwest Atlantic Basin",
|
|
162
|
+
37: "Southeast Atlantic Basin",
|
|
163
|
+
38: "Southeast Pacific Basin",
|
|
164
|
+
39: "Guatemala Basin",
|
|
165
|
+
40: "East Caroline Basin",
|
|
166
|
+
41: "Marianas Basin",
|
|
167
|
+
42: "Philippine Sea",
|
|
168
|
+
43: "Arabian Sea",
|
|
169
|
+
44: "Chile Basin",
|
|
170
|
+
45: "Somali Basin",
|
|
171
|
+
46: "Mascarene Basin",
|
|
172
|
+
47: "Crozet Basin",
|
|
173
|
+
48: "Guinea Basin",
|
|
174
|
+
49: "Brazil Basin",
|
|
175
|
+
50: "Argentine Basin",
|
|
176
|
+
51: "Tasman Sea",
|
|
177
|
+
52: "Atlantic Indian Basin",
|
|
178
|
+
53: "Caspian Sea",
|
|
179
|
+
54: "Sulu Sea II",
|
|
180
|
+
55: "Venezuela Basin",
|
|
181
|
+
56: "Bay of Bengal",
|
|
182
|
+
57: "Java Sea",
|
|
183
|
+
58: "East Indian Atlantic Basin",
|
|
184
|
+
59: "Chiloe",
|
|
185
|
+
60: "Bransfield Strait",
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
logging.info("Completed: Prepared ocean basin mask variable.")
|
|
189
|
+
|
|
190
|
+
# ========== CF Attributes ========== #
|
|
191
|
+
ds.attrs.clear()
|
|
192
|
+
ds = ds.assign_attrs({
|
|
193
|
+
"Conventions": "CF-1.6",
|
|
194
|
+
"title": f"World Ocean Atlas 2023 temperature and salinity annual climatology ({start_year}-{end_year}).",
|
|
195
|
+
"description": f"World Ocean Atlas 2023 (WOA23) temperature and salinity annual climatology for the global ocean from objectively analysed, quality controlled in-situ profile data ({start_year}-{end_year}).",
|
|
196
|
+
"source": "Numerical models: Objective Analysis. In-situ observations: World Ocean Database (WOD).",
|
|
197
|
+
"dataset_type": "observation",
|
|
198
|
+
"product_type": "climatology",
|
|
199
|
+
"product_version": "1.0",
|
|
200
|
+
"institution": "NOAA National Centers for Environmental Information (NCEI)",
|
|
201
|
+
"citation": "Reagan, James R.; Boyer, Tim P.; García, Hernán E.; Locarnini, Ricardo A.; Baranova, Olga K.; Bouchard, Courtney; Cross, Scott L.; Mishonov, Alexey V.; Paver, Christopher R.; Seidov, Dan; Wang, Zhankun; Dukhovskoy, Dmitry (2023). World Ocean Atlas 2023 (NCEI Accession 0270533). https://www.ncei.noaa.gov/archive/accession/0270533. In Reagan, James R.; Boyer, Tim P.; García, Hernán E.; Locarnini, Ricardo A.; Baranova, Olga K.; Bouchard, Courtney; Cross, Scott L.; Mishonov, Alexey V.; Paver, Christopher R.; Seidov, Dan; Wang, Zhankun; Dukhovskoy, Dmitry (2023). World Ocean Atlas 2023. NOAA National Centers for Environmental Information. Dataset. https://doi.org/10.25921/va26-hv25. Accessed 06-05-2026.",
|
|
202
|
+
"references": "Locarnini, R.A., A.V. Mishonov, O.K. Baranova, J.R. Reagan, T.P. Boyer, D. Seidov, Z. Wang, H.E. Garcia, C. Bouchard, S.L. Cross, C.R. Paver, and D. Dukhovskoy (2024). World Ocean Atlas 2023, Volume 1: Temperature. A. Mishonov Technical Editor, NOAA Atlas NESDIS 89. https://doi.org/10.25923/54bh-1613. Reagan, J.R., D. Seidov, Z. Wang, D. Dukhovskoy, T.P. Boyer, R.A. Locarnini, O.K. Baranova, A.V. Mishonov, H.E. Garcia, C. Bouchard, S.L. Cross, and C.R. Paver (2023). World Ocean Atlas 2023, Volume 2: Salinity. A. Mishonov Technical Editor, NOAA Atlas NESDIS 90, https://doi.org/10.25923/70qt-9574.",
|
|
203
|
+
"acknowledgement": "None",
|
|
204
|
+
"license": "World Ocean Atlas 2023 data were obtained from https://www.ncei.noaa.gov/access/world-ocean-atlas-2023/ and are provided under a Creative Commons CC0 1.0 Universal License https://creativecommons.org/publicdomain/zero/1.0/",
|
|
205
|
+
"doi": "https://doi.org/10.25921/va26-hv25",
|
|
206
|
+
"platform": "gr",
|
|
207
|
+
"horizontal_grid_type": "regular rectilinear",
|
|
208
|
+
"horizontal_grid_resolution": "0.25 degree",
|
|
209
|
+
"vertical_grid_type": "z",
|
|
210
|
+
"vertical_grid_coordinate": "depth",
|
|
211
|
+
"vertical_grid_levels": 57,
|
|
212
|
+
"aggregation": "mean",
|
|
213
|
+
"aggregation_frequency": "annual",
|
|
214
|
+
"status": "completed",
|
|
215
|
+
"update_frequency": "None",
|
|
216
|
+
"bbox": "[-180.0, 180.0, -90.0, 90.0]",
|
|
217
|
+
})
|
|
218
|
+
logging.info(f"Completed: Added CF-compliant global attributes to WOA23 {start_year}-{end_year} annual climatology dataset.")
|
|
219
|
+
|
|
220
|
+
# ========== Send to Icechunk Repository ========== #
|
|
221
|
+
logging.info(f"In Progress: Sending WOA23 {start_year}-{end_year} annual climatology to Icechunk Repository...")
|
|
222
|
+
bucket = "woa23"
|
|
223
|
+
prefix = f"woa23_{start_year}_{end_year}_annual_climatology"
|
|
224
|
+
exists = False
|
|
225
|
+
store_credentials_json = ".../credentials/jasmin_os_credentials.json"
|
|
226
|
+
branch = "main"
|
|
227
|
+
commit_message = f"Added WOA23 {start_year}-{end_year} annual climatology."
|
|
228
|
+
variable_commits = True
|
|
229
|
+
config_kwargs = {
|
|
230
|
+
"temporary_directory":".../OceanDataStore/OceanDataStore/data/WOA23/",
|
|
231
|
+
"local_directory":".../OceanDataStore/OceanDataStore/data/WOA23/"
|
|
232
|
+
}
|
|
233
|
+
cluster_kwargs = {
|
|
234
|
+
"n_workers" : 25,
|
|
235
|
+
"threads_per_worker" : 1,
|
|
236
|
+
"memory_limit":"3GB"
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
# Optimise chunk sizes for spatial analysis:
|
|
240
|
+
ds = ds.chunk({'longitude': 1440, 'latitude': 720, 'depth': 5})
|
|
241
|
+
|
|
242
|
+
# Update variable encodings:
|
|
243
|
+
blosccodec = zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.shuffle)
|
|
244
|
+
for var in list(ds.data_vars) + list(ds.coords):
|
|
245
|
+
ds[var].encoding['compressors'] = [blosccodec]
|
|
246
|
+
|
|
247
|
+
send_to_icechunk(
|
|
248
|
+
file=ds,
|
|
249
|
+
bucket=bucket,
|
|
250
|
+
object_prefix=prefix,
|
|
251
|
+
store_credentials_json=store_credentials_json,
|
|
252
|
+
exists=exists,
|
|
253
|
+
branch=branch,
|
|
254
|
+
commit_message=commit_message,
|
|
255
|
+
variable_commits=variable_commits,
|
|
256
|
+
dask_config_kwargs=config_kwargs,
|
|
257
|
+
dask_cluster_kwargs=cluster_kwargs,
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
logging.info(f"Completed: Sent WOA23 {start_year}-{end_year} annual climatology to Icechunk Repository.")
|
|
261
|
+
|
|
262
|
+
if __name__ == "__main__":
|
|
263
|
+
main()
|