OceanDataStore 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- OceanDataStore/__init__.py +21 -0
- OceanDataStore/catalog/__init__.py +12 -0
- OceanDataStore/catalog/oceandatacatalog.py +1242 -0
- OceanDataStore/catalog/stac/README.md +34 -0
- OceanDataStore/catalog/stac/__init__.py +30 -0
- OceanDataStore/catalog/stac/create_noc_stac.py +109 -0
- OceanDataStore/catalog/stac/npd_era5_collection.py +364 -0
- OceanDataStore/catalog/stac/npd_jra55_collection.py +196 -0
- OceanDataStore/catalog/stac/ods_obs_collection.py +534 -0
- OceanDataStore/catalog/stac/rapid_evo_collection.py +309 -0
- OceanDataStore/catalog/stac/template_collection.py +85 -0
- OceanDataStore/catalog/stac/utils.py +476 -0
- OceanDataStore/cli/__init__.py +34 -0
- OceanDataStore/cli/arg_parser.py +182 -0
- OceanDataStore/cli/cli.py +203 -0
- OceanDataStore/cli/exceptions.py +83 -0
- OceanDataStore/cli/icechunk.py +888 -0
- OceanDataStore/cli/logging.py +52 -0
- OceanDataStore/cli/object_store.py +293 -0
- OceanDataStore/cli/utils.py +275 -0
- OceanDataStore/cli/zarr.py +870 -0
- OceanDataStore/data/ARMOR3D/create_ARMOR3D_P1M-m_monthly_climatology.py +135 -0
- OceanDataStore/data/ARMOR3D/download_ARMOR3D_0.125def_P1M-m_1993_2024.py +33 -0
- OceanDataStore/data/ARMOR3D/run_create_ARMOR3D_P1M-m_monthly_climatology.slurm +32 -0
- OceanDataStore/data/ARMOR3D/run_send_ARMOR3D_P1M-m_climatology_to_os.slurm +32 -0
- OceanDataStore/data/ARMOR3D/run_send_ARMOR3D_P1M-m_monthly_to_os.slurm +32 -0
- OceanDataStore/data/ARMOR3D/run_update_ARMOR3D_P1m-m_monthly_to_os.slurm +32 -0
- OceanDataStore/data/ARMOR3D/send_ARMOR3D_P1m-m_monthly_climatology_to_os.py +99 -0
- OceanDataStore/data/ARMOR3D/send_ARMOR3D_P1m-m_monthly_to_os.py +147 -0
- OceanDataStore/data/ARMOR3D/update_ARMOR3D_P1m-m_monthly_to_os.py +143 -0
- OceanDataStore/data/EN.4.2.2/create_EN4.2.2_analysis_g10_climatology.py +162 -0
- OceanDataStore/data/EN.4.2.2/download_EN4.2.2_analysis_g10_data.sh +51 -0
- OceanDataStore/data/EN.4.2.2/run_send_EN4.2.2_analysis_g10_climatology_to_os.slurm +32 -0
- OceanDataStore/data/EN.4.2.2/run_send_EN4.2.2_analysis_g10_monthly_to_os.slurm +32 -0
- OceanDataStore/data/EN.4.2.2/run_update_EN4.2.2_analysis_g10_monthly_to_os.slurm +32 -0
- OceanDataStore/data/EN.4.2.2/send_EN4.2.2_analysis_g10_monthly_climatology_to_os.py +76 -0
- OceanDataStore/data/EN.4.2.2/send_EN4.2.2_analysis_g10_monthly_to_os.py +165 -0
- OceanDataStore/data/EN.4.2.2/update_EN4.2.2_analysis_g10_monthly_to_os.py +161 -0
- OceanDataStore/data/ERA5/create_ERA5_daily_climatology.py +110 -0
- OceanDataStore/data/ERA5/create_ERA5_daily_mean.py +69 -0
- OceanDataStore/data/ERA5/create_ERA5_monthly_mean.py +74 -0
- OceanDataStore/data/ERA5/run_create_ERA5_daily_climatology.slurm +54 -0
- OceanDataStore/data/ERA5/run_send_ERA5_daily_climatology_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/run_send_ERA5_daily_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/run_send_ERA5_monthly_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/run_update_ERA5_daily_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/run_update_ERA5_monthly_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/send_ERA5_daily_climatology_to_os.py +159 -0
- OceanDataStore/data/ERA5/send_ERA5_daily_to_os.py +141 -0
- OceanDataStore/data/ERA5/send_ERA5_monthly_to_os.py +173 -0
- OceanDataStore/data/ERA5/update_ERA5_daily_to_os.py +141 -0
- OceanDataStore/data/ERA5/update_ERA5_monthly_to_os.py +169 -0
- OceanDataStore/data/HadISST/download_HadISST1_data.sh +43 -0
- OceanDataStore/data/HadISST/run_send_HadISST1_monthly_to_os.slurm +32 -0
- OceanDataStore/data/HadISST/send_HadISST1_monthly_to_os.py +133 -0
- OceanDataStore/data/NSIDC/download_NSIDC_monthly_1979_2025_data.sh +54 -0
- OceanDataStore/data/NSIDC/process_NSIDC_SSI_Antarctic_data.py +130 -0
- OceanDataStore/data/NSIDC/process_NSIDC_SSI_Arctic_data.py +129 -0
- OceanDataStore/data/NSIDC/run_send_NSIDC_v4.0_to_OS.slurm +32 -0
- OceanDataStore/data/NSIDC/send_NSIDC_SII_v4.0_to_os.py +140 -0
- OceanDataStore/data/OISST/create_OISSTv2_daily_climatology.py +83 -0
- OceanDataStore/data/OISST/download_oisstv2_data.sh +43 -0
- OceanDataStore/data/OISST/run_create_OISSTv2_daily_climatology.slurm +44 -0
- OceanDataStore/data/OISST/run_send_OISSTv2_daily_climatology_to_os.slurm +32 -0
- OceanDataStore/data/OISST/run_send_OISSTv2_daily_to_os.slurm +32 -0
- OceanDataStore/data/OISST/run_send_OISSTv2_monthly_climatology_to_os.slurm +32 -0
- OceanDataStore/data/OISST/run_send_OISSTv2_monthly_to_os.slurm +32 -0
- OceanDataStore/data/OISST/run_update_OISSTv2_daily_to_os.slurm +32 -0
- OceanDataStore/data/OISST/send_OISSTv2_daily_climatology_to_os.py +154 -0
- OceanDataStore/data/OISST/send_OISSTv2_daily_ltm_climatology_to_os.py +151 -0
- OceanDataStore/data/OISST/send_OISSTv2_daily_to_os.py +142 -0
- OceanDataStore/data/OISST/send_OISSTv2_monthly_climatology_to_os.py +150 -0
- OceanDataStore/data/OISST/send_OISSTv2_monthly_to_os.py +145 -0
- OceanDataStore/data/OISST/update_OISSTv2_daily_to_os.py +142 -0
- OceanDataStore/data/OSTIA/create_OSTIA_daily_climatology.py +120 -0
- OceanDataStore/data/OSTIA/download_OSTIA_NRT.py +42 -0
- OceanDataStore/data/OSTIA/download_OSTIA_REP_1981_2025.py +42 -0
- OceanDataStore/data/OSTIA/run_create_OSTIA_daily_climatology.slurm +54 -0
- OceanDataStore/data/OSTIA/run_send_OSTIA_daily_climatology_to_os.slurm +32 -0
- OceanDataStore/data/OSTIA/run_send_OSTIA_nrt_daily_to_os.slurm +32 -0
- OceanDataStore/data/OSTIA/run_send_OSTIA_rep_daily_to_os.slurm +32 -0
- OceanDataStore/data/OSTIA/run_update_OSTIA_daily_to_os.slurm +33 -0
- OceanDataStore/data/OSTIA/send_OSTIA_daily_climatology_to_os.py +194 -0
- OceanDataStore/data/OSTIA/send_OSTIA_nrt_daily_to_os.py +141 -0
- OceanDataStore/data/OSTIA/send_OSTIA_rep_daily_to_os.py +145 -0
- OceanDataStore/data/OSTIA/update_OSTIA_copernicus_nrt_daily_to_os.py +144 -0
- OceanDataStore/data/OSTIA/update_OSTIA_nrt_daily_to_os.py +137 -0
- OceanDataStore/data/WOA23/download_WOA23_climatology.sh +41 -0
- OceanDataStore/data/WOA23/run_send_WOA23_annual_climatology_to_os.slurm +32 -0
- OceanDataStore/data/WOA23/run_send_WOA23_monthly_climatology_to_os.slurm +32 -0
- OceanDataStore/data/WOA23/send_WOA23_annual_climatology_to_os.py +263 -0
- OceanDataStore/data/WOA23/send_WOA23_monthly_climatology_to_os.py +292 -0
- OceanDataStore/data/update_icechunk_repo_attrs.py +76 -0
- OceanDataStore/data/update_noc_npd_era5v1_attrs.py +172 -0
- OceanDataStore/data/utils.py +506 -0
- OceanDataStore/zarr.py +993 -0
- oceandatastore-0.3.0.dist-info/METADATA +184 -0
- oceandatastore-0.3.0.dist-info/RECORD +104 -0
- oceandatastore-0.3.0.dist-info/WHEEL +5 -0
- oceandatastore-0.3.0.dist-info/entry_points.txt +2 -0
- oceandatastore-0.3.0.dist-info/licenses/LICENSE +201 -0
- oceandatastore-0.3.0.dist-info/scm_file_list.json +154 -0
- oceandatastore-0.3.0.dist-info/scm_version.json +8 -0
- oceandatastore-0.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
# =========================================================
|
|
2
|
+
# send_WOA23_monthly_climatology_to_os.py
|
|
3
|
+
#
|
|
4
|
+
# Script to write WOA23 monthly climatologies to Icechunk
|
|
5
|
+
# repository in JASMIN cloud object storage.
|
|
6
|
+
#
|
|
7
|
+
# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
|
|
8
|
+
# =========================================================
|
|
9
|
+
import glob
|
|
10
|
+
import logging
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
import pandas as pd
|
|
14
|
+
import xarray as xr
|
|
15
|
+
import zarr
|
|
16
|
+
|
|
17
|
+
from OceanDataStore.cli import initialise_logging, send_to_icechunk
|
|
18
|
+
from OceanDataStore.data.utils import (
|
|
19
|
+
compute_cell_area,
|
|
20
|
+
compute_dx,
|
|
21
|
+
compute_dy,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
logger = logging.getLogger(__name__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# -- Utility Functions -- #
|
|
28
|
+
def get_filepaths(file_prefix:str, file_directory:str) -> list:
|
|
29
|
+
"""
|
|
30
|
+
Get filepaths for World Ocean Atlas 2023 climate normals
|
|
31
|
+
in-situ temperature and practical salinity data.
|
|
32
|
+
|
|
33
|
+
Parameters
|
|
34
|
+
----------
|
|
35
|
+
file_prefix : str
|
|
36
|
+
Prefix of the WOA23 filepaths to be returned.
|
|
37
|
+
file_directory : str
|
|
38
|
+
Directory containing the WOA23 files.
|
|
39
|
+
|
|
40
|
+
Returns
|
|
41
|
+
-------
|
|
42
|
+
filepaths : list
|
|
43
|
+
List of WOA23 filepaths for practical salinity & in-situ temperature.
|
|
44
|
+
"""
|
|
45
|
+
# Define salinity and temperature filenames:
|
|
46
|
+
salinity_filename = f"{file_prefix}_s*.nc"
|
|
47
|
+
temperature_filename = f"{file_prefix}_t*.nc"
|
|
48
|
+
|
|
49
|
+
# Collect filepaths for salinity and temperature data:
|
|
50
|
+
filepaths_sal = sorted(glob.glob(f"{file_directory}/{salinity_filename}"))
|
|
51
|
+
filepaths_temp = sorted(glob.glob(f"{file_directory}/{temperature_filename}"))
|
|
52
|
+
|
|
53
|
+
return filepaths_sal, filepaths_temp
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def main():
|
|
57
|
+
# ========== Initialise OceanDataStore Logging ========== #
|
|
58
|
+
initialise_logging()
|
|
59
|
+
logging.info("In Progress: Sending WOA23 Monthly Climatologies to JASMIN Object Store.")
|
|
60
|
+
|
|
61
|
+
# ========== Prepare Data ========== #
|
|
62
|
+
filedir = "/dssgfs01/scratch/otooth/npd_data/observations/WOA2023"
|
|
63
|
+
salinity_paths, temperature_paths = [], []
|
|
64
|
+
for prefix in ['woa23_decav71A0', 'woa23_decav81B0', 'woa23_decav91C0']:
|
|
65
|
+
# Define file paths for WOA23 monthly climatologies:
|
|
66
|
+
filepaths_sal, filepaths_temp = get_filepaths(file_directory=filedir, file_prefix=prefix)
|
|
67
|
+
salinity_paths.append(filepaths_sal[1:13])
|
|
68
|
+
temperature_paths.append(filepaths_temp[1:13])
|
|
69
|
+
|
|
70
|
+
# Define year bounds for each WOA23 monthly climatology:
|
|
71
|
+
year_bound_start = [1971, 1981, 1991]
|
|
72
|
+
year_bound_end = [2000, 2010, 2020]
|
|
73
|
+
|
|
74
|
+
for temp_path, sal_path, start_year, end_year in zip(temperature_paths, salinity_paths, year_bound_start, year_bound_end):
|
|
75
|
+
logging.info(f"-> In Progress: Preparing WOA23 {start_year}-{end_year} monthly climatology...")
|
|
76
|
+
# Open WOA23 monthly climatologies dataset:
|
|
77
|
+
ds_s = xr.open_mfdataset(sal_path, decode_times=False, data_vars='all')
|
|
78
|
+
ds_t = xr.open_mfdataset(temp_path, decode_times=False, data_vars='all')
|
|
79
|
+
|
|
80
|
+
ds = xr.merge([ds_s, ds_t], compat='no_conflicts').squeeze(drop=True)
|
|
81
|
+
ds = ds.rename({'time': 'month'}).assign_coords({'month': np.arange(1, 13)})
|
|
82
|
+
ds['climatology_bounds'].data = np.array([[np.datetime64(f'{start_year}-{month:02d}', 'M'), np.datetime64(f'{end_year}-{month:02d}', 'M')] for month in range(1, 13)])
|
|
83
|
+
|
|
84
|
+
ds = ds.rename({"lon": "longitude",
|
|
85
|
+
"lat": "latitude",
|
|
86
|
+
"climatology_bounds": "time_bnds"
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
# Use OceanDataStore standard variable names:
|
|
90
|
+
for var in ds.data_vars:
|
|
91
|
+
if var.startswith("t_"):
|
|
92
|
+
ds = ds.rename({var: var.replace("t_", "thetao_")})
|
|
93
|
+
elif var.startswith("s_"):
|
|
94
|
+
ds = ds.rename({var: var.replace("s_", "so_")})
|
|
95
|
+
|
|
96
|
+
# Add ancillary variables:
|
|
97
|
+
ds['dx'] = compute_dx(ds)
|
|
98
|
+
ds['dy'] = compute_dy(ds)
|
|
99
|
+
ds['cell_area'] = compute_cell_area(ds)
|
|
100
|
+
|
|
101
|
+
# Custom ancillary variables:
|
|
102
|
+
ds['cell_thickness'] = ds['depth_bnds'].isel(nbounds=1) - ds['depth_bnds'].isel(nbounds=0)
|
|
103
|
+
ds['cell_volume'] = ds['cell_thickness'] * ds['cell_area']
|
|
104
|
+
|
|
105
|
+
# Update attributes for custom ancillary variables:
|
|
106
|
+
ds['cell_thickness'].attrs.update({
|
|
107
|
+
'long_name': "Grid-Cell Thickness",
|
|
108
|
+
'standard_name': "cell_thickness",
|
|
109
|
+
'units': "m",
|
|
110
|
+
})
|
|
111
|
+
ds['cell_volume'].attrs.update({
|
|
112
|
+
'long_name': "Grid-Cell Volume",
|
|
113
|
+
'standard_name': "cell_volume",
|
|
114
|
+
'units': "m3",
|
|
115
|
+
})
|
|
116
|
+
logging.info(f"Completed: Prepared WOA23 {start_year}-{end_year} monthly climatology dataset with ancillary variables.")
|
|
117
|
+
|
|
118
|
+
# ========== Prepare Ancillary Data ========== #
|
|
119
|
+
# Open WOA23 land sea mask:
|
|
120
|
+
df_mask = pd.read_table("/dssgfs01/working/otooth/Software/OceanDataStore/OceanDataStore/data/WOA23/data/landsea_04.msk",
|
|
121
|
+
delimiter=',',
|
|
122
|
+
header=0
|
|
123
|
+
)
|
|
124
|
+
# Define level of sea floor (i.e. bottom standard level) at each grid cell:
|
|
125
|
+
ds['bottom_level'] = xr.full_like(ds['cell_area'], fill_value=np.nan).squeeze(drop=True)
|
|
126
|
+
ds['bottom_level'].data = df_mask['Bottom_Standard_Level'].values.reshape(720, 1440)
|
|
127
|
+
ds['bottom_level'].name = "bottom_level"
|
|
128
|
+
ds['bottom_level'].attrs = {"standard_name": "model_level_number_at_sea_floor",
|
|
129
|
+
"long_name": "Model Level Number at Sea Floor",
|
|
130
|
+
"units": "1"
|
|
131
|
+
}
|
|
132
|
+
# Define land sea mask (1 for ocean grid cells, 0 for land grid cells):
|
|
133
|
+
ds['mask'] = ds['bottom_level'] > 1
|
|
134
|
+
ds['mask'].name = "sea_binary_mask"
|
|
135
|
+
ds["mask"] = ds["mask"].assign_attrs({'long_name': "Land-Sea Binary Mask",
|
|
136
|
+
"standard_name": "sea_binary_mask",
|
|
137
|
+
"comment": "1 = sea, 0 = land"
|
|
138
|
+
})
|
|
139
|
+
logging.info("Completed: Prepared land sea mask & bottom level variables.")
|
|
140
|
+
|
|
141
|
+
# Open WOA23 basin mask:
|
|
142
|
+
df_basin = pd.read_table("/dssgfs01/working/otooth/Software/OceanDataStore/OceanDataStore/data/WOA23/data/basinmask_04.msk",
|
|
143
|
+
delimiter=',',
|
|
144
|
+
header=0
|
|
145
|
+
)
|
|
146
|
+
# Define basin mask (integer values for each ocean basin, NaN for land grid cells):
|
|
147
|
+
ds['basin_mask'] = xr.full_like(ds['cell_area'], fill_value=np.nan).squeeze(drop=True)
|
|
148
|
+
for basin, longitude, latitude in df_basin[["Basin_0m", "Longitude", "Latitude"]].itertuples(index=False):
|
|
149
|
+
ds['basin_mask'].loc[dict(latitude=latitude, longitude=longitude)] = basin
|
|
150
|
+
|
|
151
|
+
ds['basin_mask'].attrs = {
|
|
152
|
+
"standard_name": "ocean_basin_mask",
|
|
153
|
+
"long_name": "Ocean Basin Mask",
|
|
154
|
+
"basin_name": {
|
|
155
|
+
1: "Atlantic Ocean",
|
|
156
|
+
2: "Pacific Ocean",
|
|
157
|
+
3: "Indian Ocean",
|
|
158
|
+
4: "Mediterranean Sea",
|
|
159
|
+
5: "Baltic Sea",
|
|
160
|
+
6: "Black Sea",
|
|
161
|
+
7: "Red Sea",
|
|
162
|
+
8: "Persian Gulf",
|
|
163
|
+
9: "Hudson Bay",
|
|
164
|
+
10: "Southern Ocean",
|
|
165
|
+
11: "Arctic Ocean",
|
|
166
|
+
12: "Sea of Japan",
|
|
167
|
+
13: "Kara Sea",
|
|
168
|
+
14: "Sulu Sea",
|
|
169
|
+
15: "Baffin Bay",
|
|
170
|
+
16: "East Mediterranean",
|
|
171
|
+
17: "West Mediterranean",
|
|
172
|
+
18: "Sea of Okhotsk",
|
|
173
|
+
19: "Banda Sea",
|
|
174
|
+
20: "Caribbean Sea",
|
|
175
|
+
21: "Andaman Basin",
|
|
176
|
+
22: "North Caribbean",
|
|
177
|
+
23: "Gulf of Mexico",
|
|
178
|
+
24: "Beaufort Sea",
|
|
179
|
+
25: "South China Sea",
|
|
180
|
+
26: "Barents Sea",
|
|
181
|
+
27: "Celebes Sea",
|
|
182
|
+
28: "Aleutian Basin",
|
|
183
|
+
29: "Fiji Basin",
|
|
184
|
+
30: "North American Basin",
|
|
185
|
+
31: "West European Basin",
|
|
186
|
+
32: "Southeast Indian Basin",
|
|
187
|
+
33: "Coral Sea",
|
|
188
|
+
34: "East Indian Basin",
|
|
189
|
+
35: "Central Indian Basin",
|
|
190
|
+
36: "Southwest Atlantic Basin",
|
|
191
|
+
37: "Southeast Atlantic Basin",
|
|
192
|
+
38: "Southeast Pacific Basin",
|
|
193
|
+
39: "Guatemala Basin",
|
|
194
|
+
40: "East Caroline Basin",
|
|
195
|
+
41: "Marianas Basin",
|
|
196
|
+
42: "Philippine Sea",
|
|
197
|
+
43: "Arabian Sea",
|
|
198
|
+
44: "Chile Basin",
|
|
199
|
+
45: "Somali Basin",
|
|
200
|
+
46: "Mascarene Basin",
|
|
201
|
+
47: "Crozet Basin",
|
|
202
|
+
48: "Guinea Basin",
|
|
203
|
+
49: "Brazil Basin",
|
|
204
|
+
50: "Argentine Basin",
|
|
205
|
+
51: "Tasman Sea",
|
|
206
|
+
52: "Atlantic Indian Basin",
|
|
207
|
+
53: "Caspian Sea",
|
|
208
|
+
54: "Sulu Sea II",
|
|
209
|
+
55: "Venezuela Basin",
|
|
210
|
+
56: "Bay of Bengal",
|
|
211
|
+
57: "Java Sea",
|
|
212
|
+
58: "East Indian Atlantic Basin",
|
|
213
|
+
59: "Chiloe",
|
|
214
|
+
60: "Bransfield Strait",
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
logging.info("Completed: Prepared ocean basin mask variable.")
|
|
218
|
+
|
|
219
|
+
# ========== CF Attributes ========== #
|
|
220
|
+
ds.attrs.clear()
|
|
221
|
+
ds = ds.assign_attrs({
|
|
222
|
+
"Conventions": "CF-1.6",
|
|
223
|
+
"title": f"World Ocean Atlas 2023 temperature and salinity monthly climatology ({start_year}-{end_year}).",
|
|
224
|
+
"description": f"World Ocean Atlas 2023 (WOA23) temperature and salinity monthly climatology for the global ocean from objectively analysed, quality controlled in-situ profile data ({start_year}-{end_year}).",
|
|
225
|
+
"source": "Numerical models: Objective Analysis. In-situ observations: World Ocean Database (WOD).",
|
|
226
|
+
"dataset_type": "observation",
|
|
227
|
+
"product_type": "climatology",
|
|
228
|
+
"product_version": "1.0",
|
|
229
|
+
"institution": "NOAA National Centers for Environmental Information (NCEI)",
|
|
230
|
+
"citation": "Reagan, James R.; Boyer, Tim P.; García, Hernán E.; Locarnini, Ricardo A.; Baranova, Olga K.; Bouchard, Courtney; Cross, Scott L.; Mishonov, Alexey V.; Paver, Christopher R.; Seidov, Dan; Wang, Zhankun; Dukhovskoy, Dmitry (2023). World Ocean Atlas 2023 (NCEI Accession 0270533). https://www.ncei.noaa.gov/archive/accession/0270533. In Reagan, James R.; Boyer, Tim P.; García, Hernán E.; Locarnini, Ricardo A.; Baranova, Olga K.; Bouchard, Courtney; Cross, Scott L.; Mishonov, Alexey V.; Paver, Christopher R.; Seidov, Dan; Wang, Zhankun; Dukhovskoy, Dmitry (2023). World Ocean Atlas 2023. NOAA National Centers for Environmental Information. Dataset. https://doi.org/10.25921/va26-hv25. Accessed 06-05-2026.",
|
|
231
|
+
"references": "Locarnini, R.A., A.V. Mishonov, O.K. Baranova, J.R. Reagan, T.P. Boyer, D. Seidov, Z. Wang, H.E. Garcia, C. Bouchard, S.L. Cross, C.R. Paver, and D. Dukhovskoy (2024). World Ocean Atlas 2023, Volume 1: Temperature. A. Mishonov Technical Editor, NOAA Atlas NESDIS 89. https://doi.org/10.25923/54bh-1613. Reagan, J.R., D. Seidov, Z. Wang, D. Dukhovskoy, T.P. Boyer, R.A. Locarnini, O.K. Baranova, A.V. Mishonov, H.E. Garcia, C. Bouchard, S.L. Cross, and C.R. Paver (2023). World Ocean Atlas 2023, Volume 2: Salinity. A. Mishonov Technical Editor, NOAA Atlas NESDIS 90, https://doi.org/10.25923/70qt-9574.",
|
|
232
|
+
"acknowledgement": "None",
|
|
233
|
+
"license": "World Ocean Atlas 2023 data were obtained from https://www.ncei.noaa.gov/access/world-ocean-atlas-2023/ and are provided under a Creative Commons CC0 1.0 Universal License https://creativecommons.org/publicdomain/zero/1.0/",
|
|
234
|
+
"doi": "https://doi.org/10.25921/va26-hv25",
|
|
235
|
+
"platform": "gr",
|
|
236
|
+
"horizontal_grid_type": "regular rectilinear",
|
|
237
|
+
"horizontal_grid_resolution": "0.25 degree",
|
|
238
|
+
"vertical_grid_type": "z",
|
|
239
|
+
"vertical_grid_coordinate": "depth",
|
|
240
|
+
"vertical_grid_levels": 57,
|
|
241
|
+
"aggregation": "mean",
|
|
242
|
+
"aggregation_frequency": "monthly",
|
|
243
|
+
"status": "completed",
|
|
244
|
+
"update_frequency": "None",
|
|
245
|
+
"bbox": "[-180.0, 180.0, -90.0, 90.0]",
|
|
246
|
+
})
|
|
247
|
+
logging.info(f"Completed: Added CF-compliant global attributes to WOA23 {start_year}-{end_year} monthly climatology dataset.")
|
|
248
|
+
|
|
249
|
+
# ========== Send to Icechunk Repository ========== #
|
|
250
|
+
logging.info(f"In Progress: Sending WOA23 {start_year}-{end_year} monthly climatology to Icechunk Repository...")
|
|
251
|
+
bucket = "woa23"
|
|
252
|
+
prefix = f"woa23_{start_year}_{end_year}_monthly_climatology"
|
|
253
|
+
exists = False
|
|
254
|
+
store_credentials_json = ".../credentials/jasmin_os_credentials.json"
|
|
255
|
+
branch = "main"
|
|
256
|
+
commit_message = f"Added WOA23 {start_year}-{end_year} monthly climatology."
|
|
257
|
+
variable_commits = True
|
|
258
|
+
config_kwargs = {
|
|
259
|
+
"temporary_directory":".../OceanDataStore/OceanDataStore/data/WOA23/",
|
|
260
|
+
"local_directory":".../OceanDataStore/OceanDataStore/data/WOA23/"
|
|
261
|
+
}
|
|
262
|
+
cluster_kwargs = {
|
|
263
|
+
"n_workers" : 25,
|
|
264
|
+
"threads_per_worker" : 1,
|
|
265
|
+
"memory_limit":"3GB"
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
# Optimise chunk sizes for spatial analysis:
|
|
269
|
+
ds = ds.chunk({'longitude': 1440, 'latitude': 720, 'depth': 5, 'month': 1})
|
|
270
|
+
|
|
271
|
+
# Update variable encodings:
|
|
272
|
+
blosccodec = zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.shuffle)
|
|
273
|
+
for var in list(ds.data_vars) + list(ds.coords):
|
|
274
|
+
ds[var].encoding['compressors'] = [blosccodec]
|
|
275
|
+
|
|
276
|
+
send_to_icechunk(
|
|
277
|
+
file=ds,
|
|
278
|
+
bucket=bucket,
|
|
279
|
+
object_prefix=prefix,
|
|
280
|
+
store_credentials_json=store_credentials_json,
|
|
281
|
+
exists=exists,
|
|
282
|
+
branch=branch,
|
|
283
|
+
commit_message=commit_message,
|
|
284
|
+
variable_commits=variable_commits,
|
|
285
|
+
dask_config_kwargs=config_kwargs,
|
|
286
|
+
dask_cluster_kwargs=cluster_kwargs,
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
logging.info(f"Completed: Sent WOA23 {start_year}-{end_year} monthly climatology to Icechunk Repository.")
|
|
290
|
+
|
|
291
|
+
if __name__ == "__main__":
|
|
292
|
+
main()
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# =========================================================
|
|
2
|
+
# update_icechunk_repo_attrs.py
|
|
3
|
+
#
|
|
4
|
+
# Script to update global and variable attributes in an
|
|
5
|
+
# Icechunk repository.
|
|
6
|
+
#
|
|
7
|
+
# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
|
|
8
|
+
# =========================================================
|
|
9
|
+
from OceanDataStore.data.utils import (
|
|
10
|
+
update_icechunk_global_attrs,
|
|
11
|
+
update_icechunk_variable_attrs,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
# ========= Shared Inputs ========= #
|
|
15
|
+
# Define credential to write to JASMIN OS:
|
|
16
|
+
credentials_filepath = '.../credentials/jasmin_os_credentials.json'
|
|
17
|
+
|
|
18
|
+
# Define time period for climatology:
|
|
19
|
+
start_year = 1991
|
|
20
|
+
end_year = 2020
|
|
21
|
+
|
|
22
|
+
# Define bucket and prefix to Icechunk repository:
|
|
23
|
+
bucket = "armor3d"
|
|
24
|
+
prefix = f"armor3d_global_my_{start_year}_{end_year}_monthly_climatogy"
|
|
25
|
+
|
|
26
|
+
# ========= Update variable attributes ========= #
|
|
27
|
+
vars = ["month"]
|
|
28
|
+
attrs = [{"units": "1", "long_name": "Month of Year"}]
|
|
29
|
+
message = f"Updated ARMOR3D Monthly Climatology ({start_year}-{end_year}) variable attributes. -> ['month']"
|
|
30
|
+
|
|
31
|
+
update_icechunk_variable_attrs(
|
|
32
|
+
credentials_filepath=credentials_filepath,
|
|
33
|
+
bucket=bucket,
|
|
34
|
+
prefix=prefix,
|
|
35
|
+
vars=vars,
|
|
36
|
+
attrs=attrs,
|
|
37
|
+
commit_message=message,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
# ========= Update global attributes ========= #
|
|
41
|
+
attrs = {
|
|
42
|
+
"Conventions": "CF-1.0",
|
|
43
|
+
"title": f"Multi Observation Global Ocean 3D Temperature Salinity Height Geostrophic Current and MLD monthly climatology ({start_year}-{end_year}).",
|
|
44
|
+
"description": f"Multi Observation Global Ocean ARMOR3D multi-year reprocessed temperature salinity, sea surface height, geostrophic current and mixed layer depth climatology on 1/8 degree regular grid and 50 depth levels ({start_year}-{end_year}).",
|
|
45
|
+
"source": "Numerical models: Multiple Linear Regression, Optimal Interpolation. In-situ observations: Copernicus In Situ TAC (including Argo, XBT, CTD and moorings) Copernicus Sea Level TAC, CNES-CLS22 Mean Dynamic Topography, OSTIA Sea Surface Temperature Analysis, Copernicus MOB TAC (Sea Surface Salinity), and World Ocean Atlas 2018 (WOA18).",
|
|
46
|
+
"dataset_type": "observation",
|
|
47
|
+
"product_type": "climatology",
|
|
48
|
+
"product_version": "2.0",
|
|
49
|
+
"institution": "Copernicus Marine Service, Mercator Ocean International, France",
|
|
50
|
+
"citation": "Multi Observation Global Ocean 3D Temperature Salinity Height Geostrophic Current and MLD. E.U. Copernicus Marine Service Information (CMEMS). Marine Data Store (MDS). DOI: 10.48670/moi-00052 (Accessed on 21 04 2026).",
|
|
51
|
+
"references": "Guinehut S., A.-L. Dhomps, G. Larnicol and P.-Y. Le Traon, 2012: High resolution 3D temperature and salinity fields derived from in situ and satellite observations. Ocean Sci., 8(5):845-857. Mulet, S., M.-H. Rio, A. Mignot, S. Guinehut and R. Morrow, 2012: A new estimate of the global 3D geostrophic ocean circulation based on satellite data and in-situ measurements. Deep Sea Research Part II : Topical Studies in Oceanography, 77-80(0):70-81.",
|
|
52
|
+
"acknowledgement": "Generated using E.U. Copernicus Marine Service Information; https://doi.org/10.48670/moi-00052.",
|
|
53
|
+
"license": "ARMOR3D data were obtained from https://doi.org/10.48670/moi-00052, and are provided under the Copernicus Marine Environment Monitoring Service Service Level Agreement (SLA) https://marine.copernicus.eu/user-corner/service-commitments-and-licence?pk_vid=42ac3e352be888641780994034c3bb6e",
|
|
54
|
+
"doi": "10.48670/moi-00052",
|
|
55
|
+
"platform": "gr",
|
|
56
|
+
"horizontal_grid_type": "regular rectilinear",
|
|
57
|
+
"horizontal_grid_resolution": "0.125 degree",
|
|
58
|
+
"vertical_grid_type": "z",
|
|
59
|
+
"vertical_grid_coordinate": "depth",
|
|
60
|
+
"vertical_grid_levels": 50,
|
|
61
|
+
"aggregation": "mean",
|
|
62
|
+
"aggregation_frequency": "monthly",
|
|
63
|
+
"status": "completed",
|
|
64
|
+
"update_frequency": "None",
|
|
65
|
+
"bbox": "[-180.0, 180.0, -90.0, 90.0]",
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
message = f"Updated ARMOR3D Monthly Climatology ({start_year}-{end_year}) -> root group attributes."
|
|
69
|
+
|
|
70
|
+
update_icechunk_global_attrs(
|
|
71
|
+
credentials_filepath=credentials_filepath,
|
|
72
|
+
bucket=bucket,
|
|
73
|
+
prefix=prefix,
|
|
74
|
+
attrs=attrs,
|
|
75
|
+
commit_message=message,
|
|
76
|
+
)
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# =========================================================
|
|
2
|
+
# update_noc_npd_era5v1_attrs.py
|
|
3
|
+
#
|
|
4
|
+
# Script to update global and variable attributes in NOC
|
|
5
|
+
# Near-Present Day ERA5v1 Icechunk repositories.
|
|
6
|
+
#
|
|
7
|
+
# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
|
|
8
|
+
# =========================================================
|
|
9
|
+
import logging
|
|
10
|
+
|
|
11
|
+
from OceanDataStore.cli import initialise_logging
|
|
12
|
+
from OceanDataStore.data.utils import update_icechunk_global_attrs
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main(credentials_filepath: str,
|
|
16
|
+
bucket: str,
|
|
17
|
+
config_name: str,
|
|
18
|
+
nemo_config_name: str,
|
|
19
|
+
platform: str,
|
|
20
|
+
agg: str,
|
|
21
|
+
prefix_list: list
|
|
22
|
+
) -> None:
|
|
23
|
+
# ========== Initialise OceanDataStore Logging ========== #
|
|
24
|
+
initialise_logging()
|
|
25
|
+
|
|
26
|
+
# ========= Update global attributes ========= #
|
|
27
|
+
for prefix in prefix_list:
|
|
28
|
+
logging.info(f"In Progress: Updating global attributes for {config_name} {prefix}...")
|
|
29
|
+
|
|
30
|
+
# Define aggregation frequency from prefix:
|
|
31
|
+
if "1y" in prefix:
|
|
32
|
+
agg_freq = "annual"
|
|
33
|
+
elif "1m" in prefix:
|
|
34
|
+
agg_freq = "monthly"
|
|
35
|
+
elif "5d" in prefix:
|
|
36
|
+
agg_freq = "5-daily"
|
|
37
|
+
else:
|
|
38
|
+
raise ValueError(f"Unable to determine aggregation frequency from prefix: {prefix}")
|
|
39
|
+
|
|
40
|
+
# Define dimensionality from prefix:
|
|
41
|
+
if "_3d" in prefix:
|
|
42
|
+
dimensionality = "3-dimensional"
|
|
43
|
+
elif "_4d" in prefix:
|
|
44
|
+
dimensionality = "4-dimensional"
|
|
45
|
+
else:
|
|
46
|
+
dimensionality = ""
|
|
47
|
+
|
|
48
|
+
# Define grid type from prefix:
|
|
49
|
+
if "T" in prefix:
|
|
50
|
+
grid = "T-grid"
|
|
51
|
+
variable_type = "scalar variables"
|
|
52
|
+
elif "U" in prefix:
|
|
53
|
+
grid = "U-grid"
|
|
54
|
+
variable_type = "vector variables"
|
|
55
|
+
elif "V" in prefix:
|
|
56
|
+
grid = "V-grid"
|
|
57
|
+
variable_type = "vector variables"
|
|
58
|
+
elif "W" in prefix:
|
|
59
|
+
grid = "W-grid"
|
|
60
|
+
variable_type = "vector variables"
|
|
61
|
+
elif "S" in prefix:
|
|
62
|
+
grid = ""
|
|
63
|
+
variable_type = "scalar variables"
|
|
64
|
+
elif "I" in prefix:
|
|
65
|
+
grid = "T-grid"
|
|
66
|
+
variable_type = "sea-ice variables"
|
|
67
|
+
else:
|
|
68
|
+
raise ValueError(f"Unable to determine grid type from prefix: {prefix}")
|
|
69
|
+
|
|
70
|
+
# Define resolution from nemo_config_name:
|
|
71
|
+
if "eORCA12" in nemo_config_name:
|
|
72
|
+
horizontal_grid_resolution = "1/12 degree"
|
|
73
|
+
elif "eORCA025" in nemo_config_name:
|
|
74
|
+
horizontal_grid_resolution = "1/4 degree"
|
|
75
|
+
elif "eORCA1" in nemo_config_name:
|
|
76
|
+
horizontal_grid_resolution = "1 degree"
|
|
77
|
+
else:
|
|
78
|
+
raise ValueError(f"Unable to determine horizontal grid resolution from NEMO configuration name: {nemo_config_name}")
|
|
79
|
+
|
|
80
|
+
attrs = {
|
|
81
|
+
"Conventions": "CF-1.6",
|
|
82
|
+
"title": f"National Oceanography Centre Near-Present Day (NPD) {horizontal_grid_resolution} global ocean physics & sea-ice hindcast.",
|
|
83
|
+
"description": f"NOC Near-Present Day {agg_freq} {agg} global ocean physics & sea-ice hindcast forced using bias-corrected ERA5 atmospheric reanalysis {dimensionality} {variable_type} stored on the native {nemo_config_name} curvilinear NEMO model {grid}.",
|
|
84
|
+
"dataset_type": "model",
|
|
85
|
+
"product_type": "timeseries",
|
|
86
|
+
"product_version": "1.0",
|
|
87
|
+
"institution": "National Oceanography Centre, UK",
|
|
88
|
+
"citation": "Blaker, A. T., Tooth, O. J., Palmiéri, J., Coward, A. C., and Mecking, J. (2025). NOC-MSM/NOC_Near_Present_Day: v0.9.0 (v0.9.0). Zenodo. https://doi.org/10.5281/zenodo.15310354.",
|
|
89
|
+
"references": "Blaker, A.T., Tooth, O.J., Palmiéri, J., Coward, A.C., & Mecking, J. (2025). NOC-MSM/NOC_Near_Present_Day: v0.9.0 (v0.9.0). Zenodo. https://doi.org/10.5281/zenodo.15310354. Guiavarc'h, C., Storkey, D., Blaker, A. T., Blockley, E., Megann, A., Hewitt, H., Bell, M. J., Calvert, D., Copsey, D., Sinha, B., Moreton, S., Mathiot, P., and An, B.: GOSI9: UK Global Ocean and Sea Ice configurations, Geosci. Model Dev., 18, 377-403, https://doi.org/10.5194/gmd-18-377-2025, 2025.",
|
|
90
|
+
"acknowledgement": "NOC Near-Present Day Documentation available at: https://noc-msm.github.io/NOC_Near_Present_Day/",
|
|
91
|
+
"license": "UK Open Government License v3.0",
|
|
92
|
+
"doi": "pending",
|
|
93
|
+
"platform": platform,
|
|
94
|
+
"horizontal_grid_type": "curvilinear",
|
|
95
|
+
"horizontal_grid_resolution": horizontal_grid_resolution,
|
|
96
|
+
"vertical_grid_type": "zps",
|
|
97
|
+
"vertical_grid_coordinate": "depth with partial step topography",
|
|
98
|
+
"vertical_grid_levels": 75,
|
|
99
|
+
"aggregation": agg,
|
|
100
|
+
"aggregation_frequency": agg_freq,
|
|
101
|
+
"status": "ongoing",
|
|
102
|
+
"update_frequency": "quarterly",
|
|
103
|
+
"bbox": "[-180.0, 180.0, -90.0, 90.0]",
|
|
104
|
+
"ocean_component": "NEMO v4.2.2",
|
|
105
|
+
"sea_ice_component": "SI3 v4.0",
|
|
106
|
+
"biogeochemistry_component": "None",
|
|
107
|
+
"atmospheric_component": "None",
|
|
108
|
+
"atmospheric_forcing": "ERA5 v1",
|
|
109
|
+
"variant": "r1i1c1f1",
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
message = f"Updated {config_name} {agg_freq} {agg} -> root group attributes."
|
|
113
|
+
|
|
114
|
+
update_icechunk_global_attrs(
|
|
115
|
+
credentials_filepath=credentials_filepath,
|
|
116
|
+
bucket=bucket,
|
|
117
|
+
prefix=prefix,
|
|
118
|
+
attrs=attrs,
|
|
119
|
+
commit_message=message,
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
logging.info(f"Completed: Updated global attributes for {config_name} {prefix}.")
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
if __name__ == "__main__":
|
|
126
|
+
# ========= Define Shared Inputs ========= #
|
|
127
|
+
# Define credential to write to JASMIN OS:
|
|
128
|
+
credentials_filepath = '.../credentials/jasmin_os_credentials.json'
|
|
129
|
+
|
|
130
|
+
# Define NPD configuration propeties:
|
|
131
|
+
bucket = "npd-eorca12-era5v1"
|
|
132
|
+
config_name = "NPD eORCA12 ERA5v1"
|
|
133
|
+
nemo_config_name = "eORCA12"
|
|
134
|
+
agg = "mean"
|
|
135
|
+
platform = "gn"
|
|
136
|
+
|
|
137
|
+
# -- eORCA1 --- #
|
|
138
|
+
# prefix_list = ["I1m", "I1y",
|
|
139
|
+
# "S1m", "S1y",
|
|
140
|
+
# "T1m", "T1y",
|
|
141
|
+
# "U1m", "U1y",
|
|
142
|
+
# "V1m", "V1y",
|
|
143
|
+
# "W1m", "W1y",
|
|
144
|
+
# ]
|
|
145
|
+
|
|
146
|
+
# -- eORCA025 --- #
|
|
147
|
+
# prefix_list = ["I1m_3d", "I1y_3d", "I5d_3d",
|
|
148
|
+
# "S1m_1d", "S1y_1d", "S5d_1d",
|
|
149
|
+
# "T1m_3d", "T1m_4d", "T1y_3d", "T1y_4d", "T5d_3d", "T5d_4d",
|
|
150
|
+
# "U1m_3d", "U1m_4d", "U1y_3d", "U1y_4d", "U5d_3d", "U5d_4d",
|
|
151
|
+
# "V1m_3d", "V1m_4d", "V1y_3d", "V1y_4d", "V5d_3d", "V5d_4d",
|
|
152
|
+
# "W1m_4d", "W1y_4d", "W5d_4d"
|
|
153
|
+
# ]
|
|
154
|
+
|
|
155
|
+
# -- eORCA12 --- #
|
|
156
|
+
prefix_list = ["I1m_3d", "I1y_3d",
|
|
157
|
+
"S1m_1d", "S1y_1d",
|
|
158
|
+
"T1m_3d", "T1m_4d", "T1y_3d", "T1y_4d",
|
|
159
|
+
"U1m_3d", "U1m_4d", "U1y_3d", "U1y_4d",
|
|
160
|
+
"V1m_3d", "V1m_4d", "V1y_3d", "V1y_4d",
|
|
161
|
+
"W1m_4d", "W1y_4d",
|
|
162
|
+
]
|
|
163
|
+
|
|
164
|
+
# ========= Run Main Function ========= #
|
|
165
|
+
main(credentials_filepath=credentials_filepath,
|
|
166
|
+
bucket=bucket,
|
|
167
|
+
config_name=config_name,
|
|
168
|
+
nemo_config_name=nemo_config_name,
|
|
169
|
+
platform=platform,
|
|
170
|
+
agg=agg,
|
|
171
|
+
prefix_list=prefix_list
|
|
172
|
+
)
|