OceanDataStore 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. OceanDataStore/__init__.py +21 -0
  2. OceanDataStore/catalog/__init__.py +12 -0
  3. OceanDataStore/catalog/oceandatacatalog.py +1242 -0
  4. OceanDataStore/catalog/stac/README.md +34 -0
  5. OceanDataStore/catalog/stac/__init__.py +30 -0
  6. OceanDataStore/catalog/stac/create_noc_stac.py +109 -0
  7. OceanDataStore/catalog/stac/npd_era5_collection.py +364 -0
  8. OceanDataStore/catalog/stac/npd_jra55_collection.py +196 -0
  9. OceanDataStore/catalog/stac/ods_obs_collection.py +534 -0
  10. OceanDataStore/catalog/stac/rapid_evo_collection.py +309 -0
  11. OceanDataStore/catalog/stac/template_collection.py +85 -0
  12. OceanDataStore/catalog/stac/utils.py +476 -0
  13. OceanDataStore/cli/__init__.py +34 -0
  14. OceanDataStore/cli/arg_parser.py +182 -0
  15. OceanDataStore/cli/cli.py +203 -0
  16. OceanDataStore/cli/exceptions.py +83 -0
  17. OceanDataStore/cli/icechunk.py +888 -0
  18. OceanDataStore/cli/logging.py +52 -0
  19. OceanDataStore/cli/object_store.py +293 -0
  20. OceanDataStore/cli/utils.py +275 -0
  21. OceanDataStore/cli/zarr.py +870 -0
  22. OceanDataStore/data/ARMOR3D/create_ARMOR3D_P1M-m_monthly_climatology.py +135 -0
  23. OceanDataStore/data/ARMOR3D/download_ARMOR3D_0.125def_P1M-m_1993_2024.py +33 -0
  24. OceanDataStore/data/ARMOR3D/run_create_ARMOR3D_P1M-m_monthly_climatology.slurm +32 -0
  25. OceanDataStore/data/ARMOR3D/run_send_ARMOR3D_P1M-m_climatology_to_os.slurm +32 -0
  26. OceanDataStore/data/ARMOR3D/run_send_ARMOR3D_P1M-m_monthly_to_os.slurm +32 -0
  27. OceanDataStore/data/ARMOR3D/run_update_ARMOR3D_P1m-m_monthly_to_os.slurm +32 -0
  28. OceanDataStore/data/ARMOR3D/send_ARMOR3D_P1m-m_monthly_climatology_to_os.py +99 -0
  29. OceanDataStore/data/ARMOR3D/send_ARMOR3D_P1m-m_monthly_to_os.py +147 -0
  30. OceanDataStore/data/ARMOR3D/update_ARMOR3D_P1m-m_monthly_to_os.py +143 -0
  31. OceanDataStore/data/EN.4.2.2/create_EN4.2.2_analysis_g10_climatology.py +162 -0
  32. OceanDataStore/data/EN.4.2.2/download_EN4.2.2_analysis_g10_data.sh +51 -0
  33. OceanDataStore/data/EN.4.2.2/run_send_EN4.2.2_analysis_g10_climatology_to_os.slurm +32 -0
  34. OceanDataStore/data/EN.4.2.2/run_send_EN4.2.2_analysis_g10_monthly_to_os.slurm +32 -0
  35. OceanDataStore/data/EN.4.2.2/run_update_EN4.2.2_analysis_g10_monthly_to_os.slurm +32 -0
  36. OceanDataStore/data/EN.4.2.2/send_EN4.2.2_analysis_g10_monthly_climatology_to_os.py +76 -0
  37. OceanDataStore/data/EN.4.2.2/send_EN4.2.2_analysis_g10_monthly_to_os.py +165 -0
  38. OceanDataStore/data/EN.4.2.2/update_EN4.2.2_analysis_g10_monthly_to_os.py +161 -0
  39. OceanDataStore/data/ERA5/create_ERA5_daily_climatology.py +110 -0
  40. OceanDataStore/data/ERA5/create_ERA5_daily_mean.py +69 -0
  41. OceanDataStore/data/ERA5/create_ERA5_monthly_mean.py +74 -0
  42. OceanDataStore/data/ERA5/run_create_ERA5_daily_climatology.slurm +54 -0
  43. OceanDataStore/data/ERA5/run_send_ERA5_daily_climatology_to_os.slurm +32 -0
  44. OceanDataStore/data/ERA5/run_send_ERA5_daily_to_os.slurm +32 -0
  45. OceanDataStore/data/ERA5/run_send_ERA5_monthly_to_os.slurm +32 -0
  46. OceanDataStore/data/ERA5/run_update_ERA5_daily_to_os.slurm +32 -0
  47. OceanDataStore/data/ERA5/run_update_ERA5_monthly_to_os.slurm +32 -0
  48. OceanDataStore/data/ERA5/send_ERA5_daily_climatology_to_os.py +159 -0
  49. OceanDataStore/data/ERA5/send_ERA5_daily_to_os.py +141 -0
  50. OceanDataStore/data/ERA5/send_ERA5_monthly_to_os.py +173 -0
  51. OceanDataStore/data/ERA5/update_ERA5_daily_to_os.py +141 -0
  52. OceanDataStore/data/ERA5/update_ERA5_monthly_to_os.py +169 -0
  53. OceanDataStore/data/HadISST/download_HadISST1_data.sh +43 -0
  54. OceanDataStore/data/HadISST/run_send_HadISST1_monthly_to_os.slurm +32 -0
  55. OceanDataStore/data/HadISST/send_HadISST1_monthly_to_os.py +133 -0
  56. OceanDataStore/data/NSIDC/download_NSIDC_monthly_1979_2025_data.sh +54 -0
  57. OceanDataStore/data/NSIDC/process_NSIDC_SSI_Antarctic_data.py +130 -0
  58. OceanDataStore/data/NSIDC/process_NSIDC_SSI_Arctic_data.py +129 -0
  59. OceanDataStore/data/NSIDC/run_send_NSIDC_v4.0_to_OS.slurm +32 -0
  60. OceanDataStore/data/NSIDC/send_NSIDC_SII_v4.0_to_os.py +140 -0
  61. OceanDataStore/data/OISST/create_OISSTv2_daily_climatology.py +83 -0
  62. OceanDataStore/data/OISST/download_oisstv2_data.sh +43 -0
  63. OceanDataStore/data/OISST/run_create_OISSTv2_daily_climatology.slurm +44 -0
  64. OceanDataStore/data/OISST/run_send_OISSTv2_daily_climatology_to_os.slurm +32 -0
  65. OceanDataStore/data/OISST/run_send_OISSTv2_daily_to_os.slurm +32 -0
  66. OceanDataStore/data/OISST/run_send_OISSTv2_monthly_climatology_to_os.slurm +32 -0
  67. OceanDataStore/data/OISST/run_send_OISSTv2_monthly_to_os.slurm +32 -0
  68. OceanDataStore/data/OISST/run_update_OISSTv2_daily_to_os.slurm +32 -0
  69. OceanDataStore/data/OISST/send_OISSTv2_daily_climatology_to_os.py +154 -0
  70. OceanDataStore/data/OISST/send_OISSTv2_daily_ltm_climatology_to_os.py +151 -0
  71. OceanDataStore/data/OISST/send_OISSTv2_daily_to_os.py +142 -0
  72. OceanDataStore/data/OISST/send_OISSTv2_monthly_climatology_to_os.py +150 -0
  73. OceanDataStore/data/OISST/send_OISSTv2_monthly_to_os.py +145 -0
  74. OceanDataStore/data/OISST/update_OISSTv2_daily_to_os.py +142 -0
  75. OceanDataStore/data/OSTIA/create_OSTIA_daily_climatology.py +120 -0
  76. OceanDataStore/data/OSTIA/download_OSTIA_NRT.py +42 -0
  77. OceanDataStore/data/OSTIA/download_OSTIA_REP_1981_2025.py +42 -0
  78. OceanDataStore/data/OSTIA/run_create_OSTIA_daily_climatology.slurm +54 -0
  79. OceanDataStore/data/OSTIA/run_send_OSTIA_daily_climatology_to_os.slurm +32 -0
  80. OceanDataStore/data/OSTIA/run_send_OSTIA_nrt_daily_to_os.slurm +32 -0
  81. OceanDataStore/data/OSTIA/run_send_OSTIA_rep_daily_to_os.slurm +32 -0
  82. OceanDataStore/data/OSTIA/run_update_OSTIA_daily_to_os.slurm +33 -0
  83. OceanDataStore/data/OSTIA/send_OSTIA_daily_climatology_to_os.py +194 -0
  84. OceanDataStore/data/OSTIA/send_OSTIA_nrt_daily_to_os.py +141 -0
  85. OceanDataStore/data/OSTIA/send_OSTIA_rep_daily_to_os.py +145 -0
  86. OceanDataStore/data/OSTIA/update_OSTIA_copernicus_nrt_daily_to_os.py +144 -0
  87. OceanDataStore/data/OSTIA/update_OSTIA_nrt_daily_to_os.py +137 -0
  88. OceanDataStore/data/WOA23/download_WOA23_climatology.sh +41 -0
  89. OceanDataStore/data/WOA23/run_send_WOA23_annual_climatology_to_os.slurm +32 -0
  90. OceanDataStore/data/WOA23/run_send_WOA23_monthly_climatology_to_os.slurm +32 -0
  91. OceanDataStore/data/WOA23/send_WOA23_annual_climatology_to_os.py +263 -0
  92. OceanDataStore/data/WOA23/send_WOA23_monthly_climatology_to_os.py +292 -0
  93. OceanDataStore/data/update_icechunk_repo_attrs.py +76 -0
  94. OceanDataStore/data/update_noc_npd_era5v1_attrs.py +172 -0
  95. OceanDataStore/data/utils.py +506 -0
  96. OceanDataStore/zarr.py +993 -0
  97. oceandatastore-0.3.0.dist-info/METADATA +184 -0
  98. oceandatastore-0.3.0.dist-info/RECORD +104 -0
  99. oceandatastore-0.3.0.dist-info/WHEEL +5 -0
  100. oceandatastore-0.3.0.dist-info/entry_points.txt +2 -0
  101. oceandatastore-0.3.0.dist-info/licenses/LICENSE +201 -0
  102. oceandatastore-0.3.0.dist-info/scm_file_list.json +154 -0
  103. oceandatastore-0.3.0.dist-info/scm_version.json +8 -0
  104. oceandatastore-0.3.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,135 @@
1
+ # =========================================================
2
+ # create_ARMOR3D_P1M-m_climatology.py
3
+ #
4
+ # Script to create ARMOR3D REP monthly climatologies for
5
+ # climate normals (1971-2000, 1981-2010, 1991-2020) and
6
+ # write to local netCDF files.
7
+ #
8
+ # Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
9
+ # =========================================================
10
+ import logging
11
+
12
+ import numpy as np
13
+ import xarray as xr
14
+ import zarr
15
+
16
+ from OceanDataStore.cli import initialise_logging
17
+ from OceanDataStore.data.utils import (
18
+ compute_dx,
19
+ compute_dy,
20
+ compute_cell_area,
21
+ compute_cell_thickness,
22
+ compute_land_sea_mask,
23
+ )
24
+
25
+ logger = logging.getLogger(__name__)
26
+
27
+ def main():
28
+ # ========== Initialize Logging and Print Banner ========== #
29
+ initialise_logging()
30
+
31
+ # ========== Prepare Data ========== #
32
+ # Open complete ARMOR3D REP monthly climatology dataset:
33
+ filepath = "/dssgfs01/scratch/otooth/npd_data/observations/ARMOR3D/armor-3d_rep_monthly_NA_*.zarr"
34
+ ds = xr.open_mfdataset(filepath, compat="no_conflicts", data_vars="all", engine="zarr")
35
+ logging.info("-> Completed: Opened ARMOR-3D REP monthly climatology dataset from Zarr stores.")
36
+
37
+ # Rename variables to standard names:
38
+ ds = ds.rename({"to": "thetao",
39
+ "so": "so",
40
+ })
41
+
42
+ # Update global attributes:
43
+ ds.attrs.clear()
44
+
45
+ ds = ds.assign_attrs({
46
+ "Conventions": "CF-1.0",
47
+ "title": "Multi Observation Global Ocean 3D Temperature Salinity Height Geostrophic Current and MLD.",
48
+ "description": "Multi Observation Global Ocean ARMOR3D multi-year reprocessed temperature salinity, sea surface height, geostrophic current and mixed layer depth climatology on 1/8 degree regular grid and 50 depth levels.",
49
+ "source": "Numerical models: Multiple Linear Regression, Optimal Interpolation. In-situ observations: Copernicus In Situ TAC (including Argo, XBT, CTD and moorings) Copernicus Sea Level TAC, CNES-CLS22 Mean Dynamic Topography, OSTIA Sea Surface Temperature Analysis, Copernicus MOB TAC (Sea Surface Salinity), and World Ocean Atlas 2018 (WOA18).",
50
+ "dataset_type": "observation",
51
+ "product_type": "climatology",
52
+ "product_version": "2.0",
53
+ "institution": "Copernicus Marine Service, Mercator Ocean International, France",
54
+ "citation": "Multi Observation Global Ocean 3D Temperature Salinity Height Geostrophic Current and MLD. E.U. Copernicus Marine Service Information (CMEMS). Marine Data Store (MDS). DOI: 10.48670/moi-00052 (Accessed on 21 04 2026).",
55
+ "references": "Guinehut S., A.-L. Dhomps, G. Larnicol and P.-Y. Le Traon, 2012: High resolution 3D temperature and salinity fields derived from in situ and satellite observations. Ocean Sci., 8(5):845-857. Mulet, S., M.-H. Rio, A. Mignot, S. Guinehut and R. Morrow, 2012: A new estimate of the global 3D geostrophic ocean circulation based on satellite data and in-situ measurements. Deep Sea Research Part II : Topical Studies in Oceanography, 77-80(0):70-81.",
56
+ "acknowledgement": "Generated using E.U. Copernicus Marine Service Information; https://doi.org/10.48670/moi-00052.",
57
+ "license": "ARMOR3D data were obtained from https://doi.org/10.48670/moi-00052, and are provided under the Copernicus Marine Environment Monitoring Service Service Level Agreement (SLA) https://marine.copernicus.eu/user-corner/service-commitments-and-licence?pk_vid=42ac3e352be888641780994034c3bb6e",
58
+ "doi": "10.48670/moi-00052",
59
+ "platform": "gr",
60
+ "horizontal_grid_type": "regular rectilinear",
61
+ "horizontal_grid_resolution": "0.125 degree",
62
+ "vertical_grid_type": "z",
63
+ "vertical_grid_coordinate": "depth",
64
+ "vertical_grid_levels": 50,
65
+ "aggregation": "mean",
66
+ "aggregation_frequency": "monthly",
67
+ "status": "completed",
68
+ "update_frequency": "None",
69
+ "bbox": "[-180.0, 180.0, -90.0, 90.0]",
70
+ })
71
+
72
+ logging.info("-> Completed: Updated ARMOR3D CF-metadata.")
73
+
74
+ # -- Calculate climate normal monthly climatologies -- #
75
+ output_dir = "/dssgfs01/scratch/otooth/npd_data/observations/ARMOR3D/climatology/"
76
+ start_years = [1971, 1981, 1991]
77
+ end_years = [2000, 2010, 2020]
78
+
79
+ for start_year, end_year in zip(start_years, end_years):
80
+ logging.info(f"In Progress: Calculating monthly climatology for {start_year}-{end_year} climate normal period...")
81
+ # Calculate monthly climatology for the specified period:
82
+ ds_climatology = ds.sel(time=slice(f'{start_year}-01', f'{end_year}-12')).groupby('time.month').mean()
83
+
84
+ # Add ancillary variables:
85
+ ds_climatology['mask'] = compute_land_sea_mask(ds['thetao'].isel(time=0, depth=0))
86
+ ds_climatology['dx'] = compute_dx(ds)
87
+ ds_climatology['dy'] = compute_dy(ds)
88
+ ds_climatology['cell_area'] = compute_cell_area(ds)
89
+ # Custom ancillary variables:
90
+ ds_climatology['cell_thickness'] = compute_cell_thickness(ds_climatology)
91
+ ds_climatology['cell_volume'] = ds_climatology['cell_thickness'] * ds_climatology['cell_area']
92
+
93
+ # Update attributes for custom ancillary variables:
94
+ ds_climatology['cell_volume'].attrs.update({
95
+ 'long_name': "Grid-Cell Volume",
96
+ 'standard_name': "cell_volume",
97
+ 'units': "m3",
98
+ })
99
+
100
+ # Update time bounds to reflect climatological period:
101
+ ds_climatology['time_bnds'] = xr.DataArray(data=np.zeros((12, 2), dtype='datetime64[M]'), dims=('month', 'bnds'))
102
+ ds_climatology['time_bnds'][:, 0] = np.arange(f'{start_year}-01', f'{start_year+1}-01', dtype='datetime64[M]')
103
+ ds_climatology['time_bnds'][:, 1] = np.arange(f'{end_year}-01', f'{end_year+1}-01', dtype='datetime64[M]')
104
+ logging.info(f"-> Completed: Calculated monthly climatology for {start_year}-{end_year} climate normal period.")
105
+
106
+ # Update title attribute to reflect climatological period:
107
+ ds_climatology.attrs['title'] = f"Multi Observation Global Ocean 3D Temperature Salinity Height Geostrophic Current and MLD monthly climatology ({start_year}-{end_year})."
108
+ ds_climatology.attrs['start_datetime'] = f"{start_year}-01-01"
109
+ ds_climatology.attrs['end_datetime'] = f"{end_year}-12-31"
110
+
111
+ # Update variable encodings:
112
+ blosccodec = zarr.codecs.BloscCodec(cname="lz4", clevel=5, shuffle=zarr.codecs.BloscShuffle.shuffle)
113
+
114
+ # Infer variable chunk sizes (preserve source chunking):
115
+ for var in list(ds_climatology.data_vars) + list(ds_climatology.coords):
116
+ source_chunks = ds_climatology[var].encoding.get('chunks', None)
117
+ ds_climatology[var].encoding.clear()
118
+ ds_climatology[var].encoding['compressors'] = [blosccodec]
119
+ # Cast float64 back to float32 to match source precision:
120
+ if ds_climatology[var].dtype == np.float64:
121
+ ds_climatology[var].encoding['dtype'] = 'float32'
122
+ if source_chunks is not None:
123
+ ds_climatology[var].encoding['chunks'] = source_chunks
124
+
125
+ # Write monthly climatology to Zarr:
126
+ output_filepath = f"{output_dir}ARMOR3D_global_{start_year}_{end_year}_monthly_climatology.zarr"
127
+ ds_climatology.to_zarr(output_filepath, zarr_format=3, mode="w")
128
+ logging.info(f"-> Completed: Saved monthly climatology for {start_year}-{end_year} climate normal period to {output_filepath}.")
129
+
130
+ # -- Close ARMOR3D analysis datasets -- #
131
+ ds_climatology.close()
132
+ ds.close()
133
+
134
+ if __name__ == "__main__":
135
+ main()
@@ -0,0 +1,33 @@
1
+ """
2
+ Download global domain of ARMOR-3D dataset from Copernicus Marine.
3
+ This script downloads the monthly reprocessed files from 1993 to 2024.
4
+
5
+ Created By: Ollie Tooth
6
+ Created On: 2026-04-21
7
+ Contact: oliver.tooth@noc.ac.uk
8
+
9
+ Note: This download script should be run using the env_oceandatastore environment.
10
+ """
11
+
12
+ # Import the Copernicus Marine API toolbox:
13
+ import copernicusmarine
14
+
15
+ # Define filepath to credentials:
16
+ credentials_fpath = ".../copernicusmarine-credentials"
17
+ # Define output directory:
18
+ out_fdir = "/dssgfs01/scratch/otooth/npd_data/observations/ARMOR3D/"
19
+
20
+ # Download the ARMOR3D analysis dataset for North Atlantic subdomain:
21
+ for year in range(2001, 2025):
22
+ print(f"In Progress: Downloading ARMOR3D analysis dataset for: {year}...")
23
+ copernicusmarine.subset(
24
+ dataset_id="cmems_obs-mob_glo_phy_my_0.125deg_P1M-m",
25
+ variables=["mlotst", "so", "to", "ugo", "vgo", "zo"],
26
+ start_datetime=f"{year}-01-01T00:00:00",
27
+ end_datetime=f"{year}-12-01T00:00:00",
28
+ credentials_file=credentials_fpath,
29
+ output_directory=out_fdir,
30
+ file_format="zarr",
31
+ output_filename=f"armor-3d_rep_monthly_NA_{year}.zarr",
32
+ )
33
+ print(f"Completed: downloading ARMOR3D analysis dataset for: {year}.")
@@ -0,0 +1,32 @@
1
+ #!/bin/bash
2
+ #SBATCH --job-name=armor3d_monthly_climatology
3
+ #SBATCH --partition=compute
4
+ #SBATCH --time=04:00:00
5
+ #SBATCH --ntasks-per-core=1
6
+ #SBATCH --ntasks-per-node=64
7
+ #SBATCH --ntasks-per-socket=32
8
+ #SBATCH --nodes=1
9
+
10
+ # ==============================================================
11
+ # run_create_ARMOR3D_P1M-m_monthly_climatology.slurm
12
+ #
13
+ # Description: SLURM script to create ARMOR3D REP monthly
14
+ # climatology datasets.
15
+ #
16
+ # Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
17
+ # Created On: 2026-06-09
18
+ #
19
+ # ==============================================================
20
+ set -euo pipefail
21
+
22
+ # -- Python Environment -- #
23
+ # Activate miniconda environment:
24
+ source .../miniforge3/bin/activate
25
+ conda activate env_ods
26
+
27
+ # -- Create ARMOR3D REP monthly climatology datasets -- #
28
+ echo "In Progress: Creating ARMOR3D REP monthly climatology datasets..."
29
+
30
+ python3 create_ARMOR3D_P1M-m_monthly_climatology.py
31
+
32
+ echo "Completed: Created ARMOR3D REP monthly climatology datasets."
@@ -0,0 +1,32 @@
1
+ #!/bin/bash
2
+ #SBATCH --job-name=send_armor3d_my_climatology
3
+ #SBATCH --partition=compute
4
+ #SBATCH --time=04:00:00
5
+ #SBATCH --ntasks-per-core=1
6
+ #SBATCH --ntasks-per-node=64
7
+ #SBATCH --ntasks-per-socket=32
8
+ #SBATCH --nodes=1
9
+
10
+ # ==============================================================
11
+ # run_send_ARMOR3D_P1M-m_climatology_to_os.slurm
12
+ #
13
+ # Description: SLURM script to send the ARMOR3D P1M-m monthly
14
+ # climatology datasets to Icechunk repository.
15
+ #
16
+ # Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
17
+ # Created On: 2026-06-09
18
+ #
19
+ # ==============================================================
20
+ set -euo pipefail
21
+
22
+ # -- Python Environment -- #
23
+ # Activate miniconda environment:
24
+ source .../miniforge3/bin/activate
25
+ conda activate env_ods
26
+
27
+ # -- Send ARMOR3D P1M-m monthly climatology datasets to JASMIN OS -- #
28
+ echo "In Progress: Sending ARMOR3D P1M-m monthly climatology to Icechunk..."
29
+
30
+ python3 send_ARMOR3D_P1m-m_monthly_climatology_to_os.py
31
+
32
+ echo "Completed: Sent ARMOR3D P1M-m monthly climatology to Icechunk."
@@ -0,0 +1,32 @@
1
+ #!/bin/bash
2
+ #SBATCH --job-name=armor3d_my_monthly
3
+ #SBATCH --partition=compute
4
+ #SBATCH --time=04:00:00
5
+ #SBATCH --ntasks-per-core=1
6
+ #SBATCH --ntasks-per-node=64
7
+ #SBATCH --ntasks-per-socket=32
8
+ #SBATCH --nodes=1
9
+
10
+ # ==============================================================
11
+ # run_send_ARMOR3D_P1M-m_monthly_to_os.slurm
12
+ #
13
+ # Description: SLURM script to send the ARMOR3D P1M-m
14
+ # monthly dataset to Icechunk repository.
15
+ #
16
+ # Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
17
+ # Created On: 2026-05-29
18
+ #
19
+ # ==============================================================
20
+ set -euo pipefail
21
+
22
+ # -- Python Environment -- #
23
+ # Activate miniconda environment:
24
+ source .../miniforge3/bin/activate
25
+ conda activate env_ods
26
+
27
+ # -- Send ARMOR3D P1M-m monthly to JASMIN OS -- #
28
+ echo "In Progress: Sending ARMOR3D P1M-m monthly to Icechunk..."
29
+
30
+ python3 send_ARMOR3D_P1m-m_monthly_to_os.py
31
+
32
+ echo "Completed: Sent ARMOR3D P1M-m monthly to Icechunk."
@@ -0,0 +1,32 @@
1
+ #!/bin/bash
2
+ #SBATCH --job-name=armor3d_monthly_update
3
+ #SBATCH --partition=compute
4
+ #SBATCH --time=01:00:00
5
+ #SBATCH --ntasks-per-core=1
6
+ #SBATCH --ntasks-per-node=64
7
+ #SBATCH --ntasks-per-socket=32
8
+ #SBATCH --nodes=1
9
+
10
+ # ==============================================================
11
+ # run_update_ARMOR3D_P1m-m_monthly_to_os.slurm
12
+ #
13
+ # Description: SLURM script to update the ARMOR3D P1m-m
14
+ # monthly dataset in Icechunk repository.
15
+ #
16
+ # Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
17
+ # Created On: 2026-05-29
18
+ #
19
+ # ==============================================================
20
+ set -euo pipefail
21
+
22
+ # -- Python Environment -- #
23
+ # Activate miniconda environment:
24
+ source .../miniforge3/bin/activate
25
+ conda activate env_ods
26
+
27
+ # -- Update ARMOR3D P1m-m monthly in JASMIN OS -- #
28
+ echo "In Progress: Updating ARMOR3D P1m-m monthly in Icechunk..."
29
+
30
+ python3 update_ARMOR3D_P1m-m_monthly_to_os.py
31
+
32
+ echo "Completed: Updated ARMOR3D P1m-m monthly in Icechunk."
@@ -0,0 +1,99 @@
1
+ # =========================================================
2
+ # send_ARMOR3D_P1m-m_monthly_climatology_to_os.py
3
+ #
4
+ # Script to write ARMOR3D monthly climatologies to
5
+ # Icechunk repositories in JASMIN cloud object storage.
6
+ #
7
+ # Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
8
+ # =========================================================
9
+ import logging
10
+
11
+ import xarray as xr
12
+ import zarr
13
+
14
+ from OceanDataStore.cli import send_to_icechunk, initialise_logging
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+
19
+ def main():
20
+ # ========== Initialise OceanDataStore Logging ========== #
21
+ initialise_logging()
22
+
23
+ # ========== Prepare Data ========== #
24
+ # Open ARMOR3D monthly climatology datasets:
25
+ filepaths = [
26
+ "/dssgfs01/scratch/otooth/npd_data/observations/ARMOR3D/climatology/ARMOR3D_global_1971_2000_monthly_climatology.zarr",
27
+ "/dssgfs01/scratch/otooth/npd_data/observations/ARMOR3D/climatology/ARMOR3D_global_1981_2010_monthly_climatology.zarr",
28
+ "/dssgfs01/scratch/otooth/npd_data/observations/ARMOR3D/climatology/ARMOR3D_global_1991_2020_monthly_climatology.zarr"
29
+ ]
30
+
31
+ # Define start & end years of climatology periods:
32
+ start_years = [1971, 1981, 1991]
33
+ end_years = [2000, 2010, 2020]
34
+
35
+ # ========== Send to Icechunk Repository ========== #
36
+ bucket = "armor3d"
37
+ exists = False
38
+ store_credentials_json = ".../credentials/jasmin_os_credentials.json"
39
+ branch = "main"
40
+ variable_commits = True
41
+ config_kwargs = {
42
+ "temporary_directory":".../OceanDataStore/OceanDataStore/data/ARMOR3D/",
43
+ "local_directory":".../OceanDataStore/OceanDataStore/data/ARMOR3D/"
44
+ }
45
+ cluster_kwargs = {
46
+ "n_workers" : 30,
47
+ "threads_per_worker" : 1,
48
+ "memory_limit":"3GB"
49
+ }
50
+
51
+ for filepath, start_yr, end_yr in zip(filepaths, start_years, end_years):
52
+ # Open ARMOR3D monthly climatology dataset:
53
+ ds = xr.open_dataset(filepath, engine='zarr')
54
+
55
+ # Optimise chunk sizes for spatial analysis:
56
+ for var in ds.data_vars:
57
+ if ds[var].ndim == 4:
58
+ ds[var] = ds[var].chunk({'month': 1, 'depth': 3, 'latitude': 689, 'longitude': 1440})
59
+ ds[var].encoding['chunks'] = (1, 3, 689, 1440)
60
+ elif ds[var].ndim == 3:
61
+ if "month" in ds[var].dims:
62
+ ds[var] = ds[var].chunk({'month': 1, 'latitude': 1378, 'longitude': 2880})
63
+ ds[var].encoding['chunks'] = (1, 1378, 2880)
64
+ elif "depth" in ds[var].dims:
65
+ ds[var] = ds[var].chunk({'depth': 10, 'latitude': 1378, 'longitude': 2880})
66
+ ds[var].encoding['chunks'] = (10, 1378, 2880)
67
+ elif (ds[var].ndim == 2):
68
+ if "latitude" in ds[var].dims and "longitude" in ds[var].dims:
69
+ ds[var] = ds[var].chunk({'latitude': 1378, 'longitude': 2880})
70
+ ds[var].encoding['chunks'] = (1378, 2880)
71
+ elif ds[var].ndim == 1:
72
+ ds[var] = ds[var].chunk({'depth': 50})
73
+ ds[var].encoding['chunks'] = (50,)
74
+
75
+ # Update variable encodings:
76
+ blosccodec = zarr.codecs.BloscCodec(cname="zstd", clevel=5, shuffle=zarr.codecs.BloscShuffle.shuffle)
77
+ for var in list(ds.data_vars) + list(ds.coords):
78
+ ds[var].encoding['compressors'] = [blosccodec]
79
+
80
+ # Define prefix and commit message based on climatology period:
81
+ prefix = f"armor3d_global_my_{start_yr}_{end_yr}_monthly_climatology"
82
+ commit_message = f"Added ARMOR3D Global monthly climatology ({start_yr}-{end_yr})."
83
+
84
+ send_to_icechunk(
85
+ file=ds.drop_vars("time"),
86
+ bucket=bucket,
87
+ object_prefix=prefix,
88
+ store_credentials_json=store_credentials_json,
89
+ exists=exists,
90
+ append_dim='month',
91
+ branch=branch,
92
+ commit_message=commit_message,
93
+ variable_commits=variable_commits,
94
+ dask_config_kwargs=config_kwargs,
95
+ dask_cluster_kwargs=cluster_kwargs,
96
+ )
97
+
98
+ if __name__ == "__main__":
99
+ main()
@@ -0,0 +1,147 @@
1
+ # =========================================================
2
+ # send_EN4.2.2_analyses_g10_to_os.py
3
+ #
4
+ # Script to write EN.4.2.2 analyses to Icechunk repository
5
+ # in JASMIN cloud object storage.
6
+ #
7
+ # Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
8
+ # =========================================================
9
+ import logging
10
+
11
+ import xarray as xr
12
+ import zarr
13
+
14
+ from OceanDataStore.cli import initialise_logging, send_to_icechunk
15
+ from OceanDataStore.data.utils import (
16
+ compute_dx,
17
+ compute_dy,
18
+ compute_cell_area,
19
+ compute_cell_thickness,
20
+ compute_land_sea_mask,
21
+ )
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+
26
+ def main():
27
+ # ========== Initialise OceanDataStore Logging ========== #
28
+ initialise_logging()
29
+
30
+ # ========== Prepare Data ========== #
31
+ # Open complete ARMOR3D REP monthly climatology dataset:
32
+ filepath = "/dssgfs01/scratch/otooth/npd_data/observations/ARMOR3D/armor-3d_rep_monthly_NA_*.zarr"
33
+ ds = xr.open_mfdataset(filepath, compat="no_conflicts", data_vars="all", engine="zarr")
34
+ logging.info("-> Completed: Opened ARMOR-3D REP monthly climatology dataset from Zarr stores.")
35
+
36
+ # Rename variables to standard names:
37
+ ds = ds.rename({"to": "thetao",
38
+ "so": "so",
39
+ })
40
+
41
+ # Update global attributes:
42
+ ds.attrs.clear()
43
+
44
+ ds = ds.assign_attrs({
45
+ "Conventions": "CF-1.0",
46
+ "title": "Multi Observation Global Ocean 3D Temperature Salinity Height Geostrophic Current and MLD.",
47
+ "description": "Multi Observation Global Ocean ARMOR3D multi-year reprocessed temperature salinity, sea surface height, geostrophic current and mixed layer depth monthly timeseries on 1/8 degree regular grid and 50 depth levels.",
48
+ "source": "Numerical models: Multiple Linear Regression, Optimal Interpolation. In-situ observations: Copernicus In Situ TAC (including Argo, XBT, CTD and moorings) Copernicus Sea Level TAC, CNES-CLS22 Mean Dynamic Topography, OSTIA Sea Surface Temperature Analysis, Copernicus MOB TAC (Sea Surface Salinity), and World Ocean Atlas 2018 (WOA18).",
49
+ "dataset_type": "observation",
50
+ "product_type": "timeseries",
51
+ "product_version": "2.0",
52
+ "institution": "Copernicus Marine Service, Mercator Ocean International, France",
53
+ "citation": "Multi Observation Global Ocean 3D Temperature Salinity Height Geostrophic Current and MLD. E.U. Copernicus Marine Service Information (CMEMS). Marine Data Store (MDS). DOI: 10.48670/moi-00052 (Accessed on 21 04 2026).",
54
+ "references": "Guinehut S., A.-L. Dhomps, G. Larnicol and P.-Y. Le Traon, 2012: High resolution 3D temperature and salinity fields derived from in situ and satellite observations. Ocean Sci., 8(5):845-857. Mulet, S., M.-H. Rio, A. Mignot, S. Guinehut and R. Morrow, 2012: A new estimate of the global 3D geostrophic ocean circulation based on satellite data and in-situ measurements. Deep Sea Research Part II : Topical Studies in Oceanography, 77-80(0):70-81.",
55
+ "acknowledgement": "Generated using E.U. Copernicus Marine Service Information; https://doi.org/10.48670/moi-00052.",
56
+ "license": "ARMOR3D data were obtained from https://doi.org/10.48670/moi-00052, and are provided under the Copernicus Marine Environment Monitoring Service Service Level Agreement (SLA) https://marine.copernicus.eu/user-corner/service-commitments-and-licence?pk_vid=42ac3e352be888641780994034c3bb6e",
57
+ "doi": "10.48670/moi-00052",
58
+ "platform": "gr",
59
+ "horizontal_grid_type": "regular rectilinear",
60
+ "horizontal_grid_resolution": "0.125 degree",
61
+ "vertical_grid_type": "z",
62
+ "vertical_grid_coordinate": "depth",
63
+ "vertical_grid_levels": 50,
64
+ "aggregation": "mean",
65
+ "aggregation_frequency": "monthly",
66
+ "status": "ongoing",
67
+ "update_frequency": "quarterly",
68
+ "bbox": "[-180.0, 180.0, -90.0, 90.0]",
69
+ })
70
+
71
+ # Add ancillary variables:
72
+ ds['mask'] = compute_land_sea_mask(ds['thetao'].isel(time=0, depth=0))
73
+ ds['dx'] = compute_dx(ds)
74
+ ds['dy'] = compute_dy(ds)
75
+ ds['cell_area'] = compute_cell_area(ds)
76
+ # Custom ancillary variables:
77
+ ds['cell_thickness'] = compute_cell_thickness(ds)
78
+ ds['cell_volume'] = ds['cell_thickness'] * ds['cell_area']
79
+
80
+ # Update attributes for custom ancillary variables:
81
+ ds['cell_volume'].attrs.update({
82
+ 'long_name': "Grid-Cell Volume",
83
+ 'standard_name': "cell_volume",
84
+ 'units': "m3",
85
+ })
86
+
87
+ # ========== Send to Icechunk Repository ========== #
88
+ bucket = "armor3d"
89
+ exists = False
90
+ store_credentials_json = ".../credentials/jasmin_os_credentials.json"
91
+ branch = "main"
92
+ variable_commits = True
93
+ config_kwargs = {
94
+ "temporary_directory":".../OceanDataStore/OceanDataStore/data/ARMOR3D/",
95
+ "local_directory":".../OceanDataStore/OceanDataStore/data/ARMOR3D/"
96
+ }
97
+ cluster_kwargs = {
98
+ "n_workers" : 25,
99
+ "threads_per_worker" : 1,
100
+ "memory_limit":"4GB"
101
+ }
102
+
103
+ # Optimise chunk sizes for spatial analysis:
104
+ for var in ds.data_vars:
105
+ if ds[var].ndim == 4:
106
+ ds[var] = ds[var].chunk({'time': 1, 'depth': 3, 'latitude': 689, 'longitude': 1440})
107
+ ds[var].encoding['chunks'] = (1, 3, 689, 1440)
108
+ elif ds[var].ndim == 3:
109
+ if "time" in ds[var].dims:
110
+ ds[var] = ds[var].chunk({'time': 1, 'latitude': 1378, 'longitude': 2880})
111
+ ds[var].encoding['chunks'] = (1, 1378, 2880)
112
+ elif "depth" in ds[var].dims:
113
+ ds[var] = ds[var].chunk({'depth': 10, 'latitude': 1378, 'longitude': 2880})
114
+ ds[var].encoding['chunks'] = (10, 1378, 2880)
115
+ elif (ds[var].ndim == 2):
116
+ if "latitude" in ds[var].dims and "longitude" in ds[var].dims:
117
+ ds[var] = ds[var].chunk({'latitude': 1378, 'longitude': 2880})
118
+ ds[var].encoding['chunks'] = (1378, 2880)
119
+ elif ds[var].ndim == 1:
120
+ ds[var] = ds[var].chunk({'depth': 50})
121
+ ds[var].encoding['chunks'] = (50,)
122
+
123
+ # Update variable encodings:
124
+ blosccodec = zarr.codecs.BloscCodec(cname="zstd", clevel=5, shuffle=zarr.codecs.BloscShuffle.shuffle)
125
+ for var in list(ds.data_vars) + list(ds.coords):
126
+ ds[var].encoding['compressors'] = [blosccodec]
127
+
128
+ # Define prefix and commit message based on period:
129
+ prefix = "armor3d_global_my_monthly"
130
+ commit_message = "Added ARMOR3D Global Monthly (1993-1999)."
131
+
132
+ send_to_icechunk(
133
+ file=ds.sel(time=slice("1993-01-01", "1999-12-31")),
134
+ bucket=bucket,
135
+ object_prefix=prefix,
136
+ store_credentials_json=store_credentials_json,
137
+ exists=exists,
138
+ append_dim='time',
139
+ branch=branch,
140
+ commit_message=commit_message,
141
+ variable_commits=variable_commits,
142
+ dask_config_kwargs=config_kwargs,
143
+ dask_cluster_kwargs=cluster_kwargs,
144
+ )
145
+
146
+ if __name__ == "__main__":
147
+ main()