OceanDataStore 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- OceanDataStore/__init__.py +21 -0
- OceanDataStore/catalog/__init__.py +12 -0
- OceanDataStore/catalog/oceandatacatalog.py +1242 -0
- OceanDataStore/catalog/stac/README.md +34 -0
- OceanDataStore/catalog/stac/__init__.py +30 -0
- OceanDataStore/catalog/stac/create_noc_stac.py +109 -0
- OceanDataStore/catalog/stac/npd_era5_collection.py +364 -0
- OceanDataStore/catalog/stac/npd_jra55_collection.py +196 -0
- OceanDataStore/catalog/stac/ods_obs_collection.py +534 -0
- OceanDataStore/catalog/stac/rapid_evo_collection.py +309 -0
- OceanDataStore/catalog/stac/template_collection.py +85 -0
- OceanDataStore/catalog/stac/utils.py +476 -0
- OceanDataStore/cli/__init__.py +34 -0
- OceanDataStore/cli/arg_parser.py +182 -0
- OceanDataStore/cli/cli.py +203 -0
- OceanDataStore/cli/exceptions.py +83 -0
- OceanDataStore/cli/icechunk.py +888 -0
- OceanDataStore/cli/logging.py +52 -0
- OceanDataStore/cli/object_store.py +293 -0
- OceanDataStore/cli/utils.py +275 -0
- OceanDataStore/cli/zarr.py +870 -0
- OceanDataStore/data/ARMOR3D/create_ARMOR3D_P1M-m_monthly_climatology.py +135 -0
- OceanDataStore/data/ARMOR3D/download_ARMOR3D_0.125def_P1M-m_1993_2024.py +33 -0
- OceanDataStore/data/ARMOR3D/run_create_ARMOR3D_P1M-m_monthly_climatology.slurm +32 -0
- OceanDataStore/data/ARMOR3D/run_send_ARMOR3D_P1M-m_climatology_to_os.slurm +32 -0
- OceanDataStore/data/ARMOR3D/run_send_ARMOR3D_P1M-m_monthly_to_os.slurm +32 -0
- OceanDataStore/data/ARMOR3D/run_update_ARMOR3D_P1m-m_monthly_to_os.slurm +32 -0
- OceanDataStore/data/ARMOR3D/send_ARMOR3D_P1m-m_monthly_climatology_to_os.py +99 -0
- OceanDataStore/data/ARMOR3D/send_ARMOR3D_P1m-m_monthly_to_os.py +147 -0
- OceanDataStore/data/ARMOR3D/update_ARMOR3D_P1m-m_monthly_to_os.py +143 -0
- OceanDataStore/data/EN.4.2.2/create_EN4.2.2_analysis_g10_climatology.py +162 -0
- OceanDataStore/data/EN.4.2.2/download_EN4.2.2_analysis_g10_data.sh +51 -0
- OceanDataStore/data/EN.4.2.2/run_send_EN4.2.2_analysis_g10_climatology_to_os.slurm +32 -0
- OceanDataStore/data/EN.4.2.2/run_send_EN4.2.2_analysis_g10_monthly_to_os.slurm +32 -0
- OceanDataStore/data/EN.4.2.2/run_update_EN4.2.2_analysis_g10_monthly_to_os.slurm +32 -0
- OceanDataStore/data/EN.4.2.2/send_EN4.2.2_analysis_g10_monthly_climatology_to_os.py +76 -0
- OceanDataStore/data/EN.4.2.2/send_EN4.2.2_analysis_g10_monthly_to_os.py +165 -0
- OceanDataStore/data/EN.4.2.2/update_EN4.2.2_analysis_g10_monthly_to_os.py +161 -0
- OceanDataStore/data/ERA5/create_ERA5_daily_climatology.py +110 -0
- OceanDataStore/data/ERA5/create_ERA5_daily_mean.py +69 -0
- OceanDataStore/data/ERA5/create_ERA5_monthly_mean.py +74 -0
- OceanDataStore/data/ERA5/run_create_ERA5_daily_climatology.slurm +54 -0
- OceanDataStore/data/ERA5/run_send_ERA5_daily_climatology_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/run_send_ERA5_daily_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/run_send_ERA5_monthly_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/run_update_ERA5_daily_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/run_update_ERA5_monthly_to_os.slurm +32 -0
- OceanDataStore/data/ERA5/send_ERA5_daily_climatology_to_os.py +159 -0
- OceanDataStore/data/ERA5/send_ERA5_daily_to_os.py +141 -0
- OceanDataStore/data/ERA5/send_ERA5_monthly_to_os.py +173 -0
- OceanDataStore/data/ERA5/update_ERA5_daily_to_os.py +141 -0
- OceanDataStore/data/ERA5/update_ERA5_monthly_to_os.py +169 -0
- OceanDataStore/data/HadISST/download_HadISST1_data.sh +43 -0
- OceanDataStore/data/HadISST/run_send_HadISST1_monthly_to_os.slurm +32 -0
- OceanDataStore/data/HadISST/send_HadISST1_monthly_to_os.py +133 -0
- OceanDataStore/data/NSIDC/download_NSIDC_monthly_1979_2025_data.sh +54 -0
- OceanDataStore/data/NSIDC/process_NSIDC_SSI_Antarctic_data.py +130 -0
- OceanDataStore/data/NSIDC/process_NSIDC_SSI_Arctic_data.py +129 -0
- OceanDataStore/data/NSIDC/run_send_NSIDC_v4.0_to_OS.slurm +32 -0
- OceanDataStore/data/NSIDC/send_NSIDC_SII_v4.0_to_os.py +140 -0
- OceanDataStore/data/OISST/create_OISSTv2_daily_climatology.py +83 -0
- OceanDataStore/data/OISST/download_oisstv2_data.sh +43 -0
- OceanDataStore/data/OISST/run_create_OISSTv2_daily_climatology.slurm +44 -0
- OceanDataStore/data/OISST/run_send_OISSTv2_daily_climatology_to_os.slurm +32 -0
- OceanDataStore/data/OISST/run_send_OISSTv2_daily_to_os.slurm +32 -0
- OceanDataStore/data/OISST/run_send_OISSTv2_monthly_climatology_to_os.slurm +32 -0
- OceanDataStore/data/OISST/run_send_OISSTv2_monthly_to_os.slurm +32 -0
- OceanDataStore/data/OISST/run_update_OISSTv2_daily_to_os.slurm +32 -0
- OceanDataStore/data/OISST/send_OISSTv2_daily_climatology_to_os.py +154 -0
- OceanDataStore/data/OISST/send_OISSTv2_daily_ltm_climatology_to_os.py +151 -0
- OceanDataStore/data/OISST/send_OISSTv2_daily_to_os.py +142 -0
- OceanDataStore/data/OISST/send_OISSTv2_monthly_climatology_to_os.py +150 -0
- OceanDataStore/data/OISST/send_OISSTv2_monthly_to_os.py +145 -0
- OceanDataStore/data/OISST/update_OISSTv2_daily_to_os.py +142 -0
- OceanDataStore/data/OSTIA/create_OSTIA_daily_climatology.py +120 -0
- OceanDataStore/data/OSTIA/download_OSTIA_NRT.py +42 -0
- OceanDataStore/data/OSTIA/download_OSTIA_REP_1981_2025.py +42 -0
- OceanDataStore/data/OSTIA/run_create_OSTIA_daily_climatology.slurm +54 -0
- OceanDataStore/data/OSTIA/run_send_OSTIA_daily_climatology_to_os.slurm +32 -0
- OceanDataStore/data/OSTIA/run_send_OSTIA_nrt_daily_to_os.slurm +32 -0
- OceanDataStore/data/OSTIA/run_send_OSTIA_rep_daily_to_os.slurm +32 -0
- OceanDataStore/data/OSTIA/run_update_OSTIA_daily_to_os.slurm +33 -0
- OceanDataStore/data/OSTIA/send_OSTIA_daily_climatology_to_os.py +194 -0
- OceanDataStore/data/OSTIA/send_OSTIA_nrt_daily_to_os.py +141 -0
- OceanDataStore/data/OSTIA/send_OSTIA_rep_daily_to_os.py +145 -0
- OceanDataStore/data/OSTIA/update_OSTIA_copernicus_nrt_daily_to_os.py +144 -0
- OceanDataStore/data/OSTIA/update_OSTIA_nrt_daily_to_os.py +137 -0
- OceanDataStore/data/WOA23/download_WOA23_climatology.sh +41 -0
- OceanDataStore/data/WOA23/run_send_WOA23_annual_climatology_to_os.slurm +32 -0
- OceanDataStore/data/WOA23/run_send_WOA23_monthly_climatology_to_os.slurm +32 -0
- OceanDataStore/data/WOA23/send_WOA23_annual_climatology_to_os.py +263 -0
- OceanDataStore/data/WOA23/send_WOA23_monthly_climatology_to_os.py +292 -0
- OceanDataStore/data/update_icechunk_repo_attrs.py +76 -0
- OceanDataStore/data/update_noc_npd_era5v1_attrs.py +172 -0
- OceanDataStore/data/utils.py +506 -0
- OceanDataStore/zarr.py +993 -0
- oceandatastore-0.3.0.dist-info/METADATA +184 -0
- oceandatastore-0.3.0.dist-info/RECORD +104 -0
- oceandatastore-0.3.0.dist-info/WHEEL +5 -0
- oceandatastore-0.3.0.dist-info/entry_points.txt +2 -0
- oceandatastore-0.3.0.dist-info/licenses/LICENSE +201 -0
- oceandatastore-0.3.0.dist-info/scm_file_list.json +154 -0
- oceandatastore-0.3.0.dist-info/scm_version.json +8 -0
- oceandatastore-0.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
## OceanDataStore Spatio-Temporal Access Catalogs (STAC)
|
|
2
|
+
|
|
3
|
+
### Creating the NOC STAC:
|
|
4
|
+
To create a new local copy of the NOC Spatio-Temporal Access Catalog, run the `create_noc_stac.py` script in a Python virtual environment:
|
|
5
|
+
|
|
6
|
+
```python
|
|
7
|
+
python3 create_noc_stac.py
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
This will create a new directory `noc-stac` inside the `catalogs/` directory storing a STAC containing NOC Near-Present Day & RAPID-Evolution model datasets. Further datasets will be added to the `noc-stac` in the future.
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
### Uploading a STAC to the JASMIN Object Store:
|
|
15
|
+
|
|
16
|
+
To upload a new or updated STAC to the `noc-msm` JASMIN Cloud Object Store tenancy, use a applicable CLI, such as `s5cmd`.
|
|
17
|
+
|
|
18
|
+
Note, this will require permission to write to the `oceandatastore` bucket within the `noc-msm` tenancy.
|
|
19
|
+
|
|
20
|
+
For example, to upload an updated `noc-stac`:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
# Remove existing noc-stac:
|
|
24
|
+
s5cmd --endpoint-url https://noc-msm-o.s3-ext.jc.rl.ac.uk --profile noc_msm rm s3://oceandatastore/noc-stac/*
|
|
25
|
+
|
|
26
|
+
# Copy local noc-stac to OceanDataStore bucket:
|
|
27
|
+
s5cmd --endpoint-url https://noc-msm-o.s3-ext.jc.rl.ac.uk --profile noc_msm cp "path/to/noc-stac/*" s3://oceandatastore/noc-stac/
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
where `noc_msm` is the profile used to provide your user credentials for the `noc-msm` tenancy
|
|
31
|
+
|
|
32
|
+
This will make the updated STAC publicly accessible via read-only access at the following URL:
|
|
33
|
+
|
|
34
|
+
https://noc-msm-o.s3-ext.jc.rl.ac.uk/oceandatastore/noc-stac/catalog.json
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""
|
|
2
|
+
OceanDataStore: STAC Catalog Module
|
|
3
|
+
|
|
4
|
+
Tools for constructing the NOC Spatio-Temporal Asset Catalog (STAC) to local JSON files.
|
|
5
|
+
"""
|
|
6
|
+
from OceanDataStore.catalog.stac.npd_era5_collection import create_npd_era5_collection
|
|
7
|
+
from OceanDataStore.catalog.stac.npd_jra55_collection import create_npd_jra55_collection
|
|
8
|
+
from OceanDataStore.catalog.stac.rapid_evo_collection import create_rapid_evo_collection
|
|
9
|
+
from OceanDataStore.catalog.stac.ods_obs_collection import (
|
|
10
|
+
create_nsidc_collection,
|
|
11
|
+
create_woa23_collection,
|
|
12
|
+
create_oisst_collection,
|
|
13
|
+
create_en4_collection,
|
|
14
|
+
create_armor3d_collection,
|
|
15
|
+
create_hadisst_collection,
|
|
16
|
+
create_era5_collection
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
__all__ = (
|
|
20
|
+
"create_npd_era5_collection",
|
|
21
|
+
"create_npd_jra55_collection",
|
|
22
|
+
"create_rapid_evo_collection",
|
|
23
|
+
"create_nsidc_collection",
|
|
24
|
+
"create_woa23_collection",
|
|
25
|
+
"create_oisst_collection",
|
|
26
|
+
"create_en4_collection",
|
|
27
|
+
"create_armor3d_collection",
|
|
28
|
+
"create_hadisst_collection",
|
|
29
|
+
"create_era5_collection"
|
|
30
|
+
)
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""
|
|
2
|
+
create_noc_stac.py
|
|
3
|
+
|
|
4
|
+
Description:
|
|
5
|
+
Script to define the National Oceanography Centre (NOC)
|
|
6
|
+
Spatio-Temporal Access Catalog and write to JSON files.
|
|
7
|
+
|
|
8
|
+
Authors:
|
|
9
|
+
- Ollie Tooth (oliver.tooth@noc.ac.uk)
|
|
10
|
+
"""
|
|
11
|
+
# -- Import Python Modules -- #
|
|
12
|
+
import datetime
|
|
13
|
+
import logging
|
|
14
|
+
import os
|
|
15
|
+
|
|
16
|
+
import pystac
|
|
17
|
+
|
|
18
|
+
from OceanDataStore.catalog.stac import (
|
|
19
|
+
create_npd_era5_collection,
|
|
20
|
+
create_npd_jra55_collection,
|
|
21
|
+
create_rapid_evo_collection,
|
|
22
|
+
create_nsidc_collection,
|
|
23
|
+
create_woa23_collection,
|
|
24
|
+
create_oisst_collection,
|
|
25
|
+
create_en4_collection,
|
|
26
|
+
create_armor3d_collection,
|
|
27
|
+
create_hadisst_collection,
|
|
28
|
+
create_era5_collection
|
|
29
|
+
)
|
|
30
|
+
from OceanDataStore.cli import initialise_logging
|
|
31
|
+
|
|
32
|
+
logger = logging.getLogger(__name__)
|
|
33
|
+
|
|
34
|
+
def create_noc_stac():
|
|
35
|
+
"""
|
|
36
|
+
Create the NOC STAC and write to JSON files.
|
|
37
|
+
"""
|
|
38
|
+
# =========== Configure OceanDataStore Logging =========== #
|
|
39
|
+
initialise_logging()
|
|
40
|
+
|
|
41
|
+
# =========== Define NOC STAC =========== #
|
|
42
|
+
noc_stac = pystac.Catalog(id="noc-stac",
|
|
43
|
+
title="NOC STAC Catalog",
|
|
44
|
+
description="National Oceanography Centre Spatio-Temporal Asset Catalog for Ocean Model and Observational Data.\n\n**About:**\n\nThe National Oceanography Centre (NOC) is one of the world's leading oceanographic institutions and has been in existence, in its various forms, for over six decades.\nWe undertake world-leading research from coastal seas to deep water, to enhance understanding of the ocean and to address critical environmental challenges.\n\n**Links:**\n- [Website](https://noc.ac.uk)\n- [OceanDataStore](https://noc-msm.github.io/OceanDataStore/)",
|
|
45
|
+
stac_extensions=None,
|
|
46
|
+
extra_fields={
|
|
47
|
+
"last_update": datetime.datetime.now().isoformat(timespec="hours"),
|
|
48
|
+
"catalog_version": "0.3.0",
|
|
49
|
+
"contacts": "Oliver Tooth (oliver.tooth@noc.ac.uk), Adam Blaker (atb299@noc.ac.uk), Andrew Coward (acc@noc.ac.uk)",
|
|
50
|
+
},
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
logging.info(f"Completed: Created NOC STAC Catalog with ID: {noc_stac.id}")
|
|
54
|
+
|
|
55
|
+
# -- Add Rapid-Evolution Collection to NOC STAC Catalog -- #
|
|
56
|
+
rapid_evo_collection = create_rapid_evo_collection(credentials_json="/dssgfs01/working/otooth/AtlantiS/credentials/rapid_evo_credentials.json")
|
|
57
|
+
noc_stac.add_child(rapid_evo_collection)
|
|
58
|
+
|
|
59
|
+
# -- Add NOC Near-Present Day JRA55-do Collection to NOC STAC Catalog -- #
|
|
60
|
+
npd_jra55v1_collection = create_npd_jra55_collection()
|
|
61
|
+
noc_stac.add_child(npd_jra55v1_collection)
|
|
62
|
+
|
|
63
|
+
# -- Add NOC Near-Present Day ERA5 Collection to NOC STAC Catalog -- #
|
|
64
|
+
npd_era5v1_collection = create_npd_era5_collection()
|
|
65
|
+
noc_stac.add_child(npd_era5v1_collection)
|
|
66
|
+
|
|
67
|
+
logging.info(f"Completed: Added NOC Near-Present Day Collections to NOC STAC: {noc_stac.id}")
|
|
68
|
+
|
|
69
|
+
# -- Add NSIDC Sea Ice Index Collection to NOC STAC Catalog -- #
|
|
70
|
+
nsidc_collection = create_nsidc_collection()
|
|
71
|
+
noc_stac.add_child(nsidc_collection)
|
|
72
|
+
|
|
73
|
+
# -- Add WOA23 Collection to NOC STAC Catalog -- #
|
|
74
|
+
woa23_collection = create_woa23_collection()
|
|
75
|
+
noc_stac.add_child(woa23_collection)
|
|
76
|
+
|
|
77
|
+
# -- Add OISST Collection to NOC STAC Catalog -- #
|
|
78
|
+
oisst_collection = create_oisst_collection()
|
|
79
|
+
noc_stac.add_child(oisst_collection)
|
|
80
|
+
|
|
81
|
+
# -- Add EN4.2.2 Collection to NOC STAC Catalog -- #
|
|
82
|
+
en4_collection = create_en4_collection()
|
|
83
|
+
noc_stac.add_child(en4_collection)
|
|
84
|
+
|
|
85
|
+
# -- Add ARMOR3D Collection to NOC STAC Catalog -- #
|
|
86
|
+
armor3d_collection = create_armor3d_collection()
|
|
87
|
+
noc_stac.add_child(armor3d_collection)
|
|
88
|
+
|
|
89
|
+
# -- Add HadISST1 Collection to NOC STAC Catalog -- #
|
|
90
|
+
hadisst_collection = create_hadisst_collection()
|
|
91
|
+
noc_stac.add_child(hadisst_collection)
|
|
92
|
+
|
|
93
|
+
# -- Add ERA5 Collection to NOC STAC Catalog -- #
|
|
94
|
+
era5_collection = create_era5_collection()
|
|
95
|
+
noc_stac.add_child(era5_collection)
|
|
96
|
+
|
|
97
|
+
logging.info(f"Completed: Added Ocean Observation Collections to NOC STAC: {noc_stac.id}")
|
|
98
|
+
|
|
99
|
+
# -- Write NOC STAC Catalog to local filesystem -- #
|
|
100
|
+
logging.info(f"NOC STAC {noc_stac.id} Summary:")
|
|
101
|
+
print(noc_stac.describe())
|
|
102
|
+
|
|
103
|
+
noc_stac.normalize_hrefs(root_href="https://noc-msm-o.s3-ext.jc.rl.ac.uk/noc-stac/")
|
|
104
|
+
noc_stac.save(catalog_type=pystac.CatalogType.RELATIVE_PUBLISHED, dest_href=os.path.join(os.getcwd(), "noc-stac"))
|
|
105
|
+
logging.info(f"Completed: Write NOC STAC to -> {os.path.join(os.getcwd(), 'noc-stac')}")
|
|
106
|
+
|
|
107
|
+
if __name__ == "__main__":
|
|
108
|
+
# -- Create NOC STAC Catalog -- #
|
|
109
|
+
create_noc_stac()
|
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
"""
|
|
2
|
+
npd_collections.py
|
|
3
|
+
|
|
4
|
+
Description:
|
|
5
|
+
Function to create the National Oceanography Centre (NOC)
|
|
6
|
+
Near-Present Day Spatio-Temporal Access Catalog Collection.
|
|
7
|
+
|
|
8
|
+
Authors:
|
|
9
|
+
- Ollie Tooth (oliver.tooth@noc.ac.uk)
|
|
10
|
+
"""
|
|
11
|
+
# -- Import Python Modules -- #
|
|
12
|
+
import datetime
|
|
13
|
+
import logging
|
|
14
|
+
|
|
15
|
+
import pystac
|
|
16
|
+
import xarray as xr
|
|
17
|
+
|
|
18
|
+
from OceanDataStore.catalog.stac.utils import (
|
|
19
|
+
create_item_with_icechunk_asset,
|
|
20
|
+
open_icechunk_store,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def description_from_prefix(prefix: str, ds: xr.Dataset) -> str:
|
|
25
|
+
"""
|
|
26
|
+
Define Item description based on the prefix and Dataset attributes.
|
|
27
|
+
|
|
28
|
+
Parameters:
|
|
29
|
+
----------
|
|
30
|
+
prefix : str
|
|
31
|
+
Prefix of Icechunk repository,
|
|
32
|
+
|
|
33
|
+
ds : xr.Dataset
|
|
34
|
+
Dataset opened from Icechunk repository, which contains global attributes
|
|
35
|
+
'aggregation_frequency' and 'aggregation' used to define Item description.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
-------
|
|
39
|
+
description : str
|
|
40
|
+
Description of NEMO model output Item.
|
|
41
|
+
"""
|
|
42
|
+
# --- Validate input arguments --- #
|
|
43
|
+
if not isinstance(prefix, str):
|
|
44
|
+
raise TypeError("'prefix' must be a string.")
|
|
45
|
+
if not isinstance(ds, xr.Dataset):
|
|
46
|
+
raise TypeError("'ds' must be an xarray.Dataset.")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# --- --- Define the item description based on the prefix --- #
|
|
50
|
+
if 'domain' in prefix:
|
|
51
|
+
description = "**Global ocean model domain and mesh mask variables.**"
|
|
52
|
+
elif 'T' in prefix:
|
|
53
|
+
description = f"**{ds.attrs.get('aggregation_frequency', 'monthly').capitalize()} {ds.attrs.get('aggregation', 'mean')} global ocean scalar outputs defined at {prefix[0]}-points.**"
|
|
54
|
+
elif 'U' in prefix:
|
|
55
|
+
description = f"**{ds.attrs.get('aggregation_frequency', 'monthly').capitalize()} {ds.attrs.get('aggregation', 'mean')} global ocean zonal vector outputs defined at {prefix[0]}-points.**"
|
|
56
|
+
elif 'V' in prefix:
|
|
57
|
+
description = f"**{ds.attrs.get('aggregation_frequency', 'monthly').capitalize()} {ds.attrs.get('aggregation', 'mean')} global ocean meridional vector outputs defined at {prefix[0]}-points.**"
|
|
58
|
+
elif 'W' in prefix:
|
|
59
|
+
description = f"**{ds.attrs.get('aggregation_frequency', 'monthly').capitalize()} {ds.attrs.get('aggregation', 'mean')} global ocean vertical vector outputs defined at {prefix[0]}-points.**"
|
|
60
|
+
elif 'I' in prefix:
|
|
61
|
+
description = f"**{ds.attrs.get('aggregation_frequency', 'monthly').capitalize()} {ds.attrs.get('aggregation', 'mean')} global sea-ice outputs defined at T-points.**"
|
|
62
|
+
elif 'S' in prefix:
|
|
63
|
+
description = f"**{ds.attrs.get('aggregation_frequency', 'monthly').capitalize()} {ds.attrs.get('aggregation', 'mean')} global ocean scalar outputs.**"
|
|
64
|
+
elif 'M' in prefix:
|
|
65
|
+
description = f"**{ds.attrs.get('aggregation_frequency', 'monthly').capitalize()} {ds.attrs.get('aggregation', 'mean')} ocean physics transect outputs defined at {prefix.split('/')[-1]}.**"
|
|
66
|
+
else:
|
|
67
|
+
raise ValueError(f"Unable to determine variable type from prefix: {prefix}")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
return description
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def create_npd_era5_collection() -> pystac.Collection:
|
|
74
|
+
"""
|
|
75
|
+
Create the NOC Near-Present Day ERA-5 STAC Collection.
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
-------
|
|
79
|
+
npd_collection : pystac.Collection
|
|
80
|
+
NOC Near-Present Day ERA-5 STAC Collection.
|
|
81
|
+
"""
|
|
82
|
+
# ==== Define NOC Near-Present Day Collection ==== #
|
|
83
|
+
# Define the spatial extent for the collection:
|
|
84
|
+
spatial_extent = pystac.SpatialExtent(bboxes=[[-180.0, -90.0, 0, 180.0, 90.0, 6000]])
|
|
85
|
+
|
|
86
|
+
# Define the current temporal extent for the collection:
|
|
87
|
+
collection_interval = sorted([datetime.datetime(year=1976, month=1, day=1), datetime.datetime(year=2026, month=5, day=15)])
|
|
88
|
+
temporal_extent = pystac.TemporalExtent(intervals=[collection_interval])
|
|
89
|
+
|
|
90
|
+
# Define the Near-Present Day Collection:
|
|
91
|
+
npd_collection = pystac.Collection(
|
|
92
|
+
id="noc-npd-era5",
|
|
93
|
+
title="NOC Near-Present Day ERA-5 Collection",
|
|
94
|
+
description="**About:**\n\nCollection of multi-decadal Near-Present Day ocean model hindcast simulations produced by the National Oceanography Centre (NOC) using bias-corrected ERA-5 atmospheric forcing as part of the Atlantic Climate and Environment Strategic Science (AtlantiS) programme.\n\n**More Information:**\n - [AtlantiS](https://atlantis.ac.uk)\n - [NOC Near-Present Day](https://noc-msm.github.io/NOC_Near_Present_Day/)",
|
|
95
|
+
extent=pystac.Extent(spatial=spatial_extent, temporal=temporal_extent),
|
|
96
|
+
# Open Government License (OGL) - UK version 3.0 - http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/
|
|
97
|
+
license="OGL-UK-3.0",
|
|
98
|
+
extra_fields=dict(contact="Ollie Tooth (oliver.tooth@noc.ac.uk)", project="AtlantiS", status="ongoing", update_frequency="biannual", last_data_update="2026-06-10"),
|
|
99
|
+
keywords=["NOC", "Near-Present Day", "AtlantiS", "hindcast", "global", "model", "ocean", "sea-ice"],
|
|
100
|
+
providers=[
|
|
101
|
+
pystac.Provider(
|
|
102
|
+
name="National Oceanography Centre (NOC)",
|
|
103
|
+
description="National Oceanography Centre (United Kingdom) - Ocean Modelling Group.",
|
|
104
|
+
roles=[pystac.ProviderRole.PRODUCER, pystac.ProviderRole.LICENSOR],
|
|
105
|
+
url="https://noc-msm.github.io/NOC_Near_Present_Day/"
|
|
106
|
+
),
|
|
107
|
+
pystac.Provider(
|
|
108
|
+
name="JASMIN",
|
|
109
|
+
description="JASMIN Environmental Data Analysis Facility (United Kingdom).",
|
|
110
|
+
roles=[pystac.ProviderRole.HOST],
|
|
111
|
+
url="https://jasmin.ac.uk"
|
|
112
|
+
)
|
|
113
|
+
],
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
logging.info(f"Completed: Created STAC Collection with ID: {npd_collection.id}")
|
|
117
|
+
|
|
118
|
+
# ==== Define NOC Near-Present Day Model Configuration Catalogs ==== #
|
|
119
|
+
npd_eorca1_era5v1 = pystac.Catalog(
|
|
120
|
+
id="npd-eorca1-era5v1",
|
|
121
|
+
title="eORCA1 ERA5v1 NPD Catalog",
|
|
122
|
+
description="Catalog of eORCA1 ERA-5 Near-Present Day ocean sea-ice simulations performed by the National Oceanography Centre."
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
logging.info(f"-> Completed: Created STAC Catalog with ID: {npd_eorca1_era5v1.id}")
|
|
126
|
+
|
|
127
|
+
npd_eorca025_era5v1 = pystac.Catalog(
|
|
128
|
+
id="npd-eorca025-era5v1",
|
|
129
|
+
title="eORCA025 ERA5v1 NPD Catalog",
|
|
130
|
+
description="Catalog of eORCA025 ERA-5 Near-Present Day ocean sea-ice simulations performed by the National Oceanography Centre.",
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
logging.info(f"-> Completed: Created STAC Catalog with ID: {npd_eorca025_era5v1.id}")
|
|
134
|
+
|
|
135
|
+
npd_eorca12_era5v1 = pystac.Catalog(
|
|
136
|
+
id="npd-eorca12-era5v1",
|
|
137
|
+
title="eORCA12 ERA5v1 NPD Catalog",
|
|
138
|
+
description="Catalog of eORCA12 ERA-5 Near-Present Day ocean sea-ice simulations performed by the National Oceanography Centre.",
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
logging.info(f"-> Completed: Created STAC Catalog with ID: {npd_eorca12_era5v1.id}")
|
|
142
|
+
|
|
143
|
+
# ==== Define NOC Near-Present Day Model Variant Catalogs ==== #
|
|
144
|
+
r1i1c1f1_eorca1_era5v1 = pystac.Catalog(
|
|
145
|
+
id="r1i1c1f1",
|
|
146
|
+
title="eORCA1 ERA5v1 NPD: r1i1c1f1 Catalog",
|
|
147
|
+
description="Catalog of eORCA1 ERA-5 Near-Present Day ocean physics & sea-ice outputs for model variant: r1i1c1f1.\n\n**Variant Label:**\n\nRealisation=1, Initialisation=1, Configuration=1, Forcing=1."
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
logging.info(f"-> Completed: Created STAC Catalog with ID: {r1i1c1f1_eorca1_era5v1.id}")
|
|
151
|
+
|
|
152
|
+
r1i1c1f1_eorca025_era5v1 = pystac.Catalog(
|
|
153
|
+
id="r1i1c1f1",
|
|
154
|
+
title="eORCA025 ERA5v1 NPD: r1i1c1f1 Catalog",
|
|
155
|
+
description="Catalog of eORCA025 ERA-5 Near-Present Day ocean physics & sea-ice outputs for model variant: r1i1c1f1.\n\n**Variant Label:**\n\nRealisation=1, Initialisation=1, Configuration=1, Forcing=1."
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
logging.info(f"-> Completed: Created STAC Catalog with ID: {r1i1c1f1_eorca025_era5v1.id}")
|
|
159
|
+
|
|
160
|
+
r1i1c1f1_eorca12_era5v1 = pystac.Catalog(
|
|
161
|
+
id="r1i1c1f1",
|
|
162
|
+
title="eORCA12 ERA5v1 NPD: r1i1c1f1 Catalog",
|
|
163
|
+
description="Catalog of eORCA12 ERA-5 Near-Present Day ocean physics & sea-ice outputs for model variant: r1i1c1f1.\n\n**Variant Label:**\n\nRealisation=1, Initialisation=1, Configuration=1, Forcing=1."
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
logging.info(f"-> Completed: Created STAC Catalog with ID: {r1i1c1f1_eorca12_era5v1.id}")
|
|
167
|
+
|
|
168
|
+
# -- Add Items to NOC Near-Present Day eORCA1 ERA5v1 {gn} Sub-Catalog -- #
|
|
169
|
+
# Define the store credentials for the eORCA1 ERA5v1 NPD data:
|
|
170
|
+
bucket = "npd-eorca1-era5v1"
|
|
171
|
+
for prefix in ["T1y", "U1y", "V1y", "W1y", "I1y", "S1y", "T1m", "U1m", "V1m", "W1m", "I1m", "S1m",
|
|
172
|
+
"domain/domain_cfg"
|
|
173
|
+
]:
|
|
174
|
+
# Open dataset from Icechunk repository:
|
|
175
|
+
ds = open_icechunk_store(bucket=bucket, prefix=prefix, branch="main")
|
|
176
|
+
|
|
177
|
+
item = create_item_with_icechunk_asset(
|
|
178
|
+
ds=ds,
|
|
179
|
+
id=f"noc-npd-era5/{bucket}/{ds.attrs.get('variant', 'r1i1c1f1')}/{prefix}",
|
|
180
|
+
bucket=bucket,
|
|
181
|
+
prefix=prefix,
|
|
182
|
+
title=f"NPD eORCA1 ERA5v1 {prefix}",
|
|
183
|
+
description=description_from_prefix(prefix=prefix, ds=ds),
|
|
184
|
+
start_date="1976-01-01",
|
|
185
|
+
end_date="2026-05-15",
|
|
186
|
+
collection=bucket
|
|
187
|
+
)
|
|
188
|
+
# Add item to the eORCA1 ERA5v1 NPD global native model grid catalog:
|
|
189
|
+
r1i1c1f1_eorca1_era5v1.add_item(item)
|
|
190
|
+
|
|
191
|
+
logging.info(f"-> Completed: Added Items to STAC Catalog with ID: {r1i1c1f1_eorca1_era5v1.id}")
|
|
192
|
+
|
|
193
|
+
# -- Add Items to NOC Near-Present Day eORCA1 ERA5v1 {tn} Sub-Catalog -- #
|
|
194
|
+
# Define the store credentials for the eORCA1 ERA5v1 NPD data:
|
|
195
|
+
for prefix in ["M1m/MOVE_16N", "M1m/SAMBA_34_5S", "M1m/RAPID_26N", "M1m/OSNAP"]:
|
|
196
|
+
# Open dataset from Icechunk repository:
|
|
197
|
+
ds = open_icechunk_store(bucket=bucket, prefix=prefix, branch="main")
|
|
198
|
+
|
|
199
|
+
item = create_item_with_icechunk_asset(
|
|
200
|
+
ds=ds,
|
|
201
|
+
id=f"noc-npd-era5/{bucket}/{ds.attrs.get('variant', 'r1i1c1f1')}/{prefix}",
|
|
202
|
+
bucket=bucket,
|
|
203
|
+
prefix=prefix,
|
|
204
|
+
title=f"NPD eORCA1 ERA5v1 {prefix}",
|
|
205
|
+
description=description_from_prefix(prefix=prefix, ds=ds),
|
|
206
|
+
platform=ds.attrs.get('platform', 'tn'),
|
|
207
|
+
start_date="1976-01-01",
|
|
208
|
+
end_date="2026-05-15",
|
|
209
|
+
collection=bucket
|
|
210
|
+
)
|
|
211
|
+
# Add item to the eORCA1 ERA5v1 NPD transect catalog:
|
|
212
|
+
r1i1c1f1_eorca1_era5v1.add_item(item)
|
|
213
|
+
|
|
214
|
+
logging.info(f"-> Completed: Added Items to STAC Catalog with ID: {r1i1c1f1_eorca1_era5v1.id}")
|
|
215
|
+
|
|
216
|
+
# -- Add Items to NOC Near-Present Day eORCA025 ERA5v1 {gn} Sub-Catalog -- #
|
|
217
|
+
# Define the store credentials for the eORCA025 ERA5v1 NPD data:
|
|
218
|
+
bucket = "npd-eorca025-era5v1"
|
|
219
|
+
for prefix in ["T1y_3d", "T1y_4d", "U1y_3d", "U1y_4d", "V1y_3d", "V1y_4d", "W1y_4d", "I1y_3d", "S1y_1d",
|
|
220
|
+
"T1m_3d", "T1m_4d", "U1m_3d", "U1m_4d", "V1m_3d", "V1m_4d", "W1m_4d", "I1m_3d", "S1m_1d",
|
|
221
|
+
"T5d_3d", "T5d_4d", "U5d_3d", "U5d_4d", "V5d_3d", "V5d_4d", "I5d_3d", "S5d_1d",
|
|
222
|
+
"domain/domain_cfg"
|
|
223
|
+
]:
|
|
224
|
+
# Open dataset from Icechunk repository:
|
|
225
|
+
ds = open_icechunk_store(bucket=bucket, prefix=prefix, branch="main")
|
|
226
|
+
|
|
227
|
+
item = create_item_with_icechunk_asset(
|
|
228
|
+
ds=ds,
|
|
229
|
+
id=f"noc-npd-era5/{bucket}/{ds.attrs.get('variant', 'r1i1c1f1')}/{prefix}",
|
|
230
|
+
bucket=bucket,
|
|
231
|
+
prefix=prefix,
|
|
232
|
+
title=f"NPD eORCA025 ERA5v1 {prefix}",
|
|
233
|
+
description=description_from_prefix(prefix=prefix, ds=ds),
|
|
234
|
+
start_date="1976-01-01",
|
|
235
|
+
end_date="2026-05-15",
|
|
236
|
+
collection=bucket
|
|
237
|
+
)
|
|
238
|
+
# Add item to the eORCA025 ERA5v1 NPD global native model grid catalog:
|
|
239
|
+
r1i1c1f1_eorca025_era5v1.add_item(item)
|
|
240
|
+
|
|
241
|
+
logging.info(f"-> Completed: Added Items to STAC Catalog with ID: {r1i1c1f1_eorca025_era5v1.id}")
|
|
242
|
+
|
|
243
|
+
# -- Add Items to NOC Near-Present Day eORCA025 ERA5v1 {tn} Sub-Catalog -- #
|
|
244
|
+
# Define the store credentials for the eORCA025 ERA5v1 NPD data:
|
|
245
|
+
for prefix in ["M1m/MOVE_16N", "M1m/SAMBA_34_5S", "M1m/RAPID_26N", "M1m/OSNAP"]:
|
|
246
|
+
# Open dataset from Icechunk repository:
|
|
247
|
+
ds = open_icechunk_store(bucket=bucket, prefix=prefix, branch="main")
|
|
248
|
+
|
|
249
|
+
item = create_item_with_icechunk_asset(
|
|
250
|
+
ds=ds,
|
|
251
|
+
id=f"noc-npd-era5/{bucket}/{ds.attrs.get('variant', 'r1i1c1f1')}/{prefix}",
|
|
252
|
+
bucket=bucket,
|
|
253
|
+
prefix=prefix,
|
|
254
|
+
title=f"NPD eORCA025 ERA5v1 {prefix}",
|
|
255
|
+
description=description_from_prefix(prefix=prefix, ds=ds),
|
|
256
|
+
platform=ds.attrs.get('platform', 'tn'),
|
|
257
|
+
start_date="1976-01-01",
|
|
258
|
+
end_date="2026-05-15",
|
|
259
|
+
collection=bucket
|
|
260
|
+
)
|
|
261
|
+
# Add item to the eORCA025 ERA5v1 NPD transect catalog:
|
|
262
|
+
r1i1c1f1_eorca025_era5v1.add_item(item)
|
|
263
|
+
|
|
264
|
+
logging.info(f"-> Completed: Added Items to STAC Catalog with ID: {r1i1c1f1_eorca025_era5v1.id}")
|
|
265
|
+
|
|
266
|
+
# -- Add Items to NOC Near-Present Day eORCA12 ERA5v1 Sub-Catalog -- #
|
|
267
|
+
# Define the store credentials for the eORCA12 ERA5v1 NPD data:
|
|
268
|
+
bucket = "npd-eorca12-era5v1"
|
|
269
|
+
|
|
270
|
+
# Add annual-mean and monthly mean Icechunk repositories:
|
|
271
|
+
for prefix in ["T1y_3d", "T1y_4d", "U1y_3d", "U1y_4d", "V1y_3d", "V1y_4d", "W1y_4d", "I1y_3d", "S1y_1d",
|
|
272
|
+
"T1m_3d", "T1m_4d", "U1m_3d", "U1m_4d", "V1m_3d", "V1m_4d", "W1m_4d", "I1m_3d", "S1m_1d",
|
|
273
|
+
"domain/domain_cfg",
|
|
274
|
+
]:
|
|
275
|
+
# Open dataset from Icechunk repository:
|
|
276
|
+
ds = open_icechunk_store(bucket=bucket, prefix=prefix, branch="main")
|
|
277
|
+
|
|
278
|
+
item = create_item_with_icechunk_asset(
|
|
279
|
+
ds=ds,
|
|
280
|
+
id=f"noc-npd-era5/{bucket}/{ds.attrs.get('variant', 'r1i1c1f1')}/{prefix}",
|
|
281
|
+
bucket=bucket,
|
|
282
|
+
prefix=prefix,
|
|
283
|
+
title=f"NPD eORCA12 ERA5v1 {prefix}",
|
|
284
|
+
description=description_from_prefix(prefix=prefix, ds=ds),
|
|
285
|
+
start_date="1976-01-01",
|
|
286
|
+
end_date="2026-05-15",
|
|
287
|
+
collection=bucket
|
|
288
|
+
)
|
|
289
|
+
# Add item to the eORCA12 ERA5v1 NPD global native model grid catalog:
|
|
290
|
+
r1i1c1f1_eorca12_era5v1.add_item(item)
|
|
291
|
+
|
|
292
|
+
# Add root group for hierarchical Icechunk repository:
|
|
293
|
+
ds = open_icechunk_store(bucket=bucket, prefix="eorca12-era5v1-5d", branch="main")
|
|
294
|
+
item = create_item_with_icechunk_asset(
|
|
295
|
+
ds=ds,
|
|
296
|
+
id=f"noc-npd-era5/{bucket}/{ds.attrs.get('variant', 'r1i1c1f1')}/eorca12-era5v1-5d",
|
|
297
|
+
bucket=bucket,
|
|
298
|
+
prefix="eorca12-era5v1-5d",
|
|
299
|
+
title="NPD eORCA12 ERA5v1 5d",
|
|
300
|
+
description="**5-day mean global ocean outputs.**",
|
|
301
|
+
start_date="1990-01-01",
|
|
302
|
+
end_date="2024-12-31",
|
|
303
|
+
collection=bucket,
|
|
304
|
+
)
|
|
305
|
+
# Add item to the eORCA12 ERA5v1 NPD global native model grid catalog:
|
|
306
|
+
r1i1c1f1_eorca12_era5v1.add_item(item)
|
|
307
|
+
|
|
308
|
+
# Add individual groups in hierarchical Icechunk repository:
|
|
309
|
+
for prefix in ["T5d", "U5d", "V5d"]:
|
|
310
|
+
# Open dataset from Icechunk repository:
|
|
311
|
+
ds = open_icechunk_store(bucket=bucket, prefix="eorca12-era5v1-5d", branch="main", group=f"grid{prefix[0]}")
|
|
312
|
+
item = create_item_with_icechunk_asset(
|
|
313
|
+
ds=ds,
|
|
314
|
+
id=f"noc-npd-era5/{bucket}/{ds.attrs.get('variant', 'r1i1c1f1')}/{prefix}",
|
|
315
|
+
bucket=bucket,
|
|
316
|
+
prefix="eorca12-era5v1-5d",
|
|
317
|
+
title=f"NPD eORCA12 ERA5v1 {prefix}",
|
|
318
|
+
description=f"**5-day mean global ocean {'scalar' if prefix[0] == 'T' else 'vector'} outputs defined at {prefix[0]}-points.**",
|
|
319
|
+
start_date="1990-01-01",
|
|
320
|
+
end_date="2024-12-31",
|
|
321
|
+
collection=bucket,
|
|
322
|
+
group=f"grid{prefix[0]}"
|
|
323
|
+
)
|
|
324
|
+
# Add item to the eORCA12 ERA5v1 NPD global native model grid catalog:
|
|
325
|
+
r1i1c1f1_eorca12_era5v1.add_item(item)
|
|
326
|
+
|
|
327
|
+
logging.info(f"-> Completed: Added Items to STAC Catalog with ID: {r1i1c1f1_eorca12_era5v1.id}")
|
|
328
|
+
|
|
329
|
+
# -- Add Items to NOC Near-Present Day eORCA12 ERA5v1 {tn} Sub-Catalog -- #
|
|
330
|
+
# Define the store credentials for the eORCA12 ERA5v1 NPD data:
|
|
331
|
+
for prefix in ["M1m/MOVE_16N", "M1m/SAMBA_34_5S", "M1m/RAPID_26N", "M1m/OSNAP"]:
|
|
332
|
+
# Open dataset from Icechunk repository:
|
|
333
|
+
ds = open_icechunk_store(bucket=bucket, prefix=prefix, branch="main")
|
|
334
|
+
|
|
335
|
+
item = create_item_with_icechunk_asset(
|
|
336
|
+
ds=ds,
|
|
337
|
+
id=f"noc-npd-era5/{bucket}/{ds.attrs.get('variant', 'r1i1c1f1')}/{prefix}",
|
|
338
|
+
bucket=bucket,
|
|
339
|
+
prefix=prefix,
|
|
340
|
+
title=f"NPD eORCA12 ERA5v1 {prefix}",
|
|
341
|
+
description=description_from_prefix(prefix=prefix, ds=ds),
|
|
342
|
+
platform=ds.attrs.get('platform', 'tn'),
|
|
343
|
+
start_date="1976-01-01",
|
|
344
|
+
end_date="2026-05-15",
|
|
345
|
+
collection=bucket
|
|
346
|
+
)
|
|
347
|
+
# Add item to the eORCA12 ERA5v1 NPD transect catalog:
|
|
348
|
+
r1i1c1f1_eorca12_era5v1.add_item(item)
|
|
349
|
+
|
|
350
|
+
logging.info(f"-> Completed: Added Items to STAC Catalog with ID: {r1i1c1f1_eorca12_era5v1.id}")
|
|
351
|
+
|
|
352
|
+
# ==== Add Nested Catalogs to NOC Near-Present Day Collection ==== #
|
|
353
|
+
|
|
354
|
+
# Model Simulation Variant Catalogs -> Model Simulation Catalogs:
|
|
355
|
+
npd_eorca1_era5v1.add_child(r1i1c1f1_eorca1_era5v1)
|
|
356
|
+
npd_eorca025_era5v1.add_child(r1i1c1f1_eorca025_era5v1)
|
|
357
|
+
npd_eorca12_era5v1.add_child(r1i1c1f1_eorca12_era5v1)
|
|
358
|
+
|
|
359
|
+
# Model Simulation Catalogs -> Near-Present Day Collection:
|
|
360
|
+
npd_collection.add_child(npd_eorca1_era5v1)
|
|
361
|
+
npd_collection.add_child(npd_eorca025_era5v1)
|
|
362
|
+
npd_collection.add_child(npd_eorca12_era5v1)
|
|
363
|
+
|
|
364
|
+
return npd_collection
|