icon4py-testing 0.0.1__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- icon4py_testing-0.2.0/PKG-INFO +36 -0
- icon4py_testing-0.2.0/README.md +1 -0
- icon4py_testing-0.2.0/pyproject.toml +93 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/__init__.py +29 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/config.py +37 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/data_handling.py +83 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/datatest_utils.py +346 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/definitions.py +235 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/filters.py +85 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/fixtures/__init__.py +12 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/fixtures/benchmark.py +127 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/fixtures/datatest.py +502 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/fixtures/stencil_tests.py +134 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/grid_utils.py +122 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/locking.py +29 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/parallel_helpers.py +160 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/py.typed +0 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/pytest_hooks.py +258 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/reference_funcs.py +49 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/serialbox.py +2256 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/stencil_tests.py +312 -0
- icon4py_testing-0.2.0/src/icon4py/model/testing/test_utils.py +114 -0
- icon4py_testing-0.2.0/src/icon4py_testing.egg-info/PKG-INFO +36 -0
- icon4py_testing-0.2.0/src/icon4py_testing.egg-info/SOURCES.txt +26 -0
- icon4py_testing-0.2.0/src/icon4py_testing.egg-info/requires.txt +9 -0
- icon4py_testing-0.2.0/src/icon4py_testing.egg-info/top_level.txt +1 -0
- icon4py_testing-0.0.1/PKG-INFO +0 -5
- icon4py_testing-0.0.1/icon4py_testing.egg-info/PKG-INFO +0 -5
- icon4py_testing-0.0.1/icon4py_testing.egg-info/SOURCES.txt +0 -5
- icon4py_testing-0.0.1/icon4py_testing.egg-info/top_level.txt +0 -1
- icon4py_testing-0.0.1/pyproject.toml +0 -12
- {icon4py_testing-0.0.1 → icon4py_testing-0.2.0}/setup.cfg +0 -0
- {icon4py_testing-0.0.1 → icon4py_testing-0.2.0/src}/icon4py_testing.egg-info/dependency_links.txt +0 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: icon4py-testing
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Testing utils for the icon4py model.
|
|
5
|
+
Author-email: ETH Zurich <gridtools@cscs.ch>
|
|
6
|
+
License: BSD-3 License
|
|
7
|
+
Project-URL: Homepage, https://github.com/C2SM/icon4py
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Science/Research
|
|
10
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
11
|
+
Classifier: Operating System :: POSIX
|
|
12
|
+
Classifier: Programming Language :: Python
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Mathematics
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Physics
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
Requires-Dist: icon4py-common[io]~=0.2.0
|
|
27
|
+
Requires-Dist: filelock<3.20,>=3.18.0
|
|
28
|
+
Requires-Dist: gt4py==1.1.10
|
|
29
|
+
Requires-Dist: numpy>=1.23.3
|
|
30
|
+
Requires-Dist: packaging>=20.0
|
|
31
|
+
Requires-Dist: pooch>=1.7.0
|
|
32
|
+
Requires-Dist: pytest>=8.0.1
|
|
33
|
+
Requires-Dist: serialbox4py>=2.6.2
|
|
34
|
+
Requires-Dist: typing-extensions>=4.11.0
|
|
35
|
+
|
|
36
|
+
# Testing utils for Python ICON port
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Testing utils for Python ICON port
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# -- Build system requirements (PEP 518) --
|
|
2
|
+
|
|
3
|
+
[build-system]
|
|
4
|
+
build-backend = "setuptools.build_meta"
|
|
5
|
+
requires = ["setuptools>=61.0", "wheel>=0.40.0"]
|
|
6
|
+
|
|
7
|
+
# -- Standard project description options (PEP 621) --
|
|
8
|
+
[project]
|
|
9
|
+
authors = [{email = "gridtools@cscs.ch", name = "ETH Zurich"}]
|
|
10
|
+
classifiers = [
|
|
11
|
+
'Development Status :: 4 - Beta',
|
|
12
|
+
'Intended Audience :: Science/Research',
|
|
13
|
+
'License :: OSI Approved :: BSD License',
|
|
14
|
+
'Operating System :: POSIX',
|
|
15
|
+
'Programming Language :: Python',
|
|
16
|
+
'Programming Language :: Python :: 3',
|
|
17
|
+
'Programming Language :: Python :: 3 :: Only',
|
|
18
|
+
'Programming Language :: Python :: 3.10',
|
|
19
|
+
'Programming Language :: Python :: 3.11',
|
|
20
|
+
'Programming Language :: Python :: 3.12',
|
|
21
|
+
'Programming Language :: Python :: 3.13',
|
|
22
|
+
'Programming Language :: Python :: 3.14',
|
|
23
|
+
'Programming Language :: Python :: Implementation :: CPython',
|
|
24
|
+
'Topic :: Scientific/Engineering :: Atmospheric Science',
|
|
25
|
+
'Topic :: Scientific/Engineering :: Mathematics',
|
|
26
|
+
'Topic :: Scientific/Engineering :: Physics'
|
|
27
|
+
]
|
|
28
|
+
dependencies = [
|
|
29
|
+
# workspace members
|
|
30
|
+
'icon4py-common[io]~=0.2.0',
|
|
31
|
+
# external dependencies
|
|
32
|
+
# TODO(msimberg): https://github.com/tox-dev/filelock/pull/408 added in 3.20.4
|
|
33
|
+
# and 3.21.0 seems to lead to hangs in CI. Investigate if other lock types are
|
|
34
|
+
# safe or if this is a bug in filelock.
|
|
35
|
+
"filelock>=3.18.0,<3.20",
|
|
36
|
+
"gt4py==1.1.10",
|
|
37
|
+
"numpy>=1.23.3",
|
|
38
|
+
'packaging>=20.0',
|
|
39
|
+
"pooch>=1.7.0",
|
|
40
|
+
"pytest>=8.0.1",
|
|
41
|
+
"serialbox4py>=2.6.2",
|
|
42
|
+
"typing-extensions>=4.11.0"
|
|
43
|
+
]
|
|
44
|
+
description = "Testing utils for the icon4py model."
|
|
45
|
+
license = {text = "BSD-3 License"}
|
|
46
|
+
name = "icon4py-testing"
|
|
47
|
+
readme = "README.md"
|
|
48
|
+
requires-python = ">=3.10"
|
|
49
|
+
# managed by bump-my-version:
|
|
50
|
+
version = "0.2.0"
|
|
51
|
+
|
|
52
|
+
[project.urls]
|
|
53
|
+
Homepage = 'https://github.com/C2SM/icon4py'
|
|
54
|
+
|
|
55
|
+
# -- bumpversion --
|
|
56
|
+
[tool.bumpversion]
|
|
57
|
+
allow_dirty = false
|
|
58
|
+
commit = false
|
|
59
|
+
current_version = "0.2.0"
|
|
60
|
+
ignore_missing_version = false
|
|
61
|
+
message = 'Bump icon4py-testing version: {current_version} → {new_version}'
|
|
62
|
+
parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(?:(?P<pre_l>a|b|rc)(?P<pre_n>\\d+))?"
|
|
63
|
+
serialize = ["{major}.{minor}.{patch}{pre_l}{pre_n}", "{major}.{minor}.{patch}"]
|
|
64
|
+
tag = false
|
|
65
|
+
|
|
66
|
+
[[tool.bumpversion.files]]
|
|
67
|
+
filename = "pyproject.toml"
|
|
68
|
+
replace = '''
|
|
69
|
+
# managed by bump-my-version:
|
|
70
|
+
version = "{new_version}"
|
|
71
|
+
'''
|
|
72
|
+
search = '''
|
|
73
|
+
# managed by bump-my-version:
|
|
74
|
+
version = "{current_version}"
|
|
75
|
+
'''
|
|
76
|
+
|
|
77
|
+
[[tool.bumpversion.files]]
|
|
78
|
+
filename = "src/icon4py/model/testing/__init__.py"
|
|
79
|
+
|
|
80
|
+
# -- ruff --
|
|
81
|
+
[tool.ruff]
|
|
82
|
+
extend = "../../pyproject.toml"
|
|
83
|
+
|
|
84
|
+
[tool.ruff.lint.isort]
|
|
85
|
+
known-first-party = ['icon4py']
|
|
86
|
+
known-third-party = ['gt4py']
|
|
87
|
+
|
|
88
|
+
# -- setuptools --
|
|
89
|
+
[tool.setuptools.package-data]
|
|
90
|
+
'*' = ['*.in', '*.md', '*.rst', '*.txt', 'LICENSE', 'py.typed']
|
|
91
|
+
|
|
92
|
+
[tool.setuptools.packages]
|
|
93
|
+
find = {namespaces = true, where = ['src']}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# ICON4Py - ICON inspired code in Python and GT4Py
|
|
2
|
+
#
|
|
3
|
+
# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss
|
|
4
|
+
# All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# Please, refer to the LICENSE file in the root directory.
|
|
7
|
+
# SPDX-License-Identifier: BSD-3-Clause
|
|
8
|
+
|
|
9
|
+
from typing import Final
|
|
10
|
+
|
|
11
|
+
from packaging import version as pkg_version
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"__author__",
|
|
16
|
+
"__copyright__",
|
|
17
|
+
"__license__",
|
|
18
|
+
"__version__",
|
|
19
|
+
"__version_info__",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
__author__: Final = "ETH Zurich, MeteoSwiss and individual contributors"
|
|
24
|
+
__copyright__: Final = "Copyright (c) 2022-2024 ETH Zurich and MeteoSwiss"
|
|
25
|
+
__license__: Final = "BSD-3-Clause"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
__version__: Final = "0.2.0"
|
|
29
|
+
__version_info__: Final = pkg_version.parse(__version__)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# ICON4Py - ICON inspired code in Python and GT4Py
|
|
2
|
+
#
|
|
3
|
+
# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss
|
|
4
|
+
# All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# Please, refer to the LICENSE file in the root directory.
|
|
7
|
+
# SPDX-License-Identifier: BSD-3-Clause
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import pathlib
|
|
13
|
+
import tempfile
|
|
14
|
+
|
|
15
|
+
from icon4py.model.common.utils import env
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _project_root() -> pathlib.Path:
|
|
19
|
+
for path in [pathlib.Path(__file__).resolve(), *pathlib.Path(__file__).resolve().parents]:
|
|
20
|
+
if (path / ".git").exists():
|
|
21
|
+
return path
|
|
22
|
+
# fallback to hardcoded relative path
|
|
23
|
+
return pathlib.Path(__file__).parents[6]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _default_download_cache() -> pathlib.Path:
|
|
27
|
+
return pathlib.Path(tempfile.gettempdir()) / "icon4py_download_cache"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
ENABLE_GRID_DOWNLOAD: bool = env.flag_to_bool("ICON4PY_ENABLE_GRID_DOWNLOAD", True)
|
|
31
|
+
ENABLE_TESTDATA_DOWNLOAD: bool = env.flag_to_bool("ICON4PY_ENABLE_TESTDATA_DOWNLOAD", True)
|
|
32
|
+
TEST_DATA_PATH: pathlib.Path = env.path("ICON4PY_TEST_DATA_PATH", _project_root() / "testdata")
|
|
33
|
+
DALLCLOSE_PRINT_INSTEAD_OF_FAIL: bool = env.flag_to_bool(
|
|
34
|
+
"ICON4PY_DALLCLOSE_PRINT_INSTEAD_OF_FAIL", False
|
|
35
|
+
)
|
|
36
|
+
DOWNLOAD_CACHE_PATH: pathlib.Path = env.path("ICON4PY_DOWNLOAD_CACHE", _default_download_cache())
|
|
37
|
+
DRIVER_LOGGING_LEVEL: str = os.environ.get("ICON4PY_DRIVER_LOGGING_LEVEL", "debug")
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# ICON4Py - ICON inspired code in Python and GT4Py
|
|
2
|
+
#
|
|
3
|
+
# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss
|
|
4
|
+
# All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# Please, refer to the LICENSE file in the root directory.
|
|
7
|
+
# SPDX-License-Identifier: BSD-3-Clause
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import os
|
|
13
|
+
import pathlib
|
|
14
|
+
import shutil
|
|
15
|
+
|
|
16
|
+
import pooch # type: ignore[import-untyped]
|
|
17
|
+
|
|
18
|
+
from icon4py.model.testing import config, locking
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def download_and_extract(
|
|
22
|
+
uri: str,
|
|
23
|
+
dst: pathlib.Path,
|
|
24
|
+
) -> None:
|
|
25
|
+
"""
|
|
26
|
+
Download and extract a tar file with locking.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
uri: download url for archived data
|
|
30
|
+
dst: the archive is extracted at this path
|
|
31
|
+
|
|
32
|
+
Downloads the archive to a temporary cache directory (configured via
|
|
33
|
+
``ICON4PY_DOWNLOAD_CACHE``, defaulting to a subdirectory of the system
|
|
34
|
+
temp directory), extracts to ``dst``, and deletes the archive. If
|
|
35
|
+
extraction fails the archive is left in the cache so that a subsequent
|
|
36
|
+
run can reuse it without re-downloading.
|
|
37
|
+
"""
|
|
38
|
+
dst.mkdir(parents=True, exist_ok=True)
|
|
39
|
+
cache = config.DOWNLOAD_CACHE_PATH
|
|
40
|
+
cache.mkdir(parents=True, exist_ok=True)
|
|
41
|
+
|
|
42
|
+
completion_marker = dst / ".extraction_complete"
|
|
43
|
+
lockfile = "filelock.lock"
|
|
44
|
+
|
|
45
|
+
if completion_marker.exists():
|
|
46
|
+
return
|
|
47
|
+
|
|
48
|
+
with locking.lock(dst, lockfile=lockfile):
|
|
49
|
+
if completion_marker.exists():
|
|
50
|
+
return
|
|
51
|
+
# Clean up any partial data from previous failed attempts
|
|
52
|
+
# (except for the lockfile)
|
|
53
|
+
for item in dst.iterdir():
|
|
54
|
+
if item.name != lockfile:
|
|
55
|
+
if item.is_file():
|
|
56
|
+
item.unlink()
|
|
57
|
+
elif item.is_dir():
|
|
58
|
+
shutil.rmtree(item)
|
|
59
|
+
# Use a URI-derived filename to avoid collisions between different downloads
|
|
60
|
+
uri_hash = hashlib.sha256(uri.encode()).hexdigest()[:16]
|
|
61
|
+
archive_fname = f"archive_{uri_hash}.tar.gz"
|
|
62
|
+
pooch.retrieve(
|
|
63
|
+
url=uri,
|
|
64
|
+
known_hash=None,
|
|
65
|
+
path=str(cache),
|
|
66
|
+
fname=archive_fname,
|
|
67
|
+
processor=pooch.Untar(extract_dir=str(dst.resolve())),
|
|
68
|
+
)
|
|
69
|
+
completion_marker.touch()
|
|
70
|
+
(cache / archive_fname).unlink(missing_ok=True)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def download_test_data(dst: pathlib.Path, uri: str) -> None:
|
|
74
|
+
if config.ENABLE_TESTDATA_DOWNLOAD:
|
|
75
|
+
download_and_extract(uri, dst)
|
|
76
|
+
else:
|
|
77
|
+
# If test data download is disabled, we check if the directory exists
|
|
78
|
+
# and isn't empty without locking. We assume the location is managed by the user
|
|
79
|
+
# and avoid locking shared directories (e.g. on CI).
|
|
80
|
+
if not dst.exists():
|
|
81
|
+
raise RuntimeError(f"Test data {dst} does not exist, and downloading is disabled.")
|
|
82
|
+
elif not any(os.scandir(dst)):
|
|
83
|
+
raise RuntimeError(f"Test data {dst} exists but is empty, and downloading is disabled.")
|
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
# ICON4Py - ICON inspired code in Python and GT4Py
|
|
2
|
+
#
|
|
3
|
+
# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss
|
|
4
|
+
# All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# Please, refer to the LICENSE file in the root directory.
|
|
7
|
+
# SPDX-License-Identifier: BSD-3-Clause
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import datetime
|
|
12
|
+
import functools
|
|
13
|
+
import json
|
|
14
|
+
import pathlib
|
|
15
|
+
import urllib.parse
|
|
16
|
+
from typing import TypeVar
|
|
17
|
+
|
|
18
|
+
import gt4py.next.typing as gtx_typing
|
|
19
|
+
|
|
20
|
+
from icon4py.model.atmosphere.diffusion import diffusion
|
|
21
|
+
from icon4py.model.atmosphere.dycore import dycore_states, solve_nonhydro as solve_nh
|
|
22
|
+
from icon4py.model.atmosphere.subgrid_scale_physics.microphysics import (
|
|
23
|
+
single_moment_six_class_gscp_graupel as graupel,
|
|
24
|
+
)
|
|
25
|
+
from icon4py.model.common import constants
|
|
26
|
+
from icon4py.model.common.decomposition import definitions as decomposition
|
|
27
|
+
from icon4py.model.common.grid import vertical as v_grid
|
|
28
|
+
from icon4py.model.common.interpolation import interpolation_factory, rbf_interpolation as rbf
|
|
29
|
+
from icon4py.model.common.metrics import metrics_factory
|
|
30
|
+
from icon4py.model.standalone_driver import config as driver_config
|
|
31
|
+
from icon4py.model.testing import data_handling, definitions, serialbox
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def get_process_properties_for_run(
|
|
35
|
+
run_instance: decomposition.RunType,
|
|
36
|
+
) -> decomposition.ProcessProperties:
|
|
37
|
+
return decomposition.get_process_properties(run_instance)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def get_experiment_name_with_version(
|
|
41
|
+
experiment_description: definitions.ExperimentDescription,
|
|
42
|
+
) -> str:
|
|
43
|
+
"""Generate experiment name with version suffix."""
|
|
44
|
+
return f"{experiment_description.name}_v{experiment_description.version:02d}"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def get_ranked_experiment_name_with_version(
|
|
48
|
+
experiment_description: definitions.ExperimentDescription, comm_size: int
|
|
49
|
+
) -> str:
|
|
50
|
+
"""Generate ranked experiment name with version suffix."""
|
|
51
|
+
return f"mpitask{comm_size}_{get_experiment_name_with_version(experiment_description)}"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def get_experiment_archive_filename(
|
|
55
|
+
experiment_description: definitions.ExperimentDescription, comm_size: int
|
|
56
|
+
) -> str:
|
|
57
|
+
"""Generate ranked archive filename for an experiment."""
|
|
58
|
+
return f"{get_ranked_experiment_name_with_version(experiment_description, comm_size)}.tar.gz"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def get_experiment_archive_url(root_url: str, filepath: str) -> str:
|
|
62
|
+
"""Build a download URL for experiment archive from root URL."""
|
|
63
|
+
return f"{root_url}/{urllib.parse.quote(filepath)}"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def get_grid_archive_filename(grid: definitions.GridDescription) -> str:
|
|
67
|
+
return f"{grid.name}.tar.gz"
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def get_grid_filename(grid: definitions.GridDescription) -> str:
|
|
71
|
+
return f"{grid.name}.nc"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def get_grid_filepath(grid: definitions.GridDescription) -> pathlib.Path:
|
|
75
|
+
return definitions.grids_path().joinpath(grid.name, get_grid_filename(grid))
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def get_grid_archive_url(root_url: str, grid: definitions.GridDescription) -> str:
|
|
79
|
+
"""Build a download URL for a grid archive from root URL."""
|
|
80
|
+
filepath = f"{definitions.GRID_DATA_DIR}/{get_grid_archive_filename(grid)}"
|
|
81
|
+
return f"{root_url}/{urllib.parse.quote(filepath)}"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def get_muphys_archive_url(root_url: str, experiment_type: str, experiment_name: str) -> str:
|
|
85
|
+
"""Build a download URL for a muphys archive from root URL."""
|
|
86
|
+
filepath = f"{definitions.MUPHYS_DATA_DIR}/{experiment_type}/{experiment_name}.tar.gz"
|
|
87
|
+
return f"{root_url}/{urllib.parse.quote(filepath)}"
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def get_datapath_for_experiment(
|
|
91
|
+
experiment_description: definitions.ExperimentDescription,
|
|
92
|
+
process_props: decomposition.ProcessProperties,
|
|
93
|
+
) -> pathlib.Path:
|
|
94
|
+
"""Get the path to serialized data for an experiment."""
|
|
95
|
+
|
|
96
|
+
experiment_dir = get_ranked_experiment_name_with_version(
|
|
97
|
+
experiment_description,
|
|
98
|
+
process_props.comm_size,
|
|
99
|
+
)
|
|
100
|
+
return definitions.serialized_data_path().joinpath(
|
|
101
|
+
experiment_dir, definitions.SERIALIZED_DATA_SUBDIR
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def create_icon_serial_data_provider(
|
|
106
|
+
datapath: pathlib.Path,
|
|
107
|
+
rank: int,
|
|
108
|
+
backend: gtx_typing.Backend | None,
|
|
109
|
+
) -> serialbox.IconSerialDataProvider:
|
|
110
|
+
return serialbox.IconSerialDataProvider(
|
|
111
|
+
backend=backend,
|
|
112
|
+
fname_prefix="icon_pydycore",
|
|
113
|
+
path=str(datapath),
|
|
114
|
+
mpi_rank=rank,
|
|
115
|
+
do_print=True,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def download_experiment(
|
|
120
|
+
experiment_description: definitions.ExperimentDescription,
|
|
121
|
+
processor_props: decomposition.ProcessProperties,
|
|
122
|
+
) -> None:
|
|
123
|
+
"""Download data and config for an experiment--if not already present."""
|
|
124
|
+
comm_size = processor_props.comm_size
|
|
125
|
+
root_url = definitions.TESTDATA_ROOT_URL
|
|
126
|
+
archive_filename = get_experiment_archive_filename(experiment_description, comm_size)
|
|
127
|
+
archive_path = definitions.EXPERIMENT_DATA_DIR + "/" + archive_filename
|
|
128
|
+
uri = get_experiment_archive_url(root_url, archive_path)
|
|
129
|
+
destination_path = get_datapath_for_experiment(experiment_description, processor_props)
|
|
130
|
+
data_handling.download_test_data(destination_path.parent, uri)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@functools.cache
|
|
134
|
+
def _read_namelist_json(json_file_path: pathlib.Path) -> dict:
|
|
135
|
+
"""
|
|
136
|
+
Read and cache the namelist JSON file.
|
|
137
|
+
|
|
138
|
+
Args:
|
|
139
|
+
json_file_path: Path to the NAMELIST_ICON_output_atm.json file
|
|
140
|
+
|
|
141
|
+
Returns:
|
|
142
|
+
Dictionary containing the parsed JSON data
|
|
143
|
+
"""
|
|
144
|
+
with json_file_path.open() as f:
|
|
145
|
+
return json.load(f)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
_T = TypeVar("_T")
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _list_to_value(obj: list[_T] | _T) -> _T:
|
|
152
|
+
# Some parameters are allocated as `max_dom`-sized lists, with one value
|
|
153
|
+
# per domain. ICON4Py tests (for now) only run on one domain.
|
|
154
|
+
# Most parameters have the same value for all elements, others (such as
|
|
155
|
+
# num_levels) have a default value different from domain[0].
|
|
156
|
+
return obj[0] if isinstance(obj, list) else obj
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def create_experiment_configuration(
|
|
160
|
+
experiment_description: definitions.ExperimentDescription,
|
|
161
|
+
processor_props: decomposition.ProcessProperties,
|
|
162
|
+
) -> definitions.ExperimentConfig:
|
|
163
|
+
"""
|
|
164
|
+
Create configuration objects from the experiment's namelist JSON file.
|
|
165
|
+
|
|
166
|
+
This function reads the NAMELIST_ICON_output_atm.json file that comes with
|
|
167
|
+
the serialized experiment data and constructs configuration objects.
|
|
168
|
+
The JSON file is cached so it's only read once per unique path.
|
|
169
|
+
|
|
170
|
+
Args:
|
|
171
|
+
experiment: The experiment definition
|
|
172
|
+
processor_props: Processor properties containing comm_size
|
|
173
|
+
|
|
174
|
+
Returns:
|
|
175
|
+
ExperimentConfig object containing the configuration for the experiment
|
|
176
|
+
"""
|
|
177
|
+
|
|
178
|
+
experiment_dir = get_ranked_experiment_name_with_version(
|
|
179
|
+
experiment_description,
|
|
180
|
+
processor_props.comm_size,
|
|
181
|
+
)
|
|
182
|
+
json_file_path = (
|
|
183
|
+
definitions.serialized_data_path()
|
|
184
|
+
/ experiment_dir
|
|
185
|
+
/ f"{definitions.NAMELIST_ATM_FNAME}.json"
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
nml_data = _read_namelist_json(json_file_path)
|
|
189
|
+
|
|
190
|
+
gridref_nml = nml_data["gridref_nml"]
|
|
191
|
+
sleve_nml = nml_data["sleve_nml"]
|
|
192
|
+
nonhydrostatic_nml = nml_data["nonhydrostatic_nml"]
|
|
193
|
+
interpol_nml = nml_data["interpol_nml"]
|
|
194
|
+
diffusion_nml = nml_data["diffusion_nml"]
|
|
195
|
+
turbdiff_nml = nml_data["turbdiff_nml"]
|
|
196
|
+
run_nml = nml_data["run_nml"]
|
|
197
|
+
nwp_phy_nml = nml_data.get("nwp_phy_nml")
|
|
198
|
+
nwp_tuning_nml = nml_data.get("nwp_tuning_nml")
|
|
199
|
+
|
|
200
|
+
# *** MetricsConfig ***
|
|
201
|
+
rayleigh_coeff = _list_to_value(nonhydrostatic_nml["rayleigh_coeff"])
|
|
202
|
+
|
|
203
|
+
metrics_config = metrics_factory.MetricsConfig(
|
|
204
|
+
exner_expol=nonhydrostatic_nml["exner_expol"],
|
|
205
|
+
vwind_offctr=nonhydrostatic_nml["vwind_offctr"],
|
|
206
|
+
thslp_zdiffu=nonhydrostatic_nml["thslp_zdiffu"],
|
|
207
|
+
thhgtd_zdiffu=nonhydrostatic_nml["thhgtd_zdiffu"],
|
|
208
|
+
rayleigh_type=constants.RayleighType(nonhydrostatic_nml["rayleigh_type"]),
|
|
209
|
+
rayleigh_coeff=rayleigh_coeff,
|
|
210
|
+
divdamp_trans_start=nonhydrostatic_nml["divdamp_trans_start"],
|
|
211
|
+
divdamp_trans_end=nonhydrostatic_nml["divdamp_trans_end"],
|
|
212
|
+
divdamp_type=nonhydrostatic_nml["divdamp_type"],
|
|
213
|
+
igradp_method=nonhydrostatic_nml["igradp_method"],
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
# *** InterpolationConfig ***
|
|
217
|
+
interpolation_config = interpolation_factory.InterpolationConfig(
|
|
218
|
+
divergence_averaging_central_cell_weight=nml_data["dynamics_nml"]["divavg_cntrwgt"],
|
|
219
|
+
_nudge_max_coeff=interpol_nml["nudge_max_coeff"],
|
|
220
|
+
nudge_efold_width=interpol_nml["nudge_efold_width"],
|
|
221
|
+
nudge_zone_width=interpol_nml["nudge_zone_width"],
|
|
222
|
+
rbf_kernel_cell=rbf.InterpolationKernel(interpol_nml["rbf_vec_kern_c"]),
|
|
223
|
+
rbf_kernel_edge=rbf.InterpolationKernel(interpol_nml["rbf_vec_kern_e"]),
|
|
224
|
+
rbf_kernel_vertex=rbf.InterpolationKernel(interpol_nml["rbf_vec_kern_v"]),
|
|
225
|
+
lsq_dim_stencil=interpol_nml["lsq_high_ord"],
|
|
226
|
+
)
|
|
227
|
+
# _nudge_max_coeff was scaled to max_nudging_coefficient in __post_init__.
|
|
228
|
+
assert interpolation_config.max_nudging_coefficient is not None
|
|
229
|
+
|
|
230
|
+
# *** VerticalGridConfig ***
|
|
231
|
+
rayleigh_damping_height = _list_to_value(nonhydrostatic_nml["damp_height"])
|
|
232
|
+
|
|
233
|
+
vertical_grid_config = v_grid.VerticalGridConfig(
|
|
234
|
+
num_levels=_list_to_value(run_nml["num_lev"]),
|
|
235
|
+
maximal_layer_thickness=sleve_nml["max_lay_thckn"],
|
|
236
|
+
top_height_limit_for_maximal_layer_thickness=sleve_nml["htop_thcknlimit"],
|
|
237
|
+
lowest_layer_thickness=sleve_nml["min_lay_thckn"],
|
|
238
|
+
model_top_height=sleve_nml["top_height"],
|
|
239
|
+
flat_height=sleve_nml["flat_height"],
|
|
240
|
+
stretch_factor=sleve_nml["stretch_fac"],
|
|
241
|
+
rayleigh_damping_height=rayleigh_damping_height,
|
|
242
|
+
htop_moist_proc=nonhydrostatic_nml["htop_moist_proc"],
|
|
243
|
+
SLEVE_decay_scale_1=sleve_nml["decay_scale_1"],
|
|
244
|
+
SLEVE_decay_scale_2=sleve_nml["decay_scale_2"],
|
|
245
|
+
SLEVE_decay_exponent=sleve_nml["decay_exp"],
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
# *** NonHydrostaticConfig ***
|
|
249
|
+
nonhydro_config = solve_nh.NonHydrostaticConfig(
|
|
250
|
+
itime_scheme=dycore_states.TimeSteppingScheme(nonhydrostatic_nml["itime_scheme"]),
|
|
251
|
+
iadv_rhotheta=dycore_states.RhoThetaAdvectionType(nonhydrostatic_nml["iadv_rhotheta"]),
|
|
252
|
+
igradp_method=dycore_states.HorizontalPressureDiscretizationType(
|
|
253
|
+
nonhydrostatic_nml["igradp_method"]
|
|
254
|
+
),
|
|
255
|
+
rayleigh_type=constants.RayleighType(nonhydrostatic_nml["rayleigh_type"]),
|
|
256
|
+
divdamp_order=dycore_states.DivergenceDampingOrder(nonhydrostatic_nml["divdamp_order"]),
|
|
257
|
+
divdamp_type=dycore_states.DivergenceDampingType(nonhydrostatic_nml["divdamp_type"]),
|
|
258
|
+
l_vert_nested=run_nml["lvert_nest"],
|
|
259
|
+
deepatmos_mode=nml_data["dynamics_nml"]["ldeepatmo"],
|
|
260
|
+
extra_diffu=nonhydrostatic_nml["lextra_diffu"],
|
|
261
|
+
rhotheta_offctr=nonhydrostatic_nml["rhotheta_offctr"],
|
|
262
|
+
veladv_offctr=nonhydrostatic_nml["veladv_offctr"],
|
|
263
|
+
max_nudging_coefficient=interpolation_config.max_nudging_coefficient,
|
|
264
|
+
fourth_order_divdamp_factor=nonhydrostatic_nml["divdamp_fac"],
|
|
265
|
+
fourth_order_divdamp_factor2=nonhydrostatic_nml["divdamp_fac2"],
|
|
266
|
+
fourth_order_divdamp_factor3=nonhydrostatic_nml["divdamp_fac3"],
|
|
267
|
+
fourth_order_divdamp_factor4=nonhydrostatic_nml["divdamp_fac4"],
|
|
268
|
+
fourth_order_divdamp_z=nonhydrostatic_nml["divdamp_z"],
|
|
269
|
+
fourth_order_divdamp_z2=nonhydrostatic_nml["divdamp_z2"],
|
|
270
|
+
fourth_order_divdamp_z3=nonhydrostatic_nml["divdamp_z3"],
|
|
271
|
+
fourth_order_divdamp_z4=nonhydrostatic_nml["divdamp_z4"],
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
# *** DiffusionConfig ***
|
|
275
|
+
hdiff_smag_w = _list_to_value(diffusion_nml["lhdiff_smag_w"])
|
|
276
|
+
smag_3d = _list_to_value(diffusion_nml["lsmag_3d"])
|
|
277
|
+
|
|
278
|
+
diffusion_config = diffusion.DiffusionConfig(
|
|
279
|
+
diffusion_type=diffusion.DiffusionType(diffusion_nml["hdiff_order"]),
|
|
280
|
+
hdiff_w=diffusion_nml["lhdiff_w"],
|
|
281
|
+
hdiff_vn=diffusion_nml["lhdiff_vn"],
|
|
282
|
+
hdiff_temp=diffusion_nml["lhdiff_temp"],
|
|
283
|
+
hdiff_smag_w=hdiff_smag_w,
|
|
284
|
+
type_vn_diffu=diffusion.SmagorinskyStencilType(diffusion_nml["itype_vn_diffu"]),
|
|
285
|
+
smag_3d=smag_3d,
|
|
286
|
+
type_t_diffu=diffusion.TemperatureDiscretizationType(diffusion_nml["itype_t_diffu"]),
|
|
287
|
+
hdiff_efdt_ratio=diffusion_nml["hdiff_efdt_ratio"],
|
|
288
|
+
hdiff_w_efdt_ratio=diffusion_nml["hdiff_w_efdt_ratio"],
|
|
289
|
+
smagorinski_scaling_factor=diffusion_nml["hdiff_smag_fac"],
|
|
290
|
+
smagorinski_scaling_factor2=diffusion_nml["hdiff_smag_fac2"],
|
|
291
|
+
smagorinski_scaling_factor3=diffusion_nml["hdiff_smag_fac3"],
|
|
292
|
+
smagorinski_scaling_factor4=diffusion_nml["hdiff_smag_fac4"],
|
|
293
|
+
smagorinski_scaling_height=diffusion_nml["hdiff_smag_z"],
|
|
294
|
+
smagorinski_scaling_height2=diffusion_nml["hdiff_smag_z2"],
|
|
295
|
+
smagorinski_scaling_height3=diffusion_nml["hdiff_smag_z3"],
|
|
296
|
+
smagorinski_scaling_height4=diffusion_nml["hdiff_smag_z4"],
|
|
297
|
+
n_substeps=nonhydrostatic_nml["ndyn_substeps"],
|
|
298
|
+
zdiffu_t=nonhydrostatic_nml["l_zdiffu_t"],
|
|
299
|
+
velocity_boundary_diffusion_denom=gridref_nml["denom_diffu_v"],
|
|
300
|
+
temperature_boundary_diffusion_denom=gridref_nml["denom_diffu_t"],
|
|
301
|
+
max_nudging_coefficient=interpolation_config.max_nudging_coefficient,
|
|
302
|
+
shear_type=diffusion.TurbulenceShearForcingType(turbdiff_nml["itype_sher"]),
|
|
303
|
+
iforcing=diffusion.ForcingType(run_nml["iforcing"]),
|
|
304
|
+
a_hshr=turbdiff_nml["a_hshr"],
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
# *** DriverConfig ***
|
|
308
|
+
driver_cfg = driver_config.DriverConfig(
|
|
309
|
+
experiment_name=experiment_description.name,
|
|
310
|
+
output_path=pathlib.Path(), # TODO (jcanton): Placeholder
|
|
311
|
+
profiling_stats=None,
|
|
312
|
+
dtime=datetime.timedelta(seconds=run_nml["dtime"]),
|
|
313
|
+
start_date=datetime.datetime(1, 1, 1, 0, 0, 0), # TODO (jcanton): Placeholder
|
|
314
|
+
end_date=datetime.datetime(1, 1, 1, 1, 0, 0), # TODO (jcanton): Placeholder
|
|
315
|
+
apply_extra_second_order_divdamp=nonhydrostatic_nml["lextra_diffu"],
|
|
316
|
+
vertical_cfl_threshold=nonhydrostatic_nml["vcfl_threshold"],
|
|
317
|
+
ndyn_substeps=nonhydrostatic_nml["ndyn_substeps"],
|
|
318
|
+
enable_statistics_output=False,
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
# *** GraupelConfig ***
|
|
322
|
+
if nwp_phy_nml is not None and nwp_tuning_nml is not None:
|
|
323
|
+
graupel_config = graupel.SingleMomentSixClassIconGraupelConfig(
|
|
324
|
+
do_latent_heat_nudging=run_nml["ldass_lhn"],
|
|
325
|
+
# ithermo_water == 0 means constant latent heat (docstring in class definition).
|
|
326
|
+
use_constant_latent_heat=_list_to_value(nwp_phy_nml["ithermo_water"]) == 0,
|
|
327
|
+
ice_stickeff_min=nwp_tuning_nml["tune_zceff_min"],
|
|
328
|
+
power_law_coeff_for_ice_mean_fall_speed=nwp_tuning_nml["tune_zvz0i"],
|
|
329
|
+
exponent_for_density_factor_in_ice_sedimentation=nwp_tuning_nml["tune_icesedi_exp"],
|
|
330
|
+
power_law_coeff_for_snow_fall_speed=nwp_tuning_nml["tune_v0snow"],
|
|
331
|
+
rain_mu=nwp_phy_nml["mu_rain"],
|
|
332
|
+
rain_n0=nwp_phy_nml["rain_n0_factor"],
|
|
333
|
+
snow2graupel_riming_coeff=nwp_tuning_nml["tune_zcsg"],
|
|
334
|
+
)
|
|
335
|
+
else:
|
|
336
|
+
graupel_config = graupel.SingleMomentSixClassIconGraupelConfig()
|
|
337
|
+
|
|
338
|
+
return definitions.ExperimentConfig(
|
|
339
|
+
driver=driver_cfg,
|
|
340
|
+
vertical_grid=vertical_grid_config,
|
|
341
|
+
nonhydrostatic=nonhydro_config,
|
|
342
|
+
diffusion=diffusion_config,
|
|
343
|
+
metrics=metrics_config,
|
|
344
|
+
interpolation=interpolation_config,
|
|
345
|
+
graupel=graupel_config,
|
|
346
|
+
)
|