model-config-tests 0.2.4__tar.gz → 0.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/PKG-INFO +8 -2
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/README.md +6 -1
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/pyproject.toml +2 -1
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/_version.py +3 -3
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/compare_exp_tests/test_repro.py +15 -8
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/config_tests/conftest.py +14 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/config_tests/qa/test_access_esm1p6_config.py +31 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/config_tests/qa/test_config.py +339 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/config_tests/test_bit_reproducibility.py +16 -5
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/exp_test_helper.py +87 -284
- model_config_tests-0.2.6/src/model_config_tests/util.py +175 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests.egg-info/PKG-INFO +8 -2
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests.egg-info/requires.txt +1 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/tests/test_exp_test_helper.py +101 -243
- model_config_tests-0.2.6/tests/test_util.py +215 -0
- model_config_tests-0.2.4/src/model_config_tests/util.py +0 -132
- model_config_tests-0.2.4/tests/test_util.py +0 -132
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/LICENSE +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/setup.cfg +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/setup.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/__init__.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/cmds/compare_exp_tests_cmd.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/cmds/config_tests_cmd.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/compare_exp_tests/conftest.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/config_tests/qa/test_access_esm1p5_config.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/config_tests/qa/test_access_om2_config.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/config_tests/qa/test_access_om3_config.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/models/__init__.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/models/accessesm1p5.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/models/accessesm1p6.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/models/accessom2.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/models/accessom3.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/models/model.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/models/mom5.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests/models/um7.py +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests.egg-info/SOURCES.txt +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests.egg-info/dependency_links.txt +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests.egg-info/entry_points.txt +0 -0
- {model_config_tests-0.2.4 → model_config_tests-0.2.6}/src/model_config_tests.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: model_config_tests
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.6
|
|
4
4
|
Summary: Test for ACCESS model (payu) configurations
|
|
5
5
|
Author: ACCESS-NRI
|
|
6
6
|
License: Apache-2.0
|
|
@@ -22,6 +22,7 @@ Requires-Dist: jsonschema>=4.21.1
|
|
|
22
22
|
Requires-Dist: payu>=1.1.3
|
|
23
23
|
Requires-Dist: pytest-sugar
|
|
24
24
|
Requires-Dist: netCDF4
|
|
25
|
+
Requires-Dist: yamanifest
|
|
25
26
|
Provides-Extra: test
|
|
26
27
|
Requires-Dist: pytest-cov; extra == "test"
|
|
27
28
|
Dynamic: license-file
|
|
@@ -49,7 +50,7 @@ Code from these pytests is adapted from COSIMAS's ACCESS-OM2's [bit reproducibil
|
|
|
49
50
|
If you need to run the model with a development version of payu, please use `payu/dev` instead:
|
|
50
51
|
|
|
51
52
|
```sh
|
|
52
|
-
module use /g/data/vk83/modules
|
|
53
|
+
module use /g/data/vk83/prerelease/modules/
|
|
53
54
|
module load payu/dev
|
|
54
55
|
```
|
|
55
56
|
|
|
@@ -358,3 +359,8 @@ If a PR was being merged into a `release-1deg_jra55do_ryf` branch, the QA tests
|
|
|
358
359
|
|
|
359
360
|
If a PR was being merged into a `dev-example-branch`, the QA tests that run automatically would use the `dev_config` tests and if repro tests were triggered manually, then it would use the `repro and (not slow)` tests.
|
|
360
361
|
For both test types, it would use the latest changes in the `model-config-tests` repository. For repro tests, it would use the `payu/dev` module on NCI for running the model.
|
|
362
|
+
|
|
363
|
+
### Release A New Version
|
|
364
|
+
|
|
365
|
+
The model-config-tests repository is actively maintained.
|
|
366
|
+
A new release should be published at least every three months to ensure that updates are available to model configurations.
|
|
@@ -21,7 +21,7 @@ Code from these pytests is adapted from COSIMAS's ACCESS-OM2's [bit reproducibil
|
|
|
21
21
|
If you need to run the model with a development version of payu, please use `payu/dev` instead:
|
|
22
22
|
|
|
23
23
|
```sh
|
|
24
|
-
module use /g/data/vk83/modules
|
|
24
|
+
module use /g/data/vk83/prerelease/modules/
|
|
25
25
|
module load payu/dev
|
|
26
26
|
```
|
|
27
27
|
|
|
@@ -330,3 +330,8 @@ If a PR was being merged into a `release-1deg_jra55do_ryf` branch, the QA tests
|
|
|
330
330
|
|
|
331
331
|
If a PR was being merged into a `dev-example-branch`, the QA tests that run automatically would use the `dev_config` tests and if repro tests were triggered manually, then it would use the `repro and (not slow)` tests.
|
|
332
332
|
For both test types, it would use the latest changes in the `model-config-tests` repository. For repro tests, it would use the `payu/dev` module on NCI for running the model.
|
|
333
|
+
|
|
334
|
+
### Release A New Version
|
|
335
|
+
|
|
336
|
+
The model-config-tests repository is actively maintained.
|
|
337
|
+
A new release should be published at least every three months to ensure that updates are available to model configurations.
|
|
@@ -8,11 +8,11 @@ import json
|
|
|
8
8
|
|
|
9
9
|
version_json = '''
|
|
10
10
|
{
|
|
11
|
-
"date": "2026-
|
|
11
|
+
"date": "2026-10-16T10:51:19+1100",
|
|
12
12
|
"dirty": false,
|
|
13
13
|
"error": null,
|
|
14
|
-
"full-revisionid": "
|
|
15
|
-
"version": "0.2.
|
|
14
|
+
"full-revisionid": "a9683d7c9f5a3a73b30e9678d94fae461c2c292d",
|
|
15
|
+
"version": "0.2.6"
|
|
16
16
|
}
|
|
17
17
|
''' # END VERSION_JSON
|
|
18
18
|
|
|
@@ -5,17 +5,18 @@ from pathlib import Path
|
|
|
5
5
|
from model_config_tests.exp_test_helper import ExpTestHelper
|
|
6
6
|
|
|
7
7
|
|
|
8
|
-
def
|
|
8
|
+
def get_lab_path_and_exp_name(experiment: Path) -> tuple[Path, str]:
|
|
9
9
|
"""
|
|
10
|
-
Derive the lab path
|
|
11
|
-
archive symlink
|
|
10
|
+
Derive the lab path and experiment name from the
|
|
11
|
+
archive symlink in the experiment configuration directory.
|
|
12
12
|
"""
|
|
13
13
|
archive_symlink = experiment / "archive"
|
|
14
14
|
if not archive_symlink.is_symlink():
|
|
15
15
|
raise ValueError(f"Archive symlink does not exist in {experiment}.")
|
|
16
16
|
|
|
17
17
|
# experiment/archive symlink points to lab_path/archive/exp_name
|
|
18
|
-
|
|
18
|
+
archive_path = archive_symlink.resolve()
|
|
19
|
+
return archive_path.parent.parent, archive_path.name
|
|
19
20
|
|
|
20
21
|
|
|
21
22
|
def test_pairwise_repro(experiment_1: Path, experiment_2: Path):
|
|
@@ -25,14 +26,20 @@ def test_pairwise_repro(experiment_1: Path, experiment_2: Path):
|
|
|
25
26
|
dynamically generate pairs of experiments to compare.
|
|
26
27
|
"""
|
|
27
28
|
|
|
28
|
-
lab_path1 =
|
|
29
|
+
lab_path1, exp_name1 = get_lab_path_and_exp_name(experiment_1)
|
|
29
30
|
exp1 = ExpTestHelper(
|
|
30
|
-
control_path=experiment_1,
|
|
31
|
+
control_path=experiment_1,
|
|
32
|
+
lab_path=lab_path1,
|
|
33
|
+
disable_payu_run=True,
|
|
34
|
+
exp_name=exp_name1,
|
|
31
35
|
)
|
|
32
36
|
|
|
33
|
-
lab_path2 =
|
|
37
|
+
lab_path2, exp_name2 = get_lab_path_and_exp_name(experiment_2)
|
|
34
38
|
exp2 = ExpTestHelper(
|
|
35
|
-
control_path=experiment_2,
|
|
39
|
+
control_path=experiment_2,
|
|
40
|
+
lab_path=lab_path2,
|
|
41
|
+
disable_payu_run=True,
|
|
42
|
+
exp_name=exp_name2,
|
|
36
43
|
)
|
|
37
44
|
|
|
38
45
|
# Compare the two experiments - compares checksums from output000
|
|
@@ -82,6 +82,13 @@ def keep_archive(request):
|
|
|
82
82
|
return request.config.getoption("--keep-archive")
|
|
83
83
|
|
|
84
84
|
|
|
85
|
+
@pytest.fixture(scope="session")
|
|
86
|
+
def walltime(request):
|
|
87
|
+
"""Set walltime for test run job submission. If not provided, defaults to DEFAULT_WALLTIME.
|
|
88
|
+
Can be in HH:MM:SS format or numeric seconds."""
|
|
89
|
+
return request.config.getoption("--walltime")
|
|
90
|
+
|
|
91
|
+
|
|
85
92
|
# Set up command line options and default for directory paths
|
|
86
93
|
def pytest_addoption(parser):
|
|
87
94
|
"""Attaches optional command line arguments"""
|
|
@@ -113,6 +120,13 @@ def pytest_addoption(parser):
|
|
|
113
120
|
help="Keep archive from previous test run and disable running payu",
|
|
114
121
|
)
|
|
115
122
|
|
|
123
|
+
parser.addoption(
|
|
124
|
+
"--walltime",
|
|
125
|
+
action="store",
|
|
126
|
+
default="",
|
|
127
|
+
help="Walltime for test run job submission (HH:MM:SS or numeric seconds). Defaults to DEFAULT_WALLTIME if not provided",
|
|
128
|
+
)
|
|
129
|
+
|
|
116
130
|
|
|
117
131
|
def pytest_configure(config):
|
|
118
132
|
config.addinivalue_line(
|
|
@@ -9,6 +9,7 @@ from typing import Any
|
|
|
9
9
|
|
|
10
10
|
import f90nml
|
|
11
11
|
import pytest
|
|
12
|
+
import yaml
|
|
12
13
|
|
|
13
14
|
from model_config_tests.config_tests.qa.test_config import (
|
|
14
15
|
check_manifest_exes_in_spack_location,
|
|
@@ -296,3 +297,33 @@ class TestAccessEsm1p6:
|
|
|
296
297
|
f" in {CICE_IN_NML_FNAME} and {ICE_HISTORY_NML_FNAME}"
|
|
297
298
|
f": {repeated_fields}"
|
|
298
299
|
)
|
|
300
|
+
|
|
301
|
+
def test_addmeta_base_configuration(self, branch, control_path):
|
|
302
|
+
"""
|
|
303
|
+
Check that the 'base_configuration' option used by addmeta matches
|
|
304
|
+
the corresponding release branch name.
|
|
305
|
+
"""
|
|
306
|
+
dataspec_path = (
|
|
307
|
+
control_path / "scripts" / "post-processing" / "addmeta" / "dataspec.yaml"
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
assert (
|
|
311
|
+
dataspec_path.is_file()
|
|
312
|
+
), "Expected configuration to contain 'dataspec.yaml' file for addmeta call during post-processing."
|
|
313
|
+
|
|
314
|
+
with open(dataspec_path) as f:
|
|
315
|
+
dataspec = yaml.safe_load(f)
|
|
316
|
+
|
|
317
|
+
assert "global" in dataspec, error_field_nonexistence("global", "dataspec.yaml")
|
|
318
|
+
|
|
319
|
+
assert "base_configuration" in dataspec["global"], error_field_nonexistence(
|
|
320
|
+
"global.base_configuration", "dataspec.yaml"
|
|
321
|
+
)
|
|
322
|
+
|
|
323
|
+
base_configuration = dataspec["global"]["base_configuration"]
|
|
324
|
+
expected_base_configuration = (
|
|
325
|
+
f"{branch.branch_name.replace(branch.config_type, 'release')}"
|
|
326
|
+
)
|
|
327
|
+
assert base_configuration == expected_base_configuration, error_field_incorrect(
|
|
328
|
+
"global.base_configuration", "dataspec.yaml", expected_base_configuration
|
|
329
|
+
)
|
|
@@ -4,6 +4,9 @@
|
|
|
4
4
|
"""Tests for checking configs and valid metadata files"""
|
|
5
5
|
|
|
6
6
|
import re
|
|
7
|
+
import shutil
|
|
8
|
+
import subprocess
|
|
9
|
+
import tempfile
|
|
7
10
|
import warnings
|
|
8
11
|
from pathlib import Path
|
|
9
12
|
from typing import Any
|
|
@@ -12,6 +15,7 @@ import jsonschema
|
|
|
12
15
|
import pytest
|
|
13
16
|
import requests
|
|
14
17
|
import yaml
|
|
18
|
+
from yamanifest import Manifest
|
|
15
19
|
|
|
16
20
|
from model_config_tests.util import get_git_branch_name
|
|
17
21
|
|
|
@@ -28,6 +32,20 @@ LICENSE_URL = "https://creativecommons.org/licenses/by/4.0/legalcode.txt"
|
|
|
28
32
|
# Release modules location on NCI
|
|
29
33
|
RELEASE_MODULE_LOCATION = "/g/data/vk83/modules"
|
|
30
34
|
|
|
35
|
+
# Model config inputs repository for input file MD5 verification
|
|
36
|
+
MODEL_CONFIG_INPUTS_URL = "https://github.com/ACCESS-NRI/model-config-inputs"
|
|
37
|
+
MODEL_CONFIG_INPUTS_CLONE_URL = "https://github.com/ACCESS-NRI/model-config-inputs.git"
|
|
38
|
+
|
|
39
|
+
# Model config input location and symlink
|
|
40
|
+
MODEL_CONFIG_INPUTS_LOCATION_SYMLINK = "/g/data/vk83/experiments/inputs/"
|
|
41
|
+
MODEL_CONFIG_INPUTS_LOCATION = "/g/data/vk83/configurations/inputs/"
|
|
42
|
+
MODEL_CONFIG_INPUTS_PRERELEASE = "/g/data/vk83/prerelease"
|
|
43
|
+
PUBLISH_DATA_LOCATION = [
|
|
44
|
+
"/g/data/qv56/replicas",
|
|
45
|
+
"/g/data/jq44",
|
|
46
|
+
"/g/data/av17",
|
|
47
|
+
]
|
|
48
|
+
|
|
31
49
|
|
|
32
50
|
def insist_array(str_or_array):
|
|
33
51
|
if isinstance(str_or_array, str):
|
|
@@ -93,6 +111,15 @@ class TestRelConfig:
|
|
|
93
111
|
+ "manifest:\n reproduce:\n exe: True"
|
|
94
112
|
)
|
|
95
113
|
|
|
114
|
+
def test_manifest_input_match_repo(
|
|
115
|
+
self, branch_type, control_path, config, cache_input_dir
|
|
116
|
+
):
|
|
117
|
+
"""Check that input file MD5 hashes in manifests/input.yaml match
|
|
118
|
+
those from model-config-inputs repository"""
|
|
119
|
+
compare_input_md5_hashes(
|
|
120
|
+
control_path, config, cache_input_dir, branch_type=branch_type
|
|
121
|
+
)
|
|
122
|
+
|
|
96
123
|
def test_metadata_is_enabled(self, config):
|
|
97
124
|
if "metadata" in config and "enable" in config["metadata"]:
|
|
98
125
|
assert config["metadata"]["enable"], (
|
|
@@ -245,6 +272,37 @@ class TestConfig:
|
|
|
245
272
|
+ "If set, branching in payu would not work."
|
|
246
273
|
)
|
|
247
274
|
|
|
275
|
+
def test_jobname_match_branch_name(
|
|
276
|
+
self, config, control_path, branch_type, target_branch
|
|
277
|
+
):
|
|
278
|
+
"""Check that jobname in config.yaml *contains* the scenario part of the branch name.
|
|
279
|
+
Scenario part is after "release-"/"dev-" and before the modifier (+wombatlite).
|
|
280
|
+
It is optional for jobname to contain the modifier, but required to contain the scenario part.
|
|
281
|
+
"""
|
|
282
|
+
jobname = config.get("jobname", "")
|
|
283
|
+
|
|
284
|
+
# Use the target branch name if provided, otherwise get the current branch name
|
|
285
|
+
branch_name = target_branch
|
|
286
|
+
if branch_name is None:
|
|
287
|
+
branch_name = get_git_branch_name(control_path)
|
|
288
|
+
|
|
289
|
+
prefix = f"{branch_type}-"
|
|
290
|
+
|
|
291
|
+
# ignore the branch type- and +modifier
|
|
292
|
+
expt_name = branch_name.split(prefix, 1)[1].split("+", 1)[0]
|
|
293
|
+
|
|
294
|
+
if "+" not in branch_name:
|
|
295
|
+
# When no modifer is in branch name, jobname must match the scenario part
|
|
296
|
+
assert (
|
|
297
|
+
jobname == expt_name
|
|
298
|
+
), f"Jobname '{jobname}' in config does not match the experiment name '{expt_name}' extracted from branch name '{branch_name}'"
|
|
299
|
+
|
|
300
|
+
else:
|
|
301
|
+
# When modifer is presented, jobname must start with the scenario part
|
|
302
|
+
assert jobname.startswith(
|
|
303
|
+
expt_name
|
|
304
|
+
), f"Jobname '{jobname}' in config does not start with the experiment name '{expt_name}' extracted from branch name '{branch_name}'"
|
|
305
|
+
|
|
248
306
|
|
|
249
307
|
def read_exe_manifest_fullpaths(control_path: Path):
|
|
250
308
|
"""Return the full paths to the executables in the executable manifest file"""
|
|
@@ -364,3 +422,284 @@ def check_manifest_exes_in_spack_location(
|
|
|
364
422
|
"Only the name of the executable is needed, as the full path is "
|
|
365
423
|
f"determined by payu (which searches PATHs added by {model_module_name} module)"
|
|
366
424
|
)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def read_input_fullpaths_from_config(config: dict[str, Any]) -> list[str]:
|
|
428
|
+
"""Extract all input file fullpaths from config.yaml.
|
|
429
|
+
|
|
430
|
+
Parameters
|
|
431
|
+
----------
|
|
432
|
+
config : Dict[str, Any]
|
|
433
|
+
The contents of the config.yaml file
|
|
434
|
+
|
|
435
|
+
Returns
|
|
436
|
+
-------
|
|
437
|
+
list[str]
|
|
438
|
+
List of fullpaths to input files
|
|
439
|
+
"""
|
|
440
|
+
fullpaths = []
|
|
441
|
+
|
|
442
|
+
# Get top-level input paths and submodel input paths
|
|
443
|
+
for model in [config] + config.get("submodels", []):
|
|
444
|
+
if "input" in model:
|
|
445
|
+
input_paths = insist_array(model["input"])
|
|
446
|
+
input_paths = [
|
|
447
|
+
p.replace(
|
|
448
|
+
MODEL_CONFIG_INPUTS_LOCATION_SYMLINK, MODEL_CONFIG_INPUTS_LOCATION
|
|
449
|
+
)
|
|
450
|
+
for p in input_paths
|
|
451
|
+
]
|
|
452
|
+
fullpaths.extend(input_paths)
|
|
453
|
+
|
|
454
|
+
return fullpaths
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def read_manifest_input_hashes(control_path: Path) -> dict[str, str]:
|
|
458
|
+
"""Read MD5 hashes from manifests/input.yaml.
|
|
459
|
+
|
|
460
|
+
Parameters
|
|
461
|
+
----------
|
|
462
|
+
control_path : Path
|
|
463
|
+
Path to the local control directory containing the manifest file
|
|
464
|
+
|
|
465
|
+
Returns
|
|
466
|
+
-------
|
|
467
|
+
dict[str, str]
|
|
468
|
+
Dictionary mapping fullpath to MD5 hash
|
|
469
|
+
"""
|
|
470
|
+
local_input = {}
|
|
471
|
+
|
|
472
|
+
manifest = Manifest(control_path / "manifests" / "input.yaml").load()
|
|
473
|
+
|
|
474
|
+
for filepath in manifest:
|
|
475
|
+
|
|
476
|
+
fullpath = manifest.fullpath(filepath)
|
|
477
|
+
fullpath = fullpath.replace(
|
|
478
|
+
MODEL_CONFIG_INPUTS_LOCATION_SYMLINK, MODEL_CONFIG_INPUTS_LOCATION
|
|
479
|
+
)
|
|
480
|
+
|
|
481
|
+
local_input[fullpath] = manifest.get(filepath, "md5")
|
|
482
|
+
|
|
483
|
+
return local_input
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def _cache_input_repo() -> Path:
|
|
487
|
+
"""Fetch and cache the entire model-config-inputs repository."""
|
|
488
|
+
cache_input_dir = Path(tempfile.mkdtemp())
|
|
489
|
+
try:
|
|
490
|
+
# Clone the model-config-inputs repository into a temporary directory
|
|
491
|
+
subprocess.run(
|
|
492
|
+
[
|
|
493
|
+
"git",
|
|
494
|
+
"clone",
|
|
495
|
+
"--branch",
|
|
496
|
+
"main",
|
|
497
|
+
"--depth",
|
|
498
|
+
"1",
|
|
499
|
+
MODEL_CONFIG_INPUTS_CLONE_URL,
|
|
500
|
+
cache_input_dir,
|
|
501
|
+
],
|
|
502
|
+
check=True,
|
|
503
|
+
timeout=100,
|
|
504
|
+
)
|
|
505
|
+
|
|
506
|
+
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
|
|
507
|
+
shutil.rmtree(cache_input_dir, ignore_errors=True)
|
|
508
|
+
raise RuntimeError(
|
|
509
|
+
f"Failed to clone model-config-inputs repository from {MODEL_CONFIG_INPUTS_CLONE_URL}: {e}"
|
|
510
|
+
) from e
|
|
511
|
+
|
|
512
|
+
return cache_input_dir
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
@pytest.fixture(scope="session")
|
|
516
|
+
def cache_input_dir():
|
|
517
|
+
"""Clone the model-config-inputs repository once per test session and
|
|
518
|
+
clean up the temporary clone once all tests using it have finished."""
|
|
519
|
+
cache_dir = _cache_input_repo()
|
|
520
|
+
yield cache_dir
|
|
521
|
+
shutil.rmtree(cache_dir, ignore_errors=True)
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def match_file_name_in_repo(
|
|
525
|
+
file_name: str,
|
|
526
|
+
data: Manifest,
|
|
527
|
+
repo_manifest_path: Path,
|
|
528
|
+
) -> str:
|
|
529
|
+
"""Match a file name to its key in the manifest.
|
|
530
|
+
The model-config-inputs repository may store file names with or without
|
|
531
|
+
a leading ``./`` (e.g. ``basin_mask.nc`` or ``./basin_mask.nc``)."""
|
|
532
|
+
if file_name in data:
|
|
533
|
+
manifest_file_name = file_name
|
|
534
|
+
elif f"./{file_name}" in data:
|
|
535
|
+
manifest_file_name = f"./{file_name}"
|
|
536
|
+
else:
|
|
537
|
+
raise ValueError(
|
|
538
|
+
f"Neither file name {file_name} and ./{file_name} not found in manifest {MODEL_CONFIG_INPUTS_URL}/tree/main/{repo_manifest_path}. "
|
|
539
|
+
)
|
|
540
|
+
|
|
541
|
+
return manifest_file_name
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def extract_input_md5_hashes_from_repo(
|
|
545
|
+
fullpaths: list[str], cache_input_dir: Path
|
|
546
|
+
) -> dict[str, dict[str, str]]:
|
|
547
|
+
"""Extract MD5 hashes for input files from local-cloned model-config-inputs repository.
|
|
548
|
+
|
|
549
|
+
Parameters
|
|
550
|
+
----------
|
|
551
|
+
fullpaths : list[str]
|
|
552
|
+
List of fullpaths to input files from config.yaml
|
|
553
|
+
cache_input_dir : Path
|
|
554
|
+
Path to the cached model-config-inputs repository
|
|
555
|
+
|
|
556
|
+
Returns
|
|
557
|
+
-------
|
|
558
|
+
dict[str, dict[str, str]]
|
|
559
|
+
Dictionary mapping fullpath to {MD5 hash, repo URL} from repo
|
|
560
|
+
"""
|
|
561
|
+
model_config_input = {}
|
|
562
|
+
|
|
563
|
+
for fullpath in fullpaths:
|
|
564
|
+
path = cache_input_dir / fullpath.split("/inputs/")[-1]
|
|
565
|
+
|
|
566
|
+
# If fullpath is a file, manifest_path should exist
|
|
567
|
+
manifest_path = path.parent / ".manifest.yaml"
|
|
568
|
+
if manifest_path.is_file():
|
|
569
|
+
repo_manifest_path = manifest_path.relative_to(
|
|
570
|
+
cache_input_dir
|
|
571
|
+
) # For error message
|
|
572
|
+
|
|
573
|
+
# Load the manifest and extract fullpath for this input file
|
|
574
|
+
data = Manifest(manifest_path).load()
|
|
575
|
+
|
|
576
|
+
# Get the file name matching the input repo
|
|
577
|
+
manifest_file_name = match_file_name_in_repo(
|
|
578
|
+
path.name, data, repo_manifest_path
|
|
579
|
+
)
|
|
580
|
+
|
|
581
|
+
# Check if the fullpath in the input repo matches the fullpath from config.yaml
|
|
582
|
+
repo_fullpath = data.fullpath(manifest_file_name)
|
|
583
|
+
assert fullpath == repo_fullpath, (
|
|
584
|
+
f'Fullpath in config.yaml "{fullpath}" does not match the one "{repo_fullpath}" in model-config-inputs repo '
|
|
585
|
+
f"in {MODEL_CONFIG_INPUTS_URL}/tree/main/{repo_manifest_path}."
|
|
586
|
+
)
|
|
587
|
+
|
|
588
|
+
model_config_input[fullpath] = {
|
|
589
|
+
"md5hash": data.get(manifest_file_name, "md5"),
|
|
590
|
+
"repo_url": f"{MODEL_CONFIG_INPUTS_URL}/tree/main/{repo_manifest_path}",
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
else:
|
|
594
|
+
# Assume that fullpath is a directory
|
|
595
|
+
# Look for all .manifest.yaml files in the directory and its subdirectories
|
|
596
|
+
manifest_paths = list(path.rglob(".manifest.yaml"))
|
|
597
|
+
|
|
598
|
+
# Raise an error if still no manifest files are found
|
|
599
|
+
assert len(manifest_paths) > 0, (
|
|
600
|
+
f"No manifest files found for input file/directory '{fullpath}'. "
|
|
601
|
+
f"Searched recursively in model-config-inputs repo at {MODEL_CONFIG_INPUTS_URL}/tree/main/{path.relative_to(cache_input_dir)}. "
|
|
602
|
+
f"Expected to find at least one .manifest.yaml file."
|
|
603
|
+
)
|
|
604
|
+
|
|
605
|
+
# Load each manifest file
|
|
606
|
+
for manifest_path in manifest_paths:
|
|
607
|
+
data = Manifest(manifest_path).load()
|
|
608
|
+
repo_manifest_path = manifest_path.relative_to(cache_input_dir)
|
|
609
|
+
|
|
610
|
+
# Extract all files' fullpath and md5hash
|
|
611
|
+
for file in data:
|
|
612
|
+
if file == ".manifest.yaml":
|
|
613
|
+
# Skip .manifest.yaml
|
|
614
|
+
continue
|
|
615
|
+
|
|
616
|
+
repo_fullpath = data.fullpath(file)
|
|
617
|
+
md5hash = data.get(file, "md5")
|
|
618
|
+
model_config_input[repo_fullpath] = {
|
|
619
|
+
"md5hash": md5hash,
|
|
620
|
+
"repo_url": f"{MODEL_CONFIG_INPUTS_URL}/tree/main/{repo_manifest_path}",
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
return model_config_input
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
def compare_input_md5_hashes(
|
|
627
|
+
control_path: Path,
|
|
628
|
+
config: dict[str, Any],
|
|
629
|
+
cache_input_dir: Path,
|
|
630
|
+
branch_type: str = "release",
|
|
631
|
+
):
|
|
632
|
+
"""Check that input file MD5 hashes match those in manifests/input.yaml
|
|
633
|
+
and from model-config-inputs repository.
|
|
634
|
+
|
|
635
|
+
Parameters
|
|
636
|
+
----------
|
|
637
|
+
control_path : Path
|
|
638
|
+
Path to the model configuration directory
|
|
639
|
+
config : Dict[str, Any]
|
|
640
|
+
The contents of the config.yaml file
|
|
641
|
+
cache_input_dir : Path
|
|
642
|
+
Path to the cached model-config-inputs repository
|
|
643
|
+
|
|
644
|
+
Raises
|
|
645
|
+
------
|
|
646
|
+
AssertionError
|
|
647
|
+
If MD5 hashes do not match or files are not found
|
|
648
|
+
"""
|
|
649
|
+
# Get input fullpaths from config.yaml
|
|
650
|
+
fullpaths = read_input_fullpaths_from_config(config)
|
|
651
|
+
|
|
652
|
+
# Check if all input files are in vk83 or published data project location
|
|
653
|
+
fullpaths = check_allowed_config_location(fullpaths, branch_type=branch_type)
|
|
654
|
+
|
|
655
|
+
# Read local MD5 hashes from manifests/input.yaml
|
|
656
|
+
local_input = read_manifest_input_hashes(control_path)
|
|
657
|
+
|
|
658
|
+
# Fetch MD5 hashes from model-config-inputs repo
|
|
659
|
+
repo_hashes = extract_input_md5_hashes_from_repo(fullpaths, cache_input_dir)
|
|
660
|
+
|
|
661
|
+
# Compare hashes for each input file
|
|
662
|
+
for fullpath, repo_response in repo_hashes.items():
|
|
663
|
+
assert (
|
|
664
|
+
fullpath in local_input
|
|
665
|
+
), f"Expected local input manifest to include input file: {fullpath}"
|
|
666
|
+
local_hash = local_input.get(fullpath)
|
|
667
|
+
|
|
668
|
+
repo_hash = repo_response.get("md5hash")
|
|
669
|
+
assert repo_hash == local_hash, (
|
|
670
|
+
f"MD5 hash mismatch for {fullpath}: "
|
|
671
|
+
f"local manifest hash: {local_hash}, repo hash: {repo_hash} from {repo_response.get('repo_url')}"
|
|
672
|
+
)
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def check_allowed_config_location(
|
|
676
|
+
fullpaths: list[str], branch_type: str = "release"
|
|
677
|
+
) -> list[str]:
|
|
678
|
+
"""
|
|
679
|
+
Picks out input files in /g/data/vk83/configurations for furture md5 hash checking.
|
|
680
|
+
It raises a RuntimeError if:
|
|
681
|
+
- any input file is not in /g/data/vk83 or a published data project location,
|
|
682
|
+
- a release branch uses prerelease input files.
|
|
683
|
+
"""
|
|
684
|
+
|
|
685
|
+
filter_fps = []
|
|
686
|
+
|
|
687
|
+
for path in fullpaths:
|
|
688
|
+
# Pick up input files in /g/data/vk83/configurations(experiments) for md5 hash checking
|
|
689
|
+
if path.startswith(MODEL_CONFIG_INPUTS_LOCATION):
|
|
690
|
+
filter_fps.append(path)
|
|
691
|
+
# Skip following checks
|
|
692
|
+
continue
|
|
693
|
+
|
|
694
|
+
# Check if a release branch is using prerelease input files
|
|
695
|
+
if path.startswith(MODEL_CONFIG_INPUTS_PRERELEASE):
|
|
696
|
+
assert (
|
|
697
|
+
branch_type != "release"
|
|
698
|
+
), f"Input file {path} is in prerelease location, which is not allowed for release branches. "
|
|
699
|
+
else:
|
|
700
|
+
# Check if input file is in a published data project location
|
|
701
|
+
assert any(
|
|
702
|
+
path.startswith(loc) for loc in PUBLISH_DATA_LOCATION
|
|
703
|
+
), f"Input file {path} is not in vk83 or a published data project location: {PUBLISH_DATA_LOCATION}. "
|
|
704
|
+
|
|
705
|
+
return filter_fps
|
|
@@ -52,7 +52,11 @@ def read_historical_checksums(
|
|
|
52
52
|
|
|
53
53
|
|
|
54
54
|
def _experiments(
|
|
55
|
-
markers: list,
|
|
55
|
+
markers: list,
|
|
56
|
+
output_path: Path,
|
|
57
|
+
control_path: Path,
|
|
58
|
+
keep_archive: Optional[bool],
|
|
59
|
+
walltime: Optional[str],
|
|
56
60
|
) -> Experiments:
|
|
57
61
|
"""
|
|
58
62
|
Run all requested experiments
|
|
@@ -70,6 +74,8 @@ def _experiments(
|
|
|
70
74
|
keep_archive: Optional[bool]
|
|
71
75
|
Whether to keep the previous archive for each experiment and
|
|
72
76
|
disable calls to payu run. This is used in testing.
|
|
77
|
+
walltime: Optional[str]
|
|
78
|
+
Walltime for test run job submission.
|
|
73
79
|
|
|
74
80
|
Returns
|
|
75
81
|
-------
|
|
@@ -102,13 +108,12 @@ def _experiments(
|
|
|
102
108
|
requested_experiments[exp_name]["n_runs"] = max(
|
|
103
109
|
requested_experiment["n_runs"], n_runs
|
|
104
110
|
)
|
|
105
|
-
|
|
106
111
|
# Cleaning up any pre-existing historical checksums output first
|
|
107
112
|
# incase there's failures in setup
|
|
108
113
|
set_checksum_output_dir(output_path=output_path)
|
|
109
114
|
|
|
110
115
|
print("Submitting all requested experiments")
|
|
111
|
-
experiments = Experiments(control_path, output_path, keep_archive)
|
|
116
|
+
experiments = Experiments(control_path, output_path, keep_archive, walltime)
|
|
112
117
|
for exp_name, config in requested_experiments.items():
|
|
113
118
|
experiments.setup_and_submit(
|
|
114
119
|
exp_name=exp_name,
|
|
@@ -126,7 +131,11 @@ def _experiments(
|
|
|
126
131
|
|
|
127
132
|
@pytest.fixture(scope="class")
|
|
128
133
|
def experiments(
|
|
129
|
-
request,
|
|
134
|
+
request,
|
|
135
|
+
output_path: Path,
|
|
136
|
+
control_path: Path,
|
|
137
|
+
keep_archive: Optional[bool],
|
|
138
|
+
walltime: Optional[str],
|
|
130
139
|
):
|
|
131
140
|
"""
|
|
132
141
|
Parse the experiments markers from the requested tests and
|
|
@@ -144,7 +153,9 @@ def experiments(
|
|
|
144
153
|
if marker:
|
|
145
154
|
experiments_markers.append(marker.args[0])
|
|
146
155
|
|
|
147
|
-
return _experiments(
|
|
156
|
+
return _experiments(
|
|
157
|
+
experiments_markers, output_path, control_path, keep_archive, walltime
|
|
158
|
+
)
|
|
148
159
|
|
|
149
160
|
|
|
150
161
|
@pytest.fixture
|