model-config-tests 0.2.2__tar.gz → 0.2.3__tar.gz

{model_config_tests-0.2.2 → model_config_tests-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: model_config_tests
-Version: 0.2.2
+Version: 0.2.3
 Summary: Test for ACCESS model (payu) configurations
 Author: ACCESS-NRI
 License: Apache-2.0

{model_config_tests-0.2.2 → model_config_tests-0.2.3}/pyproject.toml

@@ -112,6 +112,8 @@ tag_prefix = "v"
 parentdir_prefix = "model_config_tests-"
 
 [tool.coverage.run]
+patch = ["subprocess"]
 omit = [
-    "src/model_config_tests/_version.py"
+    "*/model_config_tests/_version.py",
+    "src/model_config_tests/_version.py",
 ]

{model_config_tests-0.2.2 → model_config_tests-0.2.3}/src/model_config_tests/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-21T07:48:35+1100",
+ "date": "2025-12-19T17:08:47+1100",
  "dirty": false,
  "error": null,
- "full-revisionid": "5b34e0c7b8cb513b5204a8baa5f97b22f84c2407",
- "version": "0.2.2"
+ "full-revisionid": "31685c167c305b26642a22f00796dec7d8725551",
+ "version": "0.2.3"
 }
 '''  # END VERSION_JSON
 

{model_config_tests-0.2.2 → model_config_tests-0.2.3}/src/model_config_tests/config_tests/qa/test_access_om2_config.py

@@ -116,7 +116,7 @@ class TestAccessOM2:
 
     def test_mppncombine_fast_collate_exe(self, config, branch):
         if branch.is_high_resolution:
-            pattern = r"/g/data/vk83/apps/mppnccombine-fast/.*/bin/mppnccombine-fast"
+            pattern = r".*mppnccombine-fast"
             if "collate" in config:
                 assert re.match(
                     pattern, config["collate"]["exe"]

{model_config_tests-0.2.2 → model_config_tests-0.2.3}/src/model_config_tests/config_tests/test_bit_reproducibility.py

@@ -9,7 +9,7 @@ from typing import Optional
 
 import pytest
 
-from model_config_tests.exp_test_helper import Experiments
+from model_config_tests.exp_test_helper import Experiments, ExpTestHelper
 from model_config_tests.util import DAY_IN_SECONDS, HOUR_IN_SECONDS
 
 # Names of shared experiments
@@ -147,6 +147,20 @@ def experiments(
     return _experiments(experiments_markers, output_path, control_path, keep_archive)
 
 
+@pytest.fixture
+def requested_experiments(request, experiments: Experiments):
+    """Fixture to check that requested experiments have run successfully
+    and return a dictionary of ExpTestHelper instances for each experiment."""
+    exp_marker = request.node.get_closest_marker("experiments").args[0]
+    requested_exps = {}
+    for exp_name in exp_marker:
+        # Check experiment has run successfully - this will raise an
+        # error if there are any non-zero exit codes in the outputs
+        experiments.check_experiment(exp_name)
+        requested_exps[exp_name] = experiments.get_experiment(exp_name)
+    return requested_exps
+
+
 class TestBitReproducibility:
 
     @pytest.mark.repro
@@ -160,7 +174,7 @@ class TestBitReproducibility:
         self,
         output_path: Path,
         control_path: Path,
-        experiments: Experiments,
+        requested_experiments: dict[str, ExpTestHelper],
         checksum_path: Optional[Path],
     ):
         """
@@ -178,9 +192,9 @@ class TestBitReproducibility:
             Path to the model configuration to test. This is copied for
             for control directories in experiments. Default is set in
             conftests.py.
-        experiments: Experiments
-            Class that manages the shared experiments. This is a fixture
-            defined in this file.
+        requested_experiments: dict[str, ExpTestHelper]
+            A dictionary of requested experiments, where the key is the
+            experiment name and the value is an instance of ExpTestHelper.
         checksum_path: Optional[Path]
             Path to checksums to compare model output against. Default is
             set to checksums saved on model configuration. This is a
@@ -190,12 +204,7 @@ class TestBitReproducibility:
         checksum_output_dir = set_checksum_output_dir(output_path=output_path)
 
         # Use default runtime experiment to get the historical checksums
-        experiments.check_experiments([EXP_DEFAULT_RUNTIME])
-        exp = experiments.get_experiment(EXP_DEFAULT_RUNTIME)
-
-        assert (
-            exp.model.output_exists()
-        ), "Output file required for model checksums does not exist"
+        exp = requested_experiments.get(EXP_DEFAULT_RUNTIME)
 
         # Set the checksum output filename using the model default runtime
         runtime_hours = exp.model.default_runtime_seconds // HOUR_IN_SECONDS
@@ -235,20 +244,16 @@ class TestBitReproducibility:
             EXP_1D_RUNTIME_REPEAT: {"n_runs": 1, "model_runtime": DAY_IN_SECONDS},
         }
     )
-    def test_repro_determinism(self, experiments: Experiments):
+    def test_repro_determinism(self, requested_experiments: dict[str, ExpTestHelper]):
         """
         Determinism test that confirms repeated model runs for 1 day
         give the same results
         """
-        experiments.check_experiments([EXP_1D_RUNTIME, EXP_1D_RUNTIME_REPEAT])
-        exp_1d_runtime = experiments.get_experiment(EXP_1D_RUNTIME)
-        exp_1d_runtime_repeat = experiments.get_experiment(EXP_1D_RUNTIME_REPEAT)
+        exp_1d_runtime = requested_experiments.get(EXP_1D_RUNTIME)
+        exp_1d_runtime_repeat = requested_experiments.get(EXP_1D_RUNTIME_REPEAT)
 
         # Compare expected to produced.
-        assert exp_1d_runtime.model.output_exists()
         expected = exp_1d_runtime.extract_checksums()
-
-        assert exp_1d_runtime_repeat.model.output_exists()
         produced = exp_1d_runtime_repeat.extract_checksums()
 
         assert produced == expected
@@ -262,16 +267,17 @@ class TestBitReproducibility:
             EXP_2D_RUNTIME: {"n_runs": 1, "model_runtime": 2 * DAY_IN_SECONDS},
         }
     )
-    def test_repro_restart(self, output_path: Path, experiments: Experiments):
+    def test_repro_restart(
+        self, output_path: Path, requested_experiments: dict[str, ExpTestHelper]
+    ):
         """
         Restart reproducibility test that confirms two short consecutive
         1-day model runs give the same results as a longer single 2-day model
         run.
         """
         # Get experiments with 2x1 day and 2 day runtimes
-        experiments.check_experiments([EXP_1D_RUNTIME, EXP_2D_RUNTIME])
-        exp_1d_runtime = experiments.get_experiment(EXP_1D_RUNTIME)
-        exp_2d_runtime = experiments.get_experiment(EXP_2D_RUNTIME)
+        exp_1d_runtime = requested_experiments.get(EXP_1D_RUNTIME)
+        exp_2d_runtime = requested_experiments.get(EXP_2D_RUNTIME)
 
         # Now compare the output between our two short and one long run.
         checksums_1d_0 = exp_1d_runtime.extract_checksums()
@@ -305,14 +311,15 @@ class TestBitReproducibility:
             EXP_1D_RUNTIME_REPEAT: {"n_runs": 2, "model_runtime": DAY_IN_SECONDS},
         }
     )
-    def test_repro_determinism_restart(self, experiments: Experiments):
+    def test_repro_determinism_restart(
+        self, requested_experiments: dict[str, ExpTestHelper]
+    ):
         """
         Determinism test that confirms repeated experiments with two
         consecutive 1-day model runs give the same results
         """
-        experiments.check_experiments([EXP_1D_RUNTIME, EXP_1D_RUNTIME_REPEAT])
-        exp_1d_runtime = experiments.get_experiment(EXP_1D_RUNTIME)
-        exp_1d_runtime_repeat = experiments.get_experiment(EXP_1D_RUNTIME_REPEAT)
+        exp_1d_runtime = requested_experiments.get(EXP_1D_RUNTIME)
+        exp_1d_runtime_repeat = requested_experiments.get(EXP_1D_RUNTIME_REPEAT)
 
         # Extract checksums, using the output from the second model run
         expected = exp_1d_runtime.extract_checksums(exp_1d_runtime.model.output_1)

{model_config_tests-0.2.2 → model_config_tests-0.2.3}/src/model_config_tests/exp_test_helper.py

@@ -136,9 +136,30 @@ class ExpTestHelper:
             # Change to experiment directory and run.
             os.chdir(self.control_path)
 
-            print("Running payu setup and payu sweep commands")
-            sp.run(["payu", "setup", "--lab", str(self.lab_path)], check=True)
-            sp.run(["payu", "sweep", "--lab", str(self.lab_path)], check=True)
+            print("Running payu setup")
+            result = sp.run(
+                ["payu", "setup", "--lab", str(self.lab_path)],
+                capture_output=True,
+                text=True,
+            )
+            if result.returncode != 0:
+                # Add additional error messaging for debugging
+                error_msg = (
+                    "Failed to run payu setup:\n"
+                    f"Return code: {result.returncode}\n"
+                    f"--- stdout ---\n{result.stdout}\n"
+                    f"--- stderr ---\n{result.stderr}"
+                )
+                print(error_msg)
+                raise RuntimeError(error_msg)
+
+            print("Running payu sweep")
+            sp.run(
+                ["payu", "sweep", "--lab", str(self.lab_path)],
+                capture_output=True,
+                text=True,
+                check=True,
+            )
 
             run_command = ["payu", "run", "--lab", str(self.lab_path)]
             if n_runs:
@@ -208,7 +229,7 @@ class Experiments:
         self.output_path = output_path
         self.keep_archive = keep_archive
         self.experiments = {}
-        self.successful_experiments = []
+        self.experiment_errors = {}
 
     def setup_and_submit(
         self,
@@ -282,22 +303,27 @@ class Experiments:
             try:
                 exp.wait_for_payu_run()
                 print(f"Experiment {exp_name} completed successfully")
-                self.successful_experiments.append(exp_name)
             except RuntimeError as e:
+                self.experiment_errors[exp_name] = str(e)
                 if catch_errors:
-                    print(f"Error in experiment {exp_name}: {e}")
+                    print(f"Error running experiment {exp_name}: {e}")
                 else:
-                    raise e
+                    raise
 
-    def check_experiments(self, exp_names=list[str]) -> None:
+    def check_experiment(self, exp_name: str) -> None:
         """
-        Check whether given experiments names have run successfully
+        Check whether given experiment name has run successfully
         """
-        for exp_name in exp_names:
-            # TODO: Is there other useful information to display here?
-            assert (
-                exp_name in self.successful_experiments
-            ), f"There was an error running experiment: {exp_name}"
+        if exp_name in self.experiment_errors:
+            raise RuntimeError(
+                f"There was an error running experiment {exp_name}:"
+                f" {self.experiment_errors[exp_name]}"
+            )
+
+        # Double check if the required experiment output exists
+        exp = self.experiments.get(exp_name)
+        if not exp.model.output_exists():
+            raise RuntimeError(f"Experiment {exp_name} output file does not exist.")
 
 
 def setup_exp(
@@ -519,13 +545,13 @@ def wait_for_qsub_job(
     # Check whether the run job was successful
     exit_status = parse_exit_status_from_file(stdout)
     if exit_status != 0:
-        print(
+        raise RuntimeError(
+            f"Payu {job_type} job failed with exit status {exit_status}:\n"
             f"Job_ID: {job_id}\n"
             f"Output files: {output_files}\n"
             f"--- stdout ---\n{stdout}\n"
             f"--- stderr ---\n{stderr}\n"
         )
-        raise RuntimeError(f"Payu {job_type} job failed with exit status {exit_status}")
 
     return stdout, stderr, output_files
 

{model_config_tests-0.2.2 → model_config_tests-0.2.3}/src/model_config_tests.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: model_config_tests
-Version: 0.2.2
+Version: 0.2.3
 Summary: Test for ACCESS model (payu) configurations
 Author: ACCESS-NRI
 License: Apache-2.0

{model_config_tests-0.2.2 → model_config_tests-0.2.3}/tests/test_exp_test_helper.py

@@ -1,13 +1,14 @@
 import shutil
 import subprocess
 from pathlib import Path
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 import pytest
 import yaml
 from netCDF4 import Dataset
 
 from model_config_tests.exp_test_helper import (
+    Experiments,
     ExpTestHelper,
     parse_exit_status_from_file,
     parse_gadi_pbs_ids,
@@ -127,6 +128,7 @@ def test_experiment_setup_for_test_run_remove_postprocessing(exp, tmp_path):
 @patch("subprocess.run")
 def test_experiment_submit_payu_run(mock_run, exp):
     mock_run.return_value.stdout = "1234567.gadi-pbs\nsome other output"
+    mock_run.return_value.returncode = 0
 
     current_working_dir = Path.cwd()
     exp.submit_payu_run()
@@ -150,6 +152,7 @@ def test_experiment_submit_payu_run(mock_run, exp):
 def test_experiment_submit_payu_run_n_runs(mock_run, exp):
     """Test --n-runs is added to the payu run command"""
     mock_run.return_value.stdout = "1234567.gadi-pbs\nsome other output"
+    mock_run.return_value.returncode = 0
 
     exp.submit_payu_run(n_runs=2)
 
@@ -172,13 +175,33 @@ def test_experiment_submit_payu_run_disabled(mock_run, exp):
     assert job_id is None
 
 
+@patch("subprocess.run")
+def test_experiment_submit_payu_run_setup_error(mock_run, exp):
+    """Test that an error is raised when payu setup fails"""
+    mock_run.return_value.stdout = "Some output"
+    mock_run.return_value.stderr = "Some error"
+    mock_run.return_value.returncode = 1
+
+    with pytest.raises(RuntimeError, match="Failed to run payu setup*"):
+        exp.submit_payu_run()
+
+    assert exp.run_id is None
+
+
 @patch("subprocess.run")
 def test_experiment_submit_payu_run_error(mock_run, exp):
-    """Test that an error is raised when any payu command fails"""
-    mock_run.side_effect = subprocess.CalledProcessError(
-        returncode=1, cmd="payu setup", output="Some error"
-    )
-    mock_run.return_value.stdout = "Some error"
+    """Test that an RuntimeError is raised with CalledProcessError"""
+    # Mock the first call to payu setup to succeed
+    # and subsequent payu command to fail
+    setup_success = Mock()
+    setup_success.stdout = "Setup successful"
+    setup_success.returncode = 0
+    mock_run.side_effect = [
+        setup_success,
+        subprocess.CalledProcessError(
+            returncode=1, cmd="payu run", output="Some error"
+        ),
+    ]
 
     with pytest.raises(RuntimeError, match="Failed to submit payu run.*"):
         exp.submit_payu_run()
@@ -490,3 +513,46 @@ def test_extract_checksums_split_uses_first_tile(exp_with_restarts):
 
     checksums = exp_accessom3.extract_checksums(output_directory=exp_accessom3.output_0)
     assert checksums["output"]["DTBT"][0] == "AC87F8AC28BD1436"
+
+
+def test_experiments_check_experiment_error(tmp_path):
+    with patch("model_config_tests.exp_test_helper.setup_exp") as mock_setup_exp:
+        # Create an experiment that will error later on
+        mock_error_exp = Mock(autospec=ExpTestHelper)
+        mock_setup_exp.return_value = mock_error_exp
+        mock_error_exp.wait_for_payu_run.side_effect = RuntimeError(
+            "Payu run job failed with exit status 1"
+        )
+
+        exps = Experiments(
+            control_path=tmp_path / "control",
+            output_path=tmp_path / "output",
+            keep_archive=True,
+        )
+        exps.setup_and_submit(exp_name="error_exp")
+        assert exps.experiments["error_exp"] == mock_error_exp
+
+        # Add a second experiment that will succeed
+        mock_success_exp = Mock(autospec=ExpTestHelper)
+        mock_success_exp.wait_for_payu_run.return_value = None
+        mock_setup_exp.return_value = mock_success_exp
+
+        exps.setup_and_submit(exp_name="success_exp")
+        assert exps.experiments["success_exp"] == mock_success_exp
+
+    # Check no errors are raised here
+    exps.wait_for_all_experiments(catch_errors=True)
+    assert exps.experiment_errors == {
+        "error_exp": "Payu run job failed with exit status 1"
+    }
+
+    # Check no errors with successful experiment
+    exps.check_experiment("success_exp")
+
+    # Check error raised for the failed experiment
+    error_msg = (
+        "There was an error running experiment error_exp: "
+        "Payu run job failed with exit status 1"
+    )
+    with pytest.raises(RuntimeError, match=error_msg):
+        exps.check_experiment("error_exp")