gammasimtools 0.6.1__py3-none-any.whl → 0.8.1__py3-none-any.whl

simtools/testing/assertions.py

@@ -0,0 +1,151 @@
+"""Functions asserting certain conditions are met (used e.g., in integration tests)."""
+
+import json
+import logging
+from collections import defaultdict
+from pathlib import Path
+
+import numpy as np
+import yaml
+
+_logger = logging.getLogger(__name__)
+
+
+def assert_file_type(file_type, file_name):
+    """
+    Assert that the file is of the given type.
+
+    Parameters
+    ----------
+    file_type: str
+        File type (json, yaml).
+    file_name: str
+        File name.
+
+    """
+    if file_type == "json":
+        try:
+            with open(file_name, encoding="utf-8") as file:
+                json.load(file)
+            return True
+        except (json.JSONDecodeError, FileNotFoundError):
+            return False
+    if file_type in ("yaml", "yml"):
+        if Path(file_name).suffix[1:] not in ("yaml", "yml"):
+            return False
+        try:
+            with open(file_name, encoding="utf-8") as file:
+                yaml.safe_load(file)
+            return True
+        except (yaml.YAMLError, FileNotFoundError):
+            return False
+
+    # no dedicated tests for other file types, checking suffix only
+    _logger.info(f"File type test is checking suffix only for {file_name} (suffix: {file_type}))")
+    return Path(file_name).suffix[1:] == file_type
+
+
+def assert_n_showers_and_energy_range(file):
+    """
+    Assert the number of showers and the energy range.
+
+    The number of showers should be consistent with the required one (up to 1% tolerance)
+    and the energies simulated are required to be within the configured ones.
+
+    Parameters
+    ----------
+    file: Path
+        Path to the sim_telarray file.
+
+    """
+    from eventio.simtel.simtelfile import SimTelFile  # pylint: disable=import-outside-toplevel
+
+    simulated_energies = []
+    simulation_config = {}
+    with SimTelFile(file) as f:
+        simulation_config = f.mc_run_headers[0]
+        for event in f.iter_mc_events():
+            simulated_energies.append(event["mc_shower"]["energy"])
+
+    # The relative tolerance is set to 1% because ~0.5% shower simulations do not
+    # succeed, without resulting in an error. This tolerance therefore is not an issue.
+    consistent_n_showers = np.isclose(
+        len(np.unique(simulated_energies)), simulation_config["n_showers"], rtol=1e-2
+    )
+    consistent_energy_range = all(
+        simulation_config["E_range"][0] <= energy <= simulation_config["E_range"][1]
+        for energy in simulated_energies
+    )
+
+    return consistent_n_showers and consistent_energy_range
+
+
+def assert_expected_output(file, expected_output):
+    """
+    Assert that the expected output is present in the sim_telarray file.
+
+    Parameters
+    ----------
+    file: Path
+        Path to the sim_telarray file.
+    expected_output: dict
+        Expected output values.
+
+    """
+    from eventio.simtel.simtelfile import SimTelFile  # pylint: disable=import-outside-toplevel
+
+    item_to_check = defaultdict(list)
+    with SimTelFile(file) as f:
+        for event in f:
+            if "pe_sum" in expected_output:
+                item_to_check["pe_sum"].extend(
+                    event["photoelectron_sums"]["n_pe"][event["photoelectron_sums"]["n_pe"] > 0]
+                )
+            if "trigger_time" in expected_output:
+                item_to_check["trigger_time"].extend(event["trigger_information"]["trigger_times"])
+            if "photons" in expected_output:
+                item_to_check["photons"].extend(
+                    event["photoelectron_sums"]["photons_atm_qe"][
+                        event["photoelectron_sums"]["photons"] > 0
+                    ]
+                )
+
+    for key, value in expected_output.items():
+        if len(item_to_check[key]) == 0:
+            _logger.error(f"No data found for {key}")
+            return False
+
+        if not value[0] < np.mean(item_to_check[key]) < value[1]:
+            _logger.error(
+                f"Mean of {key} is not in the expected range, got {np.mean(item_to_check[key])}"
+            )
+            return False
+
+    return True
+
+
+def check_output_from_sim_telarray(file, expected_output):
+    """
+    Check that the sim_telarray simulation result is reasonable and matches the expected output.
+
+    Parameters
+    ----------
+    file: Path
+        Path to the sim_telarray file.
+    expected_output: dict
+        Expected output values.
+
+    Raises
+    ------
+    ValueError
+        If the file is not a zstd compressed file.
+    """
+    if file.suffix != ".zst":
+        raise ValueError(
+            f"Expected output file {file} is not a zstd compressed file "
+            f"(i.e., a sim_telarray file)."
+        )
+
+    return assert_n_showers_and_energy_range(file=file) and assert_expected_output(
+        file=file, expected_output=expected_output
+    )

simtools/testing/configuration.py

@@ -0,0 +1,226 @@
+"""Integration test configuration."""
+
+import logging
+from pathlib import Path
+
+import pytest
+import yaml
+
+import simtools.utils.general as gen
+
+_logger = logging.getLogger(__name__)
+
+
+def get_list_of_test_configurations(config_files):
+    """
+    Return list of test configuration dictionaries or test names.
+
+    Read all configuration files for testing.
+    Add "--help" and "--version" calls for all applications.
+
+    Parameters
+    ----------
+    config_files: list
+        List of integration test configuration files.
+
+    Returns
+    -------
+    list:
+        list of test names or of configuration dictionaries.
+
+    """
+    _logger.debug(f"Configuration files: {config_files}")
+
+    configs = _read_configs_from_files(config_files)
+
+    # list of all applications
+    # (needs to be sorted for pytest-xdist, see Known Limitations in their website)
+    _applications = sorted({item["APPLICATION"] for item in configs if "APPLICATION" in item})
+    for app in _applications:
+        configs.extend(
+            [
+                {"APPLICATION": app, "TEST_NAME": "auto-help", "CONFIGURATION": {"HELP": True}},
+                {
+                    "APPLICATION": app,
+                    "TEST_NAME": "auto-version",
+                    "CONFIGURATION": {"VERSION": True},
+                },
+                {"APPLICATION": app, "TEST_NAME": "auto-no_config"},
+            ]
+        )
+
+    return (
+        configs,
+        [
+            f"{item.get('APPLICATION', 'no-app-name')}_{item.get('TEST_NAME', 'no-test-name')}"
+            for item in configs
+        ],
+    )
+
+
+def _read_configs_from_files(config_files):
+    """Read test configuration from files."""
+    configs = []
+    for config_file in config_files:
+        # read config file
+        # remove new line characters from config - otherwise issues
+        # with especially long file names
+        _dict = gen.remove_substring_recursively_from_dict(
+            gen.collect_data_from_file(file_name=config_file), substring="\n"
+        )
+        configs.append(_dict.get("CTA_SIMPIPE", None))
+    return configs
+
+
+def configure(config, tmp_test_directory, request):
+    """
+    Prepare configuration and command for integration tests.
+
+    Parameters
+    ----------
+    config: dict
+        Configuration dictionary.
+    tmp_test_directory: str
+        Temporary test directory (from pytest fixture).
+    request: request
+        Request object.
+
+    Returns
+    -------
+    str: command to run the application test.
+    str: config file model version.
+
+    """
+    tmp_output_path = create_tmp_output_path(tmp_test_directory, config)
+    model_version_requested = request.config.getoption("--model_version")
+
+    if "CONFIGURATION" in config:
+        _skip_test_for_model_version(config, model_version_requested)
+
+        config_file, config_string, config_file_model_version = _prepare_test_options(
+            config["CONFIGURATION"],
+            output_path=tmp_output_path,
+            model_version=model_version_requested,
+        )
+    else:
+        config_file = None
+        config_string = None
+        config_file_model_version = None
+
+    cmd = get_application_command(
+        app=config.get("APPLICATION", None),
+        config_file=config_file,
+        config_string=config_string,
+    )
+    return cmd, config_file_model_version
+
+
+def _skip_test_for_model_version(config, model_version_requested):
+    """Skip test if model version requested is not supported."""
+    if config.get("MODEL_VERSION_USE_CURRENT") is None or model_version_requested is None:
+        return
+    model_version_config = config["CONFIGURATION"]["MODEL_VERSION"]
+    if model_version_requested != model_version_config:
+        pytest.skip(
+            f"Model version requested {model_version_requested} not supported for this test"
+        )
+
+
+def _prepare_test_options(config, output_path, model_version=None):
+    """
+    Prepare test configuration.
+
+    This means either to write a temporary config file
+    or to return a single string for single boolean options (e.g., --version).
+    Change output path and file to paths provided with output_path.
+
+    Parameters
+    ----------
+    config: dict
+        Dictionary with the configuration for the application test.
+    output_path: str
+        Output path.
+    model_version: str
+        Model versions (default: use those given in config files)
+
+    Returns
+    -------
+    config_file: str
+        Path to the temporary configuration file.
+    config_string: str
+        Command line configuration as single string.
+    config_file_model_version: str
+        Configuration file model version
+
+    """
+    if len(config) == 1 and next(iter(config.values())) is True:
+        return None, "--" + next(iter(config.keys())).lower(), None
+
+    tmp_config_file = output_path / "tmp_config.yml"
+    config_file_model_version = config.get("MODEL_VERSION")
+    if model_version is not None and "MODEL_VERSION" in config:
+        config.update({"MODEL_VERSION": model_version})
+    if "OUTPUT_PATH" in config:
+        config.update({"OUTPUT_PATH": str(Path(output_path).joinpath(config["OUTPUT_PATH"]))})
+        config.update({"USE_PLAIN_OUTPUT_PATH": True})
+    if "DATA_DIRECTORY" in config:
+        config.update({"DATA_DIRECTORY": str(Path(output_path).joinpath(config["DATA_DIRECTORY"]))})
+
+    _logger.info(f"Writing config file: {tmp_config_file}")
+    with open(tmp_config_file, "w", encoding="utf-8") as file:
+        yaml.safe_dump(config, file, sort_keys=False)
+
+    return tmp_config_file, None, config_file_model_version
+
+
+def get_application_command(app, config_file=None, config_string=None):
+    """
+    Return the command to run the application with the given config file.
+
+    Parameters
+    ----------
+    app: str
+        Name of the application.
+    config_file: str
+        Configuration file.
+    config_string: str
+        Configuration string (e.g., '--version')
+
+    Returns
+    -------
+    str: command to run the application test.
+
+    """
+    cmd = app if "simtools-" in app else f"python simtools/applications/{app}.py"
+    if config_string:
+        return f"{cmd} {config_string}"
+    if config_file is not None:
+        return f"{cmd} --config {config_file}"
+    return cmd
+
+
+def create_tmp_output_path(tmp_test_directory, config):
+    """
+    Create temporary output path.
+
+    Parameters
+    ----------
+    tmp_test_directory: str
+        Temporary directory.
+    config: dict
+        Configuration dictionary.
+
+    Returns
+    -------
+    str: path to the temporary output directory.
+
+    """
+    try:
+        tmp_output_path = Path(tmp_test_directory).joinpath(
+            config["APPLICATION"] + "-" + config["TEST_NAME"]
+        )
+    except KeyError as exc:
+        raise KeyError(f"No application defined in configuration {config}.") from exc
+    tmp_output_path.mkdir(parents=True, exist_ok=True)
+    _logger.info(f"Temporary output path: {tmp_output_path}")
+    return tmp_output_path

simtools/testing/helpers.py

@@ -0,0 +1,42 @@
+"""Helper functions for integration testing."""
+
+import os
+
+import pytest
+
+
+def skip_camera_efficiency(config):
+    """Skip camera efficiency tests if the old version of testeff is used."""
+    if "camera-efficiency" in config["APPLICATION"]:
+        if not _new_testeff_version():
+            pytest.skip(
+                "Any applications calling the old version of testeff are skipped "
+                "due to a limitation of the old testeff not allowing to specify "
+                "the include directory. Please update your sim_telarray tarball."
+            )
+        full_test_name = f"{config['APPLICATION']}_{config['TEST_NAME']}"
+        if "simtools-validate-camera-efficiency_SSTS" == full_test_name:
+            pytest.skip(
+                "The test simtools-validate-camera-efficiency_SSTS is skipped "
+                "since the fake SST mirrors are not yet implemented (#1155)"
+            )
+
+
+def _new_testeff_version():
+    """
+    Testeff has been updated to allow to specify the include directory.
+
+    This test checks if the new version is used.
+    """
+    testeff_path = os.path.join(os.getenv("SIMTOOLS_SIMTEL_PATH"), "sim_telarray/testeff.c")
+    try:
+        with open(testeff_path, encoding="utf-8") as file:
+            file_content = file.read()
+            if (
+                "/* Combine the include paths such that those from '-I...' options */"
+                in file_content
+            ):
+                return True
+            return False
+    except FileNotFoundError as exc:
+        raise FileNotFoundError("The testeff executable could not be found.") from exc

simtools/testing/validate_output.py

@@ -0,0 +1,240 @@
+"""Compare application output to reference output."""
+
+import logging
+from pathlib import Path
+
+import numpy as np
+from astropy.table import Table
+
+import simtools.utils.general as gen
+from simtools.testing import assertions
+
+_logger = logging.getLogger(__name__)
+
+
+def validate_all_tests(config, request, config_file_model_version):
+    """
+    Validate test output for all integration tests.
+
+    Parameters
+    ----------
+    config: dict
+        Integration test configuration dictionary.
+    request: request
+        Request object.
+    config_file_model_version: str
+        Model version from the configuration file.
+
+    """
+    if request.config.getoption("--model_version") is None:
+        validate_application_output(config)
+    elif config_file_model_version is not None:
+        _from_command_line = request.config.getoption("--model_version")
+        _from_config_file = config_file_model_version
+        if _from_command_line == _from_config_file:
+            validate_application_output(config)
+
+
+def validate_application_output(config):
+    """
+    Validate application output against expected output.
+
+    Expected output is defined in configuration file.
+
+    Parameters
+    ----------
+    config: dict
+        dictionary with the configuration for the application test.
+
+    """
+    if "INTEGRATION_TESTS" not in config:
+        return
+
+    for integration_test in config["INTEGRATION_TESTS"]:
+        _logger.info(f"Testing application output: {integration_test}")
+        if "REFERENCE_OUTPUT_FILE" in integration_test:
+            _validate_reference_output_file(config, integration_test)
+
+        if "OUTPUT_FILE" in integration_test:
+            _validate_output_path_and_file(config, integration_test)
+
+        if "FILE_TYPE" in integration_test:
+            assert assertions.assert_file_type(
+                integration_test["FILE_TYPE"],
+                Path(config["CONFIGURATION"]["OUTPUT_PATH"]).joinpath(
+                    config["CONFIGURATION"]["OUTPUT_FILE"]
+                ),
+            )
+
+
+def _validate_reference_output_file(config, integration_test):
+    """Compare with reference output file."""
+    assert compare_files(
+        integration_test["REFERENCE_OUTPUT_FILE"],
+        Path(config["CONFIGURATION"]["OUTPUT_PATH"]).joinpath(
+            config["CONFIGURATION"]["OUTPUT_FILE"]
+        ),
+        integration_test.get("TOLERANCE", 1.0e-5),
+        integration_test.get("TEST_COLUMNS", None),
+    )
+
+
+def _validate_output_path_and_file(config, integration_test):
+    """Check if output path and file exist."""
+    _logger.info(f"PATH {config['CONFIGURATION']['OUTPUT_PATH']}")
+    _logger.info(f"File {integration_test['OUTPUT_FILE']}")
+
+    data_path = config["CONFIGURATION"].get(
+        "DATA_DIRECTORY", config["CONFIGURATION"]["OUTPUT_PATH"]
+    )
+    output_file_path = Path(data_path) / integration_test["OUTPUT_FILE"]
+
+    _logger.info(f"Checking path: {output_file_path}")
+    assert output_file_path.exists()
+
+    expected_output = [
+        d["EXPECTED_OUTPUT"] for d in config["INTEGRATION_TESTS"] if "EXPECTED_OUTPUT" in d
+    ]
+    if expected_output and "log_hist" not in integration_test["OUTPUT_FILE"]:
+        # Get the expected output from the configuration file
+        expected_output = expected_output[0]
+        _logger.info(
+            f"Checking the output of {integration_test['OUTPUT_FILE']} "
+            "complies with the expected output: "
+            f"{expected_output}"
+        )
+        assert assertions.check_output_from_sim_telarray(output_file_path, expected_output)
+
+
+def compare_files(file1, file2, tolerance=1.0e-5, test_columns=None):
+    """
+    Compare two files of file type ecsv, json or yaml.
+
+    Parameters
+    ----------
+    file1: str
+        First file to compare
+    file2: str
+        Second file to compare
+    tolerance: float
+        Tolerance for comparing numerical values.
+    test_columns: list
+        List of columns to compare. If None, all columns are compared.
+
+    Returns
+    -------
+    bool
+        True if the files are equal, False otherwise.
+
+    """
+    _file1_suffix = Path(file1).suffix
+    _file2_suffix = Path(file2).suffix
+    if _file1_suffix != _file2_suffix:
+        raise ValueError(f"File suffixes do not match: {file1} and {file2}")
+    if _file1_suffix == ".ecsv":
+        return compare_ecsv_files(file1, file2, tolerance, test_columns)
+    if _file1_suffix in (".json", ".yaml", ".yml"):
+        return compare_json_or_yaml_files(file1, file2)
+
+    _logger.warning(f"Unknown file type for files: {file1} and {file2}")
+    return False
+
+
+def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2):
+    """
+    Compare two json or yaml files.
+
+    Take into account float comparison for sim_telarray string-embedded floats.
+
+    Parameters
+    ----------
+    file1: str
+        First file to compare
+    file2: str
+        Second file to compare
+    tolerance: float
+        Tolerance for comparing numerical values.
+
+    Returns
+    -------
+    bool
+        True if the files are equal, False otherwise.
+
+    """
+    data1 = gen.collect_data_from_file(file1)
+    data2 = gen.collect_data_from_file(file2)
+
+    _logger.debug(f"Comparing json/yaml files: {file1} and {file2}")
+
+    if data1 == data2:
+        return True
+
+    if "value" in data1 and isinstance(data1["value"], str):
+        value_list_1 = gen.convert_string_to_list(data1.pop("value"))
+        value_list_2 = gen.convert_string_to_list(data2.pop("value"))
+        return np.allclose(value_list_1, value_list_2, rtol=tolerance)
+    return data1 == data2
+
+
+def compare_ecsv_files(file1, file2, tolerance=1.0e-5, test_columns=None):
+    """
+    Compare two ecsv files.
+
+    The comparison is successful if:
+
+    - same number of rows
+    - numerical values in columns are close
+
+    The comparison can be restricted to a subset of columns with some additional
+    cuts applied. This is configured through the test_columns parameter. This is
+    a list of dictionaries, where each dictionary contains the following
+    key-value pairs:
+    - TEST_COLUMN_NAME: column name to compare.
+    - CUT_COLUMN_NAME: column for filtering.
+    - CUT_CONDITION: condition for filtering.
+
+    Parameters
+    ----------
+    file1: str
+        First file to compare
+    file2: str
+        Second file to compare
+    tolerance: float
+        Tolerance for comparing numerical values.
+    test_columns: list
+        List of columns to compare. If None, all columns are compared.
+
+    """
+    _logger.info(f"Comparing files: {file1} and {file2}")
+    table1 = Table.read(file1, format="ascii.ecsv")
+    table2 = Table.read(file2, format="ascii.ecsv")
+
+    if test_columns is None:
+        test_columns = [{"TEST_COLUMN_NAME": col} for col in table1.colnames]
+
+    def generate_mask(table, column, condition):
+        """Generate a boolean mask based on the condition (note the usage of eval)."""
+        return (
+            eval(f"table['{column}'] {condition}")  # pylint: disable=eval-used
+            if condition
+            else np.ones(len(table), dtype=bool)
+        )
+
+    for col_dict in test_columns:
+        col_name = col_dict["TEST_COLUMN_NAME"]
+        mask1 = generate_mask(
+            table1, col_dict.get("CUT_COLUMN_NAME", ""), col_dict.get("CUT_CONDITION", "")
+        )
+        mask2 = generate_mask(
+            table2, col_dict.get("CUT_COLUMN_NAME", ""), col_dict.get("CUT_CONDITION", "")
+        )
+        table1_masked, table2_masked = table1[mask1], table2[mask2]
+
+        if len(table1_masked) != len(table2_masked):
+            return False
+
+        if np.issubdtype(table1_masked[col_name].dtype, np.floating):
+            if not np.allclose(table1_masked[col_name], table2_masked[col_name], rtol=tolerance):
+                return False
+
+    return True