gammasimtools 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

simtools/testing/validate_output.py

@@ -12,39 +12,22 @@ from simtools.testing import assertions
 _logger = logging.getLogger(__name__)
 
 
-def validate_all_tests(config, request, config_file_model_version):
-    """
-    Validate test output for all integration tests.
-
-    Parameters
-    ----------
-    config: dict
-        Integration test configuration dictionary.
-    request: request
-        Request object.
-    config_file_model_version: str
-        Model version from the configuration file.
-
-    """
-    if request.config.getoption("--model_version") is None:
-        validate_application_output(config)
-    elif config_file_model_version is not None:
-        _from_command_line = request.config.getoption("--model_version")
-        _from_config_file = config_file_model_version
-        if _from_command_line == _from_config_file:
-            validate_application_output(config)
-
-
-def validate_application_output(config):
+def validate_application_output(config, from_command_line=None, from_config_file=None):
     """
     Validate application output against expected output.
 
     Expected output is defined in configuration file.
+    Some tests run only if the model version from the command line
+    equals the model version from the configuration file.
 
     Parameters
     ----------
     config: dict
         dictionary with the configuration for the application test.
+    from_command_line: str
+        Model version from the command line.
+    from_config_file: str
+        Model version from the configuration file.
 
     """
     if "INTEGRATION_TESTS" not in config:
@@ -52,24 +35,40 @@ def validate_application_output(config):
 
     for integration_test in config["INTEGRATION_TESTS"]:
         _logger.info(f"Testing application output: {integration_test}")
-        if "REFERENCE_OUTPUT_FILE" in integration_test:
-            _validate_reference_output_file(config, integration_test)
-
-        if "TEST_OUTPUT_FILES" in integration_test:
-            _validate_output_path_and_file(config, integration_test["TEST_OUTPUT_FILES"])
-        if "OUTPUT_FILE" in integration_test:
-            _validate_output_path_and_file(
-                config,
-                [{"PATH_DESCRIPTOR": "OUTPUT_PATH", "FILE": integration_test["OUTPUT_FILE"]}],
-            )
 
-        if "FILE_TYPE" in integration_test:
-            assert assertions.assert_file_type(
-                integration_test["FILE_TYPE"],
-                Path(config["CONFIGURATION"]["OUTPUT_PATH"]).joinpath(
-                    config["CONFIGURATION"]["OUTPUT_FILE"]
-                ),
-            )
+        if from_command_line == from_config_file:
+            _validate_output_files(config, integration_test)
+
+            if "FILE_TYPE" in integration_test:
+                assert assertions.assert_file_type(
+                    integration_test["FILE_TYPE"],
+                    Path(config["CONFIGURATION"]["OUTPUT_PATH"]).joinpath(
+                        config["CONFIGURATION"]["OUTPUT_FILE"]
+                    ),
+                )
+        _test_simtel_cfg_files(config, integration_test, from_command_line, from_config_file)
+
+
+def _validate_output_files(config, integration_test):
+    """Validate output files."""
+    if "REFERENCE_OUTPUT_FILE" in integration_test:
+        _validate_reference_output_file(config, integration_test)
+    if "TEST_OUTPUT_FILES" in integration_test:
+        _validate_output_path_and_file(config, integration_test["TEST_OUTPUT_FILES"])
+    if "OUTPUT_FILE" in integration_test:
+        _validate_output_path_and_file(
+            config,
+            [{"PATH_DESCRIPTOR": "OUTPUT_PATH", "FILE": integration_test["OUTPUT_FILE"]}],
+        )
+
+
+def _test_simtel_cfg_files(config, integration_test, from_command_line, from_config_file):
+    """Test simtel cfg files."""
+    test_simtel_cfg_file = integration_test.get("TEST_SIMTEL_CFG_FILES", {}).get(
+        from_command_line or from_config_file
+    )
+    if test_simtel_cfg_file:
+        _validate_simtel_cfg_files(config, test_simtel_cfg_file)
 
 
 def _validate_reference_output_file(config, integration_test):
@@ -128,12 +127,13 @@ def compare_files(file1, file2, tolerance=1.0e-5, test_columns=None):
     """
     _file1_suffix = Path(file1).suffix
     _file2_suffix = Path(file2).suffix
+    _logger.info("Comparing files: %s and %s", file1, file2)
     if _file1_suffix != _file2_suffix:
         raise ValueError(f"File suffixes do not match: {file1} and {file2}")
     if _file1_suffix == ".ecsv":
         return compare_ecsv_files(file1, file2, tolerance, test_columns)
     if _file1_suffix in (".json", ".yaml", ".yml"):
-        return compare_json_or_yaml_files(file1, file2)
+        return compare_json_or_yaml_files(file1, file2, tolerance)
 
     _logger.warning(f"Unknown file type for files: {file1} and {file2}")
     return False
@@ -144,6 +144,7 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2):
     Compare two json or yaml files.
 
     Take into account float comparison for sim_telarray string-embedded floats.
+    Allow differences in 'schema_version' field.
 
     Parameters
     ----------
@@ -162,17 +163,47 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2):
     """
     data1 = gen.collect_data_from_file(file1)
     data2 = gen.collect_data_from_file(file2)
+    data1.pop("schema_version", None)
+    data2.pop("schema_version", None)
 
     _logger.debug(f"Comparing json/yaml files: {file1} and {file2}")
 
     if data1 == data2:
         return True
 
-    if "value" in data1 and isinstance(data1["value"], str):
-        value_list_1 = gen.convert_string_to_list(data1.pop("value"))
-        value_list_2 = gen.convert_string_to_list(data2.pop("value"))
-        return np.allclose(value_list_1, value_list_2, rtol=tolerance)
-    return data1 == data2
+    if data1.keys() != data2.keys():
+        _logger.error(f"Keys do not match: {data1.keys()} and {data2.keys()}")
+        return False
+    _comparison = all(
+        (
+            _compare_value_from_parameter_dict(data1[k], data2[k], tolerance)
+            if k == "value"
+            else data1[k] == data2[k]
+        )
+        for k in data1
+    )
+    if not _comparison:
+        _logger.error(f"Values do not match: {data1} and {data2} (tolerance: {tolerance})")
+    return _comparison
+
+
+def _compare_value_from_parameter_dict(data1, data2, tolerance=1.0e-5):
+    """Compare value fields given in different formats."""
+
+    def _as_list(value):
+        if isinstance(value, str):
+            return gen.convert_string_to_list(value)
+        if isinstance(value, list | np.ndarray):
+            return value
+        return [value]
+
+    _logger.info(f"Comparing values: {data1} and {data2} (tolerance: {tolerance})")
+
+    _as_list_1 = _as_list(data1)
+    _as_list_2 = _as_list(data2)
+    if isinstance(_as_list_1, str):
+        return _as_list_1 == _as_list_2
+    return np.allclose(_as_list_1, _as_list_2, rtol=tolerance)
 
 
 def compare_ecsv_files(file1, file2, tolerance=1.0e-5, test_columns=None):
@@ -238,3 +269,66 @@ def compare_ecsv_files(file1, file2, tolerance=1.0e-5, test_columns=None):
                 return False
 
     return True
+
+
+def _validate_simtel_cfg_files(config, simtel_cfg_file):
+    """
+    Check sim_telarray configuration files and compare with reference file.
+
+    Note the finetuned naming of configuration files by simtools.
+
+    """
+    reference_file = Path(simtel_cfg_file)
+    test_file = Path(config["CONFIGURATION"]["OUTPUT_PATH"]) / reference_file.name.replace(
+        "_test", f"_{config['CONFIGURATION']['LABEL']}"
+    )
+    _logger.info(
+        f"Comparing simtel cfg files: {reference_file} and {test_file} "
+        f"for model version {config['CONFIGURATION']['MODEL_VERSION']}"
+    )
+    return _compare_simtel_cfg_files(reference_file, test_file)
+
+
+def _compare_simtel_cfg_files(reference_file, test_file):
+    """
+    Compare two sim_telarray configuration files.
+
+    Line-by-line string comparison. Requires similar sequence of
+    parameters in the files. Ignore lines containing 'config_release'
+    (as it contains the simtools package version).
+
+    Parameters
+    ----------
+    reference_file: Path
+        Reference sim_telarray configuration file.
+    test_file: Path
+        Test sim_telarray configuration file.
+
+    Returns
+    -------
+    bool
+        True if the files are equal.
+
+    """
+    with open(reference_file, encoding="utf-8") as f1, open(test_file, encoding="utf-8") as f2:
+        reference_cfg = [line.rstrip() for line in f1 if line.strip()]
+        test_cfg = [line.rstrip() for line in f2 if line.strip()]
+
+    if len(reference_cfg) != len(test_cfg):
+        _logger.error(
+            f"Line counts differ: {reference_file} ({len(reference_cfg)} lines), "
+            f"{test_file} ({len(test_cfg)} lines)."
+        )
+        return False
+
+    for ref_line, test_line in zip(reference_cfg, test_cfg):
+        if any(ignore in ref_line for ignore in ("config_release", "Label")):
+            continue
+        if ref_line != test_line:
+            _logger.error(
+                f"Configuration files {reference_file} and {test_file} do not match: "
+                f"'{ref_line}' and '{test_line}'"
+            )
+            return False
+
+    return True

simtools/utils/general.py

@@ -1,6 +1,7 @@
 """General functions useful across different parts of the code."""
 
 import copy
+import datetime
 import json
 import logging
 import os
@@ -17,6 +18,7 @@ import yaml
 __all__ = [
     "InvalidConfigDataError",
     "change_dict_keys_case",
+    "clear_default_sim_telarray_cfg_directories",
     "collect_data_from_file",
     "collect_final_lines",
     "collect_kwargs",
@@ -81,6 +83,28 @@ def is_url(url):
         return False
 
 
+def url_exists(url):
+    """
+    Check if a URL exists.
+
+    Parameters
+    ----------
+    url: str
+        URL to be checked.
+
+    Returns
+    -------
+    bool
+        True if URL exists.
+    """
+    try:
+        with urllib.request.urlopen(url, timeout=5) as response:
+            return response.status == 200
+    except (urllib.error.URLError, AttributeError) as e:
+        _logger.error(f"URL {url} does not exist: {e}")
+        return False
+
+
 def collect_data_from_http(url):
     """
     Download yaml or json file from url and return it contents as dict.
@@ -135,7 +159,7 @@ def collect_data_from_http(url):
     return data
 
 
-def collect_data_from_file(file_name):
+def collect_data_from_file(file_name, yaml_document=None):
     """
     Collect data from file based on its extension.
 
@@ -143,6 +167,8 @@ def collect_data_from_file(file_name):
     ----------
     file_name: str
         Name of the yaml/json/ascii file.
+    yaml_document: None, int
+        Return list of yaml documents or a single document (for yaml files with several documents).
 
     Returns
     -------
@@ -152,21 +178,34 @@ def collect_data_from_file(file_name):
     if is_url(file_name):
         return collect_data_from_http(file_name)
 
+    suffix = Path(file_name).suffix.lower()
     with open(file_name, encoding="utf-8") as file:
-        if Path(file_name).suffix.lower() == ".json":
+        if suffix == ".json":
             return json.load(file)
+        if suffix == ".list":
+            return [line.strip() for line in file.readlines()]
+        if suffix in [".yml", ".yaml"]:
+            return _collect_data_from_yaml_file(file, file_name, yaml_document)
+    return None
 
-        if Path(file_name).suffix.lower() == ".list":
-            lines = file.readlines()
-            return [line.strip() for line in lines]
 
-        try:
-            return yaml.safe_load(file)
-        except yaml.constructor.ConstructorError:
-            return _load_yaml_using_astropy(file)
-        except yaml.composer.ComposerError:
-            file.seek(0)
-            return list(yaml.safe_load_all(file))
+def _collect_data_from_yaml_file(file, file_name, yaml_document):
+    """Collect data from a yaml file."""
+    try:
+        return yaml.safe_load(file)
+    except yaml.constructor.ConstructorError:
+        return _load_yaml_using_astropy(file)
+    except yaml.composer.ComposerError:
+        pass
+    file.seek(0)
+    if yaml_document is None:
+        return list(yaml.safe_load_all(file))
+    try:
+        return list(yaml.safe_load_all(file))[yaml_document]
+    except IndexError as exc:
+        raise InvalidConfigDataError(
+            f"YAML file {file_name} does not contain {yaml_document} documents."
+        ) from exc
 
 
 def collect_kwargs(label, in_kwargs):
@@ -705,7 +744,7 @@ def convert_list_to_string(data, comma_separated=False, shorten_list=False, coll
     return " ".join(str(item) for item in data)
 
 
-def convert_string_to_list(data_string, is_float=True):
+def convert_string_to_list(data_string, is_float=True, force_comma_separation=False):
     """
     Convert string (as used e.g. in sim_telarray) to list.
 
@@ -715,6 +754,10 @@ def convert_string_to_list(data_string, is_float=True):
     ----------
     data_string: object
         String to be converted
+    is_float: bool
+        If True, convert to float, otherwise to int.
+    force_comma_separation: bool
+        If True, force comma separation.
 
     Returns
     -------
@@ -732,7 +775,7 @@ def convert_string_to_list(data_string, is_float=True):
     if "," in data_string:
         result = data_string.split(",")
         return [item.strip() for item in result]
-    if " " in data_string:
+    if " " in data_string and not force_comma_separation:
         return data_string.split()
     return data_string
 
@@ -832,3 +875,60 @@ def convert_keys_in_dict_to_lowercase(data):
     if isinstance(data, list):
         return [convert_keys_in_dict_to_lowercase(i) for i in data]
     return data
+
+
+def clear_default_sim_telarray_cfg_directories(command):
+    """Prefix the command to clear default sim_telarray configuration directories.
+
+    Parameters
+    ----------
+    command: str
+        Command to be prefixed.
+
+    Returns
+    -------
+    str
+        Prefixed command.
+
+    """
+    return f"SIM_TELARRAY_CONFIG_PATH='' {command}"
+
+
+def get_list_of_files_from_command_line(file_names, suffix_list):
+    """
+    Get a list of files from the command line.
+
+    Files can be given as a list of file names or as a text file containing the list of files.
+    The list of suffixes restrict the files types to be returned. Note that a file list must
+    have a different suffix than those in the suffix list.
+
+    Parameters
+    ----------
+    file_names: list
+        List of file names to be checked.
+    suffix_list: list
+        List of suffixes to be checked.
+
+    Returns
+    -------
+    list
+        List of files with the given suffixes.
+    """
+    _files = []
+    for one_file in file_names:
+        path = Path(one_file)
+        try:
+            if path.suffix in suffix_list:
+                _files.append(one_file)
+            elif len(file_names) == 1:
+                with open(one_file, encoding="utf-8") as file:
+                    _files.extend(line.strip() for line in file)
+        except FileNotFoundError as exc:
+            _logger.error(f"{one_file} is not a file.")
+            raise FileNotFoundError from exc
+    return _files
+
+
+def now_date_time_in_isoformat():
+    """Return date and time in isoformat and second accuracy."""
+    return datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds")