gammasimtools 0.18.0__py3-none-any.whl → 0.20.0__py3-none-any.whl

simtools/testing/configuration.py

@@ -4,9 +4,8 @@ import logging
 import os
 from pathlib import Path
 
-import yaml
-
 import simtools.utils.general as gen
+from simtools.io import ascii_handler
 
 _logger = logging.getLogger(__name__)
 
@@ -43,24 +42,24 @@ def get_list_of_test_configurations(config_files):
 
     # list of all applications
     # (needs to be sorted for pytest-xdist, see Known Limitations in their website)
-    _applications = sorted({item["APPLICATION"] for item in configs if "APPLICATION" in item})
+    _applications = sorted({item["application"] for item in configs if "application" in item})
     for app in _applications:
         configs.extend(
             [
-                {"APPLICATION": app, "TEST_NAME": "auto-help", "CONFIGURATION": {"HELP": True}},
+                {"application": app, "test_name": "auto-help", "configuration": {"help": True}},
                 {
-                    "APPLICATION": app,
-                    "TEST_NAME": "auto-version",
-                    "CONFIGURATION": {"VERSION": True},
+                    "application": app,
+                    "test_name": "auto-version",
+                    "configuration": {"version": True},
                 },
-                {"APPLICATION": app, "TEST_NAME": "auto-no_config"},
+                {"application": app, "test_name": "auto-no_config"},
             ]
         )
 
     return (
         configs,
         [
-            f"{item.get('APPLICATION', 'no-app-name')}_{item.get('TEST_NAME', 'no-test-name')}"
+            f"{item.get('application', 'no-app-name')}_{item.get('test_name', 'no-test-name')}"
             for item in configs
         ],
     )
@@ -74,9 +73,9 @@ def _read_configs_from_files(config_files):
         # remove new line characters from config - otherwise issues
         # with especially long file names
         _dict = gen.remove_substring_recursively_from_dict(
-            gen.collect_data_from_file(file_name=config_file), substring="\n"
+            ascii_handler.collect_data_from_file(file_name=config_file), substring="\n"
         )
-        for application in _dict.get("CTA_SIMPIPE", {}).get("APPLICATIONS", []):
+        for application in _dict.get("applications", []):
             configs.append(application)
     return configs
 
@@ -108,12 +107,12 @@ def configure(config, tmp_test_directory, request):
     if isinstance(model_version_requested, list) and len(model_version_requested) == 1:
         model_version_requested = model_version_requested[0]
 
-    if "CONFIGURATION" in config:
+    if "configuration" in config:
         _skip_test_for_model_version(config, model_version_requested)
         _skip_test_for_production_db(config)
 
         config_file, config_string, config_file_model_version = _prepare_test_options(
-            config["CONFIGURATION"],
+            config["configuration"],
             output_path=tmp_output_path,
             model_version=model_version_requested,
         )
@@ -123,7 +122,7 @@ def configure(config, tmp_test_directory, request):
         config_file_model_version = None
 
     cmd = get_application_command(
-        app=config.get("APPLICATION", None),
+        app=config.get("application", None),
         config_file=config_file,
         config_string=config_string,
     )
@@ -132,9 +131,9 @@ def configure(config, tmp_test_directory, request):
 
 def _skip_test_for_model_version(config, model_version_requested):
     """Skip test if model version requested is not supported."""
-    if config.get("MODEL_VERSION_USE_CURRENT") is None or model_version_requested is None:
+    if config.get("model_version_use_current") is None or model_version_requested is None:
         return
-    model_version_config = config["CONFIGURATION"]["MODEL_VERSION"]
+    model_version_config = config["configuration"]["model_version"]
     if model_version_requested != model_version_config:
         raise VersionError(
             f"Model version requested {model_version_requested} not supported for this test"
@@ -143,7 +142,7 @@ def _skip_test_for_model_version(config, model_version_requested):
 
 def _skip_test_for_production_db(config):
     """Skip test if production db is used."""
-    if not config.get("SKIP_FOR_PRODUCTION_DB"):
+    if not config.get("skip_for_production_db"):
         return
 
     if "db.zeuthen.desy.de" in os.getenv("SIMTOOLS_DB_SERVER", ""):
@@ -184,19 +183,18 @@ def _prepare_test_options(config, output_path, model_version=None):
         return None, "--" + next(iter(config.keys())).lower(), None
 
     tmp_config_file = output_path / "tmp_config.yml"
-    config_file_model_version = config.get("MODEL_VERSION")
-    if model_version and "MODEL_VERSION" in config:
-        config.update({"MODEL_VERSION": model_version})
+    config_file_model_version = config.get("model_version")
+    if model_version and "model_version" in config:
+        config.update({"model_version": model_version})
 
-    for key in ["OUTPUT_PATH", "DATA_DIRECTORY", "PACK_FOR_GRID_REGISTER"]:
+    for key in ["output_path", "data_directory", "pack_for_grid_register"]:
         if key in config:
             config[key] = str(Path(output_path).joinpath(config[key]))
-            if key == "OUTPUT_PATH":
-                config["USE_PLAIN_OUTPUT_PATH"] = True
+            if key == "output_path":
+                config["use_plain_output_path"] = True
 
     _logger.info(f"Writing config file: {tmp_config_file}")
-    with open(tmp_config_file, "w", encoding="utf-8") as file:
-        yaml.safe_dump(config, file, sort_keys=False)
+    ascii_handler.write_data_to_file(data=config, output_file=tmp_config_file, sort_keys=False)
 
     return tmp_config_file, None, config_file_model_version
 
@@ -245,7 +243,7 @@ def create_tmp_output_path(tmp_test_directory, config):
     """
     try:
         tmp_output_path = Path(tmp_test_directory).joinpath(
-            config["APPLICATION"] + "-" + config["TEST_NAME"]
+            config["application"] + "-" + config["test_name"]
         )
     except KeyError as exc:
         raise KeyError(f"No application defined in configuration {config}.") from exc

simtools/testing/helpers.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 def skip_camera_efficiency(config):
     """Skip camera efficiency tests if the old version of testeff is used."""
-    if "camera-efficiency" in config["APPLICATION"] and not _new_testeff_version():
+    if "camera-efficiency" in config["application"] and not _new_testeff_version():
         return (
             "Any applications calling the old version of testeff are skipped "
             "due to a limitation of the old testeff not allowing to specify "
@@ -42,7 +42,7 @@ def skip_multiple_version_test(config, model_version):
     if not isinstance(model_version, list):
         return None
 
-    config_model_version = config.get("CONFIGURATION", {}).get("MODEL_VERSION", [])
+    config_model_version = config.get("configuration", {}).get("model_version", [])
 
     if not isinstance(config_model_version, list):
         config_model_version = [config_model_version]

simtools/testing/log_inspector.py

@@ -0,0 +1,52 @@
+"""Inspect logs and output for errors and warnings."""
+
+import logging
+import re
+
+_logger = logging.getLogger(__name__)
+
+
+ERROR_PATTERNS = [
+    re.compile(r"error", re.IGNORECASE),
+    re.compile(r"exception", re.IGNORECASE),
+    re.compile(r"traceback", re.IGNORECASE),
+    re.compile(r"\b(failed to|has failed)\b", re.IGNORECASE),
+    re.compile(r"runtime\s*warning", re.IGNORECASE),
+    re.compile(r"segmentation fault", re.IGNORECASE),
+]
+
+IGNORE_PATTERNS = [re.compile(r"Falling back to 'utf-8' with errors='ignore'", re.IGNORECASE)]
+
+
+def inspect(log_text):
+    """
+    Inspect log text for errors and run-time warnings.
+
+    Ignore any lines containing "INFO::" (to avoid false positives
+    like "INFO:: Job error stream ").
+
+    Parameters
+    ----------
+    log_text: str or list of str
+        Text of the log to inspect.
+
+    Returns
+    -------
+    bool
+        True if no errors or warnings are found, False otherwise.
+    """
+    log_text = log_text if isinstance(log_text, list) else [log_text]
+
+    issues = [
+        (lineno, line)
+        for txt in log_text
+        for lineno, line in enumerate(txt.splitlines(), 1)
+        if "INFO::" not in line
+        and any(p.search(line) for p in ERROR_PATTERNS)
+        and not any(p.search(line) for p in IGNORE_PATTERNS)
+    ]
+
+    for lineno, line in issues:
+        _logger.error(f"Error or warning found in log at line {lineno}: {line.strip()}")
+
+    return not issues

simtools/testing/validate_output.py

@@ -7,12 +7,16 @@ import numpy as np
 from astropy.table import Table
 
 import simtools.utils.general as gen
+from simtools.db import db_handler
+from simtools.io import ascii_handler
 from simtools.testing import assertions
 
 _logger = logging.getLogger(__name__)
 
 
-def validate_application_output(config, from_command_line=None, from_config_file=None):
+def validate_application_output(
+    config, from_command_line=None, from_config_file=None, db_config=None
+):
     """
     Validate application output against expected output.
 
@@ -30,41 +34,47 @@ def validate_application_output(config, from_command_line=None, from_config_file
         Model version from the configuration file.
 
     """
-    if "INTEGRATION_TESTS" not in config:
+    if "integration_tests" not in config:
         return
 
-    for integration_test in config["INTEGRATION_TESTS"]:
+    for integration_test in config["integration_tests"]:
         _logger.info(f"Testing application output: {integration_test}")
 
         if from_command_line == from_config_file:
-            _validate_output_files(config, integration_test)
+            _validate_output_files(config, integration_test, db_config)
 
-            if "FILE_TYPE" in integration_test:
+            if "file_type" in integration_test:
                 assert assertions.assert_file_type(
-                    integration_test["FILE_TYPE"],
-                    Path(config["CONFIGURATION"]["OUTPUT_PATH"]).joinpath(
-                        config["CONFIGURATION"]["OUTPUT_FILE"]
+                    integration_test["file_type"],
+                    Path(config["configuration"]["output_path"]).joinpath(
+                        config["configuration"]["output_file"]
                     ),
                 )
         _test_simtel_cfg_files(config, integration_test, from_command_line, from_config_file)
 
 
-def _validate_output_files(config, integration_test):
+def _validate_output_files(config, integration_test, db_config):
     """Validate output files."""
-    if "REFERENCE_OUTPUT_FILE" in integration_test:
+    if "reference_output_file" in integration_test:
         _validate_reference_output_file(config, integration_test)
-    if "TEST_OUTPUT_FILES" in integration_test:
-        _validate_output_path_and_file(config, integration_test["TEST_OUTPUT_FILES"])
-    if "OUTPUT_FILE" in integration_test:
+    if "test_output_files" in integration_test:
+        _validate_output_path_and_file(config, integration_test["test_output_files"])
+    if "output_file" in integration_test:
         _validate_output_path_and_file(
             config,
-            [{"PATH_DESCRIPTOR": "OUTPUT_PATH", "FILE": integration_test["OUTPUT_FILE"]}],
+            [{"path_descriptor": "output_path", "file": integration_test["output_file"]}],
+        )
+    if "model_parameter_validation" in integration_test:
+        _validate_model_parameter_json_file(
+            config,
+            integration_test["model_parameter_validation"],
+            db_config,
         )
 
 
 def _test_simtel_cfg_files(config, integration_test, from_command_line, from_config_file):
     """Test simtel cfg files."""
-    cfg_files = integration_test.get("TEST_SIMTEL_CFG_FILES", {})
+    cfg_files = integration_test.get("test_simtel_cfg_files", {})
     if isinstance(from_command_line, list):
         sources = from_command_line
     elif isinstance(from_config_file, list):
@@ -81,12 +91,12 @@ def _test_simtel_cfg_files(config, integration_test, from_command_line, from_con
 def _validate_reference_output_file(config, integration_test):
     """Compare with reference output file."""
     assert compare_files(
-        integration_test["REFERENCE_OUTPUT_FILE"],
-        Path(config["CONFIGURATION"]["OUTPUT_PATH"]).joinpath(
-            config["CONFIGURATION"]["OUTPUT_FILE"]
+        integration_test["reference_output_file"],
+        Path(config["configuration"]["output_path"]).joinpath(
+            config["configuration"]["output_file"]
         ),
-        integration_test.get("TOLERANCE", 1.0e-5),
-        integration_test.get("TEST_COLUMNS", None),
+        integration_test.get("tolerance", 1.0e-5),
+        integration_test.get("test_columns", None),
     )
 
 
@@ -94,26 +104,64 @@ def _validate_output_path_and_file(config, integration_file_tests):
     """Check if output paths and files exist."""
     for file_test in integration_file_tests:
         try:
-            output_path = config["CONFIGURATION"][file_test["PATH_DESCRIPTOR"]]
+            output_path = config["configuration"][file_test["path_descriptor"]]
         except KeyError as exc:
             raise KeyError(
-                f"Path {file_test['PATH_DESCRIPTOR']} not found in integration test configuration."
+                f"Path {file_test['path_descriptor']} not found in integration test configuration."
             ) from exc
 
-        output_file_path = Path(output_path) / file_test["FILE"]
+        output_file_path = Path(output_path) / file_test["file"]
         _logger.info(f"Checking path: {output_file_path}")
         try:
             assert output_file_path.exists()
         except AssertionError as exc:
             raise AssertionError(f"Output file {output_file_path} does not exist. ") from exc
 
-        if "EXPECTED_OUTPUT" in file_test:
+        if "expected_output" in file_test:
             assert assertions.check_output_from_sim_telarray(
                 output_file_path,
-                file_test["EXPECTED_OUTPUT"],
+                file_test["expected_output"],
             )
 
 
+def _validate_model_parameter_json_file(config, model_parameter_validation, db_config):
+    """
+    Validate model parameter json file and compare it with a reference parameter from the database.
+
+    Requires database connection to pull the model parameter for a given telescope or site model.
+
+    Parameters
+    ----------
+    config: dict
+        Configuration dictionary.
+    model_parameter_validation: dict
+        Dictionary with model parameter validation configuration.
+
+    """
+    _logger.info(f"Checking model parameter json file: {model_parameter_validation}")
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+
+    reference_parameter_name = model_parameter_validation.get("reference_parameter_name")
+
+    reference_model_parameter = db.get_model_parameter(
+        parameter=reference_parameter_name,
+        site=config["configuration"].get("site"),
+        array_element_name=config["configuration"].get("telescope"),
+        model_version=config["configuration"].get("model_version"),
+    )
+    parameter_file = (
+        Path(config["configuration"]["output_path"])
+        / config["configuration"].get("telescope")
+        / model_parameter_validation["parameter_file"]
+    )
+    model_parameter = ascii_handler.collect_data_from_file(parameter_file)
+    assert _compare_value_from_parameter_dict(
+        model_parameter["value"],
+        reference_model_parameter[reference_parameter_name]["value"],
+        model_parameter_validation["tolerance"],
+    )
+
+
 def compare_files(file1, file2, tolerance=1.0e-5, test_columns=None):
     """
     Compare two files of file type ecsv, json or yaml.
@@ -171,8 +219,8 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2):
         True if the files are equal.
 
     """
-    data1 = gen.collect_data_from_file(file1)
-    data2 = gen.collect_data_from_file(file2)
+    data1 = ascii_handler.collect_data_from_file(file1)
+    data2 = ascii_handler.collect_data_from_file(file2)
     data1.pop("schema_version", None)
     data2.pop("schema_version", None)
 
@@ -229,9 +277,9 @@ def compare_ecsv_files(file1, file2, tolerance=1.0e-5, test_columns=None):
     cuts applied. This is configured through the test_columns parameter. This is
     a list of dictionaries, where each dictionary contains the following
     key-value pairs:
-    - TEST_COLUMN_NAME: column name to compare.
-    - CUT_COLUMN_NAME: column for filtering.
-    - CUT_CONDITION: condition for filtering.
+    - test_column_name: column name to compare.
+    - cut_column_name: column for filtering.
+    - cut_condition: condition for filtering.
 
     Parameters
     ----------
@@ -250,7 +298,7 @@ def compare_ecsv_files(file1, file2, tolerance=1.0e-5, test_columns=None):
     table2 = Table.read(file2, format="ascii.ecsv")
 
     if test_columns is None:
-        test_columns = [{"TEST_COLUMN_NAME": col} for col in table1.colnames]
+        test_columns = [{"test_column_name": col} for col in table1.colnames]
 
     def generate_mask(table, column, condition):
         """Generate a boolean mask based on the condition (note the usage of eval)."""
@@ -261,12 +309,12 @@ def compare_ecsv_files(file1, file2, tolerance=1.0e-5, test_columns=None):
         )
 
     for col_dict in test_columns:
-        col_name = col_dict["TEST_COLUMN_NAME"]
+        col_name = col_dict["test_column_name"]
         mask1 = generate_mask(
-            table1, col_dict.get("CUT_COLUMN_NAME", ""), col_dict.get("CUT_CONDITION", "")
+            table1, col_dict.get("cut_column_name", ""), col_dict.get("cut_condition", "")
         )
         mask2 = generate_mask(
-            table2, col_dict.get("CUT_COLUMN_NAME", ""), col_dict.get("CUT_CONDITION", "")
+            table2, col_dict.get("cut_column_name", ""), col_dict.get("cut_condition", "")
         )
         table1_masked, table2_masked = table1[mask1], table2[mask2]
 
@@ -289,12 +337,14 @@ def _validate_simtel_cfg_files(config, simtel_cfg_file):
 
     """
     reference_file = Path(simtel_cfg_file)
-    test_file = Path(config["CONFIGURATION"]["OUTPUT_PATH"]) / reference_file.name.replace(
-        "_test", f"_{config['CONFIGURATION']['LABEL']}"
+    test_file = (
+        Path(config["configuration"]["output_path"])
+        / f"model/{config['configuration']['model_version']}"
+        / reference_file.name.replace("_test", f"_{config['configuration']['label']}")
     )
     _logger.info(
         f"Comparing simtel cfg files: {reference_file} and {test_file} "
-        f"for model version {config['CONFIGURATION']['MODEL_VERSION']}"
+        f"for model version {config['configuration']['model_version']}"
     )
     return _compare_simtel_cfg_files(reference_file, test_file)