gammasimtools 0.25.0__py3-none-any.whl → 0.27.0__py3-none-any.whl

simtools/telescope_trigger_rates.py

@@ -8,13 +8,13 @@ from ctao_cr_spectra.definitions import IRFDOC_PROTON_SPECTRUM
 
 from simtools.io import ascii_handler, io_handler
 from simtools.layout.array_layout_utils import get_array_elements_from_db_for_layouts
-from simtools.simtel.simtel_io_event_histograms import SimtelIOEventHistograms
+from simtools.sim_events.histograms import EventDataHistograms
 from simtools.visualization import plot_simtel_event_histograms
 
 _logger = logging.getLogger(__name__)
 
 
-def telescope_trigger_rates(args_dict, db_config):
+def telescope_trigger_rates(args_dict):
     """
     Calculate trigger rates for single telescopes or arrays of telescopes.
 
@@ -27,7 +27,6 @@ def telescope_trigger_rates(args_dict, db_config):
             args_dict["array_layout_name"],
             args_dict.get("site"),
             args_dict.get("model_version"),
-            db_config,
         )
     else:
         telescope_configs = ascii_handler.collect_data_from_file(args_dict["telescope_ids"])[
@@ -38,7 +37,7 @@ def telescope_trigger_rates(args_dict, db_config):
         _logger.info(
             f"Processing file: {args_dict['event_data_file']} with telescope config: {array_name}"
         )
-        histograms = SimtelIOEventHistograms(
+        histograms = EventDataHistograms(
             args_dict["event_data_file"], array_name=array_name, telescope_list=telescope_ids
         )
         histograms.fill()

simtools/testing/assertions.py

@@ -68,8 +68,7 @@ def assert_n_showers_and_energy_range(file):
     simulation_config = {}
     with SimTelFile(file, skip_non_triggered=False) as f:
         simulation_config = f.mc_run_headers[0]
-        for event in f:
-            simulated_energies.append(event["mc_shower"]["energy"])
+        simulated_energies.extend(event["mc_shower"]["energy"] for event in f)
 
     # The relative tolerance is set to 1% because ~0.5% shower simulations do not
     # succeed, without resulting in an error. This tolerance therefore is not an issue.
@@ -191,54 +190,94 @@ def check_output_from_sim_telarray(file, file_test):
     return assert_n_showers_and_energy_range(file=file) and assert_output and assert_metadata
 
 
-def _find_patterns(text, patterns):
-    """Find patterns in text."""
-    return {p for p in patterns if p in text}
-
-
-def _read_log(member, tar):
-    """Read and decode a gzipped log file from a tar archive."""
-    with tar.extractfile(member) as gz, gzip.open(gz, "rb") as f:
-        return f.read().decode("utf-8", "ignore")
-
-
 def check_simulation_logs(tar_file, file_test):
     """
-    Check log files of CORSIKA and sim_telarray for expected output.
+    Check simulation logs for wanted and forbidden patterns.
 
     Parameters
     ----------
-    tar_file: Path
-        Path to a log file tar package.
-    file_test: dict
-        File test description including expected log output.
-
-    Raises
-    ------
-    ValueError
-        If the file is not a tar file.
+    tar_file : str
+        Path to the tar file.
+    file_test : dict
+        Dictionary with the test configuration.
+
+    Returns
+    -------
+    bool
+        True if the logs are correct.
     """
-    expected_log = file_test.get("expected_log_output", {})
-    wanted = expected_log.get("pattern", [])
-    forbidden = expected_log.get("forbidden_pattern", [])
-
-    if not (wanted or forbidden):
-        _logger.debug(f"No expected log output provided, skipping checks {file_test}")
+    wanted, forbidden = _get_expected_patterns(file_test)
+    if wanted is None:
         return True
 
     if not tarfile.is_tarfile(tar_file):
-        raise ValueError(f"File {tar_file} is not a tar file.")
+        raise ValueError(f"{tar_file} is not a tar file")
 
-    found, bad = set(), set()
+    found_wanted = set()
+    found_forbidden = set()
     with tarfile.open(tar_file, "r:*") as tar:
         for member in tar.getmembers():
             if not member.name.endswith(".log.gz"):
                 continue
             _logger.info(f"Scanning {member.name}")
             text = _read_log(member, tar)
-            found |= _find_patterns(text, wanted)
-            bad |= _find_patterns(text, forbidden)
+            found_wanted |= _find_patterns(text, wanted)
+            found_forbidden |= _find_patterns(text, forbidden)
+
+    return _validate_patterns(found_wanted, found_forbidden, wanted)
+
+
+def check_plain_log(log_file, file_test):
+    """
+    Check plain log file for wanted and forbidden patterns.
+
+    Parameters
+    ----------
+    log_file : str
+        Path to the log file.
+    file_test : dict
+        Dictionary with the test configuration.
+
+    Returns
+    -------
+    bool
+        True if the logs are correct.
+    """
+    wanted, forbidden = _get_expected_patterns(file_test)
+    if wanted is None:
+        return True
+
+    try:
+        with open(log_file, encoding="utf-8") as f:
+            text = f.read()
+    except FileNotFoundError:
+        _logger.error(f"Log file {log_file} not found")
+        return False
+
+    found = _find_patterns(text, wanted)
+    bad = _find_patterns(text, forbidden)
+
+    return _validate_patterns(found, bad, wanted)
+
+
+def _get_expected_patterns(file_test):
+    """Get wanted and forbidden patterns from file test configuration."""
+    expected_log = file_test.get("expected_log_output")
+    if isinstance(expected_log, dict):
+        wanted = expected_log.get("pattern", [])
+        forbidden = expected_log.get("forbidden_pattern", [])
+    else:
+        wanted = file_test.get("pattern", [])
+        forbidden = file_test.get("forbidden_pattern", [])
+    if not (wanted or forbidden):
+        _logger.debug(f"No expected log output provided, skipping checks {file_test}")
+        return None, None
 
+    return wanted, forbidden
+
+
+def _validate_patterns(found, bad, wanted):
+    """Validate found patterns against wanted and forbidden ones."""
     if bad:
         _logger.error(f"Forbidden patterns found: {list(bad)}")
         return False
@@ -249,3 +288,15 @@ def check_simulation_logs(tar_file, file_test):
 
     _logger.debug(f"All expected patterns found: {wanted}")
     return True
+
+
+def _find_patterns(text, patterns):
+    """Find patterns in text (case insensitive)."""
+    text_lower = text.lower()
+    return {p for p in patterns if p.lower() in text_lower}
+
+
+def _read_log(member, tar):
+    """Read and decode a gzipped log file from a tar archive."""
+    with tar.extractfile(member) as gz, gzip.open(gz, "rb") as f:
+        return f.read().decode("utf-8", "ignore")

simtools/testing/configuration.py

@@ -76,8 +76,7 @@ def _read_configs_from_files(config_files):
         _dict = gen.remove_substring_recursively_from_dict(
             ascii_handler.collect_data_from_file(file_name=config_file), substring="\n"
         )
-        for application in _dict.get("applications", []):
-            configs.append(application)
+        configs.extend(_dict.get("applications", []))
     return configs
 
 

simtools/testing/helpers.py

@@ -1,7 +1,6 @@
 """Helper functions for integration testing."""
 
-import os
-from pathlib import Path
+from simtools import settings
 
 
 def skip_camera_efficiency(config):
@@ -21,7 +20,7 @@ def _new_testeff_version():
 
     This test checks if the new version is used.
     """
-    testeff_path = Path(os.getenv("SIMTOOLS_SIMTEL_PATH")) / "sim_telarray/testeff.c"
+    testeff_path = settings.config.sim_telarray_path / "testeff.c"
     try:
         with open(testeff_path, encoding="utf-8") as file:
             file_content = file.read()

simtools/testing/log_inspector.py

@@ -18,6 +18,7 @@ ERROR_PATTERNS = [
 IGNORE_PATTERNS = [
     re.compile(r"Falling back to 'utf-8' with errors='ignore'", re.IGNORECASE),
     re.compile(r"Failed to get user name[^\n]*setting it to UNKNOWN_USER", re.IGNORECASE),
+    re.compile(r"adjust_text::Error", re.IGNORECASE),
 ]
 
 

simtools/testing/sim_telarray_metadata.py

@@ -4,13 +4,13 @@ import logging
 
 import numpy as np
 
-from simtools.io.eventio_handler import get_corsika_run_number
+from simtools.sim_events.file_info import get_corsika_run_number
 from simtools.simtel.simtel_config_reader import SimtelConfigReader
-from simtools.simtel.simtel_config_writer import sim_telarray_random_seeds
 from simtools.simtel.simtel_io_metadata import (
     get_sim_telarray_telescope_id,
     read_sim_telarray_metadata,
 )
+from simtools.utils import random
 
 _logger = logging.getLogger(__name__)
 
@@ -30,7 +30,7 @@ def assert_sim_telarray_metadata(file, array_model):
     _logger.info(f"Found metadata in sim_telarray file for {len(telescope_meta)} telescopes")
     site_parameter_mismatch = _assert_model_parameters(global_meta, array_model.site_model)
     sim_telarray_seed_mismatch = _assert_sim_telarray_seed(
-        global_meta, array_model.sim_telarray_seeds, file
+        global_meta, array_model.sim_telarray_seed, file
     )
     if sim_telarray_seed_mismatch:
         site_parameter_mismatch.append(sim_telarray_seed_mismatch)
@@ -101,7 +101,7 @@ def _assert_model_parameters(metadata, model):
     return invalid_parameter_list
 
 
-def _assert_sim_telarray_seed(metadata, sim_telarray_seeds, file=None):
+def _assert_sim_telarray_seed(metadata, sim_telarray_seed, file=None):
     """
     Assert that sim_telarray seed matches the values in the sim_telarray metadata.
 
@@ -111,8 +111,8 @@ def _assert_sim_telarray_seed(metadata, sim_telarray_seeds, file=None):
     ----------
     metadata: dict
         Metadata dictionary.
-    sim_telarray_seeds: dict
-        Dictionary of sim_telarray seeds.
+    sim_telarray_seed: SimtelSeeds
+        sim_telarray seed.
     file : Path
         Path to the sim_telarray file.
 
@@ -122,23 +122,25 @@ def _assert_sim_telarray_seed(metadata, sim_telarray_seeds, file=None):
         Error message if sim_telarray seeds do not match.
 
     """
-    if not sim_telarray_seeds or not metadata:
+    if sim_telarray_seed is None:
         return None
 
     if "instrument_seed" in metadata.keys() and "instrument_instances" in metadata.keys():
-        if str(metadata.get("instrument_seed")) != str(sim_telarray_seeds.get("seed")):
+        if str(metadata.get("instrument_seed")) != str(sim_telarray_seed.instrument_seed):
             return (
                 "Parameter instrument_seed mismatch between sim_telarray file: "
-                f"{metadata['instrument_seed']}, and model: {sim_telarray_seeds.get('seed')}"
+                f"{metadata['instrument_seed']}, and model: {sim_telarray_seed.instrument_seed}"
             )
         _logger.info(
             f"sim_telarray_seed in sim_telarray file: {metadata['instrument_seed']}, "
-            f"and model: {sim_telarray_seeds.get('seed')}"
+            f"and model: {sim_telarray_seed.instrument_seed}"
         )
         if file:
             run_number_modified = get_corsika_run_number(file) - 1
-            test_seeds = sim_telarray_random_seeds(
-                int(metadata["instrument_seed"]), int(metadata["instrument_instances"])
+            test_seeds = random.seeds(
+                n_seeds=int(metadata["instrument_instances"]),
+                max_seed=np.iinfo(np.int32).max,
+                fixed_seed=int(metadata["instrument_seed"]),
             )
             # no +1 as in sim_telarray (as we count from 0)
             seed_used = run_number_modified % int(metadata["instrument_instances"])

simtools/testing/validate_output.py

@@ -13,30 +13,40 @@ from simtools.testing import assertions
 
 _logger = logging.getLogger(__name__)
 
+
+def _versions_match(from_command_line, from_config_file):
+    """Return True if validations should run for the given versions.
+
+    Behavior:
+    - If no version is provided from the command line, run validations.
+    - If a filter is provided from the command line, run only when it matches
+        the version(s) from the config file.
+    """
+    if from_command_line is None:
+        return True
+
+    # Normalize to collections for comparison
+    cmd_versions = from_command_line if isinstance(from_command_line, list) else [from_command_line]
+    cfg_versions = from_config_file if isinstance(from_config_file, list) else [from_config_file]
+
+    # Consider a match if any overlap exists
+    return any(cv in cmd_versions for cv in cfg_versions)
+
+
 # Keys to ignore when comparing sim_telarray configuration files
 # (e.g., version numbers, system dependent parameters, CORSIKA options)
 cfg_ignore_keys = [
     "config_release",
     "Label",
-    "simtools_version",
-    "simtools_model_production_version",
-    "simtools_build_opt",
-    "simtools_extra_def",
-    "simtools_hadronic_model",
-    "simtools_avx_flag",
-    "simtools_corsika_version",
-    "simtools_corsika_opt_patch_version",
-    "simtools_bernlohr_version",
+    "simtools_",  # ignore all simtools_ keys - version/build info dependence
 ]
 
 
-def validate_application_output(
-    config, from_command_line=None, from_config_file=None, db_config=None
-):
+def validate_application_output(config, from_command_line=None, from_config_file=None):
     """
     Validate application output against expected output.
 
-    Expected output is defined in configuration file.
+    Expected output is defined in the test configuration file.
     Some tests run only if the model version from the command line
     equals the model version from the configuration file.
 
@@ -60,8 +70,8 @@ def validate_application_output(
             f"from config file: {from_config_file}"
         )
 
-        if from_command_line == from_config_file:
-            _validate_output_files(config, integration_test, db_config)
+        if _versions_match(from_command_line, from_config_file):
+            _validate_output_files(config, integration_test)
 
             if "file_type" in integration_test:
                 assert assertions.assert_file_type(
@@ -73,11 +83,13 @@ def validate_application_output(
         _test_simtel_cfg_files(config, integration_test, from_command_line, from_config_file)
 
 
-def _validate_output_files(config, integration_test, db_config):
+def _validate_output_files(config, integration_test):
     """Validate output files."""
     if "reference_output_file" in integration_test:
         _validate_reference_output_file(config, integration_test)
     if "test_output_files" in integration_test:
+        if isinstance(integration_test["test_output_files"], dict):
+            integration_test["test_output_files"] = [integration_test["test_output_files"]]
         _validate_output_path_and_file(config, integration_test["test_output_files"])
     if "output_file" in integration_test:
         _validate_output_path_and_file(
@@ -85,11 +97,7 @@ def _validate_output_files(config, integration_test, db_config):
             [{"path_descriptor": "output_path", "file": integration_test["output_file"]}],
         )
     if "model_parameter_validation" in integration_test:
-        _validate_model_parameter_json_file(
-            config,
-            integration_test["model_parameter_validation"],
-            db_config,
-        )
+        _validate_model_parameter_json_file(config, integration_test["model_parameter_validation"])
 
 
 def _test_simtel_cfg_files(config, integration_test, from_command_line, from_config_file):
@@ -110,11 +118,10 @@ def _test_simtel_cfg_files(config, integration_test, from_command_line, from_con
 
 def _validate_reference_output_file(config, integration_test):
     """Compare with reference output file."""
+    test_file = integration_test.get("test_output_file") or config["configuration"]["output_file"]
     assert compare_files(
         integration_test["reference_output_file"],
-        Path(config["configuration"]["output_path"]).joinpath(
-            config["configuration"]["output_file"]
-        ),
+        Path(config["configuration"]["output_path"]).joinpath(test_file),
         integration_test.get("tolerance", 1.0e-5),
         integration_test.get("test_columns", None),
     )
@@ -135,15 +142,20 @@ def _validate_output_path_and_file(config, integration_file_tests):
         try:
             assert output_file_path.exists()
         except AssertionError as exc:
-            raise AssertionError(f"Output file {output_file_path} does not exist. ") from exc
+            raise AssertionError(
+                f"Output file {output_file_path} does not exist. "
+                f"Directory contents: {list(output_file_path.parent.iterdir())}"
+            ) from exc
 
         if output_file_path.name.endswith(".simtel.zst"):
             assert assertions.check_output_from_sim_telarray(output_file_path, file_test)
         elif output_file_path.name.endswith(".log_hist.tar.gz"):
             assert assertions.check_simulation_logs(output_file_path, file_test)
+        elif output_file_path.suffix == ".log":
+            assert assertions.check_plain_log(output_file_path, file_test)
 
 
-def _validate_model_parameter_json_file(config, model_parameter_validation, db_config):
+def _validate_model_parameter_json_file(config, model_parameter_validation):
     """
     Validate model parameter json file and compare it with a reference parameter from the database.
 
@@ -158,7 +170,7 @@ def _validate_model_parameter_json_file(config, model_parameter_validation, db_c
 
     """
     _logger.info(f"Checking model parameter json file: {model_parameter_validation}")
-    db = db_handler.DatabaseHandler(db_config=db_config)
+    db = db_handler.DatabaseHandler()
 
     reference_parameter_name = model_parameter_validation.get("reference_parameter_name")
 
@@ -178,6 +190,7 @@ def _validate_model_parameter_json_file(config, model_parameter_validation, db_c
         model_parameter["value"],
         reference_model_parameter[reference_parameter_name]["value"],
         model_parameter_validation["tolerance"],
+        model_parameter_validation.get("scaling", 1.0),
     )
 
 
@@ -216,12 +229,56 @@ def compare_files(file1, file2, tolerance=1.0e-5, test_columns=None):
     return False
 
 
+def _compare_nested_dicts_with_tolerance(data1, data2, tolerance, is_value_field=False):
+    """
+    Recursively compare nested dictionaries, applying allclose to "value" fields.
+
+    Parameters
+    ----------
+    data1 : dict, list, or scalar
+        First data to compare
+    data2 : dict, list, or scalar
+        Second data to compare
+    tolerance : float
+        Tolerance for comparing numerical values
+    is_value_field : bool
+        Whether this data is from a "value" key (applies tolerance to scalars)
+
+    Returns
+    -------
+    bool
+        True if the data are equal within tolerance
+    """
+    if isinstance(data1, dict) and isinstance(data2, dict):
+        return data1.keys() == data2.keys() and all(
+            _compare_nested_dicts_with_tolerance(
+                data1[k], data2[k], tolerance, is_value_field=(k == "value")
+            )
+            for k in data1
+        )
+
+    if isinstance(data1, (list, tuple)) and isinstance(data2, (list, tuple)):
+        return len(data1) == len(data2) and all(
+            _compare_nested_dicts_with_tolerance(v1, v2, tolerance, is_value_field)
+            for v1, v2 in zip(data1, data2)
+        )
+
+    # Apply tolerance if this is a "value" field, otherwise use exact equality
+    if is_value_field:
+        try:
+            return _compare_value_from_parameter_dict(data1, data2, tolerance)
+        except (TypeError, ValueError):
+            return data1 == data2
+    return data1 == data2
+
+
 def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2):
     """
     Compare two json or yaml files.
 
     Take into account float comparison for sim_telarray string-embedded floats.
     Allow differences in 'schema_version' field.
+    Works recursively for nested dicts with "value" fields on any level.
 
     Parameters
     ----------
@@ -248,24 +305,45 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2):
     if data1 == data2:
         return True
 
-    if data1.keys() != data2.keys():
-        _logger.error(f"Keys do not match: {data1.keys()} and {data2.keys()}")
-        return False
-    _comparison = all(
-        (
-            _compare_value_from_parameter_dict(data1[k], data2[k], tolerance)
-            if k == "value"
-            else data1[k] == data2[k]
-        )
-        for k in data1
+    if isinstance(data1, dict) and isinstance(data2, dict):
+        if data1.keys() != data2.keys():
+            _logger.error(f"Keys do not match: {data1.keys()} and {data2.keys()}")
+            return False
+
+    _comparison = _compare_nested_dicts_with_tolerance(
+        data1, data2, tolerance, is_value_field=False
     )
     if not _comparison:
         _logger.error(f"Values do not match: {data1} and {data2} (tolerance: {tolerance})")
     return _comparison
 
 
-def _compare_value_from_parameter_dict(data1, data2, tolerance=1.0e-5):
-    """Compare value fields given in different formats."""
+def _compare_value_from_parameter_dict(data_1, data_2, tolerance=1.0e-5, factor_1=1.0):
+    """
+    Compare value fields given in different formats.
+
+    Parameters
+    ----------
+    data_1 : float, int, str, list, numpy.ndarray
+        First value or collection of values to compare. May be a scalar,
+        a sequence, a numpy array, or a string representation of a list.
+    data_2 : float, int, str, list, numpy.ndarray
+        Second value or collection of values to compare, with the same
+        allowed formats as ``data_2``.
+    tolerance : float, optional
+        Relative tolerance used when comparing numerical values via
+        ``numpy.allclose``.
+    factor1 : float, optional
+        Multiplicative factor applied to ``data_1`` before comparison. This
+        can be used to account for unit conversions or normalisation
+        differences between ``data_1`` and ``data_2``.
+
+    Returns
+    -------
+    bool
+        True if the two values are considered equal within the given
+        tolerance, False otherwise.
+    """
 
     def _as_list(value):
         if isinstance(value, str):
@@ -274,12 +352,13 @@ def _compare_value_from_parameter_dict(data1, data2, tolerance=1.0e-5):
             return value
         return [value]
 
-    _logger.info(f"Comparing values: {data1} and {data2} (tolerance: {tolerance})")
+    _logger.info(f"Comparing values: {data_1} and {data_2} (tolerance: {tolerance})")
 
-    _as_list_1 = _as_list(data1)
-    _as_list_2 = _as_list(data2)
+    _as_list_1 = _as_list(data_1)
+    _as_list_2 = _as_list(data_2)
     if isinstance(_as_list_1, str):
         return _as_list_1 == _as_list_2
+    _as_list_1 = np.array(_as_list_1) * factor_1
     return np.allclose(_as_list_1, _as_list_2, rtol=tolerance)
 
 

simtools/utils/general.py

@@ -11,6 +11,7 @@ import urllib.request
 from pathlib import Path
 from urllib.parse import urlparse
 
+import dotenv
 import numpy as np
 
 _logger = logging.getLogger(__name__)
@@ -581,6 +582,12 @@ def validate_data_type(reference_dtype, value=None, dtype=None, allow_subtypes=T
     if reference_dtype in ("boolean", "bool"):
         return _is_valid_boolean_type(dtype, value)
 
+    if reference_dtype == "dict":
+        return isinstance(value, dict)
+
+    if reference_dtype == "list":
+        return isinstance(value, list)
+
     return _is_valid_numeric_type(dtype, reference_dtype)
 
 
@@ -795,23 +802,6 @@ def find_differences_in_json_objects(obj1, obj2, path=""):
     return diffs
 
 
-def clear_default_sim_telarray_cfg_directories(command):
-    """Prefix the command to clear default sim_telarray configuration directories.
-
-    Parameters
-    ----------
-    command: str
-        Command to be prefixed.
-
-    Returns
-    -------
-    str
-        Prefixed command.
-
-    """
-    return f"SIM_TELARRAY_CONFIG_PATH='' {command}"
-
-
 def get_list_of_files_from_command_line(file_names, suffix_list):
     """
     Get a list of files from the command line.
@@ -850,3 +840,39 @@ def get_list_of_files_from_command_line(file_names, suffix_list):
 def now_date_time_in_isoformat():
     """Return date and time in isoformat and second accuracy."""
     return datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds")
+
+
+def load_environment_variables(env_file=".env", env_list=None):
+    """
+    Load environment variables (from a .env file or directly from the environment).
+
+    Allow to read a specific list of variables or all variables from the .env file.
+
+    Parameters
+    ----------
+    env_file: str
+        Path to the .env file.
+    env_list: list, optional
+        List of environment variables to be read. If None, all variables are read.
+
+    Returns
+    -------
+    dict
+        Dictionary mapping environment variable names (lowercase, without the
+        ``SIMTOOLS_`` prefix) to their cleaned string values.
+    """
+    dotenv.load_dotenv(env_file or None)
+    keys = (
+        list(dotenv.dotenv_values(env_file).keys())
+        if env_list is None
+        else [f"SIMTOOLS_{s.upper()}" for s in env_list]
+    )
+
+    env_values = {}
+    for key in keys:
+        env_value = os.environ.get(key)
+        if env_value is None:
+            continue
+        cleaned_value = env_value.split("#")[0].strip().replace('"', "").replace("'", "")
+        env_values[key.removeprefix("SIMTOOLS_").lower()] = cleaned_value
+    return env_values