gammasimtools 0.23.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

simtools/testing/assertions.py

@@ -1,13 +1,17 @@
 """Functions asserting certain conditions are met (used e.g., in integration tests)."""
 
+import gzip
 import json
 import logging
+import tarfile
 from collections import defaultdict
 from pathlib import Path
 
 import numpy as np
 import yaml
 
+from simtools.simtel.simtel_io_metadata import read_sim_telarray_metadata
+
 _logger = logging.getLogger(__name__)
 
 
@@ -124,7 +128,35 @@ def assert_expected_output(file, expected_output):
     return True
 
 
-def check_output_from_sim_telarray(file, expected_output):
+def assert_expected_simtel_metadata(file, expected_metadata):
+    """
+    Assert that expected metadata is present in the sim_telarray file.
+
+    Parameters
+    ----------
+    file: Path
+        Path to the sim_telarray file.
+    expected_metadata: dict
+        Expected metadata values.
+
+    """
+    global_meta, telescope_meta = read_sim_telarray_metadata(file)
+
+    for key, value in expected_metadata.items():
+        if key not in global_meta and key not in telescope_meta:
+            _logger.error(f"Metadata key {key} not found in sim_telarray file {file}")
+            return False
+        if key in global_meta and global_meta[key] != value:
+            _logger.error(
+                f"Metadata key {key} has value {global_meta[key]} instead of expected {value}"
+            )
+            return False
+        _logger.debug(f"Metadata key {key} matches expected value {value}")
+
+    return True
+
+
+def check_output_from_sim_telarray(file, file_test):
     """
     Check that the sim_telarray simulation result is reasonable and matches the expected output.
 
@@ -132,20 +164,88 @@ def check_output_from_sim_telarray(file, expected_output):
     ----------
     file: Path
         Path to the sim_telarray file.
-    expected_output: dict
-        Expected output values.
+    file_test: dict
+        File test description including expected output and metadata.
 
     Raises
     ------
     ValueError
         If the file is not a zstd compressed file.
     """
-    if file.suffix != ".zst":
-        raise ValueError(
-            f"Expected output file {file} is not a zstd compressed file "
-            f"(i.e., a sim_telarray file)."
+    if "expected_output" not in file_test and "expected_simtel_metadata" not in file_test:
+        _logger.debug(f"No expected output or metadata provided, skipping checks {file_test}")
+        return True
+
+    assert_output = assert_metadata = True
+
+    if "expected_output" in file_test:
+        assert_output = assert_expected_output(
+            file=file, expected_output=file_test["expected_output"]
         )
 
-    return assert_n_showers_and_energy_range(file=file) and assert_expected_output(
-        file=file, expected_output=expected_output
-    )
+    if "expected_simtel_metadata" in file_test:
+        assert_metadata = assert_expected_simtel_metadata(
+            file=file, expected_metadata=file_test["expected_simtel_metadata"]
+        )
+
+    return assert_n_showers_and_energy_range(file=file) and assert_output and assert_metadata
+
+
+def _find_patterns(text, patterns):
+    """Find patterns in text."""
+    return {p for p in patterns if p in text}
+
+
+def _read_log(member, tar):
+    """Read and decode a gzipped log file from a tar archive."""
+    with tar.extractfile(member) as gz, gzip.open(gz, "rb") as f:
+        return f.read().decode("utf-8", "ignore")
+
+
+def check_simulation_logs(tar_file, file_test):
+    """
+    Check log files of CORSIKA and sim_telarray for expected output.
+
+    Parameters
+    ----------
+    tar_file: Path
+        Path to a log file tar package.
+    file_test: dict
+        File test description including expected log output.
+
+    Raises
+    ------
+    ValueError
+        If the file is not a tar file.
+    """
+    expected_log = file_test.get("expected_log_output", {})
+    wanted = expected_log.get("pattern", [])
+    forbidden = expected_log.get("forbidden_pattern", [])
+
+    if not (wanted or forbidden):
+        _logger.debug(f"No expected log output provided, skipping checks {file_test}")
+        return True
+
+    if not tarfile.is_tarfile(tar_file):
+        raise ValueError(f"File {tar_file} is not a tar file.")
+
+    found, bad = set(), set()
+    with tarfile.open(tar_file, "r:*") as tar:
+        for member in tar.getmembers():
+            if not member.name.endswith(".log.gz"):
+                continue
+            _logger.info(f"Scanning {member.name}")
+            text = _read_log(member, tar)
+            found |= _find_patterns(text, wanted)
+            bad |= _find_patterns(text, forbidden)
+
+    if bad:
+        _logger.error(f"Forbidden patterns found: {list(bad)}")
+        return False
+    missing = [p for p in wanted if p not in found]
+    if missing:
+        _logger.error(f"Missing expected patterns: {missing}")
+        return False
+
+    _logger.debug(f"All expected patterns found: {wanted}")
+    return True

simtools/testing/configuration.py

@@ -204,7 +204,7 @@ def _prepare_test_options(config, output_path, model_version=None):
     if model_version and "model_version" in config:
         config.update({"model_version": model_version})
 
-    for key in ["output_path", "data_directory", "pack_for_grid_register"]:
+    for key in ["output_path", "pack_for_grid_register"]:
         if key in config:
             config[key] = str(Path(output_path).joinpath(config[key]))
 

simtools/testing/log_inspector.py

@@ -15,7 +15,10 @@ ERROR_PATTERNS = [
     re.compile(r"segmentation fault", re.IGNORECASE),
 ]
 
-IGNORE_PATTERNS = [re.compile(r"Falling back to 'utf-8' with errors='ignore'", re.IGNORECASE)]
+IGNORE_PATTERNS = [
+    re.compile(r"Falling back to 'utf-8' with errors='ignore'", re.IGNORECASE),
+    re.compile(r"Failed to get user name[^\n]*setting it to UNKNOWN_USER", re.IGNORECASE),
+]
 
 
 def inspect(log_text):

simtools/testing/sim_telarray_metadata.py

@@ -4,9 +4,9 @@ import logging
 
 import numpy as np
 
+from simtools.io.eventio_handler import get_corsika_run_number
 from simtools.simtel.simtel_config_reader import SimtelConfigReader
 from simtools.simtel.simtel_config_writer import sim_telarray_random_seeds
-from simtools.simtel.simtel_io_file_info import get_corsika_run_number
 from simtools.simtel.simtel_io_metadata import (
     get_sim_telarray_telescope_id,
     read_sim_telarray_metadata,

simtools/testing/validate_output.py

@@ -13,6 +13,22 @@ from simtools.testing import assertions
 
 _logger = logging.getLogger(__name__)
 
+# Keys to ignore when comparing sim_telarray configuration files
+# (e.g., version numbers, system dependent parameters, CORSIKA options)
+cfg_ignore_keys = [
+    "config_release",
+    "Label",
+    "simtools_version",
+    "simtools_model_production_version",
+    "simtools_build_opt",
+    "simtools_extra_def",
+    "simtools_hadronic_model",
+    "simtools_avx_flag",
+    "simtools_corsika_version",
+    "simtools_corsika_opt_patch_version",
+    "simtools_bernlohr_version",
+]
+
 
 def validate_application_output(
     config, from_command_line=None, from_config_file=None, db_config=None
@@ -39,6 +55,10 @@ def validate_application_output(
 
     for integration_test in config["integration_tests"]:
         _logger.info(f"Testing application output: {integration_test}")
+        _logger.debug(
+            f"Model version from command line: {from_command_line}, "
+            f"from config file: {from_config_file}"
+        )
 
         if from_command_line == from_config_file:
             _validate_output_files(config, integration_test, db_config)
@@ -117,11 +137,10 @@ def _validate_output_path_and_file(config, integration_file_tests):
         except AssertionError as exc:
             raise AssertionError(f"Output file {output_file_path} does not exist. ") from exc
 
-        if "expected_output" in file_test:
-            assert assertions.check_output_from_sim_telarray(
-                output_file_path,
-                file_test["expected_output"],
-            )
+        if output_file_path.name.endswith(".simtel.zst"):
+            assert assertions.check_output_from_sim_telarray(output_file_path, file_test)
+        elif output_file_path.name.endswith(".log_hist.tar.gz"):
+            assert assertions.check_simulation_logs(output_file_path, file_test)
 
 
 def _validate_model_parameter_json_file(config, model_parameter_validation, db_config):
@@ -139,7 +158,7 @@ def _validate_model_parameter_json_file(config, model_parameter_validation, db_c
 
     """
     _logger.info(f"Checking model parameter json file: {model_parameter_validation}")
-    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+    db = db_handler.DatabaseHandler(db_config=db_config)
 
     reference_parameter_name = model_parameter_validation.get("reference_parameter_name")
 
@@ -346,7 +365,7 @@ def _validate_simtel_cfg_files(config, simtel_cfg_file):
         f"Comparing simtel cfg files: {reference_file} and {test_file} "
         f"for model version {config['configuration']['model_version']}"
     )
-    return _compare_simtel_cfg_files(reference_file, test_file)
+    assert _compare_simtel_cfg_files(reference_file, test_file)
 
 
 def _compare_simtel_cfg_files(reference_file, test_file):
@@ -354,8 +373,8 @@ def _compare_simtel_cfg_files(reference_file, test_file):
     Compare two sim_telarray configuration files.
 
     Line-by-line string comparison. Requires similar sequence of
-    parameters in the files. Ignore lines containing 'config_release'
-    (as it contains the simtools package version).
+    parameters in the files. Ignore lines listed in cfg_ignore_keys
+    (e.g., simtools package versions or hadronic interaction model strings).
 
     Parameters
     ----------
@@ -374,16 +393,28 @@ def _compare_simtel_cfg_files(reference_file, test_file):
         reference_cfg = [line.rstrip() for line in f1 if line.strip()]
         test_cfg = [line.rstrip() for line in f2 if line.strip()]
 
-    if len(reference_cfg) != len(test_cfg):
+    def filter_ignored(cfg_lines, file_label):
+        filtered = []
+        for line in cfg_lines:
+            ignored_key = next((ignore for ignore in cfg_ignore_keys if ignore in line), None)
+            if ignored_key:
+                _logger.debug(f"Ignoring line in {file_label} due to key '{ignored_key}': {line}")
+                continue
+            filtered.append(line)
+        return filtered
+
+    reference_cfg_filtered = filter_ignored(reference_cfg, "reference file")
+    test_cfg_filtered = filter_ignored(test_cfg, "test file")
+
+    if len(reference_cfg_filtered) != len(test_cfg_filtered):
         _logger.error(
-            f"Line counts differ: {reference_file} ({len(reference_cfg)} lines), "
-            f"{test_file} ({len(test_cfg)} lines)."
+            f"Line counts differ after filtering: {reference_file} "
+            f"({len(reference_cfg_filtered)} lines), "
+            f"{test_file} ({len(test_cfg_filtered)} lines)."
         )
         return False
 
-    for ref_line, test_line in zip(reference_cfg, test_cfg):
-        if any(ignore in ref_line for ignore in ("config_release", "Label")):
-            continue
+    for ref_line, test_line in zip(reference_cfg_filtered, test_cfg_filtered):
         if ref_line != test_line:
             _logger.error(
                 f"Configuration files {reference_file} and {test_file} do not match: "

simtools/utils/names.py

@@ -516,6 +516,8 @@ def get_site_from_array_element_name(array_element_name):
         Site name(s).
     """
     try:  # e.g. instrument is 'North' as given for the site parameters
+        if array_element_name.startswith("OBS"):
+            return validate_site_name(array_element_name.split("-")[1])
         return validate_site_name(array_element_name)
     except ValueError:  # e.g. instrument is 'LSTN' as given for the array element types
         return array_elements()[get_array_element_type_from_name(array_element_name)]["site"]
@@ -630,7 +632,6 @@ def get_simulation_software_name_from_parameter_name(
 
 def simtel_config_file_name(
     site,
-    model_version,
     array_name=None,
     telescope_model_name=None,
     label=None,
@@ -645,8 +646,6 @@ def simtel_config_file_name(
         South or North.
     telescope_model_name: str
         LST-1, MST-FlashCam, ...
-    model_version: str
-        Version of the model.
     label: str
         Instance label.
     extra_label: str
@@ -661,7 +660,6 @@ def simtel_config_file_name(
     name += f"-{array_name}" if array_name is not None else ""
     name += f"-{site}"
     name += f"-{telescope_model_name}" if telescope_model_name is not None else ""
-    name += f"-{model_version}"
     name += f"_{label}" if label is not None else ""
     name += f"_{extra_label}" if extra_label is not None else ""
     name += ".cfg"

simtools/utils/value_conversion.py

@@ -170,16 +170,21 @@ def get_value_as_quantity(value, unit):
 
     Raises
     ------
-    u.UnitConversionError
+    ValueError
         If the value cannot be converted to the given unit.
     """
     if isinstance(value, u.Quantity):
         try:
             return value.to(unit)
-        except u.UnitConversionError:
-            _logger.error(f"Cannot convert {value.unit} to {unit}.")
-            raise
-    return value * unit
+        except u.UnitConversionError as exc:
+            raise ValueError(f"Cannot convert {value} with unit {value.unit} to {unit}.") from exc
+    elif not isinstance(value, int | float):
+        return value
+
+    if unit is None or unit == "null":
+        return value * u.dimensionless_unscaled
+
+    return value * u.Unit(unit)
 
 
 def _unit_as_string(unit):

simtools/version.py

@@ -4,6 +4,9 @@
 # which is adapted from https://github.com/astropy/astropy/blob/master/astropy/version.py
 # see https://github.com/astropy/astropy/pull/10774 for a discussion on why this needed.
 
+import re
+
+from packaging.specifiers import SpecifierSet
 from packaging.version import InvalidVersion, Version
 
 MAJOR_MINOR_PATCH = "major.minor.patch"
@@ -189,3 +192,61 @@ def compare_versions(version_string_1, version_string_2, level=MAJOR_MINOR_PATCH
         raise ValueError(f"Unknown level: {level}")
 
     return (ver1 > ver2) - (ver1 < ver2)
+
+
+def is_valid_semantic_version(version_string, strict=True):
+    """
+    Check if a string is a valid semantic version.
+
+    Parameters
+    ----------
+    version_string : str
+        The version string to validate (e.g., "6.0.2", "1.0.0-alpha").
+    strict : bool, optional
+        If True, use PEP 440 validation (packaging.version.Version).
+        If False, use SemVer 2.0.0 regex pattern (allows more flexible pre-release identifiers).
+
+    Returns
+    -------
+    bool
+        True if the version string is valid, False otherwise.
+    """
+    if not version_string:
+        return False
+
+    if strict:
+        try:
+            Version(version_string)
+            return True
+        except InvalidVersion:
+            return False
+    else:
+        semver_regex = (
+            r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)"  # major.minor.patch
+            r"(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?"  # pre-release
+            r"(?:\+([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$"  # build metadata
+        )
+        return bool(re.match(semver_regex, version_string))
+
+
+def check_version_constraint(version_string, constraint):
+    """
+    Check if a version satisfies a constraint.
+
+    Parameters
+    ----------
+    version_string : str
+        The version string to check (e.g., "6.0.2").
+    constraint : str
+        The version constraint to check against (e.g., ">=6.0.0").
+
+    Returns
+    -------
+    bool
+        True if the version satisfies the constraint, False otherwise.
+    """
+    spec = SpecifierSet(constraint.strip(), prereleases=True)
+    ver = Version(version_string)
+    if ver in spec:
+        return True
+    return False

simtools/visualization/legend_handlers.py

@@ -11,12 +11,20 @@ Radii are relative to a reference radius (REFERENCE_RADIUS).
 """
 TELESCOPE_CONFIG = {
     "LST": {"color": "darkorange", "radius": 12.5, "shape": "circle", "filled": False},
-    "MST": {"color": "dodgerblue", "radius": 9.15, "shape": "circle", "filled": True},
-    "SCT": {"color": "black", "radius": 7.15, "shape": "square", "filled": True},
-    "SST": {"color": "darkgreen", "radius": 3.0, "shape": "circle", "filled": True},
+    "MST": {"color": "dodgerblue", "radius": 9.15, "shape": "circle", "filled": False},
+    "SCT": {"color": "black", "radius": 7.15, "shape": "square", "filled": False},
+    "SST": {"color": "darkgreen", "radius": 3.0, "shape": "circle", "filled": False},
     "HESS": {"color": "grey", "radius": 6.0, "shape": "hexagon", "filled": True},
     "MAGIC": {"color": "grey", "radius": 8.5, "shape": "hexagon", "filled": True},
     "VERITAS": {"color": "grey", "radius": 6.0, "shape": "hexagon", "filled": True},
+    "CEI": {"color": "purple", "radius": 2.0, "shape": "hexagon", "filled": True},
+    "RLD": {"color": "brown", "radius": 2.0, "shape": "hexagon", "filled": True},
+    "STP": {"color": "olive", "radius": 2.0, "shape": "hexagon", "filled": True},
+    "MSP": {"color": "teal", "radius": 2.0, "shape": "hexagon", "filled": True},
+    "ILL": {"color": "red", "radius": 2.0, "shape": "hexagon", "filled": False},
+    "WST": {"color": "maroon", "radius": 2.0, "shape": "hexagon", "filled": True},
+    "ASC": {"color": "cyan", "radius": 2.0, "shape": "hexagon", "filled": True},
+    "DUS": {"color": "magenta", "radius": 2.0, "shape": "hexagon", "filled": True},
 }
 
 REFERENCE_RADIUS = 12.5
@@ -30,7 +38,7 @@ def get_telescope_config(telescope_type):
 
     Parameters
     ----------
-    telescope_type : str
+    telescope_type : str, None
         The type of the telescope (e.g., "LSTN", "MSTS").
 
     Returns
@@ -38,6 +46,8 @@ def get_telescope_config(telescope_type):
     dict
         The configuration dictionary for the telescope type.
     """
+    if telescope_type is None:
+        return {"color": "blue", "radius": 2.0, "shape": "hexagon", "filled": True}
     config = TELESCOPE_CONFIG.get(telescope_type)
     if not config and len(telescope_type) >= 3:
         config = TELESCOPE_CONFIG.get(telescope_type[:3])