gammasimtools 0.22.0__py3-none-any.whl → 0.24.0__py3-none-any.whl

simtools/simulator.py

@@ -22,11 +22,6 @@ from simtools.simtel.simulator_array import SimulatorArray
 from simtools.testing.sim_telarray_metadata import assert_sim_telarray_metadata
 from simtools.version import semver_to_int
 
-__all__ = [
-    "InvalidRunsToSimulateError",
-    "Simulator",
-]
-
 
 class InvalidRunsToSimulateError(Exception):
     """Exception for invalid runs to simulate."""
@@ -130,7 +125,7 @@ class Simulator:
                 label=self.label,
                 site=self.args_dict.get("site"),
                 layout_name=self.args_dict.get("array_layout_name"),
-                mongo_db_config=self.db_config,
+                db_config=self.db_config,
                 model_version=version,
                 sim_telarray_seeds={
                     "seed": self._get_seed_for_random_instrument_instances(
@@ -145,6 +140,7 @@ class Simulator:
                 calibration_device_types=self._get_calibration_device_types(
                     self.args_dict.get("run_mode")
                 ),
+                overwrite_model_parameters=self.args_dict.get("overwrite_model_parameters", None),
             )
             for version in versions
         ]

simtools/testing/assertions.py

@@ -8,6 +8,8 @@ from pathlib import Path
 import numpy as np
 import yaml
 
+from simtools.simtel.simtel_io_metadata import read_sim_telarray_metadata
+
 _logger = logging.getLogger(__name__)
 
 
@@ -62,9 +64,9 @@ def assert_n_showers_and_energy_range(file):
 
     simulated_energies = []
     simulation_config = {}
-    with SimTelFile(file) as f:
+    with SimTelFile(file, skip_non_triggered=False) as f:
         simulation_config = f.mc_run_headers[0]
-        for event in f.iter_mc_events():
+        for event in f:
             simulated_energies.append(event["mc_shower"]["energy"])
 
     # The relative tolerance is set to 1% because ~0.5% shower simulations do not
@@ -124,7 +126,35 @@ def assert_expected_output(file, expected_output):
     return True
 
 
-def check_output_from_sim_telarray(file, expected_output):
+def assert_expected_simtel_metadata(file, expected_metadata):
+    """
+    Assert that expected metadata is present in the sim_telarray file.
+
+    Parameters
+    ----------
+    file: Path
+        Path to the sim_telarray file.
+    expected_metadata: dict
+        Expected metadata values.
+
+    """
+    global_meta, telescope_meta = read_sim_telarray_metadata(file)
+
+    for key, value in expected_metadata.items():
+        if key not in global_meta and key not in telescope_meta:
+            _logger.error(f"Metadata key {key} not found in sim_telarray file {file}")
+            return False
+        if key in global_meta and global_meta[key] != value:
+            _logger.error(
+                f"Metadata key {key} has value {global_meta[key]} instead of expected {value}"
+            )
+            return False
+        _logger.debug(f"Metadata key {key} matches expected value {value}")
+
+    return True
+
+
+def check_output_from_sim_telarray(file, file_test):
     """
     Check that the sim_telarray simulation result is reasonable and matches the expected output.
 
@@ -132,20 +162,34 @@ def check_output_from_sim_telarray(file, expected_output):
     ----------
     file: Path
         Path to the sim_telarray file.
-    expected_output: dict
-        Expected output values.
+    file_test: dict
+        File test description including expected output and metadata.
 
     Raises
     ------
     ValueError
         If the file is not a zstd compressed file.
     """
+    if "expected_output" not in file_test and "expected_simtel_metadata" not in file_test:
+        _logger.debug(f"No expected output or metadata provided, skipping checks {file_test}")
+        return True
+
     if file.suffix != ".zst":
         raise ValueError(
             f"Expected output file {file} is not a zstd compressed file "
             f"(i.e., a sim_telarray file)."
         )
 
-    return assert_n_showers_and_energy_range(file=file) and assert_expected_output(
-        file=file, expected_output=expected_output
-    )
+    assert_output = assert_metadata = True
+
+    if "expected_output" in file_test:
+        assert_output = assert_expected_output(
+            file=file, expected_output=file_test["expected_output"]
+        )
+
+    if "expected_simtel_metadata" in file_test:
+        assert_metadata = assert_expected_simtel_metadata(
+            file=file, expected_metadata=file_test["expected_simtel_metadata"]
+        )
+
+    return assert_n_showers_and_energy_range(file=file) and assert_output and assert_metadata

simtools/testing/configuration.py

@@ -5,6 +5,7 @@ import os
 from pathlib import Path
 
 import simtools.utils.general as gen
+import simtools.version as simtools_version
 from simtools.io import ascii_handler
 
 _logger = logging.getLogger(__name__)
@@ -130,11 +131,23 @@ def configure(config, tmp_test_directory, request):
 
 
 def _skip_test_for_model_version(config, model_version_requested):
-    """Skip test if model version requested is not supported."""
-    if config.get("model_version_use_current") is None or model_version_requested is None:
+    """
+    Skip test if model version requested is not supported.
+
+    Compares full and major.minor version strings.
+    """
+    if not (config.get("model_version_use_current") and model_version_requested):
         return
-    model_version_config = config["configuration"]["model_version"]
-    if model_version_requested != model_version_config:
+    model_version_config = str(config["configuration"]["model_version"])
+
+    if (
+        simtools_version.compare_versions(
+            model_version_requested,
+            model_version_config,
+            level=simtools_version.version_kind(model_version_requested),
+        )
+        != 0
+    ):
         raise VersionError(
             f"Model version requested {model_version_requested} not supported for this test"
         )

simtools/testing/validate_output.py

@@ -117,11 +117,7 @@ def _validate_output_path_and_file(config, integration_file_tests):
         except AssertionError as exc:
             raise AssertionError(f"Output file {output_file_path} does not exist. ") from exc
 
-        if "expected_output" in file_test:
-            assert assertions.check_output_from_sim_telarray(
-                output_file_path,
-                file_test["expected_output"],
-            )
+        assert assertions.check_output_from_sim_telarray(output_file_path, file_test)
 
 
 def _validate_model_parameter_json_file(config, model_parameter_validation, db_config):
@@ -139,7 +135,7 @@ def _validate_model_parameter_json_file(config, model_parameter_validation, db_c
 
     """
     _logger.info(f"Checking model parameter json file: {model_parameter_validation}")
-    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+    db = db_handler.DatabaseHandler(db_config=db_config)
 
     reference_parameter_name = model_parameter_validation.get("reference_parameter_name")
 
@@ -346,7 +342,7 @@ def _validate_simtel_cfg_files(config, simtel_cfg_file):
         f"Comparing simtel cfg files: {reference_file} and {test_file} "
         f"for model version {config['configuration']['model_version']}"
     )
-    return _compare_simtel_cfg_files(reference_file, test_file)
+    assert _compare_simtel_cfg_files(reference_file, test_file)
 
 
 def _compare_simtel_cfg_files(reference_file, test_file):
@@ -382,7 +378,7 @@ def _compare_simtel_cfg_files(reference_file, test_file):
         return False
 
     for ref_line, test_line in zip(reference_cfg, test_cfg):
-        if any(ignore in ref_line for ignore in ("config_release", "Label")):
+        if any(ignore in ref_line for ignore in ("config_release", "Label", "simtools_version")):
             continue
         if ref_line != test_line:
             _logger.error(

simtools/utils/general.py

@@ -13,17 +13,6 @@ from urllib.parse import urlparse
 
 import numpy as np
 
-__all__ = [
-    "change_dict_keys_case",
-    "clear_default_sim_telarray_cfg_directories",
-    "collect_final_lines",
-    "collect_kwargs",
-    "get_log_excerpt",
-    "get_log_level_from_user",
-    "remove_substring_recursively_from_dict",
-    "set_default_kwargs",
-]
-
 _logger = logging.getLogger(__name__)
 
 
@@ -345,7 +334,7 @@ def resolve_file_patterns(file_names):
     return _files
 
 
-def pack_tar_file(tar_file_name, file_list):
+def pack_tar_file(tar_file_name, file_list, sub_dir=None):
     """
     Pack files into a tar.gz archive.
 
@@ -355,6 +344,8 @@ def pack_tar_file(tar_file_name, file_list):
         Name of the output tar.gz file.
     file_list: list
         List of files to include in the archive.
+    sub_dir: str, optional
+        Subdirectory within the archive to place the files.
     """
     file_list = [Path(f) for f in file_list]
     base = Path(os.path.commonpath([f.resolve() for f in file_list]))
@@ -365,7 +356,8 @@ def pack_tar_file(tar_file_name, file_list):
 
     with tarfile.open(tar_file_name, "w:gz") as tar:
         for file in file_list:
-            tar.add(file, arcname=file.name)
+            arc_name = Path(sub_dir) / file.name if sub_dir else file.name
+            tar.add(file, arcname=str(arc_name))
 
 
 def get_log_excerpt(log_file, n_last_lines=30):

simtools/utils/geometry.py

@@ -7,11 +7,6 @@ import astropy.units as u
 import numpy as np
 from astropy.units import UnitsError
 
-__all__ = [
-    "convert_2d_to_radial_distr",
-    "rotate",
-]
-
 _logger = logging.getLogger(__name__)
 
 

simtools/utils/names.py

@@ -28,18 +28,6 @@ from simtools.constants import (
 
 _logger = logging.getLogger(__name__)
 
-__all__ = [
-    "generate_file_name",
-    "get_array_element_type_from_name",
-    "get_site_from_array_element_name",
-    "sanitize_name",
-    "simtel_config_file_name",
-    "simtel_single_mirror_list_file_name",
-    "validate_array_element_id_name",
-    "validate_array_element_name",
-    "validate_site_name",
-]
-
 # Mapping of db collection names to class keys
 db_collections_to_class_keys = {
     "sites": ["Site"],
@@ -122,7 +110,7 @@ def site_names():
 @cache
 def array_element_design_types(array_element_type):
     """
-    Get array element site types (e.g., 'design' or 'flashcam').
+    Get array element design type (e.g., 'design' or 'flashcam').
 
     Default values are ['design', 'test'].
 

simtools/utils/value_conversion.py

@@ -170,16 +170,21 @@ def get_value_as_quantity(value, unit):
 
     Raises
     ------
-    u.UnitConversionError
+    ValueError
         If the value cannot be converted to the given unit.
     """
     if isinstance(value, u.Quantity):
         try:
             return value.to(unit)
-        except u.UnitConversionError:
-            _logger.error(f"Cannot convert {value.unit} to {unit}.")
-            raise
-    return value * unit
+        except u.UnitConversionError as exc:
+            raise ValueError(f"Cannot convert {value} with unit {value.unit} to {unit}.") from exc
+    elif not isinstance(value, int | float):
+        return value
+
+    if unit is None or unit == "null":
+        return value * u.dimensionless_unscaled
+
+    return value * u.Unit(unit)
 
 
 def _unit_as_string(unit):

simtools/version.py

@@ -4,8 +4,12 @@
 # which is adapted from https://github.com/astropy/astropy/blob/master/astropy/version.py
 # see https://github.com/astropy/astropy/pull/10774 for a discussion on why this needed.
 
+from packaging.specifiers import SpecifierSet
 from packaging.version import InvalidVersion, Version
 
+MAJOR_MINOR_PATCH = "major.minor.patch"
+MAJOR_MINOR = "major.minor"
+
 try:
     try:
         from ._dev_version import version
@@ -128,3 +132,84 @@ def sort_versions(version_list, reverse=False):
         return [str(v) for v in sorted(map(Version, version_list), reverse=reverse)]
     except InvalidVersion as exc:
         raise ValueError(f"Invalid version in list: {version_list}") from exc
+
+
+def version_kind(version_string):
+    """
+    Determine the kind of version string.
+
+    Parameters
+    ----------
+    version_string : str
+        The version string to analyze.
+
+    Returns
+    -------
+    str
+        The kind of version string ("major.minor", "major.minor.patch", or "major").
+    """
+    try:
+        ver = Version(version_string)
+    except InvalidVersion as exc:
+        raise ValueError(f"Invalid version string: {version_string}") from exc
+    if ver.release and len(ver.release) >= 3:
+        return MAJOR_MINOR_PATCH
+    if len(ver.release) == 2:
+        return MAJOR_MINOR
+    return "major"
+
+
+def compare_versions(version_string_1, version_string_2, level=MAJOR_MINOR_PATCH):
+    """
+    Compare two versions at the given level: "major", "major.minor", "major.minor.patch".
+
+    Parameters
+    ----------
+    version_string_1 : str
+        First version string to compare.
+    version_string_2 : str
+        Second version string to compare.
+    level : str, optional
+        Level of comparison: "major", "major.minor", or "major.minor.patch"
+
+    Returns
+    -------
+    int
+        -1 if version_string_1 < version_string_2
+         0 if version_string_1 == version_string_2
+         1 if version_string_1 > version_string_2
+    """
+    ver1 = Version(version_string_1).release
+    ver2 = Version(version_string_2).release
+
+    if level == "major":
+        ver1, ver2 = ver1[:1], ver2[:1]
+    elif level == MAJOR_MINOR:
+        ver1, ver2 = ver1[:2], ver2[:2]
+    elif level != MAJOR_MINOR_PATCH:
+        raise ValueError(f"Unknown level: {level}")
+
+    return (ver1 > ver2) - (ver1 < ver2)
+
+
+def check_version_constraint(version_string, constraint):
+    """
+    Check if a version satisfies a constraint.
+
+    Parameters
+    ----------
+    version_string : str
+        The version string to check (e.g., "6.0.2").
+    constraint : str
+        The version constraint to check against (e.g., ">=6.0.0").
+
+    Returns
+    -------
+    bool
+        True if the version satisfies the constraint, False otherwise.
+    """
+    spec = SpecifierSet(constraint.strip(), prereleases=True)
+    ver = Version(version_string)
+    if ver in spec:
+        return True
+    return False