gammasimtools 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

simtools/simtel/simtel_io_histograms.py

@@ -1,10 +1,4 @@
-"""
-Reads the content of multiples files from sim_telarray.
-
-Reads the content of either multiple histogram (.hdata, or .hdata.zst) or
-simtel_array output files (.simtel or .simtel.zst). The module is built on top of the
-simtel_io_histogram module and uses its class (SimtelIOHistogram) to read the individual files.
-"""
+"""Reads the content of multiples files from sim_telarray."""
 
 import copy
 import logging
@@ -192,8 +186,9 @@ class SimtelIOHistograms:
                 energy_range=self.energy_range,
                 view_cone=self.view_cone,
             )
-            stacked_num_simulated_events += simtel_hist_instance.total_num_simulated_events
-            stacked_num_triggered_events += simtel_hist_instance.total_num_triggered_events
+            _simulated, _triggered = simtel_hist_instance.total_number_of_events
+            stacked_num_simulated_events += _simulated
+            stacked_num_triggered_events += _triggered
         return stacked_num_simulated_events, stacked_num_triggered_events
 
     def _rates_for_stacked_files(self):
@@ -245,7 +240,7 @@ class SimtelIOHistograms:
         logging.info(
             f"System trigger event rate for stacked files: "
             # pylint: disable=E1101
-            f"{triggered_event_rate.value:.4e} \u00B1 "
+            f"{triggered_event_rate.value:.4e} \u00b1 "
             # pylint: disable=E1101
             f"{triggered_event_rate_uncertainty.value:.4e} Hz"
         )
@@ -282,24 +277,17 @@ class SimtelIOHistograms:
             if print_info:
                 simtel_hist_instance.print_info()
 
+            _simulated_events, _triggered_events = simtel_hist_instance.total_number_of_events
             logging.info(f"Histogram {i_file + 1}:")
-            logging.info(
-                "Total number of simulated events: "
-                f"{simtel_hist_instance.total_num_simulated_events} events"
-            )
-            logging.info(
-                "Total number of triggered events: "
-                f"{simtel_hist_instance.total_num_triggered_events} events"
-            )
+            logging.info(f"Total number of simulated events: {_simulated_events} events")
+            logging.info(f"Total number of triggered events: {_triggered_events} events")
 
-            obs_time = simtel_hist_instance.estimate_observation_time(
-                simtel_hist_instance.total_num_simulated_events
-            )
+            obs_time = simtel_hist_instance.estimate_observation_time(_simulated_events)
             logging.info(
                 f"Estimated equivalent observation time corresponding to the number of "
                 f"events simulated: {obs_time.value} s"
             )
-            sim_event_rate = simtel_hist_instance.total_num_simulated_events / obs_time
+            sim_event_rate = _simulated_events / obs_time
             sim_event_rates.append(sim_event_rate)
             logging.info(f"Simulated event rate: {sim_event_rate.value:.4e} Hz")
 
@@ -310,7 +298,7 @@ class SimtelIOHistograms:
             logging.info(
                 f"System trigger event rate: "
                 # pylint: disable=E1101
-                f"{triggered_event_rate.value:.4e} \u00B1 "
+                f"{triggered_event_rate.value:.4e} \u00b1 "
                 # pylint: disable=E1101
                 f"{triggered_event_rate_uncertainty.value:.4e} Hz"
             )
@@ -515,15 +503,16 @@ class SimtelIOHistograms:
 
     def export_histograms(self, hdf5_file_name, overwrite=False):
         """
-        Export the histograms to hdf5 files.
+        Export sim_telarray histograms to hdf5 files.
 
         Parameters
         ----------
         hdf5_file_name: str
             Name of the file to be saved with the hdf5 tables.
         overwrite: bool
-            If True overwrites the histograms already saved in the hdf5 file.
+            If True overwrites histograms already saved in the hdf5 file.
         """
+        self._logger.info(f"Exporting histograms to {hdf5_file_name}.")
         for histogram in self.combined_hists:
             x_bin_edges_list = np.linspace(
                 histogram["lower_x"],
@@ -556,14 +545,10 @@ class SimtelIOHistograms:
                 f"Writing histogram with name {self._meta_dict['Title']} to {hdf5_file_name}."
             )
             # overwrite takes precedence over append
-            if overwrite is True:
-                append = False
-            else:
-                append = True
             write_table(
                 table,
                 hdf5_file_name,
                 f"/{self._meta_dict['Title']}",
-                append=append,
+                append=not overwrite,
                 overwrite=overwrite,
             )

simtools/simtel/simulator_array.py

@@ -4,6 +4,7 @@ import logging
 
 from simtools.io_operations import io_handler
 from simtools.runners.simtel_runner import InvalidOutputFileError, SimtelRunner
+from simtools.utils.general import clear_default_sim_telarray_cfg_directories
 
 __all__ = ["SimulatorArray"]
 
@@ -88,7 +89,7 @@ class SimulatorArray(SimtelRunner):
         command += f" {input_file}"
         command += f" > {self._log_file} 2>&1 || exit"
 
-        return command
+        return clear_default_sim_telarray_cfg_directories(command)
 
     def _check_run_result(self, run_number=None):
         """

simtools/simtel/simulator_camera_efficiency.py

@@ -135,6 +135,8 @@ class SimulatorCameraEfficiency(SimtelRunner):
         command += f" {pixel_shape_cmd} {pixel_diameter}"
         if mirror_class == 0:
             command += f" -fmir {self._telescope_model.get_parameter_value('mirror_list')}"
+        if mirror_class == 2:
+            command += f" -fmir {self._telescope_model.get_parameter_value('fake_mirror_list')}"
         command += f" -fref {mirror_reflectivity}"
         if mirror_class == 2:
             command += " -m2"
@@ -155,6 +157,9 @@ class SimulatorCameraEfficiency(SimtelRunner):
         command += f" {self._telescope_model.get_parameter_value('atmospheric_profile')}"
         command += f" {self.zenith_angle}"
 
+        # Remove the default sim_telarray configuration directories
+        command = general.clear_default_sim_telarray_cfg_directories(command)
+
         return (
             f"cd {self._simtel_path.joinpath('sim_telarray')} && {command}",
             self._file_simtel,

simtools/simtel/simulator_light_emission.py

@@ -9,6 +9,7 @@ import numpy as np
 
 from simtools.io_operations import io_handler
 from simtools.runners.simtel_runner import SimtelRunner
+from simtools.utils.general import clear_default_sim_telarray_cfg_directories
 
 __all__ = ["SimulatorLightEmission"]
 
@@ -360,7 +361,8 @@ class SimulatorLightEmission(SimtelRunner):
             f"{self.le_application[0]}_{self.le_application[1]}.ctsim.hdata\n",
         )
 
-        return command
+        # Remove the default sim_telarray configuration directories
+        return clear_default_sim_telarray_cfg_directories(command)
 
     def _remove_line_from_config(self, file_path, line_prefix):
         """

simtools/simtel/simulator_ray_tracing.py

@@ -8,6 +8,7 @@ import astropy.units as u
 from simtools.io_operations import io_handler
 from simtools.runners.simtel_runner import SimtelRunner
 from simtools.utils import names
+from simtools.utils.general import clear_default_sim_telarray_cfg_directories
 
 __all__ = ["SimulatorRayTracing"]
 
@@ -192,7 +193,7 @@ class SimulatorRayTracing(SimtelRunner):
             command += super().get_config_option("mirror_align_random_vertical", "0.,28.,0.,0.")
         command += " " + str(self._corsika_file)
 
-        return command, self._log_file, self._log_file
+        return clear_default_sim_telarray_cfg_directories(command), self._log_file, self._log_file
 
     def _check_run_result(self, run_number=None):  # pylint: disable=unused-argument
         """

simtools/testing/helpers.py

@@ -6,19 +6,12 @@ from pathlib import Path
 
 def skip_camera_efficiency(config):
     """Skip camera efficiency tests if the old version of testeff is used."""
-    if "camera-efficiency" in config["APPLICATION"]:
-        if not _new_testeff_version():
-            return (
-                "Any applications calling the old version of testeff are skipped "
-                "due to a limitation of the old testeff not allowing to specify "
-                "the include directory. Please update your sim_telarray tarball."
-            )
-        full_test_name = f"{config['APPLICATION']}_{config['TEST_NAME']}"
-        if "simtools-validate-camera-efficiency_SSTS" == full_test_name:
-            return (
-                "The test simtools-validate-camera-efficiency_SSTS is skipped "
-                "since the fake SST mirrors are not yet implemented (#1155)"
-            )
+    if "camera-efficiency" in config["APPLICATION"] and not _new_testeff_version():
+        return (
+            "Any applications calling the old version of testeff are skipped "
+            "due to a limitation of the old testeff not allowing to specify "
+            "the include directory. Please update your sim_telarray tarball."
+        )
     return None
 
 

simtools/testing/validate_output.py

@@ -12,39 +12,22 @@ from simtools.testing import assertions
 _logger = logging.getLogger(__name__)
 
 
-def validate_all_tests(config, request, config_file_model_version):
-    """
-    Validate test output for all integration tests.
-
-    Parameters
-    ----------
-    config: dict
-        Integration test configuration dictionary.
-    request: request
-        Request object.
-    config_file_model_version: str
-        Model version from the configuration file.
-
-    """
-    if request.config.getoption("--model_version") is None:
-        validate_application_output(config)
-    elif config_file_model_version is not None:
-        _from_command_line = request.config.getoption("--model_version")
-        _from_config_file = config_file_model_version
-        if _from_command_line == _from_config_file:
-            validate_application_output(config)
-
-
-def validate_application_output(config):
+def validate_application_output(config, from_command_line=None, from_config_file=None):
     """
     Validate application output against expected output.
 
     Expected output is defined in configuration file.
+    Some tests run only if the model version from the command line
+    equals the model version from the configuration file.
 
     Parameters
     ----------
     config: dict
         dictionary with the configuration for the application test.
+    from_command_line: str
+        Model version from the command line.
+    from_config_file: str
+        Model version from the configuration file.
 
     """
     if "INTEGRATION_TESTS" not in config:
@@ -52,24 +35,37 @@ def validate_application_output(config):
 
     for integration_test in config["INTEGRATION_TESTS"]:
         _logger.info(f"Testing application output: {integration_test}")
-        if "REFERENCE_OUTPUT_FILE" in integration_test:
-            _validate_reference_output_file(config, integration_test)
-
-        if "TEST_OUTPUT_FILES" in integration_test:
-            _validate_output_path_and_file(config, integration_test["TEST_OUTPUT_FILES"])
-        if "OUTPUT_FILE" in integration_test:
-            _validate_output_path_and_file(
-                config,
-                [{"PATH_DESCRIPTOR": "OUTPUT_PATH", "FILE": integration_test["OUTPUT_FILE"]}],
-            )
 
-        if "FILE_TYPE" in integration_test:
-            assert assertions.assert_file_type(
-                integration_test["FILE_TYPE"],
-                Path(config["CONFIGURATION"]["OUTPUT_PATH"]).joinpath(
-                    config["CONFIGURATION"]["OUTPUT_FILE"]
-                ),
-            )
+        if from_command_line == from_config_file:
+            if "REFERENCE_OUTPUT_FILE" in integration_test:
+                _validate_reference_output_file(config, integration_test)
+
+            if "TEST_OUTPUT_FILES" in integration_test:
+                _validate_output_path_and_file(config, integration_test["TEST_OUTPUT_FILES"])
+
+            if "OUTPUT_FILE" in integration_test:
+                _validate_output_path_and_file(
+                    config,
+                    [{"PATH_DESCRIPTOR": "OUTPUT_PATH", "FILE": integration_test["OUTPUT_FILE"]}],
+                )
+
+            if "FILE_TYPE" in integration_test:
+                assert assertions.assert_file_type(
+                    integration_test["FILE_TYPE"],
+                    Path(config["CONFIGURATION"]["OUTPUT_PATH"]).joinpath(
+                        config["CONFIGURATION"]["OUTPUT_FILE"]
+                    ),
+                )
+        _test_simtel_cfg_files(config, integration_test, from_command_line, from_config_file)
+
+
+def _test_simtel_cfg_files(config, integration_test, from_command_line, from_config_file):
+    """Test simtel cfg files."""
+    test_simtel_cfg_file = integration_test.get("TEST_SIMTEL_CFG_FILES", {}).get(
+        from_command_line or from_config_file
+    )
+    if test_simtel_cfg_file:
+        _validate_simtel_cfg_files(config, test_simtel_cfg_file)
 
 
 def _validate_reference_output_file(config, integration_test):
@@ -128,12 +124,13 @@ def compare_files(file1, file2, tolerance=1.0e-5, test_columns=None):
     """
     _file1_suffix = Path(file1).suffix
     _file2_suffix = Path(file2).suffix
+    _logger.info("Comparing files: %s and %s", file1, file2)
     if _file1_suffix != _file2_suffix:
         raise ValueError(f"File suffixes do not match: {file1} and {file2}")
     if _file1_suffix == ".ecsv":
         return compare_ecsv_files(file1, file2, tolerance, test_columns)
     if _file1_suffix in (".json", ".yaml", ".yml"):
-        return compare_json_or_yaml_files(file1, file2)
+        return compare_json_or_yaml_files(file1, file2, tolerance)
 
     _logger.warning(f"Unknown file type for files: {file1} and {file2}")
     return False
@@ -168,11 +165,35 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2):
     if data1 == data2:
         return True
 
-    if "value" in data1 and isinstance(data1["value"], str):
-        value_list_1 = gen.convert_string_to_list(data1.pop("value"))
-        value_list_2 = gen.convert_string_to_list(data2.pop("value"))
-        return np.allclose(value_list_1, value_list_2, rtol=tolerance)
-    return data1 == data2
+    if data1.keys() != data2.keys():
+        _logger.error(f"Keys do not match: {data1.keys()} and {data2.keys()}")
+        return False
+    _comparison = all(
+        (
+            _compare_value_from_parameter_dict(data1[k], data2[k], tolerance)
+            if k == "value"
+            else data1[k] == data2[k]
+        )
+        for k in data1
+    )
+    if not _comparison:
+        _logger.error(f"Values do not match: {data1} and {data2} (tolerance: {tolerance})")
+    return _comparison
+
+
+def _compare_value_from_parameter_dict(data1, data2, tolerance):
+    """Compare value fields given in different formats."""
+
+    def _as_list(value):
+        if isinstance(value, str):
+            return gen.convert_string_to_list(value)
+        if isinstance(value, list | np.ndarray):
+            return value
+        return [value]
+
+    _logger.info(f"Comparing values: {data1} and {data2} (tolerance: {tolerance})")
+
+    return np.allclose(_as_list(data1), _as_list(data2), rtol=tolerance)
 
 
 def compare_ecsv_files(file1, file2, tolerance=1.0e-5, test_columns=None):
@@ -238,3 +259,66 @@ def compare_ecsv_files(file1, file2, tolerance=1.0e-5, test_columns=None):
                 return False
 
     return True
+
+
+def _validate_simtel_cfg_files(config, simtel_cfg_file):
+    """
+    Check sim_telarray configuration files and compare with reference file.
+
+    Note the finetuned naming of configuration files by simtools.
+
+    """
+    reference_file = Path(simtel_cfg_file)
+    test_file = Path(config["CONFIGURATION"]["OUTPUT_PATH"]) / reference_file.name.replace(
+        "_test", f"_{config['CONFIGURATION']['LABEL']}"
+    )
+    _logger.info(
+        f"Comparing simtel cfg files: {reference_file} and {test_file} "
+        f"for model version {config['CONFIGURATION']['MODEL_VERSION']}"
+    )
+    return _compare_simtel_cfg_files(reference_file, test_file)
+
+
+def _compare_simtel_cfg_files(reference_file, test_file):
+    """
+    Compare two sim_telarray configuration files.
+
+    Line-by-line string comparison. Requires similar sequence of
+    parameters in the files. Ignore lines containing 'config_release'
+    (as it contains the simtools package version).
+
+    Parameters
+    ----------
+    reference_file: Path
+        Reference sim_telarray configuration file.
+    test_file: Path
+        Test sim_telarray configuration file.
+
+    Returns
+    -------
+    bool
+        True if the files are equal.
+
+    """
+    with open(reference_file, encoding="utf-8") as f1, open(test_file, encoding="utf-8") as f2:
+        reference_cfg = [line.rstrip() for line in f1 if line.strip()]
+        test_cfg = [line.rstrip() for line in f2 if line.strip()]
+
+    if len(reference_cfg) != len(test_cfg):
+        _logger.error(
+            f"Line counts differ: {reference_file} ({len(reference_cfg)} lines), "
+            f"{test_file} ({len(test_cfg)} lines)."
+        )
+        return False
+
+    for ref_line, test_line in zip(reference_cfg, test_cfg):
+        if any(ignore in ref_line for ignore in ("config_release", "Label")):
+            continue
+        if ref_line != test_line:
+            _logger.error(
+                f"Configuration files {reference_file} and {test_file} do not match: "
+                f"'{ref_line}' and '{test_line}'"
+            )
+            return False
+
+    return True

simtools/utils/general.py

@@ -17,6 +17,7 @@ import yaml
 __all__ = [
     "InvalidConfigDataError",
     "change_dict_keys_case",
+    "clear_default_sim_telarray_cfg_directories",
     "collect_data_from_file",
     "collect_final_lines",
     "collect_kwargs",
@@ -81,6 +82,28 @@ def is_url(url):
         return False
 
 
+def url_exists(url):
+    """
+    Check if a URL exists.
+
+    Parameters
+    ----------
+    url: str
+        URL to be checked.
+
+    Returns
+    -------
+    bool
+        True if URL exists.
+    """
+    try:
+        with urllib.request.urlopen(url, timeout=5) as response:
+            return response.status == 200
+    except (urllib.error.URLError, AttributeError) as e:
+        _logger.error(f"URL {url} does not exist: {e}")
+        return False
+
+
 def collect_data_from_http(url):
     """
     Download yaml or json file from url and return it contents as dict.
@@ -135,7 +158,7 @@ def collect_data_from_http(url):
     return data
 
 
-def collect_data_from_file(file_name):
+def collect_data_from_file(file_name, yaml_document=None):
     """
     Collect data from file based on its extension.
 
@@ -143,6 +166,8 @@ def collect_data_from_file(file_name):
     ----------
     file_name: str
         Name of the yaml/json/ascii file.
+    yaml_document: None, int
+        Return list of yaml documents or a single document (for yaml files with several documents).
 
     Returns
     -------
@@ -152,21 +177,34 @@ def collect_data_from_file(file_name):
     if is_url(file_name):
         return collect_data_from_http(file_name)
 
+    suffix = Path(file_name).suffix.lower()
     with open(file_name, encoding="utf-8") as file:
-        if Path(file_name).suffix.lower() == ".json":
+        if suffix == ".json":
             return json.load(file)
+        if suffix == ".list":
+            return [line.strip() for line in file.readlines()]
+        if suffix in [".yml", ".yaml"]:
+            return _collect_data_from_yaml_file(file, file_name, yaml_document)
+    return None
 
-        if Path(file_name).suffix.lower() == ".list":
-            lines = file.readlines()
-            return [line.strip() for line in lines]
 
-        try:
-            return yaml.safe_load(file)
-        except yaml.constructor.ConstructorError:
-            return _load_yaml_using_astropy(file)
-        except yaml.composer.ComposerError:
-            file.seek(0)
-            return list(yaml.safe_load_all(file))
+def _collect_data_from_yaml_file(file, file_name, yaml_document):
+    """Collect data from a yaml file."""
+    try:
+        return yaml.safe_load(file)
+    except yaml.constructor.ConstructorError:
+        return _load_yaml_using_astropy(file)
+    except yaml.composer.ComposerError:
+        pass
+    file.seek(0)
+    if yaml_document is None:
+        return list(yaml.safe_load_all(file))
+    try:
+        return list(yaml.safe_load_all(file))[yaml_document]
+    except IndexError as exc:
+        raise InvalidConfigDataError(
+            f"YAML file {file_name} does not contain {yaml_document} documents."
+        ) from exc
 
 
 def collect_kwargs(label, in_kwargs):
@@ -832,3 +870,55 @@ def convert_keys_in_dict_to_lowercase(data):
     if isinstance(data, list):
         return [convert_keys_in_dict_to_lowercase(i) for i in data]
     return data
+
+
+def clear_default_sim_telarray_cfg_directories(command):
+    """Prefix the command to clear default sim_telarray configuration directories.
+
+    Parameters
+    ----------
+    command: str
+        Command to be prefixed.
+
+    Returns
+    -------
+    str
+        Prefixed command.
+
+    """
+    return f"SIM_TELARRAY_CONFIG_PATH='' {command}"
+
+
+def get_list_of_files_from_command_line(file_names, suffix_list):
+    """
+    Get a list of files from the command line.
+
+    Files can be given as a list of file names or as a text file containing the list of files.
+    The list of suffixes restrict the files types to be returned. Note that a file list must
+    have a different suffix than those in the suffix list.
+
+    Parameters
+    ----------
+    file_names: list
+        List of file names to be checked.
+    suffix_list: list
+        List of suffixes to be checked.
+
+    Returns
+    -------
+    list
+        List of files with the given suffixes.
+    """
+    _files = []
+    for one_file in file_names:
+        path = Path(one_file)
+        try:
+            if path.suffix in suffix_list:
+                _files.append(one_file)
+            elif len(file_names) == 1:
+                with open(one_file, encoding="utf-8") as file:
+                    _files.extend(line.strip() for line in file)
+        except FileNotFoundError as exc:
+            _logger.error(f"{one_file} is not a file.")
+            raise FileNotFoundError from exc
+    return _files