gammasimtools 0.16.0__py3-none-any.whl → 0.18.0__py3-none-any.whl

simtools/simulator.py

@@ -12,11 +12,12 @@ import numpy as np
 
 import simtools.utils.general as gen
 from simtools.corsika.corsika_config import CorsikaConfig
-from simtools.io_operations import io_handler
+from simtools.io_operations import io_handler, io_table_handler
 from simtools.job_execution.job_manager import JobManager
 from simtools.model.array_model import ArrayModel
 from simtools.runners.corsika_runner import CorsikaRunner
 from simtools.runners.corsika_simtel_runner import CorsikaSimtelRunner
+from simtools.simtel.simtel_io_event_writer import SimtelIOEventDataWriter
 from simtools.simtel.simulator_array import SimulatorArray
 from simtools.testing.sim_telarray_metadata import assert_sim_telarray_metadata
 
@@ -72,8 +73,6 @@ class Simulator:
         self.runs = self._initialize_run_list()
         self._results = defaultdict(list)
         self._test = self.args_dict.get("test", False)
-        self.submit_engine = self.args_dict.get("submit_engine", "local")
-        self._submit_options = self.args_dict.get("submit_options", None)
         self._extra_commands = extra_commands
 
         self.sim_telarray_seeds = {
@@ -145,6 +144,7 @@ class Simulator:
                     ],
                     "seed_file_name": self.sim_telarray_seeds["seed_file_name"],
                 },
+                simtel_path=self.args_dict.get("simtel_path", None),
             )
             for version in versions
         ]
@@ -179,7 +179,10 @@ class Simulator:
 
     def _initialize_run_list(self):
         """
-        Initialize run list using the configuration values 'run_number_start' and 'number_of_runs'.
+        Initialize run list using the configuration values.
+
+        Uses 'run_number', 'run_number_offset' and 'number_of_runs' arguments
+        to create a list of run numbers.
 
         Returns
         -------
@@ -189,19 +192,27 @@ class Simulator:
         Raises
         ------
         KeyError
-            If 'run_number_start' or 'number_of_runs' are not found in the configuration.
+            If 'run_number', 'run_number_offset' and 'number_of_runs' are
+            not found in the configuration.
         """
         try:
+            offset_run_number = self.args_dict["run_number_offset"] + self.args_dict["run_number"]
+            if self.args_dict["number_of_runs"] <= 1:
+                return self._validate_run_list_and_range(
+                    run_list=offset_run_number,
+                    run_range=None,
+                )
             return self._validate_run_list_and_range(
                 run_list=None,
                 run_range=[
-                    self.args_dict["run_number_start"],
-                    self.args_dict["run_number_start"] + self.args_dict["number_of_runs"],
+                    offset_run_number,
+                    offset_run_number + self.args_dict["number_of_runs"],
                 ],
             )
         except KeyError as exc:
             self._logger.error(
-                "Error in initializing run list (missing 'run_number_start' or 'number_of_runs')"
+                "Error in initializing run list "
+                "(missing 'run_number', 'run_number_offset' or 'number_of_runs')."
             )
             raise exc
 
@@ -301,6 +312,8 @@ class Simulator:
             runner_args["keep_seeds"] = self.args_dict.get("corsika_test_seeds", False)
         if runner_class is not CorsikaRunner:
             runner_args["sim_telarray_seeds"] = self.sim_telarray_seeds
+        if runner_class is CorsikaSimtelRunner:
+            runner_args["sequential"] = self.args_dict.get("sequential", False)
 
         return runner_class(**runner_args)
 
@@ -330,8 +343,6 @@ class Simulator:
         input_file_list: str or list of str
             Single file or list of files of shower simulations.
         """
-        self._logger.info(f"Submission command: {self.submit_engine}")
-
         runs_and_files_to_submit = self._get_runs_and_files_to_submit(
             input_file_list=input_file_list
         )
@@ -345,11 +356,7 @@ class Simulator:
                 run_number=run_number, input_file=input_file, extra_commands=self._extra_commands
             )
 
-            job_manager = JobManager(
-                submit_engine=self.submit_engine,
-                submit_options=self._submit_options,
-                test=self._test,
-            )
+            job_manager = JobManager(test=self._test)
             job_manager.submit(
                 run_script=run_script,
                 run_out_file=self._simulation_runner.get_file_name(
@@ -454,73 +461,66 @@ class Simulator:
             run number
 
         """
-        keys = ["output", "sub_out", "log", "input", "hist", "corsika_log"]
+        keys = ["simtel_output", "sub_out", "log", "input", "hist", "corsika_log", "event_data"]
         results = {key: [] for key in keys}
 
+        def get_file_name(name, **kwargs):
+            return str(self._simulation_runner.get_file_name(file_type=name, **kwargs))
+
         if "sim_telarray" in self.simulation_software:
             results["input"].append(str(file))
 
-        results["sub_out"].append(
-            str(
-                self._simulation_runner.get_file_name(
-                    file_type="sub_log",
-                    mode="out",
-                    run_number=run_number,
-                )
-            )
-        )
+        results["sub_out"].append(get_file_name("sub_log", mode="out", run_number=run_number))
 
-        for model_version_index, _ in enumerate(self.array_models):
-            results["output"].append(
-                str(
-                    self._simulation_runner.get_file_name(
-                        file_type="output",
-                        run_number=run_number,
-                        model_version_index=model_version_index,
-                    )
-                )
+        for i in range(len(self.array_models)):
+            results["simtel_output"].append(
+                get_file_name("simtel_output", run_number=run_number, model_version_index=i)
             )
+
             if "sim_telarray" in self.simulation_software:
                 results["log"].append(
-                    str(
-                        self._simulation_runner.get_file_name(
-                            file_type="log",
-                            simulation_software="sim_telarray",
-                            run_number=run_number,
-                            model_version_index=model_version_index,
-                        )
+                    get_file_name(
+                        "log",
+                        simulation_software="sim_telarray",
+                        run_number=run_number,
+                        model_version_index=i,
                     )
                 )
                 results["hist"].append(
-                    str(
-                        self._simulation_runner.get_file_name(
-                            file_type="histogram",
-                            simulation_software="sim_telarray",
-                            run_number=run_number,
-                            model_version_index=model_version_index,
-                        )
+                    get_file_name(
+                        "histogram",
+                        simulation_software="sim_telarray",
+                        run_number=run_number,
+                        model_version_index=i,
                     )
                 )
+                results["event_data"].append(
+                    get_file_name(
+                        "event_data",
+                        simulation_software="sim_telarray",
+                        run_number=run_number,
+                        model_version_index=i,
+                    )
+                )
+
             if "corsika" in self.simulation_software:
                 results["corsika_log"].append(
-                    str(
-                        self._simulation_runner.get_file_name(
-                            file_type="corsika_log",
-                            simulation_software="corsika",
-                            run_number=run_number,
-                            model_version_index=model_version_index,
-                        )
+                    get_file_name(
+                        "corsika_log",
+                        simulation_software="corsika",
+                        run_number=run_number,
+                        model_version_index=i,
                     )
                 )
 
         for key in keys:
             self._results[key].extend(results[key])
 
-    def get_file_list(self, file_type="output"):
+    def get_file_list(self, file_type="simtel_output"):
         """
         Get list of files generated by simulations.
 
-        Options are "input", "output", "hist", "log", "corsika_log".
+        Options are "input", "simtel_output", "hist", "log", "corsika_log".
         Not all file types are available for all simulation types.
         Returns an empty list for an unknown file type.
 
@@ -538,11 +538,11 @@ class Simulator:
         self._logger.info(f"Getting list of {file_type} files")
         return self._results[file_type]
 
-    def print_list_of_files(self, file_type="output"):
+    def print_list_of_files(self, file_type="simtel_output"):
         """
         Print list of output files generated by simulations.
 
-        Options are "input", "output", "hist", "log".
+        Options are "input", "simtel_output", "hist", "log".
 
         Parameters
         ----------
@@ -636,7 +636,7 @@ class Simulator:
 
     def save_file_lists(self):
         """Save files lists for output and log files."""
-        for file_type in ["output", "log", "corsika_log", "hist"]:
+        for file_type in ["simtel_output", "log", "corsika_log", "hist"]:
             file_name = self.io_handler.get_output_directory(label=self.label).joinpath(
                 f"{file_type}_files.txt"
             )
@@ -649,6 +649,29 @@ class Simulator:
             else:
                 self._logger.debug(f"No files to save for {file_type} files.")
 
+    def save_reduced_event_lists(self):
+        """
+        Save reduced event lists with event data on simulated and triggered events.
+
+        The files are saved with the same name as the sim_telarray output file
+        but with a 'hdf5' extension.
+        """
+        if "sim_telarray" not in self.simulation_software:
+            self._logger.warning(
+                "Reduced event lists can only be saved for sim_telarray simulations."
+            )
+            return
+
+        input_files = self.get_file_list(file_type="simtel_output")
+        output_files = self.get_file_list(file_type="event_data")
+        for input_file, output_file in zip(input_files, output_files):
+            generator = SimtelIOEventDataWriter([input_file])
+            io_table_handler.write_tables(
+                tables=generator.process_files(),
+                output_file=Path(output_file),
+                overwrite_existing=True,
+            )
+
     def pack_for_register(self, directory_for_grid_upload=None):
         """
         Pack simulation output files for registering on the grid.
@@ -664,10 +687,15 @@ class Simulator:
         self._logger.info(
             f"Packing the output files for registering on the grid ({directory_for_grid_upload})"
         )
-        output_files = self.get_file_list(file_type="output")
+        output_files = self.get_file_list(file_type="simtel_output")
         log_files = self.get_file_list(file_type="log")
         corsika_log_files = self.get_file_list(file_type="corsika_log")
         histogram_files = self.get_file_list(file_type="hist")
+        reduced_event_files = (
+            self.get_file_list(file_type="event_data")
+            if self.args_dict.get("save_reduced_event_lists")
+            else []
+        )
 
         directory_for_grid_upload = (
             Path(directory_for_grid_upload)
@@ -697,11 +725,12 @@ class Simulator:
                 tar_file_path = directory_for_grid_upload.joinpath(tar_file_name)
 
                 with tarfile.open(tar_file_path, "w:gz") as tar:
+                    # Add all relevant log, histogram, and CORSIKA log files to the tarball
                     files_to_tar = model_logs + model_hists + model_corsika_logs
                     for file_to_tar in files_to_tar:
                         tar.add(file_to_tar, arcname=Path(file_to_tar).name)
 
-        for file_to_move in output_files:
+        for file_to_move in output_files + reduced_event_files:
             source_file = Path(file_to_move)
             destination_file = directory_for_grid_upload / source_file.name
             if destination_file.exists():
@@ -717,7 +746,7 @@ class Simulator:
             return
 
         for model in self.array_models:
-            files = self.get_file_list(file_type="output")
+            files = self.get_file_list(file_type="simtel_output")
             output_file = next((f for f in files if model.model_version in f), None)
             if output_file:
                 self._logger.info(f"Validating metadata for {output_file}")
@@ -738,7 +767,6 @@ class Simulator:
         ----------
         corsika_log_files: list
             List containing the original CORSIKA log file path.
-
         """
         original_log = Path(corsika_log_files[0])
         # Find which model version the original log belongs to
@@ -759,15 +787,20 @@ class Simulator:
                 original_version, model.model_version
             )
 
-            with gzip.open(new_log, "wt") as new_file:
-                new_file.write(
+            with gzip.open(new_log, "wt", encoding="utf-8") as new_file:
+                # Write the header to the new file
+                header = (
                     f"###############################################################\n"
                     f"Copy of CORSIKA log file from model version {original_version}.\n"
                     f"Applicable also for {model.model_version} (same CORSIKA configuration,\n"
                     f"different sim_telarray model versions in the same run).\n"
                     f"###############################################################\n\n"
                 )
-                with gzip.open(original_log, "rt") as orig_file:
-                    shutil.copyfileobj(orig_file, new_file)
+                new_file.write(header)
+
+                # Copy the content of the original log file, ignoring invalid characters
+                with gzip.open(original_log, "rt", encoding="utf-8", errors="ignore") as orig_file:
+                    for line in orig_file:
+                        new_file.write(line)
 
             corsika_log_files.append(str(new_log))

simtools/testing/validate_output.py

@@ -102,7 +102,10 @@ def _validate_output_path_and_file(config, integration_file_tests):
 
         output_file_path = Path(output_path) / file_test["FILE"]
         _logger.info(f"Checking path: {output_file_path}")
-        assert output_file_path.exists()
+        try:
+            assert output_file_path.exists()
+        except AssertionError as exc:
+            raise AssertionError(f"Output file {output_file_path} does not exist. ") from exc
 
         if "EXPECTED_OUTPUT" in file_test:
             assert assertions.check_output_from_sim_telarray(

simtools/utils/general.py

@@ -178,7 +178,7 @@ def _collect_data_from_different_file_types(file, file_name, suffix, yaml_docume
     """Collect data from different file types."""
     if suffix == ".json":
         return json.load(file)
-    if suffix == ".list":
+    if suffix in (".list", ".txt"):
         return [line.strip() for line in file.readlines()]
     if suffix in [".yml", ".yaml"]:
         return _collect_data_from_yaml_file(file, file_name, yaml_document)
@@ -324,10 +324,8 @@ def get_log_level_from_user(log_level):
     possible_levels = {
         "info": logging.INFO,
         "debug": logging.DEBUG,
-        "warn": logging.WARNING,
         "warning": logging.WARNING,
         "error": logging.ERROR,
-        "critical": logging.CRITICAL,
     }
     try:
         log_level_lower = log_level.lower()

simtools/utils/names.py

@@ -12,6 +12,7 @@ Naming in simtools:
 
 """
 
+import json
 import logging
 import re
 from functools import cache
@@ -22,7 +23,7 @@ import yaml
 from simtools.constants import (
     MODEL_PARAMETER_DESCRIPTION_METASCHEMA,
     MODEL_PARAMETER_SCHEMA_PATH,
-    SCHEMA_PATH,
+    RESOURCE_PATH,
 )
 
 _logger = logging.getLogger(__name__)
@@ -59,10 +60,30 @@ def array_elements():
     dict
         Array elements.
     """
-    with open(Path(SCHEMA_PATH) / "array_elements.yml", encoding="utf-8") as file:
+    # for efficiency reason, no functions from simtools.utils.general are used here
+    with open(Path(RESOURCE_PATH) / "array_elements.yml", encoding="utf-8") as file:
         return yaml.safe_load(file)["data"]
 
 
+@cache
+def array_element_common_identifiers():
+    """
+    Get array element IDs from CTAO common identifier.
+
+    Returns
+    -------
+    dict, dict
+        Dictionary mapping array element names to their IDs and vice versa.
+    """
+    # for efficiency reason, no functions from simtools.utils.general are used here
+    id_to_name = {}
+    with open(Path(RESOURCE_PATH) / "array-element-ids.json", encoding="utf-8") as file:
+        data = json.load(file)
+    id_to_name = {e["id"]: e["name"] for e in data["array_elements"]}
+    name_to_id = {e["name"]: e["id"] for e in data["array_elements"]}
+    return id_to_name, name_to_id
+
+
 @cache
 def simulation_software():
     """
@@ -415,6 +436,54 @@ def get_array_element_id_from_name(array_element_name):
         raise ValueError(f"Invalid name {array_element_name}") from exc
 
 
+def get_common_identifier_from_array_element_name(array_element_name, default_return=None):
+    """
+    Get numerical common identifier from array element name as used by CTAO.
+
+    Common identifiers are numerical IDs used by the CTAO ACADA and DPPS systems.
+
+    Parameters
+    ----------
+    array_element_name: str
+        Array element name (e.g. LSTN-01)
+
+    Returns
+    -------
+    int
+        Common identifier.
+    """
+    _, name_to_id = array_element_common_identifiers()
+    try:
+        return name_to_id[array_element_name]
+    except KeyError as exc:
+        if default_return is not None:
+            return default_return
+        raise ValueError(f"Unknown array element name {array_element_name}") from exc
+
+
+def get_array_element_name_from_common_identifier(common_identifier):
+    """
+    Get array element name from common identifier as used by CTAO.
+
+    Common identifiers are numerical IDs used by the CTAO ACADA and DPPS systems.
+
+    Parameters
+    ----------
+    common_identifier: int
+        Common identifier.
+
+    Returns
+    -------
+    str
+        Array element name.
+    """
+    id_to_name, _ = array_element_common_identifiers()
+    try:
+        return id_to_name[common_identifier]
+    except KeyError as exc:
+        raise ValueError(f"Unknown common identifier {common_identifier}") from exc
+
+
 def get_list_of_array_element_types(
     array_element_class="telescopes", site=None, observatory="CTAO"
 ):
@@ -658,7 +727,7 @@ def generate_file_name(
     """
     Generate a file name for output, config, or plotting.
 
-    Used e.g., to generate camera-efficiency and ray-tracing output files.
+    Used e.g., to generate camera_efficiency and ray_tracing output files.
 
     Parameters
     ----------
@@ -690,10 +759,10 @@ def generate_file_name(
     str
         File name.
     """
-    name = f"{file_type}-{site}-{telescope_model_name}"
-    name += f"-d{source_distance:.1f}km" if source_distance is not None else ""
-    name += f"-za{float(zenith_angle):.1f}deg"
-    name += f"-off{off_axis_angle:.3f}deg" if off_axis_angle is not None else ""
+    name = f"{file_type}_{site}_{telescope_model_name}"
+    name += f"_d{source_distance:.1f}km" if source_distance is not None else ""
+    name += f"_za{float(zenith_angle):.1f}deg"
+    name += f"_off{off_axis_angle:.3f}deg" if off_axis_angle is not None else ""
     name += f"_azm{round(azimuth_angle):03}deg" if azimuth_angle is not None else ""
     name += f"_mirror{mirror_number}" if mirror_number is not None else ""
     name += f"_{label}" if label is not None else ""