gammasimtools 0.15.0__py3-none-any.whl → 0.17.0__py3-none-any.whl

simtools/simulator.py

@@ -1,5 +1,6 @@
 """Simulator class for managing simulations of showers and array of telescopes."""
 
+import gzip
 import logging
 import re
 import shutil
@@ -11,12 +12,14 @@ import numpy as np
 
 import simtools.utils.general as gen
 from simtools.corsika.corsika_config import CorsikaConfig
-from simtools.io_operations import io_handler
+from simtools.io_operations import io_handler, io_table_handler
 from simtools.job_execution.job_manager import JobManager
 from simtools.model.array_model import ArrayModel
 from simtools.runners.corsika_runner import CorsikaRunner
 from simtools.runners.corsika_simtel_runner import CorsikaSimtelRunner
+from simtools.simtel.simtel_io_event_writer import SimtelIOEventDataWriter
 from simtools.simtel.simulator_array import SimulatorArray
+from simtools.testing.sim_telarray_metadata import assert_sim_telarray_metadata
 
 __all__ = [
     "InvalidRunsToSimulateError",
@@ -70,11 +73,16 @@ class Simulator:
         self.runs = self._initialize_run_list()
         self._results = defaultdict(list)
         self._test = self.args_dict.get("test", False)
-        self.submit_engine = self.args_dict.get("submit_engine", "local")
-        self._submit_options = self.args_dict.get("submit_options", None)
         self._extra_commands = extra_commands
 
-        self.array_model = self._initialize_array_model(mongo_db_config)
+        self.sim_telarray_seeds = {
+            "seed": self.args_dict.get("sim_telarray_instrument_seeds"),
+            "random_instrument_instances": self.args_dict.get(
+                "sim_telarray_random_instrument_instances"
+            ),
+            "seed_file_name": "sim_telarray_instrument_seeds.txt",  # name only; no directory
+        }
+        self.array_models = self._initialize_array_models(mongo_db_config)
         self._simulation_runner = self._initialize_simulation_runner(mongo_db_config)
 
     @property
@@ -89,8 +97,8 @@ class Simulator:
 
         Parameters
         ----------
-        simulation_software: choices: [simtel, corsika, corsika_simtel]
-            implemented are sim_telarray and CORSIKA or corsika_simtel
+        simulation_software: choices: [sim_telarray, corsika, corsika_sim_telarray]
+            implemented are sim_telarray and CORSIKA or corsika_sim_telarray
             (running CORSIKA and piping it directly to sim_telarray)
 
         Raises
@@ -98,14 +106,14 @@ class Simulator:
         gen.InvalidConfigDataError
 
         """
-        if simulation_software not in ["simtel", "corsika", "corsika_simtel"]:
+        if simulation_software not in ["sim_telarray", "corsika", "corsika_sim_telarray"]:
             self._logger.error(f"Invalid simulation software: {simulation_software}")
             raise gen.InvalidConfigDataError
         self._simulation_software = simulation_software.lower()
 
-    def _initialize_array_model(self, mongo_db_config):
+    def _initialize_array_models(self, mongo_db_config):
         """
-        Initialize array simulation model.
+        Initialize array simulation models.
 
         Parameters
         ----------
@@ -114,20 +122,67 @@ class Simulator:
 
         Returns
         -------
-        ArrayModel
-            ArrayModel object.
-        """
-        return ArrayModel(
-            label=self.label,
-            site=self.args_dict.get("site"),
-            layout_name=self.args_dict.get("array_layout_name"),
-            mongo_db_config=mongo_db_config,
-            model_version=self.args_dict.get("model_version", None),
-        )
+        list
+            List of ArrayModel objects.
+        """
+        model_version = self.args_dict.get("model_version", [])
+        versions = [model_version] if not isinstance(model_version, list) else model_version
+
+        return [
+            ArrayModel(
+                label=self.label,
+                site=self.args_dict.get("site"),
+                layout_name=self.args_dict.get("array_layout_name"),
+                mongo_db_config=mongo_db_config,
+                model_version=version,
+                sim_telarray_seeds={
+                    "seed": self._get_seed_for_random_instrument_instances(
+                        self.sim_telarray_seeds["seed"], version
+                    ),
+                    "random_instrument_instances": self.sim_telarray_seeds[
+                        "random_instrument_instances"
+                    ],
+                    "seed_file_name": self.sim_telarray_seeds["seed_file_name"],
+                },
+                simtel_path=self.args_dict.get("simtel_path", None),
+            )
+            for version in versions
+        ]
+
+    def _get_seed_for_random_instrument_instances(self, seed, model_version):
+        """
+        Generate seed for random instances of the instrument.
+
+        Parameters
+        ----------
+        seed : str
+            Seed string given through configuration.
+        model_version: str
+            Model version.
+
+        Returns
+        -------
+        int
+            Seed for random instances of the instrument.
+        """
+        if seed:
+            return int(seed.split(",")[0].strip())
+
+        def semver_to_int(version: str):
+            major, minor, patch = map(int, version.split("."))
+            return major * 10000 + minor * 100 + patch
+
+        seed = semver_to_int(model_version) * 10000000
+        seed = seed + 1000000 if self.args_dict.get("site") != "North" else seed + 2000000
+        seed = seed + (int)(self.args_dict["zenith_angle"].value) * 1000
+        return seed + (int)(self.args_dict["azimuth_angle"].value)
 
     def _initialize_run_list(self):
         """
-        Initialize run list using the configuration values 'run_number_start' and 'number_of_runs'.
+        Initialize run list using the configuration values.
+
+        Uses 'run_number', 'run_number_offset' and 'number_of_runs' arguments
+        to create a list of run numbers.
 
         Returns
         -------
@@ -137,19 +192,27 @@ class Simulator:
         Raises
         ------
         KeyError
-            If 'run_number_start' or 'number_of_runs' are not found in the configuration.
+            If 'run_number', 'run_number_offset' and 'number_of_runs' are
+            not found in the configuration.
         """
         try:
+            offset_run_number = self.args_dict["run_number_offset"] + self.args_dict["run_number"]
+            if self.args_dict["number_of_runs"] <= 1:
+                return self._validate_run_list_and_range(
+                    run_list=offset_run_number,
+                    run_range=None,
+                )
             return self._validate_run_list_and_range(
                 run_list=None,
                 run_range=[
-                    self.args_dict["run_number_start"],
-                    self.args_dict["run_number_start"] + self.args_dict["number_of_runs"],
+                    offset_run_number,
+                    offset_run_number + self.args_dict["number_of_runs"],
                 ],
             )
         except KeyError as exc:
             self._logger.error(
-                "Error in initializing run list (missing 'run_number_start' or 'number_of_runs')"
+                "Error in initializing run list "
+                "(missing 'run_number', 'run_number_offset' or 'number_of_runs')."
             )
             raise exc
 
@@ -214,22 +277,33 @@ class Simulator:
         CorsikaRunner or SimulatorArray or CorsikaSimtelRunner
             Simulation runner object.
         """
-        corsika_config = CorsikaConfig(
-            array_model=self.array_model,
-            label=self.label,
-            args_dict=self.args_dict,
-            db_config=db_config,
-        )
+        corsika_configurations = []
+        for array_model in self.array_models:
+            corsika_configurations.append(
+                CorsikaConfig(
+                    array_model=array_model,
+                    label=self.label,
+                    args_dict=self.args_dict,
+                    db_config=db_config,
+                )
+            )
 
         runner_class = {
             "corsika": CorsikaRunner,
-            "simtel": SimulatorArray,
-            "corsika_simtel": CorsikaSimtelRunner,
+            "sim_telarray": SimulatorArray,
+            "corsika_sim_telarray": CorsikaSimtelRunner,
         }.get(self.simulation_software)
 
+        # In case of CorsikaSimtelRunner we should pass all corsika_configurations
+        # to the runner, since we define multiple configurations to run in a single job
+        # using multipipe. In other cases we pass only the first one (there's only one anyway).
         runner_args = {
             "label": self.label,
-            "corsika_config": corsika_config,
+            "corsika_config": (
+                corsika_configurations
+                if runner_class is CorsikaSimtelRunner
+                else corsika_configurations[0]
+            ),
             "simtel_path": self.args_dict.get("simtel_path"),
             "use_multipipe": runner_class is CorsikaSimtelRunner,
         }
@@ -237,7 +311,9 @@ class Simulator:
         if runner_class is not SimulatorArray:
             runner_args["keep_seeds"] = self.args_dict.get("corsika_test_seeds", False)
         if runner_class is not CorsikaRunner:
-            runner_args["sim_telarray_seeds"] = self.args_dict.get("sim_telarray_seeds")
+            runner_args["sim_telarray_seeds"] = self.sim_telarray_seeds
+        if runner_class is CorsikaSimtelRunner:
+            runner_args["sequential"] = self.args_dict.get("sequential", False)
 
         return runner_class(**runner_args)
 
@@ -267,8 +343,6 @@ class Simulator:
         input_file_list: str or list of str
             Single file or list of files of shower simulations.
         """
-        self._logger.info(f"Submission command: {self.submit_engine}")
-
         runs_and_files_to_submit = self._get_runs_and_files_to_submit(
             input_file_list=input_file_list
         )
@@ -282,11 +356,7 @@ class Simulator:
                 run_number=run_number, input_file=input_file, extra_commands=self._extra_commands
             )
 
-            job_manager = JobManager(
-                submit_engine=self.submit_engine,
-                submit_options=self._submit_options,
-                test=self._test,
-            )
+            job_manager = JobManager(test=self._test)
             job_manager.submit(
                 run_script=run_script,
                 run_out_file=self._simulation_runner.get_file_name(
@@ -325,10 +395,10 @@ class Simulator:
         _runs_and_files = {}
         self._logger.debug(f"Getting runs and files to submit ({input_file_list})")
 
-        if self.simulation_software == "simtel":
+        if self.simulation_software == "sim_telarray":
             input_file_list = self._enforce_list_type(input_file_list)
             _runs_and_files = {self._guess_run_from_file(file): file for file in input_file_list}
-        elif self.simulation_software in ["corsika", "corsika_simtel"]:
+        elif self.simulation_software in ["corsika", "corsika_sim_telarray"]:
             _runs_and_files = dict.fromkeys(self._get_runs_to_simulate())
         if len(_runs_and_files) == 0:
             raise ValueError("No runs to submit.")
@@ -391,46 +461,66 @@ class Simulator:
             run number
 
         """
-        keys = ["output", "sub_out", "log", "input", "hist", "corsika_log"]
-        defaults = dict.fromkeys(keys)
-        results = {key: defaults[key] for key in keys}
-        results["output"] = str(
-            self._simulation_runner.get_file_name(file_type="output", run_number=run_number)
-        )
-        results["sub_out"] = str(
-            self._simulation_runner.get_file_name(
-                file_type="sub_log", mode="out", run_number=run_number
+        keys = ["simtel_output", "sub_out", "log", "input", "hist", "corsika_log", "event_data"]
+        results = {key: [] for key in keys}
+
+        def get_file_name(name, **kwargs):
+            return str(self._simulation_runner.get_file_name(file_type=name, **kwargs))
+
+        if "sim_telarray" in self.simulation_software:
+            results["input"].append(str(file))
+
+        results["sub_out"].append(get_file_name("sub_log", mode="out", run_number=run_number))
+
+        for i in range(len(self.array_models)):
+            results["simtel_output"].append(
+                get_file_name("simtel_output", run_number=run_number, model_version_index=i)
             )
-        )
 
-        if "simtel" in self.simulation_software:
-            results["log"] = str(
-                self._simulation_runner.get_file_name(
-                    file_type="log", simulation_software="simtel", run_number=run_number
+            if "sim_telarray" in self.simulation_software:
+                results["log"].append(
+                    get_file_name(
+                        "log",
+                        simulation_software="sim_telarray",
+                        run_number=run_number,
+                        model_version_index=i,
+                    )
                 )
-            )
-            results["input"] = str(file)
-            results["hist"] = str(
-                self._simulation_runner.get_file_name(
-                    file_type="histogram", simulation_software="simtel", run_number=run_number
+                results["hist"].append(
+                    get_file_name(
+                        "histogram",
+                        simulation_software="sim_telarray",
+                        run_number=run_number,
+                        model_version_index=i,
+                    )
+                )
+                results["event_data"].append(
+                    get_file_name(
+                        "event_data",
+                        simulation_software="sim_telarray",
+                        run_number=run_number,
+                        model_version_index=i,
+                    )
                 )
-            )
 
-        if "corsika" in self.simulation_software:
-            results["corsika_log"] = str(
-                self._simulation_runner.get_file_name(
-                    file_type="corsika_log", simulation_software="corsika", run_number=run_number
+            if "corsika" in self.simulation_software:
+                results["corsika_log"].append(
+                    get_file_name(
+                        "corsika_log",
+                        simulation_software="corsika",
+                        run_number=run_number,
+                        model_version_index=i,
+                    )
                 )
-            )
 
         for key in keys:
-            self._results[key].append(results[key])
+            self._results[key].extend(results[key])
 
-    def get_file_list(self, file_type="output"):
+    def get_file_list(self, file_type="simtel_output"):
         """
         Get list of files generated by simulations.
 
-        Options are "input", "output", "hist", "log", "corsika_log".
+        Options are "input", "simtel_output", "hist", "log", "corsika_log".
         Not all file types are available for all simulation types.
         Returns an empty list for an unknown file type.
 
@@ -448,11 +538,11 @@ class Simulator:
         self._logger.info(f"Getting list of {file_type} files")
         return self._results[file_type]
 
-    def print_list_of_files(self, file_type="output"):
+    def print_list_of_files(self, file_type="simtel_output"):
         """
         Print list of output files generated by simulations.
 
-        Options are "input", "output", "hist", "log".
+        Options are "input", "simtel_output", "hist", "log".
 
         Parameters
         ----------
@@ -546,7 +636,7 @@ class Simulator:
 
     def save_file_lists(self):
         """Save files lists for output and log files."""
-        for file_type in ["output", "log", "corsika_log", "hist"]:
+        for file_type in ["simtel_output", "log", "corsika_log", "hist"]:
             file_name = self.io_handler.get_output_directory(label=self.label).joinpath(
                 f"{file_type}_files.txt"
             )
@@ -559,10 +649,35 @@ class Simulator:
             else:
                 self._logger.debug(f"No files to save for {file_type} files.")
 
+    def save_reduced_event_lists(self):
+        """
+        Save reduced event lists with event data on simulated and triggered events.
+
+        The files are saved with the same name as the sim_telarray output file
+        but with a 'hdf5' extension.
+        """
+        if "sim_telarray" not in self.simulation_software:
+            self._logger.warning(
+                "Reduced event lists can only be saved for sim_telarray simulations."
+            )
+            return
+
+        input_files = self.get_file_list(file_type="simtel_output")
+        output_files = self.get_file_list(file_type="event_data")
+        for input_file, output_file in zip(input_files, output_files):
+            generator = SimtelIOEventDataWriter([input_file])
+            io_table_handler.write_tables(
+                tables=generator.process_files(),
+                output_file=Path(output_file),
+                overwrite_existing=True,
+            )
+
     def pack_for_register(self, directory_for_grid_upload=None):
         """
         Pack simulation output files for registering on the grid.
 
+        Creates separate tarballs for each model version's log files.
+
         Parameters
         ----------
         directory_for_grid_upload: str
@@ -572,11 +687,16 @@ class Simulator:
         self._logger.info(
             f"Packing the output files for registering on the grid ({directory_for_grid_upload})"
         )
-        output_files = self.get_file_list(file_type="output")
+        output_files = self.get_file_list(file_type="simtel_output")
         log_files = self.get_file_list(file_type="log")
         corsika_log_files = self.get_file_list(file_type="corsika_log")
         histogram_files = self.get_file_list(file_type="hist")
-        tar_file_name = Path(log_files[0]).name.replace("log.gz", "log_hist.tar.gz")
+        reduced_event_files = (
+            self.get_file_list(file_type="event_data")
+            if self.args_dict.get("save_reduced_event_lists")
+            else []
+        )
+
         directory_for_grid_upload = (
             Path(directory_for_grid_upload)
             if directory_for_grid_upload
@@ -586,24 +706,101 @@ class Simulator:
         )
         directory_for_grid_upload.mkdir(parents=True, exist_ok=True)
 
-        tar_file_name = directory_for_grid_upload.joinpath(tar_file_name)
+        # If there are more than one model version,
+        # duplicate the corsika log file to have one for each model version with the "right name".
+        if len(self.array_models) > 1 and corsika_log_files:
+            self._copy_corsika_log_file_for_all_versions(corsika_log_files)
 
-        with tarfile.open(tar_file_name, "w:gz") as tar:
-            files_to_tar = (
-                (log_files[:1] if log_files else [])
-                + (histogram_files[:1] if histogram_files else [])
-                + (corsika_log_files[:1] if corsika_log_files else [])
-            )
-            for file_to_tar in files_to_tar:
-                tar.add(file_to_tar, arcname=Path(file_to_tar).name)
+        # Group files by model version
+        for model in self.array_models:
+            model_version = model.model_version
+
+            # Filter files for this model version
+            model_logs = [f for f in log_files if model_version in f]
+            model_hists = [f for f in histogram_files if model_version in f]
+            model_corsika_logs = [f for f in corsika_log_files if model_version in f]
+
+            if model_logs:
+                tar_file_name = Path(model_logs[0]).name.replace("log.gz", "log_hist.tar.gz")
+                tar_file_path = directory_for_grid_upload.joinpath(tar_file_name)
 
-        for file_to_move in [*output_files]:
+                with tarfile.open(tar_file_path, "w:gz") as tar:
+                    # Add all relevant log, histogram, and CORSIKA log files to the tarball
+                    files_to_tar = model_logs + model_hists + model_corsika_logs
+                    for file_to_tar in files_to_tar:
+                        tar.add(file_to_tar, arcname=Path(file_to_tar).name)
+
+        for file_to_move in output_files + reduced_event_files:
             source_file = Path(file_to_move)
             destination_file = directory_for_grid_upload / source_file.name
             if destination_file.exists():
                 self._logger.warning(f"Overwriting existing file: {destination_file}")
-            # Note that this will overwrite previous files which exist in the directory
-            # It should be fine for normal production since each run is on a separate node
-            # so no files are expected there.
             shutil.move(source_file, destination_file)
+
         self._logger.info(f"Output files for the grid placed in {directory_for_grid_upload!s}")
+
+    def validate_metadata(self):
+        """Validate metadata in the sim_telarray output files."""
+        if "sim_telarray" not in self.simulation_software:
+            self._logger.info("No sim_telarray files to validate.")
+            return
+
+        for model in self.array_models:
+            files = self.get_file_list(file_type="simtel_output")
+            output_file = next((f for f in files if model.model_version in f), None)
+            if output_file:
+                self._logger.info(f"Validating metadata for {output_file}")
+                assert_sim_telarray_metadata(output_file, model)
+                self._logger.info(f"Metadata for sim_telarray file {output_file} is valid.")
+            else:
+                self._logger.warning(
+                    f"No sim_telarray file found for model version {model.model_version}: {files}"
+                )
+
+    def _copy_corsika_log_file_for_all_versions(self, corsika_log_files):
+        """
+        Create copies of the CORSIKA log file for each model version.
+
+        Adds a header comment to each copy explaining its relationship to the original.
+
+        Parameters
+        ----------
+        corsika_log_files: list
+            List containing the original CORSIKA log file path.
+        """
+        original_log = Path(corsika_log_files[0])
+        # Find which model version the original log belongs to
+        original_version = next(
+            model.model_version
+            for model in self.array_models
+            if re.search(
+                rf"(?<![0-9A-Za-z]){re.escape(model.model_version)}(?![0-9A-Za-z])",
+                original_log.name,
+            )
+        )
+
+        for model in self.array_models:
+            if model.model_version == original_version:
+                continue
+
+            new_log = original_log.parent / original_log.name.replace(
+                original_version, model.model_version
+            )
+
+            with gzip.open(new_log, "wt", encoding="utf-8") as new_file:
+                # Write the header to the new file
+                header = (
+                    f"###############################################################\n"
+                    f"Copy of CORSIKA log file from model version {original_version}.\n"
+                    f"Applicable also for {model.model_version} (same CORSIKA configuration,\n"
+                    f"different sim_telarray model versions in the same run).\n"
+                    f"###############################################################\n\n"
+                )
+                new_file.write(header)
+
+                # Copy the content of the original log file, ignoring invalid characters
+                with gzip.open(original_log, "rt", encoding="utf-8", errors="ignore") as orig_file:
+                    for line in orig_file:
+                        new_file.write(line)
+
+            corsika_log_files.append(str(new_log))

simtools/testing/configuration.py

@@ -102,6 +102,11 @@ def configure(config, tmp_test_directory, request):
     """
     tmp_output_path = create_tmp_output_path(tmp_test_directory, config)
     model_version_requested = request.config.getoption("--model_version")
+    model_version_requested = (
+        model_version_requested.split(",") if model_version_requested else None
+    )
+    if isinstance(model_version_requested, list) and len(model_version_requested) == 1:
+        model_version_requested = model_version_requested[0]
 
     if "CONFIGURATION" in config:
         _skip_test_for_model_version(config, model_version_requested)

simtools/testing/helpers.py

@@ -33,3 +33,21 @@ def _new_testeff_version():
             return False
     except FileNotFoundError as exc:
         raise FileNotFoundError("The testeff executable could not be found.") from exc
+
+
+def skip_multiple_version_test(config, model_version):
+    """Skip a test which is not meant for multiple versions if multiple versions are given."""
+    message = "Skipping test not meant for multiple model versions."
+
+    if not isinstance(model_version, list):
+        return None
+
+    config_model_version = config.get("CONFIGURATION", {}).get("MODEL_VERSION", [])
+
+    if not isinstance(config_model_version, list):
+        config_model_version = [config_model_version]
+
+    if 1 < len(model_version) != len(config_model_version):
+        return message
+
+    return None