gammasimtools 0.18.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

simtools/simtel/simulator_ray_tracing.py

@@ -5,7 +5,7 @@ from collections import namedtuple
 
 import astropy.units as u
 
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.runners.simtel_runner import SimtelRunner
 from simtools.utils import names
 from simtools.utils.general import clear_default_sim_telarray_cfg_directories

simtools/simulator.py

@@ -4,7 +4,6 @@ import gzip
 import logging
 import re
 import shutil
-import tarfile
 from collections import defaultdict
 from pathlib import Path
 
@@ -12,7 +11,7 @@ import numpy as np
 
 import simtools.utils.general as gen
 from simtools.corsika.corsika_config import CorsikaConfig
-from simtools.io_operations import io_handler, io_table_handler
+from simtools.io import io_handler, table_handler
 from simtools.job_execution.job_manager import JobManager
 from simtools.model.array_model import ArrayModel
 from simtools.runners.corsika_runner import CorsikaRunner
@@ -49,8 +48,8 @@ class Simulator:
         Instance label.
     extra_commands: str or list of str
         Extra commands to be added to the run script before the run command.
-    mongo_db_config: dict
-        MongoDB configuration.
+    db_config: dict
+        Database configuration.
     """
 
     def __init__(
@@ -58,15 +57,18 @@ class Simulator:
         args_dict,
         label=None,
         extra_commands=None,
-        mongo_db_config=None,
+        db_config=None,
     ):
         """Initialize Simulator class."""
-        self._logger = logging.getLogger(__name__)
+        self.logger = logging.getLogger(__name__)
+        self.label = label
+
         self.args_dict = args_dict
+        self.db_config = db_config
 
-        self.simulation_software = self.args_dict["simulation_software"]
-        self._logger.debug(f"Init Simulator {self.simulation_software}")
-        self.label = label
+        self.simulation_software = self.args_dict.get("simulation_software", "corsika_sim_telarray")
+        self.run_mode = self.args_dict.get("run_mode")
+        self.logger.debug(f"Init Simulator {self.simulation_software}")
 
         self.io_handler = io_handler.IOHandler()
 
@@ -82,8 +84,8 @@ class Simulator:
             ),
             "seed_file_name": "sim_telarray_instrument_seeds.txt",  # name only; no directory
         }
-        self.array_models = self._initialize_array_models(mongo_db_config)
-        self._simulation_runner = self._initialize_simulation_runner(mongo_db_config)
+        self.array_models = self._initialize_array_models()
+        self._simulation_runner = self._initialize_simulation_runner()
 
     @property
     def simulation_software(self):
@@ -103,37 +105,30 @@ class Simulator:
 
         Raises
         ------
-        gen.InvalidConfigDataError
+        ValueError
 
         """
         if simulation_software not in ["sim_telarray", "corsika", "corsika_sim_telarray"]:
-            self._logger.error(f"Invalid simulation software: {simulation_software}")
-            raise gen.InvalidConfigDataError
+            raise ValueError(f"Invalid simulation software: {simulation_software}")
         self._simulation_software = simulation_software.lower()
 
-    def _initialize_array_models(self, mongo_db_config):
+    def _initialize_array_models(self):
         """
         Initialize array simulation models.
 
-        Parameters
-        ----------
-        mongo_db_config: dict
-            Database configuration.
-
         Returns
         -------
         list
             List of ArrayModel objects.
         """
-        model_version = self.args_dict.get("model_version", [])
-        versions = [model_version] if not isinstance(model_version, list) else model_version
+        versions = gen.ensure_iterable(self.args_dict.get("model_version", []))
 
         return [
             ArrayModel(
                 label=self.label,
                 site=self.args_dict.get("site"),
                 layout_name=self.args_dict.get("array_layout_name"),
-                mongo_db_config=mongo_db_config,
+                mongo_db_config=self.db_config,
                 model_version=version,
                 sim_telarray_seeds={
                     "seed": self._get_seed_for_random_instrument_instances(
@@ -188,35 +183,24 @@ class Simulator:
         -------
         list
             List of run numbers.
-
-        Raises
-        ------
-        KeyError
-            If 'run_number', 'run_number_offset' and 'number_of_runs' are
-            not found in the configuration.
         """
-        try:
-            offset_run_number = self.args_dict["run_number_offset"] + self.args_dict["run_number"]
-            if self.args_dict["number_of_runs"] <= 1:
-                return self._validate_run_list_and_range(
-                    run_list=offset_run_number,
-                    run_range=None,
-                )
-            return self._validate_run_list_and_range(
-                run_list=None,
-                run_range=[
-                    offset_run_number,
-                    offset_run_number + self.args_dict["number_of_runs"],
-                ],
-            )
-        except KeyError as exc:
-            self._logger.error(
-                "Error in initializing run list "
-                "(missing 'run_number', 'run_number_offset' or 'number_of_runs')."
+        offset_run_number = self.args_dict.get("run_number_offset", 0) + self.args_dict.get(
+            "run_number", 1
+        )
+        if self.args_dict.get("number_of_runs", 1) <= 1:
+            return self._prepare_run_list_and_range(
+                run_list=offset_run_number,
+                run_range=None,
             )
-            raise exc
+        return self._prepare_run_list_and_range(
+            run_list=None,
+            run_range=[
+                offset_run_number,
+                offset_run_number + self.args_dict["number_of_runs"],
+            ],
+        )
 
-    def _validate_run_list_and_range(self, run_list, run_range):
+    def _prepare_run_list_and_range(self, run_list, run_range):
         """
         Prepare list of run numbers from a list or from a range.
 
@@ -233,49 +217,40 @@ class Simulator:
         -------
         list
             list of unique run numbers (integers)
-
         """
         if run_list is None and run_range is None:
-            self._logger.debug("Nothing to validate - run_list and run_range not given.")
+            self.logger.debug("Nothing to prepare - run_list and run_range not given.")
             return None
 
         validated_runs = []
         if run_list is not None:
-            if not isinstance(run_list, list):
-                run_list = [run_list]
-            if not all(isinstance(r, int) for r in run_list):
-                msg = "run_list must contain only integers."
-                self._logger.error(msg)
-                raise InvalidRunsToSimulateError
-            validated_runs = list(run_list)
+            validated_runs = gen.ensure_iterable(run_list)
+            if not all(isinstance(r, int) for r in validated_runs):
+                raise InvalidRunsToSimulateError(f"Run list must contain only integers: {run_list}")
 
         if run_range is not None:
             if not all(isinstance(r, int) for r in run_range) or len(run_range) != 2:
-                msg = "run_range must contain two integers only."
-                self._logger.error(msg)
-                raise InvalidRunsToSimulateError
-
+                raise InvalidRunsToSimulateError(
+                    f"Run_range must contain two integers only: {run_range}"
+                )
             run_range = np.arange(run_range[0], run_range[1])
-            self._logger.debug(f"run_range: {run_range}")
             validated_runs.extend(list(run_range))
 
         validated_runs_unique = sorted(set(validated_runs))
-        self._logger.info(f"run_list: {validated_runs_unique}")
-        return list(validated_runs_unique)
+        self.logger.info(f"Runlist: {validated_runs_unique}")
+        return validated_runs_unique
 
-    def _initialize_simulation_runner(self, db_config):
+    def _corsika_configuration(self):
         """
-        Initialize corsika configuration and simulation runners.
+        Define CORSIKA configurations based on the simulation model.
 
-        Parameters
-        ----------
-        db_config: dict
-            Database configuration.
+        For 'corsika_sim_telarray', this is a list since multiple configurations
+        might be defined to run in a single job using multipipe.
 
         Returns
         -------
-        CorsikaRunner or SimulatorArray or CorsikaSimtelRunner
-            Simulation runner object.
+        CorsikaConfig or list of CorsikaConfig
+            CORSIKA configuration(s) based on the simulation model.
         """
         corsika_configurations = []
         for array_model in self.array_models:
@@ -284,9 +259,26 @@ class Simulator:
                     array_model=array_model,
                     label=self.label,
                     args_dict=self.args_dict,
-                    db_config=db_config,
+                    db_config=self.db_config,
+                    dummy_simulations=self._is_calibration_run(),
                 )
             )
+        return (
+            corsika_configurations
+            if self.simulation_software == "corsika_sim_telarray"
+            else corsika_configurations[0]
+        )
+
+    def _initialize_simulation_runner(self):
+        """
+        Initialize corsika configuration and simulation runners.
+
+        Returns
+        -------
+        CorsikaRunner or SimulatorArray or CorsikaSimtelRunner
+            Simulation runner object.
+        """
+        corsika_configurations = self._corsika_configuration()
 
         runner_class = {
             "corsika": CorsikaRunner,
@@ -294,16 +286,9 @@ class Simulator:
             "corsika_sim_telarray": CorsikaSimtelRunner,
         }.get(self.simulation_software)
 
-        # In case of CorsikaSimtelRunner we should pass all corsika_configurations
-        # to the runner, since we define multiple configurations to run in a single job
-        # using multipipe. In other cases we pass only the first one (there's only one anyway).
         runner_args = {
             "label": self.label,
-            "corsika_config": (
-                corsika_configurations
-                if runner_class is CorsikaSimtelRunner
-                else corsika_configurations[0]
-            ),
+            "corsika_config": corsika_configurations,
             "simtel_path": self.args_dict.get("simtel_path"),
             "use_multipipe": runner_class is CorsikaSimtelRunner,
         }
@@ -314,6 +299,8 @@ class Simulator:
             runner_args["sim_telarray_seeds"] = self.sim_telarray_seeds
         if runner_class is CorsikaSimtelRunner:
             runner_args["sequential"] = self.args_dict.get("sequential", False)
+            if self._is_calibration_run():
+                runner_args["calibration_runner_args"] = self.args_dict
 
         return runner_class(**runner_args)
 
@@ -326,9 +313,7 @@ class Simulator:
         input_file_list: str or list of str
             Single file or list of files of shower simulations.
         """
-        input_file_list = self._enforce_list_type(input_file_list)
-
-        for file in input_file_list:
+        for file in gen.ensure_iterable(input_file_list):
             run = self._guess_run_from_file(file)
             self._fill_results(file, run)
             if run not in self.runs:
@@ -346,7 +331,7 @@ class Simulator:
         runs_and_files_to_submit = self._get_runs_and_files_to_submit(
             input_file_list=input_file_list
         )
-        self._logger.info(
+        self.logger.info(
             f"Starting submission for {len(runs_and_files_to_submit)} "
             f"run{'s' if len(runs_and_files_to_submit) > 1 else ''}"
         )
@@ -373,7 +358,7 @@ class Simulator:
         """
         Return a dictionary with run numbers and simulation files.
 
-        The latter are expected to be given for the simtel simulator.
+        The latter are expected to be given for the sim_telarray simulator.
 
         Parameters
         ----------
@@ -390,39 +375,19 @@ class Simulator:
         ------
         ValueError
             If no runs are to be submitted.
-
         """
         _runs_and_files = {}
-        self._logger.debug(f"Getting runs and files to submit ({input_file_list})")
+        self.logger.debug(f"Getting runs and files to submit ({input_file_list})")
 
-        if self.simulation_software == "sim_telarray":
-            input_file_list = self._enforce_list_type(input_file_list)
+        if self.simulation_software == "sim_telarray" and self.run_mode is None:
+            input_file_list = gen.ensure_iterable(input_file_list)
             _runs_and_files = {self._guess_run_from_file(file): file for file in input_file_list}
-        elif self.simulation_software in ["corsika", "corsika_sim_telarray"]:
+        else:
             _runs_and_files = dict.fromkeys(self._get_runs_to_simulate())
         if len(_runs_and_files) == 0:
             raise ValueError("No runs to submit.")
         return _runs_and_files
 
-    @staticmethod
-    def _enforce_list_type(input_file_list):
-        """
-        Enforce the input list to be a list.
-
-        Parameters
-        ----------
-        input_file_list: str or list of str
-            Single file or list of files of shower simulations.
-
-        Returns
-        -------
-        list
-            List of input files.
-        """
-        if not input_file_list:
-            return []
-        return input_file_list if isinstance(input_file_list, list) else [input_file_list]
-
     def _guess_run_from_file(self, file):
         """
         Extract the run number from the given file name.
@@ -446,7 +411,7 @@ class Simulator:
             run_str = re.search(r"run\d*", file_name).group()
             return int(run_str[3:])
         except (ValueError, AttributeError):
-            self._logger.warning(f"Run number could not be guessed from {file_name} using run = 1")
+            self.logger.warning(f"Run number could not be guessed from {file_name} using run = 1")
             return 1
 
     def _fill_results(self, file, run_number):
@@ -535,25 +500,9 @@ class Simulator:
             List with the full path of all output files.
 
         """
-        self._logger.info(f"Getting list of {file_type} files")
+        self.logger.info(f"Getting list of {file_type} files")
         return self._results[file_type]
 
-    def print_list_of_files(self, file_type="simtel_output"):
-        """
-        Print list of output files generated by simulations.
-
-        Options are "input", "simtel_output", "hist", "log".
-
-        Parameters
-        ----------
-        file_type : str
-            File type to be listed.
-
-        """
-        self._logger.info(f"Printing list of {file_type} files")
-        for file in self._results[file_type]:
-            print(file)
-
     def _make_resources_report(self, input_file_list):
         """
         Prepare a simple report on computing wall clock time used in the simulations.
@@ -632,7 +581,7 @@ class Simulator:
         """
         if run_list is None and run_range is None:
             return [] if self.runs is None else self.runs
-        return self._validate_run_list_and_range(run_list, run_range)
+        return self._prepare_run_list_and_range(run_list, run_range)
 
     def save_file_lists(self):
         """Save files lists for output and log files."""
@@ -642,12 +591,12 @@ class Simulator:
             )
             file_list = self.get_file_list(file_type=file_type)
             if all(element is not None for element in file_list) and len(file_list) > 0:
-                self._logger.info(f"Saving list of {file_type} files to {file_name}")
+                self.logger.info(f"Saving list of {file_type} files to {file_name}")
                 with open(file_name, "w", encoding="utf-8") as f:
                     for line in self.get_file_list(file_type=file_type):
                         f.write(f"{line}\n")
             else:
-                self._logger.debug(f"No files to save for {file_type} files.")
+                self.logger.debug(f"No files to save for {file_type} files.")
 
     def save_reduced_event_lists(self):
         """
@@ -657,7 +606,7 @@ class Simulator:
         but with a 'hdf5' extension.
         """
         if "sim_telarray" not in self.simulation_software:
-            self._logger.warning(
+            self.logger.warning(
                 "Reduced event lists can only be saved for sim_telarray simulations."
             )
             return
@@ -666,7 +615,7 @@ class Simulator:
         output_files = self.get_file_list(file_type="event_data")
         for input_file, output_file in zip(input_files, output_files):
             generator = SimtelIOEventDataWriter([input_file])
-            io_table_handler.write_tables(
+            table_handler.write_tables(
                 tables=generator.process_files(),
                 output_file=Path(output_file),
                 overwrite_existing=True,
@@ -684,7 +633,7 @@ class Simulator:
             Directory for the tarball with output files.
 
         """
-        self._logger.info(
+        self.logger.info(
             f"Packing the output files for registering on the grid ({directory_for_grid_upload})"
         )
         output_files = self.get_file_list(file_type="simtel_output")
@@ -714,6 +663,7 @@ class Simulator:
         # Group files by model version
         for model in self.array_models:
             model_version = model.model_version
+            model_files = gen.ensure_iterable(model.pack_model_files())
 
             # Filter files for this model version
             model_logs = [f for f in log_files if model_version in f]
@@ -723,37 +673,34 @@ class Simulator:
             if model_logs:
                 tar_file_name = Path(model_logs[0]).name.replace("log.gz", "log_hist.tar.gz")
                 tar_file_path = directory_for_grid_upload.joinpath(tar_file_name)
-
-                with tarfile.open(tar_file_path, "w:gz") as tar:
-                    # Add all relevant log, histogram, and CORSIKA log files to the tarball
-                    files_to_tar = model_logs + model_hists + model_corsika_logs
-                    for file_to_tar in files_to_tar:
-                        tar.add(file_to_tar, arcname=Path(file_to_tar).name)
+                # Add all relevant model, log, histogram, and CORSIKA log files to the tarball
+                files_to_tar = model_logs + model_hists + model_corsika_logs + model_files
+                gen.pack_tar_file(tar_file_path, files_to_tar)
 
         for file_to_move in output_files + reduced_event_files:
             source_file = Path(file_to_move)
             destination_file = directory_for_grid_upload / source_file.name
             if destination_file.exists():
-                self._logger.warning(f"Overwriting existing file: {destination_file}")
+                self.logger.warning(f"Overwriting existing file: {destination_file}")
             shutil.move(source_file, destination_file)
 
-        self._logger.info(f"Output files for the grid placed in {directory_for_grid_upload!s}")
+        self.logger.info(f"Output files for the grid placed in {directory_for_grid_upload!s}")
 
     def validate_metadata(self):
         """Validate metadata in the sim_telarray output files."""
         if "sim_telarray" not in self.simulation_software:
-            self._logger.info("No sim_telarray files to validate.")
+            self.logger.info("No sim_telarray files to validate.")
             return
 
         for model in self.array_models:
             files = self.get_file_list(file_type="simtel_output")
             output_file = next((f for f in files if model.model_version in f), None)
             if output_file:
-                self._logger.info(f"Validating metadata for {output_file}")
+                self.logger.info(f"Validating metadata for {output_file}")
                 assert_sim_telarray_metadata(output_file, model)
-                self._logger.info(f"Metadata for sim_telarray file {output_file} is valid.")
+                self.logger.info(f"Metadata for sim_telarray file {output_file} is valid.")
             else:
-                self._logger.warning(
+                self.logger.warning(
                     f"No sim_telarray file found for model version {model.model_version}: {files}"
                 )
 
@@ -804,3 +751,14 @@ class Simulator:
                         new_file.write(line)
 
             corsika_log_files.append(str(new_log))
+
+    def _is_calibration_run(self):
+        """
+        Check if this simulation is a calibration run.
+
+        Returns
+        -------
+        bool
+            True if it is a calibration run, False otherwise.
+        """
+        return self.run_mode in ["pedestals", "dark_pedestals", "nsb_only_pedestals", "flasher"]

simtools/testing/configuration.py

@@ -4,9 +4,8 @@ import logging
 import os
 from pathlib import Path
 
-import yaml
-
 import simtools.utils.general as gen
+from simtools.io import ascii_handler
 
 _logger = logging.getLogger(__name__)
 
@@ -43,24 +42,24 @@ def get_list_of_test_configurations(config_files):
 
     # list of all applications
     # (needs to be sorted for pytest-xdist, see Known Limitations in their website)
-    _applications = sorted({item["APPLICATION"] for item in configs if "APPLICATION" in item})
+    _applications = sorted({item["application"] for item in configs if "application" in item})
     for app in _applications:
         configs.extend(
             [
-                {"APPLICATION": app, "TEST_NAME": "auto-help", "CONFIGURATION": {"HELP": True}},
+                {"application": app, "test_name": "auto-help", "configuration": {"help": True}},
                 {
-                    "APPLICATION": app,
-                    "TEST_NAME": "auto-version",
-                    "CONFIGURATION": {"VERSION": True},
+                    "application": app,
+                    "test_name": "auto-version",
+                    "configuration": {"version": True},
                 },
-                {"APPLICATION": app, "TEST_NAME": "auto-no_config"},
+                {"application": app, "test_name": "auto-no_config"},
             ]
         )
 
     return (
         configs,
         [
-            f"{item.get('APPLICATION', 'no-app-name')}_{item.get('TEST_NAME', 'no-test-name')}"
+            f"{item.get('application', 'no-app-name')}_{item.get('test_name', 'no-test-name')}"
             for item in configs
         ],
     )
@@ -74,9 +73,9 @@ def _read_configs_from_files(config_files):
         # remove new line characters from config - otherwise issues
         # with especially long file names
         _dict = gen.remove_substring_recursively_from_dict(
-            gen.collect_data_from_file(file_name=config_file), substring="\n"
+            ascii_handler.collect_data_from_file(file_name=config_file), substring="\n"
         )
-        for application in _dict.get("CTA_SIMPIPE", {}).get("APPLICATIONS", []):
+        for application in _dict.get("applications", []):
             configs.append(application)
     return configs
 
@@ -108,12 +107,12 @@ def configure(config, tmp_test_directory, request):
     if isinstance(model_version_requested, list) and len(model_version_requested) == 1:
         model_version_requested = model_version_requested[0]
 
-    if "CONFIGURATION" in config:
+    if "configuration" in config:
         _skip_test_for_model_version(config, model_version_requested)
         _skip_test_for_production_db(config)
 
         config_file, config_string, config_file_model_version = _prepare_test_options(
-            config["CONFIGURATION"],
+            config["configuration"],
             output_path=tmp_output_path,
             model_version=model_version_requested,
         )
@@ -123,7 +122,7 @@ def configure(config, tmp_test_directory, request):
         config_file_model_version = None
 
     cmd = get_application_command(
-        app=config.get("APPLICATION", None),
+        app=config.get("application", None),
         config_file=config_file,
         config_string=config_string,
     )
@@ -132,9 +131,9 @@ def configure(config, tmp_test_directory, request):
 
 def _skip_test_for_model_version(config, model_version_requested):
     """Skip test if model version requested is not supported."""
-    if config.get("MODEL_VERSION_USE_CURRENT") is None or model_version_requested is None:
+    if config.get("model_version_use_current") is None or model_version_requested is None:
         return
-    model_version_config = config["CONFIGURATION"]["MODEL_VERSION"]
+    model_version_config = config["configuration"]["model_version"]
     if model_version_requested != model_version_config:
         raise VersionError(
             f"Model version requested {model_version_requested} not supported for this test"
@@ -143,7 +142,7 @@ def _skip_test_for_model_version(config, model_version_requested):
 
 def _skip_test_for_production_db(config):
     """Skip test if production db is used."""
-    if not config.get("SKIP_FOR_PRODUCTION_DB"):
+    if not config.get("skip_for_production_db"):
         return
 
     if "db.zeuthen.desy.de" in os.getenv("SIMTOOLS_DB_SERVER", ""):
@@ -184,19 +183,18 @@ def _prepare_test_options(config, output_path, model_version=None):
         return None, "--" + next(iter(config.keys())).lower(), None
 
     tmp_config_file = output_path / "tmp_config.yml"
-    config_file_model_version = config.get("MODEL_VERSION")
-    if model_version and "MODEL_VERSION" in config:
-        config.update({"MODEL_VERSION": model_version})
+    config_file_model_version = config.get("model_version")
+    if model_version and "model_version" in config:
+        config.update({"model_version": model_version})
 
-    for key in ["OUTPUT_PATH", "DATA_DIRECTORY", "PACK_FOR_GRID_REGISTER"]:
+    for key in ["output_path", "data_directory", "pack_for_grid_register"]:
         if key in config:
             config[key] = str(Path(output_path).joinpath(config[key]))
-            if key == "OUTPUT_PATH":
-                config["USE_PLAIN_OUTPUT_PATH"] = True
+            if key == "output_path":
+                config["use_plain_output_path"] = True
 
     _logger.info(f"Writing config file: {tmp_config_file}")
-    with open(tmp_config_file, "w", encoding="utf-8") as file:
-        yaml.safe_dump(config, file, sort_keys=False)
+    ascii_handler.write_data_to_file(data=config, output_file=tmp_config_file, sort_keys=False)
 
     return tmp_config_file, None, config_file_model_version
 
@@ -245,7 +243,7 @@ def create_tmp_output_path(tmp_test_directory, config):
     """
     try:
         tmp_output_path = Path(tmp_test_directory).joinpath(
-            config["APPLICATION"] + "-" + config["TEST_NAME"]
+            config["application"] + "-" + config["test_name"]
         )
     except KeyError as exc:
         raise KeyError(f"No application defined in configuration {config}.") from exc

simtools/testing/helpers.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 def skip_camera_efficiency(config):
     """Skip camera efficiency tests if the old version of testeff is used."""
-    if "camera-efficiency" in config["APPLICATION"] and not _new_testeff_version():
+    if "camera-efficiency" in config["application"] and not _new_testeff_version():
         return (
             "Any applications calling the old version of testeff are skipped "
             "due to a limitation of the old testeff not allowing to specify "
@@ -42,7 +42,7 @@ def skip_multiple_version_test(config, model_version):
     if not isinstance(model_version, list):
         return None
 
-    config_model_version = config.get("CONFIGURATION", {}).get("MODEL_VERSION", [])
+    config_model_version = config.get("configuration", {}).get("model_version", [])
 
     if not isinstance(config_model_version, list):
         config_model_version = [config_model_version]