gammasimtools 0.25.0__py3-none-any.whl → 0.27.0__py3-none-any.whl

simtools/runners/corsika_runner.py

@@ -1,37 +1,24 @@
 """Generate run scripts and directories for CORSIKA simulations."""
 
 import logging
-import stat
 from pathlib import Path
 
-from simtools.io import io_handler
+from simtools import settings
 from simtools.runners.runner_services import RunnerServices
 
 
-class MissingRequiredEntryInCorsikaConfigError(Exception):
-    """Exception for missing required entry in corsika config."""
-
-
 class CorsikaRunner:
     """
-    Generate run scripts and directories for CORSIKA simulations. Run simulations if requested.
-
-    CorsikaRunner is responsible for configuring and running CORSIKA, using corsika_autoinputs
-    provided by the sim_telarray package. CorsikaRunner generates shell scripts to be run
-    externally or by the simulator module simulator.
+    Prepare and run CORSIKA simulations.
 
-    CorsikaRunner is configured through a CorsikaConfig instance.
+    Generate run scripts and directories for CORSIKA simulations. Run simulations if requested.
 
     Parameters
     ----------
     corsika_config_data: CorsikaConfig
         CORSIKA configuration.
-    simtel_path: str or Path
-        Location of source of the sim_telarray/CORSIKA package.
     label: str
         Instance label.
-    keep_seeds: bool
-        Use seeds based on run number and primary particle. If False, use sim_telarray seeds.
     use_multipipe: bool
         Use multipipe to run CORSIKA and sim_telarray.
     curved_atmosphere_min_zenith_angle: Quantity
@@ -41,9 +28,7 @@ class CorsikaRunner:
     def __init__(
         self,
         corsika_config,
-        simtel_path,
         label=None,
-        keep_seeds=False,
         use_multipipe=False,
         curved_atmosphere_min_zenith_angle=None,
     ):
@@ -53,74 +38,63 @@ class CorsikaRunner:
         self.label = label
 
         self.corsika_config = corsika_config
-        self._keep_seeds = keep_seeds
         self._use_multipipe = use_multipipe
         self.curved_atmosphere_min_zenith_angle = curved_atmosphere_min_zenith_angle
 
-        self._simtel_path = Path(simtel_path)
-        self.io_handler = io_handler.IOHandler()
-
-        self.runner_service = RunnerServices(corsika_config, label)
-        self._directory = self.runner_service.load_data_directories("corsika")
+        self.runner_service = RunnerServices(corsika_config, run_type="corsika", label=label)
+        self.file_list = None
 
-    def prepare_run_script(
-        self, run_number=None, extra_commands=None, input_file=None, use_pfp=True
-    ):
+    def prepare_run(self, run_number, sub_script, extra_commands=None, corsika_file=None):
         """
-        Prepare and write CORSIKA run script.
+        Prepare CORSIKA run script and run directory.
+
+        The CORSIKA run directory includes all input files needed for the simulation.
 
         Parameters
         ----------
-        use_pfp: bool
-            Whether to use the preprocessor in preparing the CORSIKA input file
         run_number: int
             Run number.
+        sub_script: str or Path
+            Path to the CORSIKA run script to be created.
+        corsika_file: str or Path
+            Path to the multipipe script (used only if use_multipipe is True).
         extra_commands: str
             Additional commands for running simulations.
-
-        Returns
-        -------
-        Path:
-            Full path of the run script file.
         """
-        if input_file is not None:
-            self._logger.warning(
-                "input_file parameter is not used in CorsikaRunner.prepare_run_script"
-            )
-        self.corsika_config.run_number = run_number
-
-        script_file_path = self.get_file_name(
-            file_type="sub_script", run_number=self.corsika_config.run_number
-        )
-        corsika_input_file = self.corsika_config.generate_corsika_input_file(
-            use_multipipe=self._use_multipipe, use_test_seeds=self._keep_seeds
+        self.file_list = self.runner_service.load_files(run_number=run_number)
+
+        self.corsika_config.generate_corsika_input_file(
+            self._use_multipipe,
+            self.runner_service.get_file_name("corsika_input", run_number=run_number),
+            self.runner_service.get_file_name("corsika_output", run_number=run_number)
+            if not self._use_multipipe
+            else corsika_file,
         )
 
-        # CORSIKA input file for a specific run, created by the preprocessor pfp
-        corsika_input_tmp_name = self.corsika_config.get_corsika_config_file_name(
-            file_type="config_tmp", run_number=self.corsika_config.run_number
-        )
-        corsika_input_tmp_file = self._directory["inputs"].joinpath(corsika_input_tmp_name)
-        # CORSIKA log file naming (temporary and final)
-        corsika_log_tmp_file = (
-            self._directory["data"]
-            .joinpath(f"run{self.corsika_config.run_number:06}")
-            .joinpath(f"run{self.corsika_config.run_number}.log")
-        )
-        corsika_log_file = self.get_file_name(
-            file_type="corsika_log", run_number=self.corsika_config.run_number
-        )
+        self._logger.debug(f"Extra commands to be added to the run script: {extra_commands}")
 
-        if use_pfp:
-            pfp_command = self._get_pfp_command(corsika_input_tmp_file, corsika_input_file)
-        autoinputs_command = self._get_autoinputs_command(
-            self.corsika_config.run_number, corsika_input_tmp_file
-        )
+        corsika_run_dir = self.runner_service.get_file_name(
+            "corsika_output", run_number=run_number
+        ).parent
 
-        self._logger.debug(f"Extra commands to be added to the run script: {extra_commands}")
-        self._logger.debug(f"CORSIKA data will be set to {self._directory['data']}")
+        self._export_run_script(run_number, sub_script, corsika_run_dir, extra_commands)
 
-        with open(script_file_path, "w", encoding="utf-8") as file:
+    def _corsika_executable(self):
+        """Get the CORSIKA executable path."""
+        if self.corsika_config.use_curved_atmosphere:
+            self._logger.debug("Using curved-atmosphere CORSIKA binary.")
+            return Path(settings.config.corsika_exe_curved)
+        self._logger.debug("Using flat-atmosphere CORSIKA binary.")
+        return Path(settings.config.corsika_exe)
+
+    def _export_run_script(self, run_number, sub_script, corsika_run_dir, extra_commands):
+        """Export CORSIKA run script."""
+        corsika_log_file = self.runner_service.get_file_name(
+            "corsika_log", run_number=run_number
+        ).with_suffix("")  # remove .gz from log file
+        corsika_input = self.runner_service.get_file_name("corsika_input", run_number=run_number)
+        sub_script = Path(sub_script)
+        with open(sub_script, "w", encoding="utf-8") as file:
             file.write("#!/usr/bin/env bash\n")
             file.write("set -e\n")
             file.write("set -o pipefail\n")
@@ -133,136 +107,19 @@ class CorsikaRunner:
                 file.write(f"{extra_commands}\n")
                 file.write("# End of extras\n\n")
 
-            file.write(f"export CORSIKA_DATA={self._directory['data']}\n")
+            file.write(f"export CORSIKA_DATA={corsika_run_dir}\n")
             file.write('mkdir -p "$CORSIKA_DATA"\n')
             file.write('cd "$CORSIKA_DATA" || exit 2\n')
-            if use_pfp:
-                file.write("\n# Running pfp\n")
-                file.write(pfp_command)
-                file.write("\n# Replacing the XXXXXX placeholder with the run number\n")
-                file.write(
-                    f"sed -i 's/XXXXXX/{self.corsika_config.run_number:06}/g' "
-                    f"{corsika_input_tmp_file}\n"
-                )
-            else:
-                file.write("\n# Copying CORSIKA input file to run location\n")
-                file.write(f"cp {corsika_input_file} {corsika_input_tmp_file}")
-            file.write("\n# Running corsika_autoinputs\n")
-            file.write(autoinputs_command)
-            file.write("\n# Moving log files to the corsika log directory\n")
-            file.write(f"gzip {corsika_log_tmp_file}\n")
-            file.write(f"mv -v {corsika_log_tmp_file}.gz {corsika_log_file}\n")
-
-            file.write('\necho "RUNTIME: $SECONDS"\n')
-
-        script_file_path.chmod(script_file_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP)
-        return script_file_path
 
-    def get_resources(self, run_number=None):
-        """Return computing resources used."""
-        return self.runner_service.get_resources(run_number)
-
-    def _get_pfp_command(self, input_tmp_file, corsika_input_file):
-        """
-        Get pfp pre-processor command.
-
-        pfp is a pre-processor tool and part of sim_telarray.
-
-        Parameters
-        ----------
-        input_tmp_file: Path
-            Temporary input file.
-
-        Returns
-        -------
-        str
-            pfp command.
-        """
-        cmd = self._simtel_path.joinpath("sim_telarray/bin/pfp")
-        cmd = str(cmd) + f" -V -DWITHOUT_MULTIPIPE - < {corsika_input_file}"
-        cmd += f" > {input_tmp_file} || exit\n"
-        return cmd
-
-    def _get_autoinputs_command(self, run_number, input_tmp_file):
-        """
-        Get autoinputs command.
-
-        corsika_autoinputs is a tool to generate random seeds and user/host dependent
-        parameters for CORSIKA configuration.
-
-        Parameters
-        ----------
-        run_number: int
-            Run number.
-        input_tmp_file: Path
-            Temporary input file.
-
-        Returns
-        -------
-        str
-            autoinputs command.
-        """
-        if self.corsika_config.use_curved_atmosphere:
-            corsika_bin_path = self._simtel_path.joinpath("corsika-run/corsika-curved")
-            self._logger.debug("Using curved-atmosphere CORSIKA binary.")
-        else:
-            corsika_bin_path = self._simtel_path.joinpath("corsika-run/corsika")
-            self._logger.debug("Using flat-atmosphere CORSIKA binary.")
-
-        log_file = self.get_file_name(file_type="log", run_number=run_number)
-        if self._use_multipipe:
-            log_file = log_file.with_name(f"multipipe_{log_file.name}")
-
-        cmd = self._simtel_path.joinpath("sim_telarray/bin/corsika_autoinputs")
-        cmd = str(cmd) + f" --run {corsika_bin_path}"
-        cmd += f" -R {run_number}"
-        cmd += ' -p "$CORSIKA_DATA"'
-        if self._keep_seeds:
-            logging.warning(
-                "Using --keep-seeds option in corsika_autoinputs is not recommended. "
-                "It should only be used for testing purposes."
+            file.write("\n# Running corsika\n")
+            file.write(
+                f"{self._corsika_executable()} < {corsika_input} > {corsika_log_file} 2>&1\n"
             )
-            cmd += " --keep-seeds"
-        cmd += f" {input_tmp_file} | gzip > {log_file} 2>&1"
-        cmd += " || exit 1\n"
-        return cmd
+            file.write("\n# Cleanup\n")
+            file.write(f"gzip {corsika_log_file}\n")
 
-    def get_file_name(
-        self,
-        simulation_software="corsika",
-        file_type=None,
-        run_number=None,
-        mode="",
-        model_version_index=0,
-    ):
-        """
-        Get the full path of a file for a given run number.
-
-        Parameters
-        ----------
-        simulation_software: str
-            Simulation software.
-        file_type: str
-            File type.
-        run_number: int
-            Run number.
-        model_version_index: int
-            Index of the model version.
-            This is used to select the correct simulator_array instance in case
-            multiple array models are simulated.
+            file.write('\necho "RUNTIME: $SECONDS"\n')
 
-        Returns
-        -------
-        str
-            File name with full path.
-        """
-        if simulation_software.lower() != "corsika":
-            raise ValueError(
-                f"simulation_software ({simulation_software}) is not supported in CorsikaRunner"
-            )
-        return self.runner_service.get_file_name(
-            file_type=file_type,
-            run_number=run_number,
-            mode=mode,
-            _model_version_index=model_version_index,
-        )
+    def get_resources(self, sub_out_file):
+        """Return computing resources used."""
+        return self.runner_service.get_resources(sub_out_file)

simtools/runners/corsika_simtel_runner.py

@@ -2,10 +2,10 @@
 
 import logging
 import stat
-from pathlib import Path
 
 import simtools.utils.general as gen
-from simtools.runners.corsika_runner import CorsikaRunner
+from simtools import settings
+from simtools.runners import corsika_runner, runner_services, simtel_runner
 from simtools.simtel.simulator_array import SimulatorArray
 
 
@@ -21,49 +21,40 @@ class CorsikaSimtelRunner:
     corsika_config : CorsikaConfig or list of CorsikaConfig
         A list of "CorsikaConfig" instances which
         contain the CORSIKA configuration parameters.
-    simtel_path : str or Path
-        Location of the sim_telarray package.
     label : str
         Label.
-    keep_seeds : bool
-        Use seeds based on run number and primary particle. If False, use sim_telarray seeds.
     use_multipipe : bool
         Use multipipe to run CORSIKA and sim_telarray.
-    sim_telarray_seeds : dict
         Dictionary with configuration for sim_telarray random instrument setup.
-    calibration_config : dict
-        Configuration for the calibration of the sim_telarray data.
+    is_calibration_run : bool
+        Flag to indicate if this is a calibration run.
     """
 
     def __init__(
         self,
         corsika_config,
-        simtel_path,
         label=None,
-        keep_seeds=False,
-        use_multipipe=False,
-        sim_telarray_seeds=None,
         sequential=False,
-        calibration_config=None,
         curved_atmosphere_min_zenith_angle=None,
+        is_calibration_run=False,
     ):
         self._logger = logging.getLogger(__name__)
         self.corsika_config = gen.ensure_iterable(corsika_config)
         # the base corsika config is the one used to define the CORSIKA specific parameters.
         # The others are used for the array configurations.
         self.base_corsika_config = self.corsika_config[0]
-        self._simtel_path = simtel_path
-        self.sim_telarray_seeds = sim_telarray_seeds
         self.label = label
         self.sequential = "--sequential" if sequential else ""
 
-        self.base_corsika_config.set_output_file_and_directory(use_multipipe)
-        self.corsika_runner = CorsikaRunner(
+        self.runner_service = runner_services.RunnerServices(
+            self.base_corsika_config, run_type="multi_pipe", label=label
+        )
+        self.file_list = None
+
+        self.corsika_runner = corsika_runner.CorsikaRunner(
             corsika_config=self.base_corsika_config,
-            simtel_path=simtel_path,
             label=label,
-            keep_seeds=keep_seeds,
-            use_multipipe=use_multipipe,
+            use_multipipe=True,
             curved_atmosphere_min_zenith_angle=curved_atmosphere_min_zenith_angle,
         )
         # The simulator array should be defined for every CORSIKA configuration
@@ -73,17 +64,12 @@ class CorsikaSimtelRunner:
             self.simulator_array.append(
                 SimulatorArray(
                     corsika_config=_corsika_config,
-                    simtel_path=simtel_path,
                     label=label,
-                    use_multipipe=use_multipipe,
-                    sim_telarray_seeds=sim_telarray_seeds,
-                    calibration_config=calibration_config,
+                    is_calibration_run=is_calibration_run,
                 )
             )
 
-    def prepare_run_script(
-        self, run_number=None, input_file=None, extra_commands=None, use_pfp=False
-    ):
+    def prepare_run(self, run_number=None, sub_script=None, corsika_file=None, extra_commands=None):
         """
         Get the full path of the run script file for a given run number.
 
@@ -91,21 +77,24 @@ class CorsikaSimtelRunner:
         ----------
         run_number: int
             Run number.
-        use_pfp: bool
-            Whether to use the preprocessor in preparing the CORSIKA input file
-
-        Returns
-        -------
-        Path:
-            Full path of the run script file.
+        corsika_file: str or Path
+            Path to the CORSIKA input file.
+        extra_commands: str
+            Additional commands for running simulations.
         """
+        self.file_list = self.runner_service.load_files(run_number=run_number)
         self._export_multipipe_script(run_number)
-        return self.corsika_runner.prepare_run_script(
+        self.corsika_runner.prepare_run(
             run_number=run_number,
-            input_file=input_file,
+            sub_script=sub_script,
+            corsika_file=self.runner_service.get_file_name(
+                file_type="multi_pipe_script", run_number=run_number
+            )
+            if not corsika_file
+            else corsika_file,
             extra_commands=extra_commands,
-            use_pfp=use_pfp,
         )
+        self.update_file_list_from_runners()
 
     def _export_multipipe_script(self, run_number):
         """
@@ -121,44 +110,29 @@ class CorsikaSimtelRunner:
         Path:
             Full path of the run script file.
         """
-        multipipe_file = Path(self.base_corsika_config.config_file_path.parent).joinpath(
-            self.base_corsika_config.get_corsika_config_file_name("multipipe")
+        multipipe_file = self.runner_service.get_file_name(
+            "multi_pipe_config", run_number=run_number
         )
-
         with open(multipipe_file, "w", encoding="utf-8") as file:
             for simulator_array in self.simulator_array:
+                log_file = simulator_array.runner_service.get_file_name(
+                    file_type="sim_telarray_log", run_number=run_number
+                )
                 run_command = simulator_array.make_run_command(
                     run_number=run_number,
-                    input_file="-",  # instruct sim_telarray to take input from standard output
-                    weak_pointing=self._determine_pointing_option(self.label),
+                    input_file="-",  # instruct sim_telarray to take input from stdout
+                )
+                file.write(
+                    f"{simtel_runner.sim_telarray_env_as_string()} "
+                    + " ".join(run_command)
+                    + f" | gzip > {log_file} 2>&1\n"
                 )
-                file.write(f"{run_command}")
                 file.write("\n")
-        self._logger.info(f"Multipipe script: {multipipe_file}")
-        self._write_multipipe_script(multipipe_file)
-
-    @staticmethod
-    def _determine_pointing_option(label):
-        """
-        Determine the pointing option for sim_telarray.
 
-        Parameters
-        ----------
-        label: str
-            Label of the simulation.
-
-        Returns
-        -------
-        str:
-            Pointing option.
-        """
-        try:
-            return any(pointing in label for pointing in ["divergent", "convergent"])
-        except TypeError:  # allow for pointing_option to be None
-            pass
-        return False
+        self._logger.info(f"Multipipe script: {multipipe_file}")
+        self._write_multipipe_script(multipipe_file, run_number)
 
-    def _write_multipipe_script(self, multipipe_file):
+    def _write_multipipe_script(self, multipipe_file, run_number):
         """
         Write script used to call the multipipe_corsika command.
 
@@ -166,61 +140,56 @@ class CorsikaSimtelRunner:
         ----------
         multipipe_file: str or Path
             The name of the multipipe file which contains all of the multipipe commands.
+        run_number: int
+            Run number.
         """
-        multipipe_script = Path(self.base_corsika_config.config_file_path.parent).joinpath(
-            "run_cta_multipipe"
+        multipipe_script = self.runner_service.get_file_name(
+            "multi_pipe_script", run_number=run_number
         )
         with open(multipipe_script, "w", encoding="utf-8") as file:
-            multipipe_command = Path(self._simtel_path).joinpath(
-                f"sim_telarray/bin/multipipe_corsika -c {multipipe_file} {self.sequential} "
+            multipipe_command = settings.config.sim_telarray_path.joinpath(
+                f"bin/multipipe_corsika -c {multipipe_file} {self.sequential} "
                 "|| echo 'Fan-out failed'"
             )
             file.write(f"{multipipe_command}")
 
         multipipe_script.chmod(multipipe_script.stat().st_mode | stat.S_IEXEC)
 
-    def get_file_name(
-        self,
-        simulation_software=None,
-        file_type=None,
-        run_number=None,
-        mode=None,
-        model_version_index=0,
-    ):
+    def get_resources(self, run_number=None):
+        """Return computing resources used."""
+        return self.corsika_runner.get_resources(run_number)
+
+    def update_file_list_from_runners(self):
         """
-        Get the full path of a file for a given run number.
+        Get list of generated files (independent of type).
+
+        Includes file lists from all runners.
 
         Parameters
         ----------
-        simulation_software: str
-            Simulation software.
-        file_type: str
-            File type.
-        run_number: int
-            Run number.
-        mode: str
-            Mode to use for the file name.
-        model_version_index: int
-            Index of the model version.
-            This is used to select the correct simulator_array instance
-            in case multiple array models are simulated.
+        file_type : str
+            File type to be listed.
 
         Returns
         -------
-        str
-            File name with full path.
-        """
-        if simulation_software is None:
-            # preference to sim_telarray output (multipipe)
-            simulation_software = "sim_telarray" if self.simulator_array else "corsika"
-
-        runner = (
-            self.corsika_runner
-            if simulation_software == "corsika"
-            else self.simulator_array[model_version_index]
-        )
-        return runner.get_file_name(file_type=file_type, run_number=run_number, mode=mode)
+        list
+            List with the full path of all output files.
 
-    def get_resources(self, run_number=None):
-        """Return computing resources used."""
-        return self.corsika_runner.get_resources(run_number)
+        """
+        if self.file_list is None:
+            self.file_list = self.corsika_runner.file_list
+        else:
+            self.file_list.update(self.corsika_runner.file_list)
+
+        for simulator_array in self.simulator_array:
+            _tmp_list = simulator_array.file_list
+            for key, data in _tmp_list.items():
+                if key in self.file_list:
+                    # in case of multiple sim_telarray instances, make list of files
+                    if not isinstance(self.file_list[key], list):
+                        self.file_list[key] = [self.file_list[key]]
+                    self.file_list[key].append(data)
+                else:
+                    self.file_list[key] = [data]
+
+        return self.file_list