gammasimtools 0.6.1__py3-none-any.whl → 0.8.2__py3-none-any.whl

simtools/runners/corsika_runner.py

@@ -0,0 +1,240 @@
+"""Generate run scripts and directories for CORSIKA simulations."""
+
+import logging
+import os
+from pathlib import Path
+
+from simtools.io_operations import io_handler
+from simtools.runners.runner_services import RunnerServices
+
+__all__ = ["CorsikaRunner", "MissingRequiredEntryInCorsikaConfigError"]
+
+
+class MissingRequiredEntryInCorsikaConfigError(Exception):
+    """Exception for missing required entry in corsika config."""
+
+
+class CorsikaRunner:
+    """
+    Generate run scripts and directories for CORSIKA simulations. Run simulations if requested.
+
+    CorsikaRunner is responsible for configuring and running CORSIKA, using corsika_autoinputs
+    provided by the sim_telarray package. CorsikaRunner generates shell scripts to be run
+    externally or by the simulator module simulator.
+
+    CorsikaRunner is configured through a CorsikaConfig instance.
+
+    Parameters
+    ----------
+    corsika_config_data: CorsikaConfig
+        CORSIKA configuration.
+    simtel_path: str or Path
+        Location of source of the sim_telarray/CORSIKA package.
+    label: str
+        Instance label.
+    keep_seeds: bool
+        Use seeds based on run number and primary particle. If False, use sim_telarray seeds.
+    use_multipipe: bool
+        Use multipipe to run CORSIKA and sim_telarray.
+    """
+
+    def __init__(
+        self,
+        corsika_config,
+        simtel_path,
+        label=None,
+        keep_seeds=False,
+        use_multipipe=False,
+    ):
+        """Initialize CorsikaRunner."""
+        self._logger = logging.getLogger(__name__)
+        self._logger.debug("Init CorsikaRunner")
+        self.label = label
+
+        self.corsika_config = corsika_config
+        self._keep_seeds = keep_seeds
+        self._use_multipipe = use_multipipe
+
+        self._simtel_path = Path(simtel_path)
+        self.io_handler = io_handler.IOHandler()
+
+        self.runner_service = RunnerServices(corsika_config, label)
+        self._directory = self.runner_service.load_data_directories("corsika")
+
+    def prepare_run_script(
+        self, run_number=None, extra_commands=None, input_file=None, use_pfp=True
+    ):
+        """
+        Get the full path of the run script file for a given run number.
+
+        Parameters
+        ----------
+        use_pfp: bool
+            Whether to use the preprocessor in preparing the CORSIKA input file
+        run_number: int
+            Run number.
+        extra_commands: str
+            Additional commands for running simulations.
+
+        Returns
+        -------
+        Path:
+            Full path of the run script file.
+        """
+        if input_file is not None:
+            self._logger.warning(
+                "input_file parameter is not used in CorsikaRunner.prepare_run_script"
+            )
+        self.corsika_config.run_number = run_number
+
+        script_file_path = self.get_file_name(
+            file_type="sub_script", run_number=self.corsika_config.run_number
+        )
+        corsika_input_file = self.corsika_config.generate_corsika_input_file(
+            use_multipipe=self._use_multipipe, use_test_seeds=self._keep_seeds
+        )
+
+        # CORSIKA input file for a specific run, created by the preprocessor pfp
+        corsika_input_tmp_name = self.corsika_config.get_corsika_config_file_name(
+            file_type="config_tmp", run_number=self.corsika_config.run_number
+        )
+        corsika_input_tmp_file = self._directory["inputs"].joinpath(corsika_input_tmp_name)
+
+        if use_pfp:
+            pfp_command = self._get_pfp_command(corsika_input_tmp_file, corsika_input_file)
+        autoinputs_command = self._get_autoinputs_command(
+            self.corsika_config.run_number, corsika_input_tmp_file
+        )
+
+        self._logger.debug(f"Extra commands to be added to the run script: {extra_commands}")
+        self._logger.debug(f"CORSIKA data will be set to {self._directory['data']}")
+
+        with open(script_file_path, "w", encoding="utf-8") as file:
+            file.write("#!/usr/bin/env bash\n")
+            file.write("set -e\n")
+            file.write("set -o pipefail\n")
+
+            # Setting SECONDS variable to measure runtime
+            file.write("\nSECONDS=0\n")
+
+            if extra_commands is not None:
+                file.write("\n# Writing extras\n")
+                file.write(f"{extra_commands}\n")
+                file.write("# End of extras\n\n")
+
+            file.write(f"export CORSIKA_DATA={self._directory['data']}\n")
+            file.write('mkdir -p "$CORSIKA_DATA"\n')
+            file.write('cd "$CORSIKA_DATA" || exit 2\n')
+            if use_pfp:
+                file.write("\n# Running pfp\n")
+                file.write(pfp_command)
+                file.write("\n# Replacing the XXXXXX placeholder with the run number\n")
+                file.write(
+                    f"sed -i 's/XXXXXX/{self.corsika_config.run_number:06}/g' "
+                    f"{corsika_input_tmp_file}\n"
+                )
+            else:
+                file.write("\n# Copying CORSIKA input file to run location\n")
+                file.write(f"cp {corsika_input_file} {corsika_input_tmp_file}")
+            file.write("\n# Running corsika_autoinputs\n")
+            file.write(autoinputs_command)
+
+            file.write('\necho "RUNTIME: $SECONDS"\n')
+
+        os.system(f"chmod ug+x {script_file_path}")
+
+        return script_file_path
+
+    def get_resources(self, run_number=None):
+        """Return computing resources used."""
+        return self.runner_service.get_resources(run_number)
+
+    def _get_pfp_command(self, input_tmp_file, corsika_input_file):
+        """
+        Get pfp pre-processor command.
+
+        pfp is a pre-processor tool and part of sim_telarray.
+
+        Parameters
+        ----------
+        input_tmp_file: Path
+            Temporary input file.
+
+        Returns
+        -------
+        str
+            pfp command.
+        """
+        cmd = self._simtel_path.joinpath("sim_telarray/bin/pfp")
+        cmd = str(cmd) + f" -V -DWITHOUT_MULTIPIPE - < {corsika_input_file}"
+        cmd += f" > {input_tmp_file} || exit\n"
+        return cmd
+
+    def _get_autoinputs_command(self, run_number, input_tmp_file):
+        """
+        Get autoinputs command.
+
+        corsika_autoinputs is a tool to generate random and user/host dependent
+        parameters for CORSIKA configuration.
+
+        Parameters
+        ----------
+        run_number: int
+            Run number.
+        input_tmp_file: Path
+            Temporary input file.
+
+        Returns
+        -------
+        str
+            autoinputs command.
+        """
+        corsika_bin_path = self._simtel_path.joinpath("corsika-run/corsika")
+
+        log_file = self.get_file_name(file_type="log", run_number=run_number)
+        if self._use_multipipe:
+            log_file = log_file.with_name(f"multipipe_{log_file.name}")
+
+        cmd = self._simtel_path.joinpath("sim_telarray/bin/corsika_autoinputs")
+        cmd = str(cmd) + f" --run {corsika_bin_path}"
+        cmd += f" -R {run_number}"
+        cmd += ' -p "$CORSIKA_DATA"'
+        if self._keep_seeds:
+            logging.warning(
+                "Using --keep-seeds option in corsika_autoinputs is not recommended. "
+                "It should only be used for testing purposes."
+            )
+            cmd += " --keep-seeds"
+        cmd += f" {input_tmp_file} | gzip > {log_file} 2>&1"
+        cmd += " || exit 1\n"
+        return cmd
+
+    def get_file_name(
+        self, simulation_software="corsika", file_type=None, run_number=None, mode=""
+    ):
+        """
+        Get the full path of a file for a given run number.
+
+        Parameters
+        ----------
+        simulation_software: str
+            Simulation software.
+        file_type: str
+            File type.
+        run_number: int
+            Run number.
+
+        Returns
+        -------
+        str
+            File name with full path.
+        """
+        if simulation_software.lower() != "corsika":
+            raise ValueError(
+                f"simulation_software ({simulation_software}) is not supported in CorsikaRunner"
+            )
+        return self.runner_service.get_file_name(
+            file_type=file_type,
+            run_number=run_number,
+            mode=mode,
+        )

simtools/runners/corsika_simtel_runner.py

@@ -0,0 +1,225 @@
+"""Run simulations with CORSIKA and pipe it to sim_telarray using the multipipe functionality."""
+
+import logging
+import stat
+from pathlib import Path
+
+from simtools.runners.corsika_runner import CorsikaRunner
+from simtools.simtel.simulator_array import SimulatorArray
+
+__all__ = ["CorsikaSimtelRunner"]
+
+
+class CorsikaSimtelRunner:
+    """
+    Run simulations with CORSIKA and pipe it to sim_telarray using the multipipe functionality.
+
+    Uses CorsikaConfig to manage the CORSIKA configuration and SimulatorArray
+    for the sim_telarray configuration.
+
+    Parameters
+    ----------
+    corsika_config : CorsikaConfig
+        Configuration parameters for CORSIKA.
+    simtel_path : str or Path
+        Location of the sim_telarray package.
+    label : str
+        Label.
+    keep_seeds : bool
+        Use seeds based on run number and primary particle. If False, use sim_telarray seeds.
+    use_multipipe : bool
+        Use multipipe to run CORSIKA and sim_telarray.
+    """
+
+    def __init__(
+        self,
+        corsika_config,
+        simtel_path,
+        label=None,
+        keep_seeds=False,
+        use_multipipe=False,
+        sim_telarray_seeds=None,
+    ):
+        self._logger = logging.getLogger(__name__)
+        self.corsika_config = corsika_config
+        self._simtel_path = simtel_path
+        self.sim_telarray_seeds = sim_telarray_seeds
+        self.label = label
+
+        self.corsika_config.set_output_file_and_directory(use_multipipe)
+        self.corsika_runner = CorsikaRunner(
+            corsika_config=corsika_config,
+            simtel_path=simtel_path,
+            label=label,
+            keep_seeds=keep_seeds,
+            use_multipipe=use_multipipe,
+        )
+        self.simulator_array = SimulatorArray(
+            corsika_config=corsika_config,
+            simtel_path=simtel_path,
+            label=label,
+            use_multipipe=use_multipipe,
+            sim_telarray_seeds=sim_telarray_seeds,
+        )
+
+    def prepare_run_script(
+        self, run_number=None, input_file=None, extra_commands=None, use_pfp=False
+    ):
+        """
+        Get the full path of the run script file for a given run number.
+
+        Parameters
+        ----------
+        run_number: int
+            Run number.
+        use_pfp: bool
+            Whether to use the preprocessor in preparing the CORSIKA input file
+
+        Returns
+        -------
+        Path:
+            Full path of the run script file.
+        """
+        self._export_multipipe_script(run_number)
+        return self.corsika_runner.prepare_run_script(
+            run_number=run_number,
+            input_file=input_file,
+            extra_commands=extra_commands,
+            use_pfp=use_pfp,
+        )
+
+    def _export_multipipe_script(self, run_number):
+        """
+        Write the multipipe script used in piping CORSIKA to sim_telarray.
+
+        Parameters
+        ----------
+        run_number: int
+            Run number.
+
+        Returns
+        -------
+        Path:
+            Full path of the run script file.
+        """
+        run_command = self._make_run_command(
+            run_number=run_number,
+            input_file="-",  # instruct sim_telarray to take input from standard output
+        )
+        multipipe_file = Path(self.corsika_config.config_file_path.parent).joinpath(
+            self.corsika_config.get_corsika_config_file_name("multipipe")
+        )
+        with open(multipipe_file, "w", encoding="utf-8") as file:
+            file.write(f"{run_command}")
+        self._logger.info(f"Multipipe script: {multipipe_file}")
+        self._write_multipipe_script(multipipe_file)
+
+    def _write_multipipe_script(self, multipipe_file):
+        """
+        Write script used to call the multipipe_corsika command.
+
+        Parameters
+        ----------
+        multipipe_file: str or Path
+            The name of the multipipe file which contains all of the multipipe commands.
+        """
+        multipipe_script = Path(self.corsika_config.config_file_path.parent).joinpath(
+            "run_cta_multipipe"
+        )
+        with open(multipipe_script, "w", encoding="utf-8") as file:
+            multipipe_command = Path(self._simtel_path).joinpath(
+                "sim_telarray/bin/multipipe_corsika "
+                f"-c {multipipe_file}"
+                " || echo 'Fan-out failed'"
+            )
+            file.write(f"{multipipe_command}")
+
+        multipipe_script.chmod(multipipe_script.stat().st_mode | stat.S_IEXEC)
+
+    def _make_run_command(self, run_number=None, input_file=None):
+        """
+        Build and return the command to run simtel_array.
+
+        Parameters
+        ----------
+        run_number: int
+            Run number.
+        input_file: str
+            Full path of the input CORSIKA file.
+            Use '-' to tell sim_telarray to read from standard output
+
+        Returns
+        -------
+        str:
+            Command to run sim_telarray.
+        """
+        try:
+            weak_pointing = any(pointing in self.label for pointing in ["divergent", "convergent"])
+        except TypeError:  # allow for self.label to be None
+            weak_pointing = False
+
+        command = str(self._simtel_path.joinpath("sim_telarray/bin/sim_telarray"))
+        command += f" -c {self.corsika_config.array_model.get_config_file()}"
+        command += f" -I{self.corsika_config.array_model.get_config_directory()}"
+        command += self.simulator_array.get_config_option(
+            "telescope_theta", self.corsika_config.zenith_angle, weak_option=weak_pointing
+        )
+        command += self.simulator_array.get_config_option(
+            "telescope_phi", self.corsika_config.azimuth_angle, weak_option=weak_pointing
+        )
+        command += self.simulator_array.get_config_option(
+            "power_law",
+            SimulatorArray.get_power_law_for_sim_telarray_histograms(
+                self.corsika_config.primary_particle
+            ),
+        )
+        command += self.simulator_array.get_config_option(
+            "histogram_file",
+            self.get_file_name(
+                simulation_software="simtel", file_type="histogram", run_number=run_number
+            ),
+        )
+        command += self.simulator_array.get_config_option("random_state", "none")
+        if self.sim_telarray_seeds:
+            command += self.simulator_array.get_config_option(
+                "random_seed", self.sim_telarray_seeds
+            )
+        command += self.simulator_array.get_config_option("show", "all")
+        command += self.simulator_array.get_config_option(
+            "output_file",
+            self.simulator_array.get_file_name(
+                simulation_software="simtel", file_type="output", run_number=run_number
+            ),
+        )
+        command += f" {input_file}"
+        _log_file = self.simulator_array.get_file_name(
+            simulation_software="simtel", file_type="log", run_number=run_number
+        )
+        command += f" | gzip > {_log_file} 2>&1 || exit"
+
+        return command
+
+    def get_file_name(self, simulation_software=None, file_type=None, run_number=None, mode=None):
+        """
+        Get the full path of a file for a given run number.
+
+        Parameters
+        ----------
+        simulation_software: str
+            Simulation software.
+        file_type: str
+            File type.
+        run_number: int
+            Run number.
+
+        Returns
+        -------
+        str
+            File name with full path.
+        """
+        if simulation_software is None:
+            # preference to simtel output (multipipe)
+            simulation_software = "simtel" if self.simulator_array else "corsika"
+
+        runner = self.corsika_runner if simulation_software == "corsika" else self.simulator_array
+        return runner.get_file_name(file_type=file_type, run_number=run_number, mode=mode)