gammasimtools 0.6.1__py3-none-any.whl → 0.8.1__py3-none-any.whl

simtools/runners/runner_services.py

@@ -0,0 +1,307 @@
+"""Base service methods for simulation runners."""
+
+import logging
+from pathlib import Path
+
+from simtools.io_operations import io_handler
+
+_logger = logging.getLogger(__name__)
+
+
+class RunnerServices:
+    """
+    Base services for simulation runners.
+
+    Parameters
+    ----------
+    corsika_config : CorsikaConfig
+        Configuration parameters for CORSIKA.
+    label : str
+        Label.
+    """
+
+    def __init__(self, corsika_config, label=None):
+        """Initialize RunnerServices."""
+        self._logger = logging.getLogger(__name__)
+        self._logger.debug("Init RunnerServices")
+        self.label = label
+        self.corsika_config = corsika_config
+        self.directory = {}
+
+    def _get_info_for_file_name(self, run_number):
+        """
+        Return dictionary for building names for simulation output files.
+
+        Parameters
+        ----------
+        run_number : int
+            Run number.
+
+        Returns
+        -------
+        dict
+            Dictionary with the keys or building the file names for simulation output files.
+        """
+        _vc_high = self.corsika_config.get_config_parameter("VIEWCONE")[1]
+        primary_name = self.corsika_config.primary
+        if primary_name == "gamma" and _vc_high > 0:
+            primary_name = "gamma_diffuse"
+        return {
+            "run_number": self.corsika_config.validate_run_number(run_number),
+            "primary": primary_name,
+            "array_name": self.corsika_config.array_model.layout_name,
+            "site": self.corsika_config.array_model.site,
+            "label": self.label,
+            "zenith": self.corsika_config.zenith_angle,
+            "azimuth": self.corsika_config.azimuth_angle,
+        }
+
+    @staticmethod
+    def _get_simulation_software_list(simulation_software):
+        """
+        Return a list of simulation software based on the input string.
+
+        Args:
+            simulation_software: String representing the desired software.
+
+        Returns
+        -------
+            List of simulation software names.
+        """
+        software_map = {
+            "corsika": ["corsika"],
+            "simtel": ["simtel"],
+            "corsika_simtel": ["corsika", "simtel"],
+        }
+        return software_map.get(simulation_software.lower(), [])
+
+    def load_data_directories(self, simulation_software):
+        """
+        Create and return directories for output, data, log and input.
+
+        Parameters
+        ----------
+        simulation_software : str
+            Simulation software to be used.
+
+        Returns
+        -------
+        dict
+            Dictionary containing paths requires for simulation configuration.
+        """
+        self.directory["output"] = io_handler.IOHandler().get_output_directory(self.label)
+        _logger.debug(f"Creating output dir {self.directory['output']}")
+        for dir_name in ["sub_scripts", "sub_logs"]:
+            self.directory[dir_name] = self.directory["output"].joinpath(dir_name)
+            self.directory[dir_name].mkdir(parents=True, exist_ok=True)
+        for _simulation_software in self._get_simulation_software_list(simulation_software):
+            for dir_name in ["data", "inputs", "logs"]:
+                self.directory[dir_name] = self.directory["output"].joinpath(
+                    _simulation_software, dir_name
+                )
+                self.directory[dir_name].mkdir(parents=True, exist_ok=True)
+        self._logger.debug(f"Data directories for {simulation_software}: {self.directory}")
+        return self.directory
+
+    def has_file(self, file_type, run_number=None, mode="out"):
+        """
+        Check that the file of file_type for the specified run number exists.
+
+        Parameters
+        ----------
+        file_type: str
+            File type to check.
+        run_number: int
+            Run number.
+
+        """
+        run_sub_file = self.get_file_name(file_type, run_number=run_number, mode=mode)
+        _logger.debug(f"Checking if {run_sub_file} exists")
+        return Path(run_sub_file).is_file()
+
+    def _get_file_basename(self, run_number):
+        """
+        Get the base name for the simulation files.
+
+        Parameters
+        ----------
+        run_number: int
+            Run number.
+
+        Returns
+        -------
+        str
+            Base name for the simulation files.
+        """
+        info_for_file_name = self._get_info_for_file_name(run_number)
+        file_label = f"_{info_for_file_name['label']}" if info_for_file_name.get("label") else ""
+        zenith = self.corsika_config.get_config_parameter("THETAP")[0]
+        azimuth = self.corsika_config.azimuth_angle
+        run_dir = self._get_run_number_string(info_for_file_name["run_number"])
+        return (
+            f"{run_dir}_{info_for_file_name['primary']}_"
+            f"za{round(zenith):02}deg_azm{azimuth:03}deg_"
+            f"{info_for_file_name['site']}_{info_for_file_name['array_name']}{file_label}"
+        )
+
+    def _get_log_file_path(self, file_type, file_name):
+        """
+        Return path for log files.
+
+        Parameters
+        ----------
+        file_type : str
+            File type.
+        file_name : str
+            File name.
+
+        Returns
+        -------
+        Path
+            Path for log files.
+        """
+        log_suffixes = {
+            "log": ".log.gz",
+            "histogram": ".hdata.zst",
+        }
+        return self.directory["logs"].joinpath(f"{file_name}{log_suffixes[file_type]}")
+
+    def _get_data_file_path(self, file_type, file_name, run_number):
+        """
+        Return path for data files.
+
+        Parameters
+        ----------
+        file_type : str
+            File type.
+        file_name : str
+            File name.
+        run_number : int
+            Run number.
+
+        Returns
+        -------
+        Path
+            Path for data files.
+        """
+        data_suffixes = {
+            "output": ".zst",
+            "corsika_output": ".zst",
+            "corsika_log": ".log",
+            "simtel_output": ".simtel.zst",
+        }
+        run_dir = self._get_run_number_string(run_number)
+        data_run_dir = self.directory["data"].joinpath(run_dir)
+        data_run_dir.mkdir(parents=True, exist_ok=True)
+        return data_run_dir.joinpath(f"{file_name}{data_suffixes[file_type]}")
+
+    def _get_sub_file_path(self, file_type, file_name, mode):
+        """
+        Return path for submission files.
+
+        Parameters
+        ----------
+        file_type : str
+            File type.
+        file_name : str
+            File name.
+        mode : str
+            Mode (out or err).
+
+        Returns
+        -------
+        Path
+            Path for submission files.
+        """
+        suffix = ".log" if file_type == "sub_log" else ".sh"
+        if mode and mode != "":
+            suffix = f".{mode}"
+        sub_log_file_dir = self.directory["output"].joinpath(f"{file_type}s")
+        sub_log_file_dir.mkdir(parents=True, exist_ok=True)
+        return sub_log_file_dir.joinpath(f"sub_{file_name}{suffix}")
+
+    def get_file_name(self, file_type, run_number=None, mode=None):
+        """
+        Get a CORSIKA/sim_telarray style file name for various log and data file types.
+
+        Parameters
+        ----------
+        file_type : str
+            The type of file (determines the file suffix).
+        run_number : int
+            Run number.
+        mode: str
+            out or err (optional, relevant only for sub_log).
+
+        Returns
+        -------
+        str
+            File name with full path.
+
+        Raises
+        ------
+        ValueError
+            If file_type is unknown.
+        """
+        file_name = self._get_file_basename(run_number)
+
+        if file_type in ["log", "histogram"]:
+            return self._get_log_file_path(file_type, file_name)
+
+        if file_type in ["output", "corsika_output", "corsika_log", "simtel_output"]:
+            return self._get_data_file_path(file_type, file_name, run_number)
+
+        if file_type in ("sub_log", "sub_script"):
+            return self._get_sub_file_path(file_type, file_name, mode)
+
+        raise ValueError(f"The requested file type ({file_type}) is unknown")
+
+    @staticmethod
+    def _get_run_number_string(run_number):
+        """
+        Get run number string as used for the simulation file names(ex. run000014).
+
+        Parameters
+        ----------
+        run_number: int
+            Run number.
+
+        Returns
+        -------
+        str
+            Run number string.
+        """
+        nn = str(run_number)
+        if len(nn) > 6:
+            raise ValueError("Run number cannot have more than 6 digits")
+        return "run" + nn.zfill(6)
+
+    def get_resources(self, run_number=None):
+        """
+        Read run time of job from last line of submission log file.
+
+        Parameters
+        ----------
+        run_number: int
+            Run number.
+
+        Returns
+        -------
+        dict
+            run time and number of simulated events
+        """
+        sub_log_file = self.get_file_name(file_type="sub_log", run_number=run_number, mode="out")
+        _logger.debug(f"Reading resources from {sub_log_file}")
+
+        _resources = {}
+        _resources["runtime"] = None
+        with open(sub_log_file, encoding="utf-8") as file:
+            for line in reversed(list(file)):
+                if "RUNTIME" in line:
+                    _resources["runtime"] = int(line.split()[1])
+                    break
+
+        if _resources["runtime"] is None:
+            _logger.debug("RUNTIME was not found in run log file")
+        _resources["n_events"] = int(self.corsika_config.get_config_parameter("NSHOW"))
+        return _resources

simtools/runners/simtel_runner.py

@@ -0,0 +1,224 @@
+"""Base class for running sim_telarray simulations."""
+
+import logging
+import os
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.runners.runner_services import RunnerServices
+
+__all__ = ["InvalidOutputFileError", "SimtelExecutionError", "SimtelRunner"]
+
+
+class SimtelExecutionError(Exception):
+    """Exception for simtel_array execution error."""
+
+
+class InvalidOutputFileError(Exception):
+    """Exception for invalid output file."""
+
+
+class SimtelRunner:
+    """
+    Base class for running sim_telarray simulations.
+
+    Parameters
+    ----------
+    simtel_path: str or Path
+        Location of sim_telarray installation.
+    label: str
+        Instance label. Important for output file naming.
+    """
+
+    def __init__(self, simtel_path, label=None, corsika_config=None, use_multipipe=False):
+        """Initialize SimtelRunner."""
+        self._logger = logging.getLogger(__name__)
+
+        self._simtel_path = Path(simtel_path)
+        self.label = label
+        self._base_directory = None
+
+        self.runs_per_set = 1
+
+        self.runner_service = RunnerServices(corsika_config, label)
+        self._directory = self.runner_service.load_data_directories(
+            "corsika_simtel" if use_multipipe else "simtel"
+        )
+
+    def __repr__(self):
+        """Return a string representation of the SimtelRunner object."""
+        return f"SimtelRunner(label={self.label})\n"
+
+    def prepare_run_script(self, test=False, input_file=None, run_number=None, extra_commands=None):
+        """
+        Build and return the full path of the bash run script containing the sim_telarray command.
+
+        Parameters
+        ----------
+        test: bool
+            Test flag for faster execution.
+        input_file: str or Path
+            Full path of the input CORSIKA file.
+        run_number: int
+            Run number.
+        extra_commands: str
+            Additional commands for running simulations given in config.yml.
+
+        Returns
+        -------
+        Path
+            Full path of the run script.
+        """
+        self._logger.debug("Creating run bash script")
+
+        script_file_path = self.get_file_name(file_type="sub_script", run_number=run_number)
+
+        self._logger.debug(f"Run bash script - {script_file_path}")
+
+        self._logger.debug(f"Extra commands to be added to the run script {extra_commands}")
+
+        command = self._make_run_command(run_number=run_number, input_file=input_file)
+        with script_file_path.open("w", encoding="utf-8") as file:
+            file.write("#!/usr/bin/env bash\n\n")
+            file.write("set -e\n")
+            file.write("set -o pipefail\n")
+            file.write("\nSECONDS=0\n")
+
+            if extra_commands is not None:
+                file.write("# Writing extras\n")
+                for line in extra_commands:
+                    file.write(f"{line}\n")
+                file.write("# End of extras\n\n")
+
+            n = 1 if test else self.runs_per_set
+            for _ in range(n):
+                file.write(f"{command}\n\n")
+
+            file.write('\necho "RUNTIME: $SECONDS"\n')
+
+        os.system(f"chmod ug+x {script_file_path}")
+        return script_file_path
+
+    def run(self, test=False, input_file=None, run_number=None):
+        """
+        Make run command and run sim_telarray.
+
+        Parameters
+        ----------
+        test: bool
+            If True, make simulations faster.
+        input_file: str or Path
+            Full path of the input CORSIKA file.
+        run_number: int
+            Run number.
+        """
+        self._logger.debug("Running sim_telarray")
+
+        command = self._make_run_command(run_number=run_number, input_file=input_file)
+
+        if test:
+            self._logger.info(f"Running (test) with command: {command}")
+            self._run_simtel_and_check_output(command)
+        else:
+            self._logger.debug(f"Running ({self.runs_per_set}x) with command: {command}")
+            for _ in range(self.runs_per_set):
+                self._run_simtel_and_check_output(command)
+
+        self._check_run_result(run_number=run_number)
+
+    def _check_run_result(self, run_number=None):  # pylint: disable=all
+        """Check if simtel output file exists."""
+        pass
+
+    def _raise_simtel_error(self):
+        """
+        Raise sim_telarray execution error.
+
+        Final 30 lines from the log file are collected and printed.
+
+        Raises
+        ------
+        SimtelExecutionError
+        """
+        if hasattr(self, "_log_file"):
+            msg = gen.get_log_excerpt(self._log_file)
+        else:
+            msg = "Simtel log file does not exist."
+
+        self._logger.error(msg)
+        raise SimtelExecutionError(msg)
+
+    def _run_simtel_and_check_output(self, command):
+        """
+        Run the sim_telarray command and check the exit code.
+
+        Raises
+        ------
+        SimtelExecutionError
+            if run was not successful.
+        """
+        if os.system(command) != 0:
+            self._raise_simtel_error()
+
+    def _make_run_command(self, run_number=None, input_file=None):
+        self._logger.debug(
+            "make_run_command is being called from the base class - "
+            "it should be implemented in the sub class"
+        )
+        input_file = input_file if input_file else "nofile"
+        run_number = run_number if run_number else 1
+        return f"{input_file}-{run_number}"
+
+    @staticmethod
+    def get_config_option(par, value=None, weak_option=False):
+        """
+        Build sim_telarray command.
+
+        Parameters
+        ----------
+        par: str
+            Parameter name.
+        value: str
+            Parameter value.
+        weak_option: bool
+            If True, use -W option instead of -C.
+
+        Returns
+        -------
+        str
+            Command for sim_telarray.
+        """
+        option_syntax = "-W" if weak_option else "-C"
+        c = f" {option_syntax} {par}"
+        c += f"={value}" if value is not None else ""
+        return c
+
+    def get_resources(self, run_number=None):
+        """Return computing resources used."""
+        return self.runner_service.get_resources(run_number)
+
+    def get_file_name(self, simulation_software="simtel", file_type=None, run_number=None, mode=""):
+        """
+        Get the full path of a file for a given run number.
+
+        Parameters
+        ----------
+        simulation_software: str
+            Simulation software.
+        file_type: str
+            File type.
+        run_number: int
+            Run number.
+
+        Returns
+        -------
+        str
+            File name with full path.
+        """
+        if simulation_software.lower() != "simtel":
+            raise ValueError(
+                f"simulation_software ({simulation_software}) is not supported in SimulatorArray"
+            )
+        return self.runner_service.get_file_name(
+            file_type=file_type, run_number=run_number, mode=mode
+        )

simtools/schemas/array_elements.yml

@@ -0,0 +1,137 @@
+%YAML 1.2
+---
+title: List of allowed array elements
+version: 0.1.0
+description: |
+  List of array elements known to simtools and its
+  corresponding database collections.
+  Also used as part of the schema validation process.
+data:
+  LSTN:
+    collection: "telescopes"
+    observatory: "CTAO"
+    site: "North"
+  MSTN:
+    collection: "telescopes"
+    observatory: "CTAO"
+    site: "North"
+  LSTS:
+    collection: "telescopes"
+    observatory: "CTAO"
+    site: "South"
+  MSTS:
+    collection: "telescopes"
+    observatory: "CTAO"
+    site: "South"
+  SSTS:
+    collection: "telescopes"
+    observatory: "CTAO"
+    site: "South"
+    description: "Small-sized telescope"
+  SCTS:
+    collection: "telescopes"
+    observatory: "CTAO"
+    site: "South"
+    description: "Mid-sized Schwarzschild-Couder telescope"
+  ILLN:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "North"
+    description: "Illuminator"
+  RLDN:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "North"
+    description: "Raman lidar"
+  STPN:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "North"
+    description: "Wide-field stellar photometer"
+  MSPN:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "North"
+    description: "Moon/sun photometer"
+  CEIN:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "North"
+    description: "Ceilometer"
+  WSTN:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "North"
+    description: "Weather station"
+  ASCN:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "North"
+    description: "All-sky camera"
+  DUSN:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "North"
+    description: "Dust sensor"
+  LISN:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "North"
+    description: "Lightning sensor"
+  ILLS:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "South"
+    description: "Illuminator"
+  RLDS:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "South"
+    description: "Raman lidar"
+  STPS:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "South"
+    description: "Wide-field stellar photometer"
+  MSPS:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "South"
+    description: "Moon/sun photometer"
+  CEIS:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "South"
+    description: "Ceilometer"
+  WSTS:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "South"
+    description: "Weather station"
+  ASCS:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "South"
+    description: "All-sky camera"
+  DUSS:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "South"
+    description: "Dust sensor"
+  LISS:
+    collection: "calibration_devices"
+    observatory: "CTAO"
+    site: "South"
+    description: "Lightning sensor"
+  HESS:
+    collection: "telescopes"
+    site: "South"
+    observatory: "HESS"
+  MAGIC:
+    collection: "telescopes"
+    site: "North"
+    observatory: "MAGIC"
+  VERITAS:
+    collection: "telescopes"
+    site: "North"
+    observatory: "VERITAS"