gammasimtools 0.25.0__py3-none-any.whl → 0.27.0__py3-none-any.whl

simtools/runners/simtel_runner.py

@@ -1,19 +1,18 @@
 """Base class for running sim_telarray simulations."""
 
 import logging
-import subprocess
-from pathlib import Path
 
-import simtools.utils.general as gen
+from simtools.job_execution import job_manager
 from simtools.runners.runner_services import RunnerServices
 
+SIM_TELARRAY_ENV = {
+    "SIM_TELARRAY_CONFIG_PATH": "",
+}
 
-class SimtelExecutionError(Exception):
-    """Exception for sim_telarray execution error."""
 
-
-class InvalidOutputFileError(Exception):
-    """Exception for invalid output file."""
+def sim_telarray_env_as_string():
+    """Return the sim_telarray environment variables as a string."""
+    return " ".join(f'{key}="{value}" ' for key, value in SIM_TELARRAY_ENV.items())
 
 
 class SimtelRunner:
@@ -25,40 +24,26 @@ class SimtelRunner:
 
     Parameters
     ----------
-    simtel_path: str or Path
-        Location of sim_telarray installation.
     label: str
         Instance label. Important for output file naming.
-    corsika_config: CorsikaConfig
-        CORSIKA configuration.
-    use_multipipe: bool
-        Use multipipe to run CORSIKA and sim_telarray.
-    calibration_run_mode: str
-        Calibration run mode.
+    config: CorsikaConfig or dict
+        Configuration parameters.
+    is_calibration_run: bool
+        Flag to indicate if this is a calibration run.
     """
 
-    def __init__(
-        self,
-        simtel_path,
-        label=None,
-        corsika_config=None,
-        use_multipipe=False,
-        calibration_run_mode=None,
-    ):
+    def __init__(self, label=None, config=None, is_calibration_run=False):
         """Initialize SimtelRunner."""
         self._logger = logging.getLogger(__name__)
 
-        self._simtel_path = Path(simtel_path)
         self.label = label
         self._base_directory = None
-        self.calibration_run_mode = calibration_run_mode
+        self.is_calibration_run = is_calibration_run
 
         self.runs_per_set = 1
 
-        self.runner_service = RunnerServices(corsika_config, label)
-        self._directory = self.runner_service.load_data_directories(
-            "corsika_sim_telarray" if use_multipipe else "sim_telarray"
-        )
+        self.runner_service = RunnerServices(config, run_type="sim_telarray", label=label)
+        self.file_list = None
 
     def run(self, test=False, input_file=None, run_number=None):
         """
@@ -75,142 +60,24 @@ class SimtelRunner:
         """
         self._logger.debug("Running sim_telarray")
 
-        command, stdout_file, stderr_file = self._make_run_command(
+        command, stdout_file, stderr_file = self.make_run_command(
             run_number=run_number, input_file=input_file
         )
-
-        if test:
-            self._logger.info(f"Running (test) with command: {command}")
-            self._run_simtel_and_check_output(command, stdout_file, stderr_file)
-        else:
-            self._logger.debug(f"Running ({self.runs_per_set}x) with command: {command}")
-            for _ in range(self.runs_per_set):
-                self._run_simtel_and_check_output(command, stdout_file, stderr_file)
-
-        self._check_run_result(run_number=run_number)
-
-    def _check_run_result(self, run_number=None):  # pylint: disable=all
-        """Check if simtel output file exists."""
-        pass
-
-    def _raise_simtel_error(self):
-        """
-        Raise sim_telarray execution error.
-
-        Final 30 lines from the log file are collected and printed.
-
-        Raises
-        ------
-        SimtelExecutionError
-        """
-        if hasattr(self, "_log_file"):
-            msg = gen.get_log_excerpt(self._log_file)
-        else:
-            msg = "Simtel log file does not exist."
-        raise SimtelExecutionError(msg)
-
-    def _run_simtel_and_check_output(self, command, stdout_file, stderr_file):
-        """
-        Run the sim_telarray command and check the exit code.
-
-        Raises
-        ------
-        SimtelExecutionError
-            if run was not successful.
-        """
-        stdout_file = stdout_file if stdout_file else "/dev/null"
-        stderr_file = stderr_file if stderr_file else "/dev/null"
-        with (
-            open(f"{stdout_file}", "w", encoding="utf-8") as stdout,
-            open(f"{stderr_file}", "w", encoding="utf-8") as stderr,
-        ):
-            result = subprocess.run(
+        runs = 1 if test else self.runs_per_set
+        label = "test" if test else f"{self.runs_per_set}x"
+        self._logger.info(f"Running ({label}) with command: {command}")
+        for _ in range(runs):
+            job_manager.submit(
                 command,
-                shell=True,
-                text=True,
-                stdout=stdout,
-                stderr=stderr,
+                out_file=stdout_file,
+                err_file=stderr_file,
+                env=SIM_TELARRAY_ENV,
             )
 
-        if result.returncode != 0:
-            self._logger.error(result.stderr)
-            self._raise_simtel_error()
-        return result.returncode
-
-    def _make_run_command(self, run_number=None, input_file=None):
-        self._logger.debug(
-            "make_run_command is being called from the base class - "
-            "it should be implemented in the sub class"
-        )
-        input_file = input_file if input_file else "nofile"
-        run_number = run_number if run_number else 1
-        return f"{input_file}-{run_number}", None, None
-
-    @staticmethod
-    def get_config_option(par, value=None, weak_option=False):
-        """
-        Build sim_telarray command.
-
-        Parameters
-        ----------
-        par: str
-            Parameter name.
-        value: str
-            Parameter value.
-        weak_option: bool
-            If True, use -W option instead of -C.
-
-        Returns
-        -------
-        str
-            Command for sim_telarray.
-        """
-        option_syntax = "-W" if weak_option else "-C"
-        c = f" {option_syntax} {par}"
-        c += f"={value}" if value is not None else ""
-        return c
+    def make_run_command(self, run_number=None, input_file=None):
+        """Make the sim_telarray run command (to implemented in subclasses)."""
+        raise NotImplementedError("Must be implemented in concrete subclass")
 
     def get_resources(self, run_number=None):
         """Return computing resources used."""
         return self.runner_service.get_resources(run_number)
-
-    def get_file_name(
-        self,
-        simulation_software="sim_telarray",
-        file_type=None,
-        run_number=None,
-        mode="",
-        model_version_index=0,
-    ):
-        """
-        Get the full path of a file for a given run number.
-
-        Parameters
-        ----------
-        simulation_software: str
-            Simulation software.
-        file_type: str
-            File type.
-        run_number: int
-            Run number.
-        model_version_index: int
-            Index of the model version.
-            This is used to select the correct simulator_array instance in case
-            multiple array models are simulated.
-
-        Returns
-        -------
-        str
-            File name with full path.
-        """
-        if simulation_software.lower() != "sim_telarray":
-            raise ValueError(
-                f"simulation_software ({simulation_software}) is not supported in SimulatorArray"
-            )
-        return self.runner_service.get_file_name(
-            file_type=file_type,
-            run_number=run_number,
-            mode=mode,
-            _model_version_index=model_version_index,
-            calibration_run_mode=self.calibration_run_mode,
-        )

simtools/runners/simtools_runner.py

@@ -1,15 +1,15 @@
 """Tools for running applications in the simtools framework."""
 
 import shutil
-import subprocess
 from pathlib import Path
 
 import simtools.utils.general as gen
 from simtools import dependencies
 from simtools.io import ascii_handler
+from simtools.job_execution import job_manager
 
 
-def run_applications(args_dict, db_config, logger):
+def run_applications(args_dict, logger):
     """
     Run simtools applications step-by-step as defined in a configuration file.
 
@@ -17,8 +17,6 @@ def run_applications(args_dict, db_config, logger):
     ----------
     args_dict : dict
         Dictionary containing command line arguments.
-    db_config : dict
-        Database configuration
     logger : logging.Logger
         Logger for logging application output.
     """
@@ -33,7 +31,7 @@ def run_applications(args_dict, db_config, logger):
 
     with log_file.open("w", encoding="utf-8") as file:
         file.write("Running simtools applications\n")
-        file.write(dependencies.get_version_string(db_config, run_time))
+        file.write(dependencies.get_version_string(run_time))
 
         for config in configurations:
             app = config.get("application")
@@ -41,75 +39,15 @@ def run_applications(args_dict, db_config, logger):
                 logger.info(f"Skipping application: {app}")
                 continue
             logger.info(f"Running application: {app}")
-            stdout, stderr = run_application(run_time, app, config.get("configuration"), logger)
+            result = job_manager.submit(
+                app,
+                out_file=None,
+                err_file=None,
+                configuration=config.get("configuration"),
+                runtime_environment=run_time,
+            )
             file.write("=" * 80 + "\n")
-            file.write(f"Application: {app}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}\n")
-
-
-def run_application(runtime_environment, application, configuration, logger):
-    """
-    Run a simtools application and return stdout and stderr.
-
-    Allow to specify a runtime environment (e.g., Docker) and a working directory.
-
-    Parameters
-    ----------
-    runtime_environment : list
-        Command to run the application in the specified runtime environment.
-    application : str
-        Name of the application to run.
-    configuration : dict
-        Configuration for the application.
-    logger : logging.Logger
-        Logger for logging application output.
-
-    Returns
-    -------
-    tuple
-        stdout and stderr from the application run.
-
-    """
-    command = [application, *_convert_dict_to_args(configuration)]
-    if runtime_environment:
-        command = runtime_environment + command
-    try:
-        result = subprocess.run(
-            command,
-            check=True,
-            capture_output=True,
-            text=True,
-        )
-    except subprocess.CalledProcessError as exc:
-        logger.error(f"Error running application {application}: {exc.stderr}")
-        raise exc
-
-    return result.stdout, result.stderr
-
-
-def _convert_dict_to_args(parameters):
-    """
-    Convert a dictionary of parameters to a list of command line arguments.
-
-    Parameters
-    ----------
-    parameters : dict
-        Dictionary containing parameters to convert.
-
-    Returns
-    -------
-    list
-        List of command line arguments.
-    """
-    args = []
-    for key, value in parameters.items():
-        if isinstance(value, bool):
-            if value:
-                args.append(f"--{key}")
-        elif isinstance(value, list):
-            args.extend([f"--{key}", *(str(item) for item in value)])
-        else:
-            args.extend([f"--{key}", str(value)])
-    return args
+            file.write(f"Application: {app}\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}\n")
 
 
 def _read_application_configuration(configuration_file, steps, logger):

simtools/schemas/application_workflow.metaschema.yml

@@ -142,6 +142,10 @@ definitions:
             description: |
               "Allowed tolerance for floating point comparison."
             type: number
+          scaling:
+            description: |
+              "Scaling factor to apply to the model parameter value before comparison."
+            type: number
       test_simtel_cfg_files:
         description: |
           "Reference file used for comparison of sim_telarray configuration files."
@@ -152,6 +156,10 @@ definitions:
             type: string
             description: Path to the configuration file for the given version.
         minProperties: 1
+      test_output_file:
+        description: |
+          "Output file generated by the integration test for comparison with the reference file."
+        type: string
       tolerance:
         description: "Allowed tolerance for floating point comparison."
         type: number

simtools/settings.py

@@ -0,0 +1,173 @@
+"""Centralized settings object with command line and environment variables."""
+
+import os
+import socket
+from pathlib import Path
+from types import MappingProxyType
+
+
+class _Config:
+    """Centralized settings object with command line and environment variables."""
+
+    def __init__(self):
+        """Initialize empty config."""
+        self._args = {}
+        self._db_config = {}
+        self._sim_telarray_path = None
+        self._sim_telarray_exe = None
+        self._corsika_path = None
+        self._corsika_interaction_table_path = None
+        self._corsika_exe = None
+        self.user = os.getenv("USER", "unknown")
+        self.hostname = socket.gethostname()
+
+    def load(self, args=None, db_config=None):
+        """
+        Load configuration from command line arguments and environment variables.
+
+        For paths, first check for environment variables, then command line arguments.
+
+        Parameters
+        ----------
+        args : dict, optional
+            Command line arguments.
+        db_config : dict, optional
+            Database configuration.
+
+        """
+        self._args = MappingProxyType(args) if args is not None else {}
+        self._db_config = MappingProxyType(db_config) if db_config is not None else {}
+        self._sim_telarray_path = (
+            args.get("sim_telarray_path")
+            if args is not None and "sim_telarray_path" in args
+            else os.getenv("SIMTOOLS_SIM_TELARRAY_PATH")
+        )
+
+        self._sim_telarray_exe = (
+            args.get("sim_telarray_executable")
+            if args is not None and "sim_telarray_executable" in args
+            else os.getenv("SIMTOOLS_SIM_TELARRAY_EXECUTABLE", "sim_telarray")
+        )
+
+        self._corsika_path = (
+            args.get("corsika_path")
+            if args is not None and "corsika_path" in args
+            else os.getenv("SIMTOOLS_CORSIKA_PATH")
+        )
+
+        self._corsika_interaction_table_path = (
+            args.get("corsika_interaction_table_path")
+            if args is not None and "corsika_interaction_table_path" in args
+            else os.getenv("SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH")
+        )
+
+        self._corsika_exe = self._get_corsika_exec() if self._corsika_path is not None else None
+
+    def _get_corsika_exec(self):
+        """
+        Get the CORSIKA executable from environment variable or command line argument.
+
+        Build the executable name based on configured interaction models. Fall back to
+        legacy naming (simply "corsika") if models are not specified.
+        """
+        he_model = (
+            self._args.get("corsika_he_interaction")
+            if self._args is not None and "corsika_he_interaction" in self._args
+            else os.getenv("SIMTOOLS_CORSIKA_HE_INTERACTION")
+        )
+
+        le_model = (
+            self._args.get("corsika_le_interaction")
+            if self._args is not None and "corsika_le_interaction" in self._args
+            else os.getenv("SIMTOOLS_CORSIKA_LE_INTERACTION")
+        )
+
+        if he_model and le_model:
+            corsika_exe = self.corsika_path / f"corsika_{he_model}_{le_model}_flat"
+            if corsika_exe.exists():
+                return corsika_exe
+
+        # legacy naming
+        return self.corsika_path / "corsika"
+
+    @property
+    def args(self):
+        """Command line arguments."""
+        return self._args
+
+    @property
+    def db_config(self):
+        """Database configuration."""
+        return self._db_config
+
+    @property
+    def sim_telarray_path(self):
+        """Path to the sim_telarray installation directory."""
+        return Path(self._sim_telarray_path) if self._sim_telarray_path is not None else None
+
+    @property
+    def sim_telarray_exe(self):
+        """Path to the sim_telarray executable."""
+        return (
+            Path(self._sim_telarray_path) / "bin" / self._sim_telarray_exe
+            if self._sim_telarray_path is not None
+            else None
+        )
+
+    @property
+    def sim_telarray_exe_debug_trace(self):
+        """Path to the debug trace version of the sim_telarray executable."""
+        return (
+            Path(self._sim_telarray_path) / "bin" / (self._sim_telarray_exe + "_debug_trace")
+            if self._sim_telarray_path is not None
+            else None
+        )
+
+    @property
+    def corsika_path(self):
+        """Path to the CORSIKA installation directory."""
+        return Path(self._corsika_path) if self._corsika_path is not None else None
+
+    @property
+    def corsika_interaction_table_path(self):
+        """Path to the CORSIKA interaction table directory."""
+        return (
+            Path(self._corsika_interaction_table_path)
+            if self._corsika_interaction_table_path is not None
+            else self.corsika_path
+        )
+
+    @property
+    def corsika_exe(self):
+        """Path to the CORSIKA executable."""
+        return (
+            Path(self._corsika_path) / self._corsika_exe if self._corsika_path is not None else None
+        )
+
+    @property
+    def corsika_exe_curved(self):
+        """Path to the curved version of the CORSIKA executable."""
+        if self._corsika_exe is None:
+            return None
+        corsika_curved = (
+            self._corsika_exe.name.replace("_flat", "_curved")
+            if "_flat" in self._corsika_exe.name
+            else self._corsika_exe.name + "-curved"  # legacy naming convention
+        )
+        return Path(self._corsika_path) / corsika_curved if self._corsika_path is not None else None
+
+    @property
+    def corsika_dummy_file(self):
+        """
+        Path to a dummy CORSIKA file required by sim_telarray for ray-tracing simulations.
+
+        This file does not need to exist; sim_telarray only requires a file path.
+        """
+        return (
+            self.sim_telarray_path / "run9991.corsika.gz"
+            if self._sim_telarray_path is not None
+            else None
+        )
+
+
+config = _Config()

simtools/{io/eventio_handler.py → sim_events/file_info.py}

@@ -24,7 +24,7 @@ def get_corsika_run_number(file):
     int, None
         CORSIKA run number. Returns None if not found.
     """
-    run_header = get_combined_corsika_run_header(file)
+    run_header = get_combined_eventio_run_header(file)
     if run_header and "run" in run_header:
         return run_header["run"]
     run_header, _ = get_corsika_run_and_event_headers(file)
@@ -34,9 +34,9 @@ def get_corsika_run_number(file):
         return None
 
 
-def get_combined_corsika_run_header(sim_telarray_file):
+def get_combined_eventio_run_header(sim_telarray_file):
     """
-    Return the CORSIKA run header information from an sim_telarray file.
+    Return the CORSIKA run header information from an eventio (sim_telarray) file.
 
     Reads both RunHeader and MCRunHeader object from file and returns a merged dictionary.
     Adds primary id from the first event.

simtools/{simtel/simtel_io_event_histograms.py → sim_events/histograms.py}

@@ -1,4 +1,4 @@
-"""Histograms for shower and triggered events."""
+"""Histograms for shower and (if available) triggered events."""
 
 import copy
 import logging
@@ -6,12 +6,12 @@ import logging
 import astropy.units as u
 import numpy as np
 
-from simtools.simtel.simtel_io_event_reader import SimtelIOEventDataReader
+from simtools.sim_events.reader import EventDataReader
 
 
-class SimtelIOEventHistograms:
+class EventDataHistograms:
     """
-    Generate and fill histograms for shower and triggered events.
+    Generate and fill histograms for shower and (if available) triggered events.
 
     Event data is read from the reduced MC event data file.
     Calculate cumulative and relative (efficiency) distributions.
@@ -35,7 +35,7 @@ class SimtelIOEventHistograms:
         self.histograms = {}
         self.file_info = {}
 
-        self.reader = SimtelIOEventDataReader(event_data_file, telescope_list=telescope_list)
+        self.reader = EventDataReader(event_data_file, telescope_list=telescope_list)
 
     def fill(self):
         """
@@ -63,12 +63,10 @@ class SimtelIOEventHistograms:
 
             self.histograms = self._define_histograms(event_data, triggered_data, shower_data)
 
-            for name, data in self.histograms.items():
-                self._logger.debug(f"Filling histogram {name}")
+            for data in self.histograms.values():
                 self._fill_histogram_and_bin_edges(data)
 
         self.print_summary()
-
         self.calculate_efficiency_data()
         self.calculate_cumulative_data()
 
@@ -202,11 +200,14 @@ class SimtelIOEventHistograms:
         Adds to existing histogram if present, otherwise initializes it.
         """
         if data["1d"]:
+            if data["event_data"] is None:
+                return
             hist, _ = np.histogram(
-                getattr(data["event_data"], data["event_data_column"]),
-                bins=data["bin_edges"],
+                getattr(data["event_data"], data["event_data_column"]), bins=data["bin_edges"]
             )
         else:
+            if data["event_data"][0] is None or data["event_data"][1] is None:
+                return
             hist, _, _ = np.histogram2d(
                 getattr(data["event_data"][0], data["event_data_column"][0]),
                 getattr(data["event_data"][1], data["event_data_column"][1]),
@@ -227,6 +228,8 @@ class SimtelIOEventHistograms:
         dict
             Dictionary containing the efficiency histograms.
         """
+        if not any(isinstance(ds, dict) and "TRIGGERS" in ds for ds in self.reader.data_sets):
+            return None
 
         def calculate_efficiency(trig_hist, mc_hist):
             with np.errstate(divide="ignore", invalid="ignore"):
@@ -295,11 +298,15 @@ class SimtelIOEventHistograms:
         """Return bins for the viewcone histogram."""
         if "viewcone_bin_edges" in self.histograms:
             return self.histograms["viewcone_bin_edges"]
-        return np.linspace(
-            self.file_info.get("viewcone_min", 0.0 * u.deg).to("deg").value,
-            self.file_info.get("viewcone_max", 20.0 * u.deg).to("deg").value,
-            100,
-        )
+
+        viewcone_min = self.file_info.get("viewcone_min", 0.0 * u.deg).to("deg").value
+        viewcone_max = self.file_info.get("viewcone_max", 20.0 * u.deg).to("deg").value
+
+        # avoid zero-width bins
+        if viewcone_min == viewcone_max:
+            viewcone_max = viewcone_min + 0.5
+
+        return np.linspace(viewcone_min, viewcone_max, 100)
 
     def calculate_cumulative_data(self):
         """
@@ -310,6 +317,9 @@ class SimtelIOEventHistograms:
         dict
             Dictionary containing the cumulative histograms.
         """
+        if not any(isinstance(ds, dict) and "TRIGGERS" in ds for ds in self.reader.data_sets):
+            return None
+
         cumulative_data = {}
         suffix = "_cumulative"