gammasimtools 0.26.0__py3-none-any.whl → 0.27.1__py3-none-any.whl

simtools/runners/simtel_runner.py

@@ -1,18 +1,18 @@
 """Base class for running sim_telarray simulations."""
 
 import logging
-import subprocess
 
-import simtools.utils.general as gen
+from simtools.job_execution import job_manager
 from simtools.runners.runner_services import RunnerServices
 
+SIM_TELARRAY_ENV = {
+    "SIM_TELARRAY_CONFIG_PATH": "",
+}
 
-class SimtelExecutionError(Exception):
-    """Exception for sim_telarray execution error."""
 
-
-class InvalidOutputFileError(Exception):
-    """Exception for invalid output file."""
+def sim_telarray_env_as_string():
+    """Return the sim_telarray environment variables as a string."""
+    return " ".join(f'{key}="{value}" ' for key, value in SIM_TELARRAY_ENV.items())
 
 
 class SimtelRunner:
@@ -26,34 +26,24 @@ class SimtelRunner:
     ----------
     label: str
         Instance label. Important for output file naming.
-    corsika_config: CorsikaConfig
-        CORSIKA configuration.
-    use_multipipe: bool
-        Use multipipe to run CORSIKA and sim_telarray.
-    calibration_run_mode: str
-        Calibration run mode.
+    config: CorsikaConfig or dict
+        Configuration parameters.
+    is_calibration_run: bool
+        Flag to indicate if this is a calibration run.
     """
 
-    def __init__(
-        self,
-        label=None,
-        corsika_config=None,
-        use_multipipe=False,
-        calibration_run_mode=None,
-    ):
+    def __init__(self, label=None, config=None, is_calibration_run=False):
         """Initialize SimtelRunner."""
         self._logger = logging.getLogger(__name__)
 
         self.label = label
         self._base_directory = None
-        self.calibration_run_mode = calibration_run_mode
+        self.is_calibration_run = is_calibration_run
 
         self.runs_per_set = 1
 
-        self.runner_service = RunnerServices(corsika_config, label)
-        self._directory = self.runner_service.load_data_directories(
-            "corsika_sim_telarray" if use_multipipe else "sim_telarray"
-        )
+        self.runner_service = RunnerServices(config, run_type="sim_telarray", label=label)
+        self.file_list = None
 
     def run(self, test=False, input_file=None, run_number=None):
         """
@@ -70,142 +60,24 @@ class SimtelRunner:
         """
         self._logger.debug("Running sim_telarray")
 
-        command, stdout_file, stderr_file = self._make_run_command(
+        command, stdout_file, stderr_file = self.make_run_command(
             run_number=run_number, input_file=input_file
         )
-
-        if test:
-            self._logger.info(f"Running (test) with command: {command}")
-            self._run_simtel_and_check_output(command, stdout_file, stderr_file)
-        else:
-            self._logger.debug(f"Running ({self.runs_per_set}x) with command: {command}")
-            for _ in range(self.runs_per_set):
-                self._run_simtel_and_check_output(command, stdout_file, stderr_file)
-
-        self._check_run_result(run_number=run_number)
-
-    def _check_run_result(self, run_number=None):  # pylint: disable=all
-        """Check if simtel output file exists."""
-        pass
-
-    def _raise_simtel_error(self):
-        """
-        Raise sim_telarray execution error.
-
-        Final 30 lines from the log file are collected and printed.
-
-        Raises
-        ------
-        SimtelExecutionError
-        """
-        if hasattr(self, "_log_file"):
-            msg = gen.get_log_excerpt(self._log_file)
-        else:
-            msg = "Simtel log file does not exist."
-        raise SimtelExecutionError(msg)
-
-    def _run_simtel_and_check_output(self, command, stdout_file, stderr_file):
-        """
-        Run the sim_telarray command and check the exit code.
-
-        Raises
-        ------
-        SimtelExecutionError
-            if run was not successful.
-        """
-        stdout_file = stdout_file if stdout_file else "/dev/null"
-        stderr_file = stderr_file if stderr_file else "/dev/null"
-        with (
-            open(f"{stdout_file}", "w", encoding="utf-8") as stdout,
-            open(f"{stderr_file}", "w", encoding="utf-8") as stderr,
-        ):
-            result = subprocess.run(
+        runs = 1 if test else self.runs_per_set
+        label = "test" if test else f"{self.runs_per_set}x"
+        self._logger.info(f"Running ({label}) with command: {command}")
+        for _ in range(runs):
+            job_manager.submit(
                 command,
-                shell=True,
-                text=True,
-                stdout=stdout,
-                stderr=stderr,
+                out_file=stdout_file,
+                err_file=stderr_file,
+                env=SIM_TELARRAY_ENV,
             )
 
-        if result.returncode != 0:
-            self._logger.error(result.stderr)
-            self._raise_simtel_error()
-        return result.returncode
-
-    def _make_run_command(self, run_number=None, input_file=None):
-        self._logger.debug(
-            "make_run_command is being called from the base class - "
-            "it should be implemented in the sub class"
-        )
-        input_file = input_file if input_file else "nofile"
-        run_number = run_number if run_number else 1
-        return f"{input_file}-{run_number}", None, None
-
-    @staticmethod
-    def get_config_option(par, value=None, weak_option=False):
-        """
-        Build sim_telarray command.
-
-        Parameters
-        ----------
-        par: str
-            Parameter name.
-        value: str
-            Parameter value.
-        weak_option: bool
-            If True, use -W option instead of -C.
-
-        Returns
-        -------
-        str
-            Command for sim_telarray.
-        """
-        option_syntax = "-W" if weak_option else "-C"
-        c = f" {option_syntax} {par}"
-        c += f"={value}" if value is not None else ""
-        return c
+    def make_run_command(self, run_number=None, input_file=None):
+        """Make the sim_telarray run command (to implemented in subclasses)."""
+        raise NotImplementedError("Must be implemented in concrete subclass")
 
     def get_resources(self, run_number=None):
         """Return computing resources used."""
         return self.runner_service.get_resources(run_number)
-
-    def get_file_name(
-        self,
-        simulation_software="sim_telarray",
-        file_type=None,
-        run_number=None,
-        mode="",
-        model_version_index=0,
-    ):
-        """
-        Get the full path of a file for a given run number.
-
-        Parameters
-        ----------
-        simulation_software: str
-            Simulation software.
-        file_type: str
-            File type.
-        run_number: int
-            Run number.
-        model_version_index: int
-            Index of the model version.
-            This is used to select the correct simulator_array instance in case
-            multiple array models are simulated.
-
-        Returns
-        -------
-        str
-            File name with full path.
-        """
-        if simulation_software.lower() != "sim_telarray":
-            raise ValueError(
-                f"simulation_software ({simulation_software}) is not supported in SimulatorArray"
-            )
-        return self.runner_service.get_file_name(
-            file_type=file_type,
-            run_number=run_number,
-            mode=mode,
-            _model_version_index=model_version_index,
-            calibration_run_mode=self.calibration_run_mode,
-        )

simtools/runners/simtools_runner.py

@@ -1,12 +1,12 @@
 """Tools for running applications in the simtools framework."""
 
 import shutil
-import subprocess
 from pathlib import Path
 
 import simtools.utils.general as gen
 from simtools import dependencies
 from simtools.io import ascii_handler
+from simtools.job_execution import job_manager
 
 
 def run_applications(args_dict, logger):
@@ -39,75 +39,15 @@ def run_applications(args_dict, logger):
                 logger.info(f"Skipping application: {app}")
                 continue
             logger.info(f"Running application: {app}")
-            stdout, stderr = run_application(run_time, app, config.get("configuration"), logger)
+            result = job_manager.submit(
+                app,
+                out_file=None,
+                err_file=None,
+                configuration=config.get("configuration"),
+                runtime_environment=run_time,
+            )
             file.write("=" * 80 + "\n")
-            file.write(f"Application: {app}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}\n")
-
-
-def run_application(runtime_environment, application, configuration, logger):
-    """
-    Run a simtools application and return stdout and stderr.
-
-    Allow to specify a runtime environment (e.g., Docker) and a working directory.
-
-    Parameters
-    ----------
-    runtime_environment : list
-        Command to run the application in the specified runtime environment.
-    application : str
-        Name of the application to run.
-    configuration : dict
-        Configuration for the application.
-    logger : logging.Logger
-        Logger for logging application output.
-
-    Returns
-    -------
-    tuple
-        stdout and stderr from the application run.
-
-    """
-    command = [application, *_convert_dict_to_args(configuration)]
-    if runtime_environment:
-        command = runtime_environment + command
-    try:
-        result = subprocess.run(
-            command,
-            check=True,
-            capture_output=True,
-            text=True,
-        )
-    except subprocess.CalledProcessError as exc:
-        logger.error(f"Error running application {application}: {exc.stderr}")
-        raise exc
-
-    return result.stdout, result.stderr
-
-
-def _convert_dict_to_args(parameters):
-    """
-    Convert a dictionary of parameters to a list of command line arguments.
-
-    Parameters
-    ----------
-    parameters : dict
-        Dictionary containing parameters to convert.
-
-    Returns
-    -------
-    list
-        List of command line arguments.
-    """
-    args = []
-    for key, value in parameters.items():
-        if isinstance(value, bool):
-            if value:
-                args.append(f"--{key}")
-        elif isinstance(value, list):
-            args.extend([f"--{key}", *(str(item) for item in value)])
-        else:
-            args.extend([f"--{key}", str(value)])
-    return args
+            file.write(f"Application: {app}\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}\n")
 
 
 def _read_application_configuration(configuration_file, steps, logger):

simtools/schemas/application_workflow.metaschema.yml

@@ -142,6 +142,10 @@ definitions:
             description: |
               "Allowed tolerance for floating point comparison."
             type: number
+          scaling:
+            description: |
+              "Scaling factor to apply to the model parameter value before comparison."
+            type: number
       test_simtel_cfg_files:
         description: |
           "Reference file used for comparison of sim_telarray configuration files."
@@ -152,6 +156,10 @@ definitions:
             type: string
             description: Path to the configuration file for the given version.
         minProperties: 1
+      test_output_file:
+        description: |
+          "Output file generated by the integration test for comparison with the reference file."
+        type: string
       tolerance:
         description: "Allowed tolerance for floating point comparison."
         type: number

simtools/settings.py

@@ -1,6 +1,7 @@
 """Centralized settings object with command line and environment variables."""
 
 import os
+import socket
 from pathlib import Path
 from types import MappingProxyType
 
@@ -15,7 +16,10 @@ class _Config:
         self._sim_telarray_path = None
         self._sim_telarray_exe = None
         self._corsika_path = None
+        self._corsika_interaction_table_path = None
         self._corsika_exe = None
+        self.user = os.getenv("USER", "unknown")
+        self.hostname = socket.gethostname()
 
     def load(self, args=None, db_config=None):
         """
@@ -51,6 +55,12 @@ class _Config:
             else os.getenv("SIMTOOLS_CORSIKA_PATH")
         )
 
+        self._corsika_interaction_table_path = (
+            args.get("corsika_interaction_table_path")
+            if args is not None and "corsika_interaction_table_path" in args
+            else os.getenv("SIMTOOLS_CORSIKA_INTERACTION_TABLE_PATH")
+        )
+
         self._corsika_exe = self._get_corsika_exec() if self._corsika_path is not None else None
 
     def _get_corsika_exec(self):
@@ -118,6 +128,15 @@ class _Config:
         """Path to the CORSIKA installation directory."""
         return Path(self._corsika_path) if self._corsika_path is not None else None
 
+    @property
+    def corsika_interaction_table_path(self):
+        """Path to the CORSIKA interaction table directory."""
+        return (
+            Path(self._corsika_interaction_table_path)
+            if self._corsika_interaction_table_path is not None
+            else self.corsika_path
+        )
+
     @property
     def corsika_exe(self):
         """Path to the CORSIKA executable."""

simtools/simtel/simtel_config_writer.py

@@ -17,27 +17,6 @@ from simtools.utils import names
 logger = logging.getLogger(__name__)
 
 
-def sim_telarray_random_seeds(seed, number):
-    """
-    Generate random seeds to be used in sim_telarray.
-
-    Parameters
-    ----------
-    seed: int
-        Seed for the random number generator.
-    number: int
-        Number of random seeds to generate.
-
-    Returns
-    -------
-    list
-        List of random seeds.
-    """
-    rng = np.random.default_rng(seed)
-    max_int32 = np.iinfo(np.int32).max  # sim_telarray requires 32 bit integers
-    return list(rng.integers(low=1, high=max_int32, size=number, dtype=np.int32))
-
-
 class SimtelConfigWriter:
     """
     SimtelConfigWriter writes sim_telarray configuration files.
@@ -71,7 +50,6 @@ class SimtelConfigWriter:
     ):
         """Initialize SimtelConfigWriter."""
         self._logger = logging.getLogger(__name__)
-        self._logger.debug("Init SimtelConfigWriter")
 
         self._site = site
         self._model_version = model_version
@@ -492,39 +470,6 @@ class SimtelConfigWriter:
                 file.write(f"# include <{tel_config_file}>\n\n")
             file.write("#endif \n\n")  # configuration files need to end with \n\n
 
-        if additional_metadata and additional_metadata.get("random_instrument_instances"):
-            self._write_random_seeds_file(additional_metadata, config_file_directory)
-
-    def _write_random_seeds_file(self, sim_telarray_seeds, config_file_directory):
-        """
-        Write list of random number used to generate random instances of instrument.
-
-        Parameters
-        ----------
-        random_instrument_instances: int
-            Number of random instances of the instrument.
-        """
-        self._logger.info(
-            "Writing random seed file "
-            f"{config_file_directory}/{sim_telarray_seeds['seed_file_name']}"
-            f" (global seed {sim_telarray_seeds['seed']})"
-        )
-        if sim_telarray_seeds["random_instrument_instances"] > 1024:
-            raise ValueError("Number of random instances of instrument must be less than 1024")
-        random_integers = sim_telarray_random_seeds(
-            sim_telarray_seeds["seed"], sim_telarray_seeds["random_instrument_instances"]
-        )
-        with open(
-            config_file_directory / sim_telarray_seeds["seed_file_name"], "w", encoding="utf-8"
-        ) as file:
-            file.write(
-                "# Random seeds for instrument configuration generated with seed "
-                f"{sim_telarray_seeds['seed']}"
-                f" (model version {self._model_version}, site {self._site})\n"
-            )
-            for number in random_integers:
-                file.write(f"{number}\n")
-
     def write_single_mirror_list_file(
         self, mirror_number, mirrors, single_mirror_list_file, set_focal_length_to_zero=False
     ):

simtools/simtel/simtel_seeds.py

@@ -0,0 +1,184 @@
+"""Seeds for sim_telarray simulations."""
+
+import logging
+from pathlib import Path
+
+from simtools import settings
+from simtools.constants import SIMTEL_MAX_SEED
+from simtools.io import ascii_handler
+from simtools.utils import names, random
+from simtools.version import semver_to_int
+
+
+class SimtelSeeds:
+    """Manage seeds for sim_telarray simulations."""
+
+    def __init__(
+        self, output_path=None, site=None, model_version=None, zenith_angle=None, azimuth_angle=None
+    ):
+        """
+        Initialize seeds for sim_telarray simulations.
+
+        Two seeds are set for sim_telarray simulations:
+
+            - Instrument seed: used to randomize the instrument setup
+            - Simulation seed: used for the shower simulation
+
+        Parameters
+        ----------
+        output_path : str or Path or None
+            Output path for the seed file.
+        site : str or None
+            Site name.
+        model_version : str or None
+            Model version.
+        zenith_angle : float or None
+            Zenith angle.
+        azimuth_angle : float or None
+            Azimuth angle.
+        """
+        self._logger = logging.getLogger(__name__)
+
+        self.instrument_seed = settings.config.args.get("sim_telarray_instrument_seed", None)
+        self.instruments = settings.config.args.get(
+            "sim_telarray_random_instrument_instances", None
+        )
+        self.simulation_seed = settings.config.args.get("sim_telarray_seed", None)
+        self.seed_file = settings.config.args.get("sim_telarray_seed_file", None)
+        if output_path is not None:
+            self.seed_file = Path(output_path) / self.seed_file
+
+        self.seed_string = self.initialize_seeds(site, model_version, zenith_angle, azimuth_angle)
+
+    def initialize_seeds(self, site, model_version, zenith_angle, azimuth_angle):
+        """Initialize seeds based on provided parameters."""
+        if isinstance(self.simulation_seed, list):
+            return self._set_fixed_seeds()
+
+        if not self.simulation_seed:
+            self.simulation_seed = random.seeds(max_seed=SIMTEL_MAX_SEED)
+
+        if not self.instruments or self.instruments <= 1:
+            return self._generate_seed_pair()
+
+        return self._generate_seeds_with_file(site, model_version, zenith_angle, azimuth_angle)
+
+    def _set_fixed_seeds(self):
+        """
+        Set fixed seeds to be using for testing purposes only.
+
+        Fixes both instrument and simulation seeds.
+        """
+        try:
+            seed_string = f"{self.simulation_seed[0]},{self.simulation_seed[1]}"
+        except IndexError as exc:
+            raise IndexError(
+                "Two seeds must be provided for testing purposes: "
+                "first for instrument, second for shower simulation."
+            ) from exc
+        self._logger.warning(f"Using fixed test seeds: {seed_string}")
+        return seed_string
+
+    def _generate_seed_pair(self):
+        """Generate seed string."""
+        if not self.instrument_seed:
+            self.instrument_seed = random.seeds(max_seed=SIMTEL_MAX_SEED)
+
+        self._logger.info(
+            f"Generated sim_telarray seeds - Instrument: {self.instrument_seed}, "
+            f"Shower simulation: {self.simulation_seed}"
+        )
+        return f"{self.instrument_seed},{self.simulation_seed}"
+
+    def _generate_seeds_with_file(self, site, model_version, zenith_angle, azimuth_angle):
+        """Generate a seed file for the instrument seeds and return the seed string."""
+        self.instrument_seed = self._get_instrument_seed(
+            site, model_version, zenith_angle, azimuth_angle
+        )
+
+        self._logger.info(
+            f"Writing random instrument seed file {self.seed_file}"
+            f" (instrument seed {self.instrument_seed})"
+        )
+        if self.instruments > 1024:
+            raise ValueError("Number of random instances of instrument must be less than 1024")
+        random_integers = random.seeds(
+            n_seeds=self.instruments,
+            max_seed=SIMTEL_MAX_SEED,
+            fixed_seed=self.instrument_seed,
+        )
+        with open(self.seed_file, "w", encoding="utf-8") as file:
+            file.write(
+                "# Random seeds for instrument configuration generated with seed "
+                f"{self.instrument_seed} (model version {model_version}, site {site})\n"
+                f"# Zenith angle: {zenith_angle}, Azimuth angle: {azimuth_angle}\n"
+            )
+            for number in random_integers:
+                file.write(f"{number}\n")
+
+        return f"file-by-run:{self.seed_file},{self.simulation_seed}"
+
+    def _get_instrument_seed(self, site, model_version, zenith_angle, azimuth_angle):
+        """
+        Get configuration dependent instrument seed.
+
+        Three different scenarios are possible:
+
+            - instrument seed provided through configuration: use it
+            - site, model_version, zenith_angle, azimuth_angle provided:
+              generate a seed based on these parameters
+            - none of the above: generate a random seed
+
+        Parameters
+        ----------
+        site : str or None
+            Site name.
+        model_version : str or None
+            Model version.
+        zenith_angle : float or None
+            Zenith angle.
+        azimuth_angle : float or None
+            Azimuth angle.
+
+        Returns
+        -------
+        int
+            Instrument seed.
+        """
+        # Use the instrument seed from the configuration if provided
+        if self.instrument_seed:
+            return self.instrument_seed
+
+        # Generate a seed based on site, model_version, zenith_angle, and azimuth_angle
+        if model_version and zenith_angle is not None and azimuth_angle is not None:
+            try:
+                key_index = next(
+                    i + 1
+                    for i, (_, values) in enumerate(names.site_names().items())
+                    if site.lower() in values
+                )
+            except StopIteration as exc:
+                raise ValueError(f"Unknown site: {site!r}") from exc
+
+            seed = semver_to_int(model_version) * 10000000
+            seed = seed + key_index * 1000000
+            seed = seed + int(zenith_angle) * 1000
+            return seed + int(azimuth_angle)
+
+        # Generate a random instrument seed
+        return random.seeds(max_seed=SIMTEL_MAX_SEED)
+
+    def save_seeds(self, path):
+        """
+        Save the seeds to a file.
+
+        Parameters
+        ----------
+        path : str or Path
+            Path to the seed file.
+        """
+        seed_dict = {
+            "instrument_seed": self.instrument_seed,
+            "simulation_seed": self.simulation_seed,
+        }
+        ascii_handler.write_data_to_file(path, seed_dict)