gammasimtools 0.8.2__py3-none-any.whl → 0.9.0__py3-none-any.whl

simtools/job_execution/htcondor_script_generator.py

@@ -0,0 +1,133 @@
+"""HT Condor script generator for simulation production."""
+
+import logging
+from pathlib import Path
+
+import astropy.units as u
+
+_logger = logging.getLogger(__name__)
+
+
+def generate_submission_script(args_dict):
+    """
+    Generate the HT Condor submission script.
+
+    Parameters
+    ----------
+    args_dict: dict
+        Arguments dictionary.
+    """
+    _logger.info("Generating HT Condor submission scripts ")
+
+    work_dir = Path(args_dict["output_path"])
+    log_dir = work_dir / "logs"
+    work_dir.mkdir(parents=True, exist_ok=True)
+    log_dir.mkdir(parents=True, exist_ok=True)
+    submit_file_name = "simulate_prod.submit"
+
+    with open(work_dir / f"{submit_file_name}.condor", "w", encoding="utf-8") as submit_file_handle:
+        submit_file_handle.write(
+            _get_submit_file(
+                f"{submit_file_name}.sh",
+                args_dict["apptainer_image"],
+                args_dict["priority"],
+                +args_dict["number_of_runs"],
+            )
+        )
+
+    with open(work_dir / f"{submit_file_name}.sh", "w", encoding="utf-8") as submit_script_handle:
+        submit_script_handle.write(_get_submit_script(args_dict))
+
+    Path(work_dir / f"{submit_file_name}.sh").chmod(0o755)
+
+
+def _get_submit_file(executable, apptainer_image, priority, n_jobs):
+    """
+    Return HT Condor submit file.
+
+    Database access variables are passed through the environment file.
+
+    Parameters
+    ----------
+    executable: str
+        Name of the executable script.
+    apptainer_image: str
+        Path to the Apptainer image.
+    priority: int
+        Priority of the job.
+    n_jobs: int
+        Number of jobs to queue.
+
+    Returns
+    -------
+    str
+        HT Condor submit file content.
+    """
+    return f"""universe = container
+container_image = {apptainer_image}
+transfer_container = false
+
+executable = {executable}
+error      = logs/err.$(cluster)_$(process)
+output     = logs/out.$(cluster)_$(process)
+log        = logs/log.$(cluster)_$(process)
+
+priority = {priority}
+arguments = "$(process) env.txt"
+
+queue {n_jobs}
+"""
+
+
+def _get_submit_script(args_dict):
+    """
+    Return HT Condor submit script.
+
+    Parameters
+    ----------
+    args_dict: dict
+        Arguments dictionary.
+
+    Returns
+    -------
+    str
+        HT Condor submit script content.
+    """
+    azimuth_angle_string = f"{args_dict['azimuth_angle'].to(u.deg).value}"
+    zenith_angle_string = f"{args_dict['zenith_angle'].to(u.deg).value}"
+    energy_range = args_dict["energy_range"]
+    energy_range_string = (
+        f'"{energy_range[0].to(u.GeV).value} GeV {energy_range[1].to(u.GeV).value} GeV"'
+    )
+    core_scatter = args_dict["core_scatter"]
+    core_scatter_string = f'"{core_scatter[0]} {core_scatter[1].to(u.m).value} m"'
+
+    label = args_dict["label"] if args_dict["label"] else "simulate-prod"
+
+    return f"""#!/usr/bin/env bash
+
+# Process ID used to generate run number
+process_id="$1"
+# Load environment variables (for DB access)
+set -a; source "$2"
+
+simtools-simulate-prod \\
+    --simulation_software {args_dict["simulation_software"]} \\
+    --label {label} \\
+    --model_version {args_dict["model_version"]} \\
+    --site {args_dict["site"]} \\
+    --array_layout_name {args_dict["array_layout_name"]} \\
+    --primary {args_dict["primary"]} \\
+    --azimuth_angle {azimuth_angle_string} \\
+    --zenith_angle {zenith_angle_string} \\
+    --nshow {args_dict["nshow"]} \\
+    --energy_range {energy_range_string} \\
+    --core_scatter {core_scatter_string} \\
+    --run_number_start $((process_id + {args_dict["run_number_start"]})) \\
+    --number_of_runs 1 \\
+    --submit_engine \"local\" \\
+    --data_directory /tmp/simtools-data \\
+    --output_path /tmp/simtools-output \\
+    --log_level {args_dict["log_level"]} \\
+    --pack_for_grid_register simtools-output
+"""

simtools/job_execution/job_manager.py

@@ -1,7 +1,7 @@
 """Interface to workload managers like gridengine or HTCondor."""
 
 import logging
-import os
+import subprocess
 from pathlib import Path
 
 import simtools.utils.general as gen
@@ -65,11 +65,9 @@ class JobManager:
         ValueError
             if invalid submit engine.
         """
-        if value is None:
-            value = "local"
-        if value not in self.engines:
-            raise ValueError(f"Invalid submit command: {value}")
-        self._submit_engine = value
+        self._submit_engine = value or "local"
+        if self._submit_engine not in self.engines:
+            raise ValueError(f"Invalid submit command: {self._submit_engine}")
 
     def check_submission_system(self):
         """
@@ -77,14 +75,17 @@ class JobManager:
 
         Raises
         ------
-        MissingWorkloadManagerError
+        JobExecutionError
             if workflow manager is not found.
         """
-        if self.submit_engine is None or self.submit_engine == "local":
-            return
-
-        if gen.program_is_executable(self.engines[self.submit_engine]):
-            return
+        try:
+            if self.submit_engine in (None, "local") or gen.program_is_executable(
+                self.engines[self.submit_engine]
+            ):
+                return
+        except KeyError:
+            pass
+        raise JobExecutionError(f"Submit engine {self.submit_engine} not found")
 
     def submit(self, run_script=None, run_out_file=None, log_file=None):
         """
@@ -109,12 +110,14 @@ class JobManager:
         self._logger.info(f"Job error stream {self.run_out_file + '.err'}")
         self._logger.info(f"Job log stream {self.run_out_file + '.job'}")
 
-        if self.submit_engine == "gridengine":
-            self._submit_gridengine()
-        elif self.submit_engine == "htcondor":
-            self._submit_htcondor()
-        elif self.submit_engine == "local":
-            self._submit_local(log_file)
+        submit_result = 0
+        if self.submit_engine == "local":
+            submit_result = self._submit_local(log_file)
+        else:
+            submit_result = getattr(self, f"_submit_{self.submit_engine}")()
+
+        if submit_result != 0:
+            raise JobExecutionError(f"Job submission failed with return code {submit_result}")
 
     def _submit_local(self, log_file):
         """
@@ -125,50 +128,72 @@ class JobManager:
         log_file: str or Path
             The log file of the actual simulator (CORSIKA or sim_telarray).
             Provided in order to print the log excerpt in case of run time error.
+
+        Returns
+        -------
+        int
+            Return code of the executed script
         """
         self._logger.info("Running script locally")
 
-        shell_command = f"{self.run_script} > {self.run_out_file}.out 2> {self.run_out_file}.err"
-
-        if not self.test:
-            sys_output = os.system(shell_command)
-            if sys_output != 0:
-                msg = gen.get_log_excerpt(f"{self.run_out_file}.err")
-                self._logger.error(msg)
-                if log_file.exists() and gen.get_file_age(log_file) < 5:
-                    msg = gen.get_log_excerpt(log_file)
-                    self._logger.error(msg)
-                raise JobExecutionError("See excerpt from log file above\n")
-        else:
+        if self.test:
             self._logger.info("Testing (local)")
+            return 0
+
+        result = None
+        try:
+            with (
+                open(f"{self.run_out_file}.out", "w", encoding="utf-8") as stdout,
+                open(f"{self.run_out_file}.err", "w", encoding="utf-8") as stderr,
+            ):
+                result = subprocess.run(
+                    f"{self.run_script}",
+                    shell=True,
+                    check=True,
+                    text=True,
+                    stdout=stdout,
+                    stderr=stderr,
+                )
+        except subprocess.CalledProcessError as exc:
+            self._logger.error(gen.get_log_excerpt(f"{self.run_out_file}.err"))
+            if log_file.exists() and gen.get_file_age(log_file) < 5:
+                self._logger.error(gen.get_log_excerpt(log_file))
+            raise JobExecutionError("See excerpt from log file above\n") from exc
+
+        return result.returncode if result else 0
 
     def _submit_htcondor(self):
         """Submit a job described by a shell script to HTcondor."""
         _condor_file = self.run_script + ".condor"
+        lines = [
+            f"Executable = {self.run_script}",
+            f"Output = {self.run_out_file}.out",
+            f"Error = {self.run_out_file}.err",
+            f"Log = {self.run_out_file}.job",
+        ]
+        if self.submit_options:
+            lines.extend(option.lstrip() for option in self.submit_options.split(","))
+        lines.append("queue 1")
         try:
             with open(_condor_file, "w", encoding="utf-8") as file:
-                file.write(f"Executable = {self.run_script}\n")
-                file.write(f"Output = {self.run_out_file + '.out'}\n")
-                file.write(f"Error = {self.run_out_file + '.err'}\n")
-                file.write(f"Log = {self.run_out_file + '.job'}\n")
-                if self.submit_options:
-                    submit_option_list = self.submit_options.split(",")
-                    for option in submit_option_list:
-                        file.write(option.lstrip() + "\n")
-                file.write("queue 1\n")
+                file.write("\n".join(lines) + "\n")
         except FileNotFoundError as exc:
             self._logger.error(f"Failed creating condor submission file {_condor_file}")
             raise JobExecutionError from exc
 
-        self._execute(self.submit_engine, self.engines[self.submit_engine] + " " + _condor_file)
+        return self._execute(self.submit_engine, [self.engines[self.submit_engine], _condor_file])
 
     def _submit_gridengine(self):
         """Submit a job described by a shell script to gridengine."""
-        this_sub_cmd = self.engines[self.submit_engine]
-        this_sub_cmd = this_sub_cmd + " -o " + self.run_out_file + ".out"
-        this_sub_cmd = this_sub_cmd + " -e " + self.run_out_file + ".err"
-
-        self._execute(self.submit_engine, this_sub_cmd + " " + self.run_script)
+        this_sub_cmd = [
+            self.engines[self.submit_engine],
+            "-o",
+            self.run_out_file + ".out",
+            "-e",
+            self.run_out_file + ".err",
+            self.run_script,
+        ]
+        return self._execute(self.submit_engine, this_sub_cmd)
 
     def _execute(self, engine, shell_command):
         """
@@ -178,13 +203,15 @@ class JobManager:
         ----------
         engine : str
             Engine to use.
-        shell_command : str
-            Shell command to execute.
+        shell_command : list
+            List of shell command plus arguments.
         """
         self._logger.info(f"Submitting script to {engine}")
         self._logger.debug(shell_command)
+        result = None
         if not self.test:
-            os.system(shell_command)
+            result = subprocess.run(shell_command, shell=True, check=True)
         else:
-            self._logger.info(f"Testing ({engine})")
-            self._logger.info(shell_command)
+            self._logger.info(f"Testing ({engine}: {shell_command})")
+
+        return result.returncode if result else 0

simtools/model/camera.py

@@ -5,8 +5,6 @@ from pathlib import Path
 
 import astropy.units as u
 import numpy as np
-from scipy.spatial import cKDTree as KDTree
-from scipy.spatial import distance
 
 from simtools.utils.geometry import rotate
 
@@ -304,6 +302,8 @@ class Camera:
         float
             The camera fill factor.
         """
+        from scipy.spatial import distance  # pylint: disable=import-outside-toplevel
+
         if self.pixels["pixel_spacing"] == 9999:
             points = np.array([self.pixels["x"], self.pixels["y"]]).T
             pixel_distances = distance.cdist(points, points, "euclidean")
@@ -403,6 +403,8 @@ class Camera:
         list of lists
             Array of neighbor indices in a list for each pixel
         """
+        from scipy.spatial import cKDTree as KDTree  # pylint: disable=import-outside-toplevel
+
         tree = KDTree(np.column_stack([x_pos, y_pos]))
         neighbors = tree.query_ball_tree(tree, radius)
         return [list(np.setdiff1d(neigh, [i])) for i, neigh in enumerate(neighbors)]

simtools/model/model_parameter.py

@@ -6,10 +6,12 @@ import shutil
 from copy import copy
 
 import astropy.units as u
+from astropy.table import Table
 
 import simtools.utils.general as gen
 from simtools.db import db_handler
 from simtools.io_operations import io_handler
+from simtools.simtel import simtel_table_reader
 from simtools.simtel.simtel_config_writer import SimtelConfigWriter
 from simtools.utils import names
 
@@ -273,7 +275,7 @@ class ModelParameter:
 
     def _set_config_file_directory_and_name(self):
         """Set and create the directory and the name of the config file."""
-        if self.name is None:
+        if self.name is None and self.site is None:
             return
 
         self._config_file_directory = self.io_handler.get_output_directory(
@@ -281,15 +283,14 @@ class ModelParameter:
         )
 
         # Setting file name and the location
-        if self.site is not None and self.name is not None:
-            config_file_name = names.simtel_config_file_name(
-                self.site,
-                self.model_version,
-                telescope_model_name=self.name,
-                label=self.label,
-                extra_label=self._extra_label,
-            )
-            self._config_file_path = self.config_file_directory.joinpath(config_file_name)
+        config_file_name = names.simtel_config_file_name(
+            self.site,
+            self.model_version,
+            telescope_model_name=self.name,
+            label=self.label,
+            extra_label=self._extra_label,
+        )
+        self._config_file_path = self.config_file_directory.joinpath(config_file_name)
 
         self._logger.debug(f"Config file path: {self._config_file_path}")
 
@@ -515,6 +516,35 @@ class ModelParameter:
         self.db.export_model_files(pars_from_db, self.config_file_directory)
         self._is_exported_model_files_up_to_date = True
 
+    def get_model_file_as_table(self, par_name):
+        """
+        Return tabular data from file as astropy table.
+
+        Parameters
+        ----------
+        par_name: str
+            Name of the parameter.
+
+        Returns
+        -------
+        Table
+            Astropy table.
+        """
+        _par_entry = {}
+        try:
+            _par_entry[par_name] = self._parameters[par_name]
+        except KeyError as exc:
+            raise ValueError(f"Parameter {par_name} not found in the model.") from exc
+        self.db.export_model_files(_par_entry, self.config_file_directory)
+        if _par_entry[par_name]["value"].endswith("ecsv"):
+            return Table.read(
+                self.config_file_directory.joinpath(_par_entry[par_name]["value"]),
+                format="ascii.ecsv",
+            )
+        return simtel_table_reader.read_simtel_table(
+            par_name, self.config_file_directory.joinpath(_par_entry[par_name]["value"])
+        )
+
     def export_config_file(self):
         """Export the config file used by sim_telarray."""
         # Exporting model file

simtools/model/site_model.py

@@ -80,7 +80,7 @@ class SiteModel(ModelParameter):
             Site-related CORSIKA parameters as dict
         """
         if config_file_style:
-            model_directory = model_directory or Path("")
+            model_directory = model_directory or Path()
             return {
                 "OBSLEV": [
                     self.get_parameter_value_with_unit("corsika_observation_level").to_value("cm")

simtools/ray_tracing/mirror_panel_psf.py

@@ -138,43 +138,63 @@ class MirrorPanelPSF:
             self.rnda_opt, save_figures=save_figures
         )
 
-    def _optimize_reflection_angle(self, step_size=0.1):
-        """Optimize the random reflection angle to minimize the difference in D80 containment."""
+    def _optimize_reflection_angle(self, step_size=0.1, max_iteration=100):
+        """
+        Optimize the random reflection angle to minimize the difference in D80 containment.
+
+        Parameters
+        ----------
+        step_size: float
+            Initial step size for optimization.
+        max_iteration: int
+            Maximum number of iterations.
+
+        Raises
+        ------
+        ValueError
+            If the optimization reaches the maximum number of iterations without converging.
+
+        """
+        relative_tolerance_d80 = self.args_dict["rtol_psf_containment"]
+        self._logger.info(
+            "Optimizing random reflection angle "
+            f"(relative tolerance = {relative_tolerance_d80}, "
+            f"step size = {step_size}, max iteration = {max_iteration})"
+        )
 
         def collect_results(rnda, mean, sig):
             self.results_rnda.append(rnda)
             self.results_mean.append(mean)
             self.results_sig.append(sig)
 
-        stop = False
-        mean_d80, sig_d80 = self.run_simulations_and_analysis(self.rnda_start)
+        reference_d80 = self.args_dict["psf_measurement_containment_mean"]
         rnda = self.rnda_start
-        sign_delta = np.sign(mean_d80 - self.args_dict["psf_measurement_containment_mean"])
-        collect_results(rnda, mean_d80, sig_d80)
-        while not stop:
-            rnda = rnda - (step_size * self.rnda_start * sign_delta)
-            if rnda < 0:
-                rnda = 0
-                collect_results(rnda, mean_d80, sig_d80)
-                break
+        prev_error_d80 = float("inf")
+        iteration = 0
+
+        while True:
             mean_d80, sig_d80 = self.run_simulations_and_analysis(rnda)
-            new_sign_delta = np.sign(mean_d80 - self.args_dict["psf_measurement_containment_mean"])
-            stop = new_sign_delta != sign_delta
-            sign_delta = new_sign_delta
+            error_d80 = abs(1 - mean_d80 / reference_d80)
             collect_results(rnda, mean_d80, sig_d80)
 
-        self._interpolate_optimal_rnda()
+            if error_d80 < relative_tolerance_d80:
+                break
+
+            if mean_d80 < reference_d80:
+                rnda += step_size * self.rnda_start
+            else:
+                rnda -= step_size * self.rnda_start
 
-    def _interpolate_optimal_rnda(self):
-        """Interpolate to find the optimal random reflection angle."""
-        self.results_rnda, self.results_mean, self.results_sig = gen.sort_arrays(
-            self.results_rnda, self.results_mean, self.results_sig
-        )
-        self.rnda_opt = np.interp(
-            x=self.args_dict["psf_measurement_containment_mean"],
-            xp=self.results_mean,
-            fp=self.results_rnda,
-        )
+            if error_d80 >= prev_error_d80:
+                step_size = step_size / 2
+            prev_error_d80 = error_d80
+            iteration += 1
+            if iteration > max_iteration:
+                raise ValueError(
+                    f"Maximum iterations ({max_iteration}) reached without convergence."
+                )
+
+        self.rnda_opt = rnda
 
     def _get_starting_value(self):
         """Get optimization starting value from command line or previous model."""
@@ -275,6 +295,6 @@ class MirrorPanelPSF:
         )
         writer.ModelDataWriter.dump(
             args_dict=self.args_dict,
-            metadata=MetadataCollector(args_dict=self.args_dict).top_level_meta,
+            metadata=MetadataCollector(args_dict=self.args_dict).get_top_level_metadata(),
             product_data=result_table,
         )

simtools/runners/corsika_runner.py

@@ -1,7 +1,7 @@
 """Generate run scripts and directories for CORSIKA simulations."""
 
 import logging
-import os
+import stat
 from pathlib import Path
 
 from simtools.io_operations import io_handler
@@ -99,6 +99,15 @@ class CorsikaRunner:
             file_type="config_tmp", run_number=self.corsika_config.run_number
         )
         corsika_input_tmp_file = self._directory["inputs"].joinpath(corsika_input_tmp_name)
+        # CORSIKA log file naming (temporary and final)
+        corsika_log_tmp_file = (
+            self._directory["data"]
+            .joinpath(f"run{self.corsika_config.run_number:06}")
+            .joinpath(f"run{self.corsika_config.run_number}.log")
+        )
+        corsika_log_file = self.get_file_name(
+            file_type="corsika_log", run_number=self.corsika_config.run_number
+        )
 
         if use_pfp:
             pfp_command = self._get_pfp_command(corsika_input_tmp_file, corsika_input_file)
@@ -138,11 +147,13 @@ class CorsikaRunner:
                 file.write(f"cp {corsika_input_file} {corsika_input_tmp_file}")
             file.write("\n# Running corsika_autoinputs\n")
             file.write(autoinputs_command)
+            file.write("\n# Moving log files to the corsika log directory\n")
+            file.write(f"gzip {corsika_log_tmp_file}\n")
+            file.write(f"mv -v {corsika_log_tmp_file}.gz {corsika_log_file}\n")
 
             file.write('\necho "RUNTIME: $SECONDS"\n')
 
-        os.system(f"chmod ug+x {script_file_path}")
-
+        script_file_path.chmod(script_file_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP)
         return script_file_path
 
     def get_resources(self, run_number=None):

simtools/runners/runner_services.py

@@ -163,6 +163,7 @@ class RunnerServices:
         log_suffixes = {
             "log": ".log.gz",
             "histogram": ".hdata.zst",
+            "corsika_log": ".corsika.log.gz",
         }
         return self.directory["logs"].joinpath(f"{file_name}{log_suffixes[file_type]}")
 
@@ -187,7 +188,6 @@ class RunnerServices:
         data_suffixes = {
             "output": ".zst",
             "corsika_output": ".zst",
-            "corsika_log": ".log",
             "simtel_output": ".simtel.zst",
         }
         run_dir = self._get_run_number_string(run_number)
@@ -245,10 +245,10 @@ class RunnerServices:
         """
         file_name = self._get_file_basename(run_number)
 
-        if file_type in ["log", "histogram"]:
+        if file_type in ["log", "histogram", "corsika_log"]:
             return self._get_log_file_path(file_type, file_name)
 
-        if file_type in ["output", "corsika_output", "corsika_log", "simtel_output"]:
+        if file_type in ["output", "corsika_output", "simtel_output"]:
             return self._get_data_file_path(file_type, file_name, run_number)
 
         if file_type in ("sub_log", "sub_script"):