gammasimtools 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

simtools/job_execution/htcondor_script_generator.py

@@ -0,0 +1,133 @@
+"""HT Condor script generator for simulation production."""
+
+import logging
+from pathlib import Path
+
+import astropy.units as u
+
+_logger = logging.getLogger(__name__)
+
+
+def generate_submission_script(args_dict):
+    """
+    Generate the HT Condor submission script.
+
+    Parameters
+    ----------
+    args_dict: dict
+        Arguments dictionary.
+    """
+    _logger.info("Generating HT Condor submission scripts ")
+
+    work_dir = Path(args_dict["output_path"])
+    log_dir = work_dir / "logs"
+    work_dir.mkdir(parents=True, exist_ok=True)
+    log_dir.mkdir(parents=True, exist_ok=True)
+    submit_file_name = "simulate_prod.submit"
+
+    with open(work_dir / f"{submit_file_name}.condor", "w", encoding="utf-8") as submit_file_handle:
+        submit_file_handle.write(
+            _get_submit_file(
+                f"{submit_file_name}.sh",
+                args_dict["apptainer_image"],
+                args_dict["priority"],
+                +args_dict["number_of_runs"],
+            )
+        )
+
+    with open(work_dir / f"{submit_file_name}.sh", "w", encoding="utf-8") as submit_script_handle:
+        submit_script_handle.write(_get_submit_script(args_dict))
+
+    Path(work_dir / f"{submit_file_name}.sh").chmod(0o755)
+
+
+def _get_submit_file(executable, apptainer_image, priority, n_jobs):
+    """
+    Return HT Condor submit file.
+
+    Database access variables are passed through the environment file.
+
+    Parameters
+    ----------
+    executable: str
+        Name of the executable script.
+    apptainer_image: str
+        Path to the Apptainer image.
+    priority: int
+        Priority of the job.
+    n_jobs: int
+        Number of jobs to queue.
+
+    Returns
+    -------
+    str
+        HT Condor submit file content.
+    """
+    return f"""universe = container
+container_image = {apptainer_image}
+transfer_container = false
+
+executable = {executable}
+error      = logs/err.$(cluster)_$(process)
+output     = logs/out.$(cluster)_$(process)
+log        = logs/log.$(cluster)_$(process)
+
+priority = {priority}
+arguments = "$(process) env.txt"
+
+queue {n_jobs}
+"""
+
+
+def _get_submit_script(args_dict):
+    """
+    Return HT Condor submit script.
+
+    Parameters
+    ----------
+    args_dict: dict
+        Arguments dictionary.
+
+    Returns
+    -------
+    str
+        HT Condor submit script content.
+    """
+    azimuth_angle_string = f"{args_dict['azimuth_angle'].to(u.deg).value}"
+    zenith_angle_string = f"{args_dict['zenith_angle'].to(u.deg).value}"
+    energy_range = args_dict["energy_range"]
+    energy_range_string = (
+        f'"{energy_range[0].to(u.GeV).value} GeV {energy_range[1].to(u.GeV).value} GeV"'
+    )
+    core_scatter = args_dict["core_scatter"]
+    core_scatter_string = f'"{core_scatter[0]} {core_scatter[1].to(u.m).value} m"'
+
+    label = args_dict["label"] if args_dict["label"] else "simulate-prod"
+
+    return f"""#!/usr/bin/env bash
+
+# Process ID used to generate run number
+process_id="$1"
+# Load environment variables (for DB access)
+set -a; source "$2"
+
+simtools-simulate-prod \\
+    --simulation_software {args_dict["simulation_software"]} \\
+    --label {label} \\
+    --model_version {args_dict["model_version"]} \\
+    --site {args_dict["site"]} \\
+    --array_layout_name {args_dict["array_layout_name"]} \\
+    --primary {args_dict["primary"]} \\
+    --azimuth_angle {azimuth_angle_string} \\
+    --zenith_angle {zenith_angle_string} \\
+    --nshow {args_dict["nshow"]} \\
+    --energy_range {energy_range_string} \\
+    --core_scatter {core_scatter_string} \\
+    --run_number_start $((process_id + {args_dict["run_number_start"]})) \\
+    --number_of_runs 1 \\
+    --submit_engine \"local\" \\
+    --data_directory /tmp/simtools-data \\
+    --output_path /tmp/simtools-output \\
+    --log_level {args_dict["log_level"]} \\
+    --pack_for_grid_register simtools-output
+"""

simtools/job_execution/job_manager.py

@@ -1,7 +1,7 @@
 """Interface to workload managers like gridengine or HTCondor."""
 
 import logging
-import os
+import subprocess
 from pathlib import Path
 
 import simtools.utils.general as gen
@@ -65,11 +65,9 @@ class JobManager:
         ValueError
             if invalid submit engine.
         """
-        if value is None:
-            value = "local"
-        if value not in self.engines:
-            raise ValueError(f"Invalid submit command: {value}")
-        self._submit_engine = value
+        self._submit_engine = value or "local"
+        if self._submit_engine not in self.engines:
+            raise ValueError(f"Invalid submit command: {self._submit_engine}")
 
     def check_submission_system(self):
         """
@@ -77,14 +75,17 @@ class JobManager:
 
         Raises
         ------
-        MissingWorkloadManagerError
+        JobExecutionError
             if workflow manager is not found.
         """
-        if self.submit_engine is None or self.submit_engine == "local":
-            return
-
-        if gen.program_is_executable(self.engines[self.submit_engine]):
-            return
+        try:
+            if self.submit_engine in (None, "local") or gen.program_is_executable(
+                self.engines[self.submit_engine]
+            ):
+                return
+        except KeyError:
+            pass
+        raise JobExecutionError(f"Submit engine {self.submit_engine} not found")
 
     def submit(self, run_script=None, run_out_file=None, log_file=None):
         """
@@ -109,12 +110,14 @@ class JobManager:
         self._logger.info(f"Job error stream {self.run_out_file + '.err'}")
         self._logger.info(f"Job log stream {self.run_out_file + '.job'}")
 
-        if self.submit_engine == "gridengine":
-            self._submit_gridengine()
-        elif self.submit_engine == "htcondor":
-            self._submit_htcondor()
-        elif self.submit_engine == "local":
-            self._submit_local(log_file)
+        submit_result = 0
+        if self.submit_engine == "local":
+            submit_result = self._submit_local(log_file)
+        else:
+            submit_result = getattr(self, f"_submit_{self.submit_engine}")()
+
+        if submit_result != 0:
+            raise JobExecutionError(f"Job submission failed with return code {submit_result}")
 
     def _submit_local(self, log_file):
         """
@@ -125,50 +128,72 @@ class JobManager:
         log_file: str or Path
             The log file of the actual simulator (CORSIKA or sim_telarray).
             Provided in order to print the log excerpt in case of run time error.
+
+        Returns
+        -------
+        int
+            Return code of the executed script
         """
         self._logger.info("Running script locally")
 
-        shell_command = f"{self.run_script} > {self.run_out_file}.out 2> {self.run_out_file}.err"
-
-        if not self.test:
-            sys_output = os.system(shell_command)
-            if sys_output != 0:
-                msg = gen.get_log_excerpt(f"{self.run_out_file}.err")
-                self._logger.error(msg)
-                if log_file.exists() and gen.get_file_age(log_file) < 5:
-                    msg = gen.get_log_excerpt(log_file)
-                    self._logger.error(msg)
-                raise JobExecutionError("See excerpt from log file above\n")
-        else:
+        if self.test:
             self._logger.info("Testing (local)")
+            return 0
+
+        result = None
+        try:
+            with (
+                open(f"{self.run_out_file}.out", "w", encoding="utf-8") as stdout,
+                open(f"{self.run_out_file}.err", "w", encoding="utf-8") as stderr,
+            ):
+                result = subprocess.run(
+                    f"{self.run_script}",
+                    shell=True,
+                    check=True,
+                    text=True,
+                    stdout=stdout,
+                    stderr=stderr,
+                )
+        except subprocess.CalledProcessError as exc:
+            self._logger.error(gen.get_log_excerpt(f"{self.run_out_file}.err"))
+            if log_file.exists() and gen.get_file_age(log_file) < 5:
+                self._logger.error(gen.get_log_excerpt(log_file))
+            raise JobExecutionError("See excerpt from log file above\n") from exc
+
+        return result.returncode if result else 0
 
     def _submit_htcondor(self):
         """Submit a job described by a shell script to HTcondor."""
         _condor_file = self.run_script + ".condor"
+        lines = [
+            f"Executable = {self.run_script}",
+            f"Output = {self.run_out_file}.out",
+            f"Error = {self.run_out_file}.err",
+            f"Log = {self.run_out_file}.job",
+        ]
+        if self.submit_options:
+            lines.extend(option.lstrip() for option in self.submit_options.split(","))
+        lines.append("queue 1")
         try:
             with open(_condor_file, "w", encoding="utf-8") as file:
-                file.write(f"Executable = {self.run_script}\n")
-                file.write(f"Output = {self.run_out_file + '.out'}\n")
-                file.write(f"Error = {self.run_out_file + '.err'}\n")
-                file.write(f"Log = {self.run_out_file + '.job'}\n")
-                if self.submit_options:
-                    submit_option_list = self.submit_options.split(",")
-                    for option in submit_option_list:
-                        file.write(option.lstrip() + "\n")
-                file.write("queue 1\n")
+                file.write("\n".join(lines) + "\n")
         except FileNotFoundError as exc:
             self._logger.error(f"Failed creating condor submission file {_condor_file}")
             raise JobExecutionError from exc
 
-        self._execute(self.submit_engine, self.engines[self.submit_engine] + " " + _condor_file)
+        return self._execute(self.submit_engine, [self.engines[self.submit_engine], _condor_file])
 
     def _submit_gridengine(self):
         """Submit a job described by a shell script to gridengine."""
-        this_sub_cmd = self.engines[self.submit_engine]
-        this_sub_cmd = this_sub_cmd + " -o " + self.run_out_file + ".out"
-        this_sub_cmd = this_sub_cmd + " -e " + self.run_out_file + ".err"
-
-        self._execute(self.submit_engine, this_sub_cmd + " " + self.run_script)
+        this_sub_cmd = [
+            self.engines[self.submit_engine],
+            "-o",
+            self.run_out_file + ".out",
+            "-e",
+            self.run_out_file + ".err",
+            self.run_script,
+        ]
+        return self._execute(self.submit_engine, this_sub_cmd)
 
     def _execute(self, engine, shell_command):
         """
@@ -178,13 +203,15 @@ class JobManager:
         ----------
         engine : str
             Engine to use.
-        shell_command : str
-            Shell command to execute.
+        shell_command : list
+            List of shell command plus arguments.
         """
         self._logger.info(f"Submitting script to {engine}")
         self._logger.debug(shell_command)
+        result = None
         if not self.test:
-            os.system(shell_command)
+            result = subprocess.run(shell_command, shell=True, check=True)
         else:
-            self._logger.info(f"Testing ({engine})")
-            self._logger.info(shell_command)
+            self._logger.info(f"Testing ({engine}: {shell_command})")
+
+        return result.returncode if result else 0

simtools/model/array_model.py

@@ -27,25 +27,25 @@ class ArrayModel:
         MongoDB configuration.
     model_version: str
         Model version.
-    label: str
+    label: str, optional
         Instance label. Used for output file naming.
-    site: str
+    site: str, optional
         Site name.
-    layout_name: str
+    layout_name: str, optional
         Layout name.
-    array_elements: str, Path, list
+    array_elements: Union[str, Path, List[str]], optional
         Array element definitions (list of array element or path to file with
         the array element positions).
     """
 
     def __init__(
         self,
-        mongo_db_config,
-        model_version,
-        label=None,
-        site=None,
-        layout_name=None,
-        array_elements=None,
+        mongo_db_config: dict,
+        model_version: str,
+        label: str | None = None,
+        site: str | None = None,
+        layout_name: str | None = None,
+        array_elements: str | Path | list[str] | None = None,
     ):
         """Initialize ArrayModel."""
         self._logger = logging.getLogger(__name__)
@@ -53,7 +53,7 @@ class ArrayModel:
         self.mongo_db_config = mongo_db_config
         self.model_version = model_version
         self.label = label
-        self.layout_name = layout_name if layout_name else None
+        self.layout_name = layout_name
         self._config_file_path = None
         self._config_file_directory = None
         self.io_handler = io_handler.IOHandler()
@@ -66,7 +66,7 @@ class ArrayModel:
         self._telescope_model_files_exported = False
         self._array_model_file_exported = False
 
-    def _initialize(self, site, array_elements_config):
+    def _initialize(self, site: str, array_elements_config: str | Path | list[str]):
         """
         Initialize ArrayModel taking different configuration options into account.
 
@@ -74,7 +74,7 @@ class ArrayModel:
         ----------
         site: str
             Site name.
-        array_elements_config: str, Path, list
+        array_elements_config: Union[str, Path, List[str]]
             Array element definitions.
 
         Returns
@@ -108,7 +108,7 @@ class ArrayModel:
             array_elements = self._get_array_elements_from_list(
                 site_model.get_array_elements_for_layout(self.layout_name)
             )
-        if array_elements == {}:
+        if not array_elements:
             raise ValueError(
                 "No array elements found. "
                 "Possibly missing valid layout name or missing telescope list."
@@ -117,7 +117,7 @@ class ArrayModel:
         return array_elements, site_model, telescope_model
 
     @property
-    def number_of_telescopes(self):
+    def number_of_telescopes(self) -> int:
         """
         Return the number of telescopes.
 
@@ -129,7 +129,7 @@ class ArrayModel:
         return len(self.telescope_model)
 
     @property
-    def site(self):
+    def site(self) -> str:
         """
         Return site.
 
@@ -140,7 +140,7 @@ class ArrayModel:
         """
         return self.site_model.site
 
-    def _build_telescope_models(self, site_model, array_elements):
+    def _build_telescope_models(self, site_model: SiteModel, array_elements: dict) -> dict:
         """
         Build the the telescope models for all telescopes of this array.
 
@@ -150,7 +150,7 @@ class ArrayModel:
 
         Parameters
         ----------
-        site_model: str
+        site_model: SiteModel
             Site model.
         array_elements: dict
             Dict with array elements.
@@ -233,7 +233,7 @@ class ArrayModel:
         if not self._array_model_file_exported:
             self.export_simtel_array_config_file()
 
-    def get_config_file(self):
+    def get_config_file(self) -> Path:
         """
         Return the path of the array config file for sim_telarray.
 
@@ -247,7 +247,7 @@ class ArrayModel:
         self.export_all_simtel_config_files()
         return self._config_file_path
 
-    def get_config_directory(self):
+    def get_config_directory(self) -> Path:
         """
         Get the path of the array config directory for sim_telarray.
 
@@ -260,7 +260,9 @@ class ArrayModel:
             self._config_file_directory = self.io_handler.get_output_directory(self.label, "model")
         return self._config_file_directory
 
-    def _load_array_element_positions_from_file(self, array_elements_file, site):
+    def _load_array_element_positions_from_file(
+        self, array_elements_file: str | Path, site: str
+    ) -> dict:
         """
         Load array element (e.g. telescope) positions from a file into a dict.
 
@@ -268,7 +270,7 @@ class ArrayModel:
 
         Parameters
         ----------
-        array_elements_file: str
+        array_elements_file: Union[str, Path]
             Path to the file with the array element positions.
         site: str
             Site name.
@@ -287,7 +289,9 @@ class ArrayModel:
             for row in table
         }
 
-    def _get_telescope_position_parameter(self, telescope_name, site, x, y, z):
+    def _get_telescope_position_parameter(
+        self, telescope_name: str, site: str, x: u.Quantity, y: u.Quantity, z: u.Quantity
+    ) -> dict:
         """
         Return dictionary with telescope position parameters (following DB model database format).
 
@@ -323,7 +327,7 @@ class ArrayModel:
             "file": False,
         }
 
-    def _get_array_elements_from_list(self, array_elements_list):
+    def _get_array_elements_from_list(self, array_elements_list: list[str]) -> dict:
         """
         Return dictionary with array elements from a list of telescope names.
 
@@ -349,7 +353,7 @@ class ArrayModel:
                 array_elements_dict.update(self._get_all_array_elements_of_type(name))
         return array_elements_dict
 
-    def _get_all_array_elements_of_type(self, array_element_type):
+    def _get_all_array_elements_of_type(self, array_element_type: str) -> dict:
         """
         Return all array elements of a specific type using the database.
 
@@ -371,7 +375,7 @@ class ArrayModel:
         )
         return self._get_array_elements_from_list(all_elements)
 
-    def export_array_elements_as_table(self, coordinate_system="ground"):
+    def export_array_elements_as_table(self, coordinate_system: str = "ground") -> QTable:
         """
         Export array elements positions to astropy table.
 

simtools/model/calibration_model.py

@@ -23,23 +23,20 @@ class CalibrationModel(ModelParameter):
         MongoDB configuration.
     model_version: str
         Model version.
-    label: str
+    label: str, optional
         Instance label. Important for output file naming.
     """
 
     def __init__(
         self,
-        site,
-        calibration_device_model_name,
-        mongo_db_config,
-        model_version,
-        label=None,
+        site: str,
+        calibration_device_model_name: str,
+        mongo_db_config: dict,
+        model_version: str,
+        label: str | None = None,
     ):
         """Initialize CalibrationModel."""
-        self._logger = logging.getLogger(__name__)
-        self._logger.debug("Init CalibrationModel %s %s", site, calibration_device_model_name)
-        ModelParameter.__init__(
-            self,
+        super().__init__(
             site=site,
             array_element_name=calibration_device_model_name,
             collection="calibration_devices",
@@ -48,3 +45,6 @@ class CalibrationModel(ModelParameter):
             db=None,
             label=label,
         )
+
+        self._logger = logging.getLogger(__name__)
+        self._logger.debug("Init CalibrationModel %s %s", site, calibration_device_model_name)