gammasimtools 0.6.1__py3-none-any.whl → 0.8.2__py3-none-any.whl

simtools/io_operations/hdf5_handler.py

@@ -1,3 +1,5 @@
+"""Helper module for reading and writing in hd5 format."""
+
 import logging
 from pathlib import PosixPath
 
@@ -19,6 +21,7 @@ _logger = logging.getLogger(__name__)
 def fill_hdf5_table(hist, x_bin_edges, y_bin_edges, x_label, y_label, meta_data):
     """
     Create and fill an hdf5 table with the histogram information.
+
     It works for both 1D and 2D distributions.
 
     Parameters
@@ -38,7 +41,6 @@ def fill_hdf5_table(hist, x_bin_edges, y_bin_edges, x_label, y_label, meta_data)
     meta_data: dict
         Dictionary with the histogram metadata.
     """
-
     # Complement metadata
     if x_label is not None:
         meta_data["x_bin_edges"] = sanitize_name(x_label)

simtools/io_operations/io_handler.py

@@ -1,38 +1,37 @@
+"""Handle input and output paths."""
+
 import datetime
 import logging
-import re
 from pathlib import Path
 
-__all__ = ["IOHandlerSingleton", "IOHandler"]
+__all__ = ["IOHandler", "IOHandlerSingleton"]
 
 
-class IncompleteIOHandlerInit(Exception):
-    """Exception raised when IOHandler is not initialized"""
+class IncompleteIOHandlerInitError(Exception):
+    """Exception raised when IOHandler is not initialized."""
 
 
 class IOHandlerSingleton(type):
-    """
-    Singleton base class
-    """
+    """Singleton base class."""
 
     _instances = {}
 
     def __call__(cls, *args, **kwargs):
+        """
+        Ensure a single instance of the IOHandlerSingleton class.
+
+        Creates a new instance if it doesn't exist, otherwise returns the existing instance.
+        """
         if cls not in cls._instances:
-            cls._instances[cls] = super(IOHandlerSingleton, cls).__call__(*args, **kwargs)
+            cls._instances[cls] = super().__call__(*args, **kwargs)
         return cls._instances[cls]
 
 
 class IOHandler(metaclass=IOHandlerSingleton):
-    """
-    Handle input and output paths.
-    """
+    """Handle input and output paths."""
 
     def __init__(self):
-        """
-        Initialize IOHandler.
-
-        """
+        """Initialize IOHandler."""
         self._logger = logging.getLogger(__name__)
         self._logger.debug("Init IOHandler")
 
@@ -64,9 +63,9 @@ class IOHandler(metaclass=IOHandlerSingleton):
         self.data_path = data_path
         self.model_path = model_path
 
-    def get_output_directory(self, label=None, sub_dir=None, dir_type="simtools"):
+    def get_output_directory(self, label=None, sub_dir=None):
         """
-        Return path to output directory
+        Return path to output directory.
 
         Parameters
         ----------
@@ -74,12 +73,6 @@ class IOHandler(metaclass=IOHandlerSingleton):
             Instance label.
         sub_dir: str
             Name of the subdirectory (ray-tracing, model etc)
-        dir_type: str
-            The type of directory (e.g., 'simtools', 'test', 'simtools-result').
-            If 'simtools-result' is used, the output directory will be returned
-            without appending a subdirectory string when using the
-            use_plain_output_path option. For the cause of not using use_plain_output_path,
-            output paths appended by 'simtools-output'.
 
         Returns
         -------
@@ -92,37 +85,27 @@ class IOHandler(metaclass=IOHandlerSingleton):
         TypeError
             raised for errors while creating directory name
         """
-
-        if self.use_plain_output_path:
-            path = Path(self.output_path)
-        else:
-            if str(self.output_path).endswith("-output"):
-                output_directory_prefix = Path(self.output_path)
-            else:
-                try:
-                    output_directory_prefix = Path(self.output_path).joinpath(
-                        re.sub(r"\-result$", "", dir_type) + "-output"
-                    )
-                except TypeError:
-                    self._logger.error(f"Error creating output directory name from {dir_type}")
-                    raise
+        path = Path(self.output_path)
+        if not self.use_plain_output_path:
+            path = (
+                path
+                if str(self.output_path).endswith("-output")
+                else path.joinpath("simtools-output")
+            )
             label_dir = label if label is not None else "d-" + str(datetime.date.today())
-            path = output_directory_prefix.joinpath(label_dir)
-        if sub_dir is not None:
-            if not self.use_plain_output_path:
-                path = path.joinpath(sub_dir)
-            elif dir_type != "simtools-result":
-                path = path.joinpath(sub_dir)
+            path = (
+                path.joinpath(label_dir) if sub_dir is None else path.joinpath(label_dir, sub_dir)
+            )
 
         try:
             path.mkdir(parents=True, exist_ok=True)
         except FileNotFoundError:
-            self._logger.error(f"Error creating directory {str(path)}")
+            self._logger.error(f"Error creating directory {path!s}")
             raise
 
         return path.absolute()
 
-    def get_output_file(self, file_name, label=None, sub_dir=None, dir_type="simtools"):
+    def get_output_file(self, file_name, label=None, sub_dir=None):
         """
         Get path of an output file.
 
@@ -134,25 +117,18 @@ class IOHandler(metaclass=IOHandlerSingleton):
             Instance label.
         sub_dir: str
             Name of the subdirectory (ray-tracing, model etc)
-        dir_type: str
-            Directory type (e.g., 'simtools', 'test', 'simtools-result')
-            If 'simtools-result' is used, the output directory will be returned
-            without appending a subdirectory string when using the
-            use_plain_output_path option.
 
         Returns
         -------
         Path
         """
         return (
-            self.get_output_directory(label=label, sub_dir=sub_dir, dir_type=dir_type)
-            .joinpath(file_name)
-            .absolute()
+            self.get_output_directory(label=label, sub_dir=sub_dir).joinpath(file_name).absolute()
         )
 
     def get_input_data_file(self, parent_dir=None, file_name=None, test=False):
         """
-        Get path of a data file, using data_path
+        Get path of a data file, using data_path.
 
         Parameters
         ----------
@@ -169,15 +145,14 @@ class IOHandler(metaclass=IOHandlerSingleton):
 
         Raises
         ------
-        IncompleteIOHandlerInit
+        IncompleteIOHandlerInitError
             if data_path is not set
 
         """
-
         if test:
             file_prefix = Path("tests/resources/")
         elif self.data_path is not None:
             file_prefix = Path(self.data_path).joinpath(parent_dir)
         else:
-            raise IncompleteIOHandlerInit
+            raise IncompleteIOHandlerInitError
         return file_prefix.joinpath(file_name).absolute()

simtools/job_execution/job_manager.py

@@ -1,78 +1,90 @@
+"""Interface to workload managers like gridengine or HTCondor."""
+
 import logging
 import os
-from copy import copy
 from pathlib import Path
 
 import simtools.utils.general as gen
 
-__all__ = ["JobManager", "MissingWorkloadManager", "JobExecutionError"]
-
-
-class MissingWorkloadManager(Exception):
-    """Exception for missing work load manager."""
+__all__ = ["JobExecutionError", "JobManager"]
 
 
 class JobExecutionError(Exception):
-    """Exception for job execution error (usually CORSIKA or sim_telarray)."""
+    """Job execution error."""
 
 
 class JobManager:
     """
-    JobManager provides an interface to workload managers like gridengine or HTCondor.
+    Interface to workload managers like gridengine or HTCondor.
+
+    Expects that jobs are described by shell scripts.
 
     Parameters
     ----------
-    submit_command: str
-        Job submission command.
-    test: bool
+    submit_engine : str
+        Job submission system. Default is local.
+    test : bool
         Testing mode without sub submission.
-
-    Raises
-    ------
-    MissingWorkloadManager
-        if requested workflow manager not found.
     """
 
-    def __init__(self, submit_command=None, test=False):
-        """
-        Initialize JobManager
-        """
+    engines = {
+        "gridengine": "qsub",
+        "htcondor": "condor_submit",
+        "local": "",
+        "test_wms": "test_wms",  # used for testing only
+    }
+
+    def __init__(self, submit_engine=None, submit_options=None, test=False):
+        """Initialize JobManager."""
         self._logger = logging.getLogger(__name__)
-        self.submit_command = submit_command
+        self.submit_engine = submit_engine
+        self.submit_options = submit_options
         self.test = test
         self.run_script = None
         self.run_out_file = None
 
-        try:
-            self.test_submission_system()
-        except MissingWorkloadManager:
-            self._logger.error(f"Requested workflow manager not found: {self.submit_command}")
-            raise
+        self.check_submission_system()
+
+    @property
+    def submit_engine(self):
+        """Get the submit command."""
+        return self._submit_engine
 
-    def test_submission_system(self):
+    @submit_engine.setter
+    def submit_engine(self, value):
         """
-        Check that the requested workload manager exist on the system this script is executed.
+        Set the submit command.
+
+        Parameters
+        ----------
+        value : str
+            Name of submit engine.
 
         Raises
         ------
-        MissingWorkloadManager
-            if workflow manager is not found.
+        ValueError
+            if invalid submit engine.
         """
+        if value is None:
+            value = "local"
+        if value not in self.engines:
+            raise ValueError(f"Invalid submit command: {value}")
+        self._submit_engine = value
 
-        if self.submit_command is None:
-            return
-        if self.submit_command.find("qsub") >= 0:
-            if gen.program_is_executable("qsub"):
-                return
-            raise MissingWorkloadManager
-        if self.submit_command.find("condor_submit") >= 0:
-            if gen.program_is_executable("condor_submit"):
-                return
-            raise MissingWorkloadManager
-        if self.submit_command.find("local") >= 0:
+    def check_submission_system(self):
+        """
+        Check that the requested workload manager exist on the system.
+
+        Raises
+        ------
+        MissingWorkloadManagerError
+            if workflow manager is not found.
+        """
+        if self.submit_engine is None or self.submit_engine == "local":
             return
 
-        raise MissingWorkloadManager
+        if gen.program_is_executable(self.engines[self.submit_engine]):
+            return
 
     def submit(self, run_script=None, run_out_file=None, log_file=None):
         """
@@ -97,17 +109,16 @@ class JobManager:
         self._logger.info(f"Job error stream {self.run_out_file + '.err'}")
         self._logger.info(f"Job log stream {self.run_out_file + '.job'}")
 
-        if self.submit_command.find("qsub") >= 0:
+        if self.submit_engine == "gridengine":
             self._submit_gridengine()
-        elif self.submit_command.find("condor_submit") >= 0:
+        elif self.submit_engine == "htcondor":
             self._submit_htcondor()
-        elif self.submit_command.find("local") >= 0:
+        elif self.submit_engine == "local":
             self._submit_local(log_file)
 
     def _submit_local(self, log_file):
         """
-        Run a job script on the command line
-        (no submission to a workload manager)
+        Run a job script on the command line (no submission to a workload manager).
 
         Parameters
         ----------
@@ -115,7 +126,6 @@ class JobManager:
             The log file of the actual simulator (CORSIKA or sim_telarray).
             Provided in order to print the log excerpt in case of run time error.
         """
-
         self._logger.info("Running script locally")
 
         shell_command = f"{self.run_script} > {self.run_out_file}.out 2> {self.run_out_file}.err"
@@ -133,45 +143,48 @@ class JobManager:
             self._logger.info("Testing (local)")
 
     def _submit_htcondor(self):
-        """
-        Submit a job described by a shell script to HTcondor
-
-        """
-
+        """Submit a job described by a shell script to HTcondor."""
         _condor_file = self.run_script + ".condor"
-        self._logger.info(f"Submitting script to HTCondor ({_condor_file})")
         try:
             with open(_condor_file, "w", encoding="utf-8") as file:
                 file.write(f"Executable = {self.run_script}\n")
                 file.write(f"Output = {self.run_out_file + '.out'}\n")
                 file.write(f"Error = {self.run_out_file + '.err'}\n")
                 file.write(f"Log = {self.run_out_file + '.job'}\n")
+                if self.submit_options:
+                    submit_option_list = self.submit_options.split(",")
+                    for option in submit_option_list:
+                        file.write(option.lstrip() + "\n")
                 file.write("queue 1\n")
-        except FileNotFoundError:
+        except FileNotFoundError as exc:
             self._logger.error(f"Failed creating condor submission file {_condor_file}")
+            raise JobExecutionError from exc
 
-        shell_command = self.submit_command + " " + _condor_file
-        if not self.test:
-            os.system(shell_command)
-        else:
-            self._logger.info("Testing (HTcondor)")
+        self._execute(self.submit_engine, self.engines[self.submit_engine] + " " + _condor_file)
 
     def _submit_gridengine(self):
-        """
-        Submit a job described by a shell script to gridengine
-
-        """
-
-        this_sub_cmd = copy(self.submit_command)
+        """Submit a job described by a shell script to gridengine."""
+        this_sub_cmd = self.engines[self.submit_engine]
         this_sub_cmd = this_sub_cmd + " -o " + self.run_out_file + ".out"
         this_sub_cmd = this_sub_cmd + " -e " + self.run_out_file + ".err"
 
-        self._logger.info("Submitting script to gridengine")
+        self._execute(self.submit_engine, this_sub_cmd + " " + self.run_script)
 
-        shell_command = this_sub_cmd + " " + self.run_script
+    def _execute(self, engine, shell_command):
+        """
+        Execute a shell command using a specific engine.
+
+        Parameters
+        ----------
+        engine : str
+            Engine to use.
+        shell_command : str
+            Shell command to execute.
+        """
+        self._logger.info(f"Submitting script to {engine}")
         self._logger.debug(shell_command)
         if not self.test:
             os.system(shell_command)
         else:
-            self._logger.info("Testing (gridengine)")
+            self._logger.info(f"Testing ({engine})")
             self._logger.info(shell_command)