gammasimtools 0.20.0__py3-none-any.whl → 0.22.0__py3-none-any.whl

simtools/io/io_handler.py

@@ -1,15 +1,8 @@
-"""Handle input and output paths."""
+"""Handle input and output directories and file paths."""
 
-import datetime
 import logging
 from pathlib import Path
 
-__all__ = ["IOHandler", "IOHandlerSingleton"]
-
-
-class IncompleteIOHandlerInitError(Exception):
-    """Exception raised when IOHandler is not initialized."""
-
 
 class IOHandlerSingleton(type):
     """Singleton base class."""
@@ -17,32 +10,24 @@ class IOHandlerSingleton(type):
     _instances = {}
 
     def __call__(cls, *args, **kwargs):
-        """
-        Ensure a single instance of the IOHandlerSingleton class.
-
-        Creates a new instance if it doesn't exist, otherwise returns the existing instance.
-        """
+        """Create a new instance if it doesn't exist, otherwise returns the existing instance."""
         if cls not in cls._instances:
             cls._instances[cls] = super().__call__(*args, **kwargs)
         return cls._instances[cls]
 
 
 class IOHandler(metaclass=IOHandlerSingleton):
-    """Handle input and output paths."""
+    """Handle input and output directories and file paths."""
 
     def __init__(self):
         """Initialize IOHandler."""
-        self._logger = logging.getLogger(__name__)
-        self._logger.debug("Init IOHandler")
+        self.logger = logging.getLogger(__name__)
+        self.logger.debug("Init IOHandler")
 
         self.output_path = None
-        self.use_plain_output_path = False
-        self.data_path = None
         self.model_path = None
 
-    def set_paths(
-        self, output_path=None, data_path=None, model_path=None, use_plain_output_path=False
-    ):
+    def set_paths(self, output_path=None, model_path=None):
         """
         Set paths for input and output.
 
@@ -50,28 +35,19 @@ class IOHandler(metaclass=IOHandlerSingleton):
         ----------
         output_path: str or Path
             Path pointing to the output directory.
-        data_path: str or Path
-            Path pointing to the data files (e.g., CORSIKA or sim_telarray output).
         model_path: str or Path
             Path pointing to the model file directory.
-        use_plain_output_path: bool
-            Use plain output path without adding tool name and date
-
         """
         self.output_path = output_path
-        self.use_plain_output_path = use_plain_output_path
-        self.data_path = data_path
         self.model_path = model_path
 
-    def get_output_directory(self, label=None, sub_dir=None):
+    def get_output_directory(self, sub_dir=None):
         """
-        Return path to output directory.
+        Create and get path of an output directory.
 
         Parameters
         ----------
-        label: str
-            Instance label.
-        sub_dir: str
+        sub_dir: str or list of str, optional
             Name of the subdirectory (ray_tracing, model etc)
 
         Returns
@@ -81,34 +57,24 @@ class IOHandler(metaclass=IOHandlerSingleton):
         Raises
         ------
         FileNotFoundError
-            if error creating directory
-        TypeError
-            raised for errors while creating directory name
+            if the directory cannot be created
         """
-        path = Path(self.output_path)
-        if not self.use_plain_output_path:
-            path = (
-                path
-                if str(self.output_path).endswith("-output")
-                else path.joinpath("simtools-output")
-            )
-            label_dir = label if label is not None else "d-" + str(datetime.date.today())
-            path = (
-                path.joinpath(label_dir) if sub_dir is None else path.joinpath(label_dir, sub_dir)
-            )
-
-        return self._mkdir(path)
-
-    def _mkdir(self, path):
-        """Create a directory and return path."""
+        if sub_dir is None:
+            parts = []
+        elif isinstance(sub_dir, list | tuple):
+            parts = sub_dir
+        else:
+            parts = [sub_dir]
+        path = Path(self.output_path, *parts)
+
         try:
             path.mkdir(parents=True, exist_ok=True)
         except FileNotFoundError as exc:
             raise FileNotFoundError(f"Error creating directory {path!s}") from exc
 
-        return path.absolute()
+        return path.resolve()
 
-    def get_output_file(self, file_name, label=None, sub_dir=None):
+    def get_output_file(self, file_name, sub_dir=None):
         """
         Get path of an output file.
 
@@ -116,51 +82,31 @@ class IOHandler(metaclass=IOHandlerSingleton):
         ----------
         files_name: str
             File name.
-        label: str
-            Instance label.
-        sub_dir: str
+        sub_dir: sub_dir: str or list of str, optional
             Name of the subdirectory (ray_tracing, model etc)
 
         Returns
         -------
         Path
         """
-        return (
-            self.get_output_directory(label=label, sub_dir=sub_dir).joinpath(file_name).absolute()
-        )
+        return self.get_output_directory(sub_dir).joinpath(file_name).absolute()
 
-    def get_input_data_file(self, parent_dir=None, file_name=None, test=False):
+    def get_test_data_file(self, file_name=None):
         """
-        Get path of a data file, using data_path.
+        Get path of a data file in the test resources directory.
 
         Parameters
         ----------
-        parent_dir: str
-            Parent directory of the file.
         files_name: str
             File name.
-        test: bool
-            If true, return test resources location
 
         Returns
         -------
         Path
-
-        Raises
-        ------
-        IncompleteIOHandlerInitError
-            if data_path is not set
-
         """
-        if test:
-            file_prefix = Path("tests/resources/")
-        elif self.data_path is not None:
-            file_prefix = Path(self.data_path).joinpath(parent_dir)
-        else:
-            raise IncompleteIOHandlerInitError
-        return file_prefix.joinpath(file_name).absolute()
+        return Path("tests/resources", file_name).resolve()
 
-    def get_model_configuration_directory(self, label, model_version):
+    def get_model_configuration_directory(self, model_version, sub_dir=None):
         """
         Get path of the simulation model configuration directory.
 
@@ -168,13 +114,15 @@ class IOHandler(metaclass=IOHandlerSingleton):
 
         Parameters
         ----------
-        label: str
-            Instance label.
         model_version: str
             Model version.
+        sub_dir: str
+            subdirectory
 
         Returns
         -------
         Path
         """
-        return self._mkdir(self.get_output_directory(label=label).joinpath("model", model_version))
+        return self.get_output_directory(
+            sub_dir=[sub_dir, "model", model_version] if sub_dir else ["model", model_version]
+        )

simtools/job_execution/job_manager.py

@@ -2,6 +2,7 @@
 
 import logging
 import subprocess
+import time
 from pathlib import Path
 
 import simtools.utils.general as gen
@@ -9,10 +10,54 @@ import simtools.utils.general as gen
 __all__ = ["JobExecutionError", "JobManager"]
 
 
+logger = logging.getLogger(__name__)
+
+
 class JobExecutionError(Exception):
     """Job execution error."""
 
 
+def retry_command(command, max_attempts=3, delay=10):
+    """
+    Execute a shell command with retry logic for network-related failures.
+
+    Parameters
+    ----------
+    command : str
+        Shell command to execute.
+    max_attempts : int
+        Maximum number of retry attempts (default: 3).
+    delay : int
+        Delay in seconds between attempts (default: 10).
+
+    Returns
+    -------
+    bool
+        True if command succeeded, False if all attempts failed.
+
+    Raises
+    ------
+    subprocess.CalledProcessError
+        If command fails after all retry attempts.
+    """
+    for attempt in range(1, max_attempts + 1):
+        logger.info(f"Attempt {attempt} of {max_attempts}: {command}")
+        try:
+            subprocess.run(command, shell=True, check=True, text=True)
+            logger.info(f"Command succeeded on attempt {attempt}")
+            return True
+        except subprocess.CalledProcessError as exc:
+            logger.warning(f"Command failed on attempt {attempt}")
+            if attempt < max_attempts:
+                logger.info(f"Waiting {delay}s before retry...")
+                time.sleep(delay)
+            else:
+                logger.error(f"Command failed after {max_attempts} attempts")
+                raise exc from None
+
+    return False
+
+
 class JobManager:
     """
     Job manager for submitting jobs to a compute node.

simtools/layout/array_layout_utils.py

@@ -154,10 +154,7 @@ def write_array_layouts(array_layouts, args_dict, db_config):
     _logger.info(f"Writing updated array layouts to the database for site {site}.")
 
     io_handler_instance = io_handler.IOHandler()
-    io_handler_instance.set_paths(
-        output_path=args_dict["output_path"],
-        use_plain_output_path=args_dict["use_plain_output_path"],
-    )
+    io_handler_instance.set_paths(output_path=args_dict["output_path"])
     output_file = io_handler_instance.get_output_file(
         f"array-layouts-{args_dict['updated_parameter_version']}.json"
     )
@@ -168,7 +165,6 @@ def write_array_layouts(array_layouts, args_dict, db_config):
         instrument=site,
         parameter_version=args_dict.get("updated_parameter_version"),
         output_file=output_file,
-        use_plain_output_path=args_dict["use_plain_output_path"],
         db_config=db_config,
     )
     MetadataCollector.dump(

simtools/model/array_model.py

@@ -9,6 +9,7 @@ from astropy.table import QTable
 from simtools.data_model import data_reader, schema
 from simtools.db import db_handler
 from simtools.io import io_handler
+from simtools.model.calibration_model import CalibrationModel
 from simtools.model.site_model import SiteModel
 from simtools.model.telescope_model import TelescopeModel
 from simtools.simtel.simtel_config_writer import SimtelConfigWriter
@@ -40,18 +41,21 @@ class ArrayModel:
         Dictionary with configuration for sim_telarray random instrument setup.
     simtel_path: str, Path, optional
         Path to the sim_telarray installation directory.
+    calibration_device_types: List[str], optional
+        List of calibration device types (e.g., 'flat_fielding') attached to each telescope.
     """
 
     def __init__(
         self,
-        mongo_db_config: dict,
-        model_version: str,
-        label: str | None = None,
-        site: str | None = None,
-        layout_name: str | None = None,
-        array_elements: str | Path | list[str] | None = None,
-        sim_telarray_seeds: dict | None = None,
-        simtel_path: str | Path | None = None,
+        mongo_db_config,
+        model_version,
+        label=None,
+        site=None,
+        layout_name=None,
+        array_elements=None,
+        sim_telarray_seeds=None,
+        simtel_path=None,
+        calibration_device_types=None,
     ):
         """Initialize ArrayModel."""
         self._logger = logging.getLogger(__name__)
@@ -69,8 +73,8 @@ class ArrayModel:
         self.io_handler = io_handler.IOHandler()
         self.db = db_handler.DatabaseHandler(mongo_db_config=mongo_db_config)
 
-        self.array_elements, self.site_model, self.telescope_model = self._initialize(
-            site, array_elements
+        self.array_elements, self.site_model, self.telescope_models, self.calibration_models = (
+            self._initialize(site, array_elements, calibration_device_types)
         )
 
         self._telescope_model_files_exported = False
@@ -78,7 +82,7 @@ class ArrayModel:
         self.sim_telarray_seeds = sim_telarray_seeds
         self.simtel_path = simtel_path
 
-    def _initialize(self, site: str, array_elements_config: str | Path | list[str]):
+    def _initialize(self, site, array_elements_config, calibration_device_types):
         """
         Initialize ArrayModel taking different configuration options into account.
 
@@ -88,6 +92,8 @@ class ArrayModel:
             Site name.
         array_elements_config: Union[str, Path, List[str]]
             Array element definitions.
+        calibration_device_types: str
+            Calibration device types.
 
         Returns
         -------
@@ -106,7 +112,6 @@ class ArrayModel:
             label=self.label,
         )
 
-        array_elements = {}
         # Case 1: array_elements is a file name
         if isinstance(array_elements_config, str | Path):
             array_elements = self._load_array_element_positions_from_file(
@@ -120,16 +125,20 @@ class ArrayModel:
             array_elements = self._get_array_elements_from_list(
                 site_model.get_array_elements_for_layout(self.layout_name)
             )
-        if not array_elements:
+        else:
             raise ValueError(
                 "No array elements found. "
                 "Possibly missing valid layout name or missing telescope list."
             )
-        telescope_model = self._build_telescope_models(site_model, array_elements)
-        return array_elements, site_model, telescope_model
+
+        telescope_models, calibration_models = self._build_telescope_models(
+            site_model, array_elements, calibration_device_types
+        )
+
+        return array_elements, site_model, telescope_models, calibration_models
 
     @property
-    def config_file_path(self) -> Path:
+    def config_file_path(self):
         """
         Return the path of the array config file for sim_telarray.
 
@@ -149,7 +158,7 @@ class ArrayModel:
         return self._config_file_path
 
     @property
-    def number_of_telescopes(self) -> int:
+    def number_of_telescopes(self):
         """
         Return the number of telescopes.
 
@@ -158,7 +167,7 @@ class ArrayModel:
         int
             Number of telescopes.
         """
-        return len(self.telescope_model)
+        return len(self.telescope_models)
 
     @property
     def site(self) -> str:
@@ -200,12 +209,16 @@ class ArrayModel:
             )
         self._model_version = model_version
 
-    def _build_telescope_models(self, site_model: SiteModel, array_elements: dict) -> dict:
+    def _build_telescope_models(self, site_model, array_elements, calibration_device_types):
         """
-        Build the the telescope models for all telescopes of this array.
+        Build telescope models for all telescopes of this array.
 
-        Includes reading of telescope model parameters from the DB.
-        The array is defined in the telescopes dictionary. Array element positions
+        Adds calibration device models, if requested through the calibration_device_types argument.
+        Calibration device models are stored in a dictionary with the telescope name as key (to
+        identify the calibration device model on a given telescope).
+
+        Includes reading of telescope model parameters from the database.
+        The array is defined in the array_elements dictionary. Array element positions
         are read from the database if no values are given in this dictionary.
 
         Parameters
@@ -214,40 +227,78 @@ class ArrayModel:
             Site model.
         array_elements: dict
             Dict with array elements.
+        calibration_device_types: List[str]
+            List of calibration device types (e.g., 'flat_fielding')
+
+        Returns
+        -------
+        dict, dict
+            Dictionaries with telescope and calibration models.
+        """
+        telescope_models, calibration_models = {}, {}
+
+        for element_name in array_elements:
+            if names.get_collection_name_from_array_element_name(element_name) != "telescopes":
+                continue
+
+            telescope_models[element_name] = TelescopeModel(
+                site=site_model.site,
+                telescope_name=element_name,
+                model_version=self.model_version,
+                mongo_db_config=self.mongo_db_config,
+                label=self.label,
+            )
+            calibration_models[element_name] = self._build_calibration_models(
+                telescope_models[element_name],
+                site_model,
+                calibration_device_types,
+            )
+
+        return telescope_models, calibration_models
+
+    def _build_calibration_models(self, telescope_model, site_model, calibration_device_types):
+        """
+        Build calibration device models for all telescopes in the array.
+
+        A telescope can have multiple calibration devices of different types.
 
         Returns
         -------
         dict
-            Dictionary with telescope models.
-        """
-        telescope_model = {}
-        for element_name, _ in array_elements.items():
-            collection = names.get_collection_name_from_array_element_name(element_name)
-            if collection == "telescopes":
-                telescope_model[element_name] = TelescopeModel(
-                    site=site_model.site,
-                    telescope_name=element_name,
-                    model_version=self.model_version,
-                    mongo_db_config=self.mongo_db_config,
-                    label=self.label,
-                )
-        return telescope_model
+            Dict with calibration device models.
+        """
+        calibration_models = {}
+        for calibration_device_type in calibration_device_types or []:
+            device_name = telescope_model.get_calibration_device_name(calibration_device_type)
+            if device_name is None:
+                continue
+
+            calibration_models[device_name] = CalibrationModel(
+                site=site_model.site,
+                calibration_device_model_name=device_name,
+                mongo_db_config=self.mongo_db_config,
+                model_version=self.model_version,
+                label=self.label,
+            )
+        return calibration_models
 
     def print_telescope_list(self):
         """Print list of telescopes."""
-        for tel_name, data in self.telescope_model.items():
+        for tel_name, data in self.telescope_models.items():
             print(f"Name: {tel_name}\t Model: {data.name}")
 
     def export_simtel_telescope_config_files(self):
         """Export sim_telarray configuration files for all telescopes into the model directory."""
         exported_models = []
-        for _, tel_model in self.telescope_model.items():
+        for tel_model in self.telescope_models.values():
             name = tel_model.name + (
                 "_" + tel_model.extra_label if tel_model.extra_label != "" else ""
             )
             if name not in exported_models:
                 self._logger.debug(f"Exporting configuration file for telescope {name}")
-                tel_model.write_sim_telarray_config_file()
+                tel_model.write_sim_telarray_config_file(
+                    additional_models=self.calibration_models.get(tel_model.name)
+                )
                 exported_models.append(name)
             else:
                 self._logger.debug(
@@ -270,7 +321,7 @@ class ArrayModel:
         )
         simtel_writer.write_array_config_file(
             config_file_path=self.config_file_path,
-            telescope_model=self.telescope_model,
+            telescope_model=self.telescope_models,
             site_model=self.site_model,
             additional_metadata=self._get_additional_simtel_metadata(),
         )
@@ -298,7 +349,7 @@ class ArrayModel:
         """
         if self._config_file_directory is None:
             self._config_file_directory = self.io_handler.get_model_configuration_directory(
-                self.label, self.model_version
+                model_version=self.model_version
             )
         return self._config_file_directory
 
@@ -459,7 +510,7 @@ class ArrayModel:
         table = QTable(meta={"array_name": self.layout_name, "site": self.site_model.site})
 
         name, pos_x, pos_y, pos_z, tel_r = [], [], [], [], []
-        for tel_name, data in self.telescope_model.items():
+        for tel_name, data in self.telescope_models.items():
             name.append(tel_name)
             xyz = data.position(coordinate_system=coordinate_system)
             pos_x.append(xyz[0])

simtools/model/model_parameter.py

@@ -234,7 +234,9 @@ class ModelParameter:
             return [_value[i] * _astropy_units[i] for i in range(len(_value))]
 
         except (KeyError, TypeError, AttributeError) as exc:
-            self._logger.debug(f"{exc} encountered, returning only value without units.")
+            self._logger.debug(
+                f"{exc} encountered for parameter {par_name}, returning only value without units."
+            )
             return _value  # if unit is NoneType
 
     def get_parameter_type(self, par_name):
@@ -308,7 +310,7 @@ class ModelParameter:
             return
 
         self._config_file_directory = self.io_handler.get_model_configuration_directory(
-            label=self.label, model_version=self.model_version
+            model_version=self.model_version
         )
 
         # Setting file name and the location
@@ -495,23 +497,19 @@ class ModelParameter:
         )
         self._is_exported_model_files_up_to_date = True
 
-    def write_sim_telarray_config_file(self, additional_model=None):
+    def write_sim_telarray_config_file(self, additional_models=None):
         """
         Write the sim_telarray configuration file.
 
         Parameters
         ----------
-        additional_model: TelescopeModel or SiteModel
+        additional_models: TelescopeModel or SiteModel
             Model object for additional parameter to be written to the config file.
         """
         self.parameters.update(self._simulation_config_parameters.get("sim_telarray", {}))
         self.export_model_files(update_if_necessary=True)
 
-        if additional_model:
-            self.parameters.update(additional_model.parameters)
-            additional_model.export_model_files(
-                self.config_file_directory, update_if_necessary=True
-            )
+        self._add_additional_models(additional_models)
 
         self._load_simtel_config_writer()
         self.simtel_config_writer.write_telescope_config_file(
@@ -519,6 +517,19 @@ class ModelParameter:
             parameters=self.parameters,
         )
 
+    def _add_additional_models(self, additional_models):
+        """Add additional models to the current model parameters."""
+        if additional_models is None:
+            return
+
+        if isinstance(additional_models, dict):
+            for additional_model in additional_models.values():
+                self._add_additional_models(additional_model)
+            return
+
+        self.parameters.update(additional_models.parameters)
+        additional_models.export_model_files(self.config_file_directory, update_if_necessary=True)
+
     @property
     def config_file_directory(self):
         """Directory for configuration files. Configure if not yet set."""