gammasimtools 0.21.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

simtools/job_execution/job_manager.py

@@ -2,17 +2,59 @@
 
 import logging
 import subprocess
+import time
 from pathlib import Path
 
 import simtools.utils.general as gen
 
-__all__ = ["JobExecutionError", "JobManager"]
+logger = logging.getLogger(__name__)
 
 
 class JobExecutionError(Exception):
     """Job execution error."""
 
 
+def retry_command(command, max_attempts=3, delay=10):
+    """
+    Execute a shell command with retry logic for network-related failures.
+
+    Parameters
+    ----------
+    command : str
+        Shell command to execute.
+    max_attempts : int
+        Maximum number of retry attempts (default: 3).
+    delay : int
+        Delay in seconds between attempts (default: 10).
+
+    Returns
+    -------
+    bool
+        True if command succeeded, False if all attempts failed.
+
+    Raises
+    ------
+    subprocess.CalledProcessError
+        If command fails after all retry attempts.
+    """
+    for attempt in range(1, max_attempts + 1):
+        logger.info(f"Attempt {attempt} of {max_attempts}: {command}")
+        try:
+            subprocess.run(command, shell=True, check=True, text=True)
+            logger.info(f"Command succeeded on attempt {attempt}")
+            return True
+        except subprocess.CalledProcessError as exc:
+            logger.warning(f"Command failed on attempt {attempt}")
+            if attempt < max_attempts:
+                logger.info(f"Waiting {delay}s before retry...")
+                time.sleep(delay)
+            else:
+                logger.error(f"Command failed after {max_attempts} attempts")
+                raise exc from None
+
+    return False
+
+
 class JobManager:
     """
     Job manager for submitting jobs to a compute node.

simtools/layout/array_layout.py

@@ -17,8 +17,6 @@ from simtools.model.site_model import SiteModel
 from simtools.model.telescope_model import TelescopeModel
 from simtools.utils import names, value_conversion
 
-__all__ = ["ArrayLayout", "InvalidTelescopeListFileError"]
-
 
 class InvalidTelescopeListFileError(Exception):
     """Exception for invalid telescope list file."""

simtools/layout/array_layout_utils.py

@@ -154,10 +154,7 @@ def write_array_layouts(array_layouts, args_dict, db_config):
     _logger.info(f"Writing updated array layouts to the database for site {site}.")
 
     io_handler_instance = io_handler.IOHandler()
-    io_handler_instance.set_paths(
-        output_path=args_dict["output_path"],
-        use_plain_output_path=args_dict["use_plain_output_path"],
-    )
+    io_handler_instance.set_paths(output_path=args_dict["output_path"])
     output_file = io_handler_instance.get_output_file(
         f"array-layouts-{args_dict['updated_parameter_version']}.json"
     )
@@ -168,7 +165,6 @@ def write_array_layouts(array_layouts, args_dict, db_config):
         instrument=site,
         parameter_version=args_dict.get("updated_parameter_version"),
         output_file=output_file,
-        use_plain_output_path=args_dict["use_plain_output_path"],
         db_config=db_config,
     )
     MetadataCollector.dump(

simtools/layout/telescope_position.py

@@ -6,8 +6,6 @@ import astropy.units as u
 import numpy as np
 import pyproj
 
-__all__ = ["InvalidCoordSystemErrorError", "TelescopePosition"]
-
 
 class InvalidCoordSystemErrorError(Exception):
     """Exception for invalid coordinate system."""

simtools/model/array_model.py

@@ -9,13 +9,12 @@ from astropy.table import QTable
 from simtools.data_model import data_reader, schema
 from simtools.db import db_handler
 from simtools.io import io_handler
+from simtools.model.calibration_model import CalibrationModel
 from simtools.model.site_model import SiteModel
 from simtools.model.telescope_model import TelescopeModel
 from simtools.simtel.simtel_config_writer import SimtelConfigWriter
 from simtools.utils import general, names
 
-__all__ = ["ArrayModel"]
-
 
 class ArrayModel:
     """
@@ -40,18 +39,21 @@ class ArrayModel:
         Dictionary with configuration for sim_telarray random instrument setup.
     simtel_path: str, Path, optional
         Path to the sim_telarray installation directory.
+    calibration_device_types: List[str], optional
+        List of calibration device types (e.g., 'flat_fielding') attached to each telescope.
     """
 
     def __init__(
         self,
-        mongo_db_config: dict,
-        model_version: str,
-        label: str | None = None,
-        site: str | None = None,
-        layout_name: str | None = None,
-        array_elements: str | Path | list[str] | None = None,
-        sim_telarray_seeds: dict | None = None,
-        simtel_path: str | Path | None = None,
+        mongo_db_config,
+        model_version,
+        label=None,
+        site=None,
+        layout_name=None,
+        array_elements=None,
+        sim_telarray_seeds=None,
+        simtel_path=None,
+        calibration_device_types=None,
     ):
         """Initialize ArrayModel."""
         self._logger = logging.getLogger(__name__)
@@ -69,8 +71,8 @@ class ArrayModel:
         self.io_handler = io_handler.IOHandler()
         self.db = db_handler.DatabaseHandler(mongo_db_config=mongo_db_config)
 
-        self.array_elements, self.site_model, self.telescope_model = self._initialize(
-            site, array_elements
+        self.array_elements, self.site_model, self.telescope_models, self.calibration_models = (
+            self._initialize(site, array_elements, calibration_device_types)
         )
 
         self._telescope_model_files_exported = False
@@ -78,7 +80,7 @@ class ArrayModel:
         self.sim_telarray_seeds = sim_telarray_seeds
         self.simtel_path = simtel_path
 
-    def _initialize(self, site: str, array_elements_config: str | Path | list[str]):
+    def _initialize(self, site, array_elements_config, calibration_device_types):
         """
         Initialize ArrayModel taking different configuration options into account.
 
@@ -88,6 +90,8 @@ class ArrayModel:
             Site name.
         array_elements_config: Union[str, Path, List[str]]
             Array element definitions.
+        calibration_device_types: str
+            Calibration device types.
 
         Returns
         -------
@@ -106,7 +110,6 @@ class ArrayModel:
             label=self.label,
         )
 
-        array_elements = {}
         # Case 1: array_elements is a file name
         if isinstance(array_elements_config, str | Path):
             array_elements = self._load_array_element_positions_from_file(
@@ -120,16 +123,20 @@ class ArrayModel:
             array_elements = self._get_array_elements_from_list(
                 site_model.get_array_elements_for_layout(self.layout_name)
             )
-        if not array_elements:
+        else:
             raise ValueError(
                 "No array elements found. "
                 "Possibly missing valid layout name or missing telescope list."
             )
-        telescope_model = self._build_telescope_models(site_model, array_elements)
-        return array_elements, site_model, telescope_model
+
+        telescope_models, calibration_models = self._build_telescope_models(
+            site_model, array_elements, calibration_device_types
+        )
+
+        return array_elements, site_model, telescope_models, calibration_models
 
     @property
-    def config_file_path(self) -> Path:
+    def config_file_path(self):
         """
         Return the path of the array config file for sim_telarray.
 
@@ -149,7 +156,7 @@ class ArrayModel:
         return self._config_file_path
 
     @property
-    def number_of_telescopes(self) -> int:
+    def number_of_telescopes(self):
         """
         Return the number of telescopes.
 
@@ -158,7 +165,7 @@ class ArrayModel:
         int
             Number of telescopes.
         """
-        return len(self.telescope_model)
+        return len(self.telescope_models)
 
     @property
     def site(self) -> str:
@@ -200,12 +207,16 @@ class ArrayModel:
             )
         self._model_version = model_version
 
-    def _build_telescope_models(self, site_model: SiteModel, array_elements: dict) -> dict:
+    def _build_telescope_models(self, site_model, array_elements, calibration_device_types):
         """
-        Build the the telescope models for all telescopes of this array.
+        Build telescope models for all telescopes of this array.
 
-        Includes reading of telescope model parameters from the DB.
-        The array is defined in the telescopes dictionary. Array element positions
+        Adds calibration device models, if requested through the calibration_device_types argument.
+        Calibration device models are stored in a dictionary with the telescope name as key (to
+        identify the calibration device model on a given telescope).
+
+        Includes reading of telescope model parameters from the database.
+        The array is defined in the array_elements dictionary. Array element positions
         are read from the database if no values are given in this dictionary.
 
         Parameters
@@ -214,40 +225,78 @@ class ArrayModel:
             Site model.
         array_elements: dict
             Dict with array elements.
+        calibration_device_types: List[str]
+            List of calibration device types (e.g., 'flat_fielding')
+
+        Returns
+        -------
+        dict, dict
+            Dictionaries with telescope and calibration models.
+        """
+        telescope_models, calibration_models = {}, {}
+
+        for element_name in array_elements:
+            if names.get_collection_name_from_array_element_name(element_name) != "telescopes":
+                continue
+
+            telescope_models[element_name] = TelescopeModel(
+                site=site_model.site,
+                telescope_name=element_name,
+                model_version=self.model_version,
+                mongo_db_config=self.mongo_db_config,
+                label=self.label,
+            )
+            calibration_models[element_name] = self._build_calibration_models(
+                telescope_models[element_name],
+                site_model,
+                calibration_device_types,
+            )
+
+        return telescope_models, calibration_models
+
+    def _build_calibration_models(self, telescope_model, site_model, calibration_device_types):
+        """
+        Build calibration device models for all telescopes in the array.
+
+        A telescope can have multiple calibration devices of different types.
 
         Returns
         -------
         dict
-            Dictionary with telescope models.
-        """
-        telescope_model = {}
-        for element_name, _ in array_elements.items():
-            collection = names.get_collection_name_from_array_element_name(element_name)
-            if collection == "telescopes":
-                telescope_model[element_name] = TelescopeModel(
-                    site=site_model.site,
-                    telescope_name=element_name,
-                    model_version=self.model_version,
-                    mongo_db_config=self.mongo_db_config,
-                    label=self.label,
-                )
-        return telescope_model
+            Dict with calibration device models.
+        """
+        calibration_models = {}
+        for calibration_device_type in calibration_device_types or []:
+            device_name = telescope_model.get_calibration_device_name(calibration_device_type)
+            if device_name is None:
+                continue
+
+            calibration_models[device_name] = CalibrationModel(
+                site=site_model.site,
+                calibration_device_model_name=device_name,
+                mongo_db_config=self.mongo_db_config,
+                model_version=self.model_version,
+                label=self.label,
+            )
+        return calibration_models
 
     def print_telescope_list(self):
         """Print list of telescopes."""
-        for tel_name, data in self.telescope_model.items():
+        for tel_name, data in self.telescope_models.items():
             print(f"Name: {tel_name}\t Model: {data.name}")
 
     def export_simtel_telescope_config_files(self):
         """Export sim_telarray configuration files for all telescopes into the model directory."""
         exported_models = []
-        for _, tel_model in self.telescope_model.items():
+        for tel_model in self.telescope_models.values():
             name = tel_model.name + (
                 "_" + tel_model.extra_label if tel_model.extra_label != "" else ""
             )
             if name not in exported_models:
                 self._logger.debug(f"Exporting configuration file for telescope {name}")
-                tel_model.write_sim_telarray_config_file()
+                tel_model.write_sim_telarray_config_file(
+                    additional_models=self.calibration_models.get(tel_model.name)
+                )
                 exported_models.append(name)
             else:
                 self._logger.debug(
@@ -270,7 +319,7 @@ class ArrayModel:
         )
         simtel_writer.write_array_config_file(
             config_file_path=self.config_file_path,
-            telescope_model=self.telescope_model,
+            telescope_model=self.telescope_models,
             site_model=self.site_model,
             additional_metadata=self._get_additional_simtel_metadata(),
         )
@@ -298,7 +347,7 @@ class ArrayModel:
         """
         if self._config_file_directory is None:
             self._config_file_directory = self.io_handler.get_model_configuration_directory(
-                self.label, self.model_version
+                model_version=self.model_version
             )
         return self._config_file_directory
 
@@ -316,8 +365,8 @@ class ArrayModel:
             self._logger.warning("No model files found to pack.")
             return None
 
-        archive_name = self.get_config_directory() / "model_files.tar.gz"
-        general.pack_tar_file(archive_name, model_files)
+        archive_name = self.get_config_directory() / f"model_files_{self.model_version}.tar.gz"
+        general.pack_tar_file(archive_name, model_files, sub_dir=f"model/{self.model_version}")
         self._logger.info(f"Packed model files into {archive_name}")
         return archive_name
 
@@ -459,7 +508,7 @@ class ArrayModel:
         table = QTable(meta={"array_name": self.layout_name, "site": self.site_model.site})
 
         name, pos_x, pos_y, pos_z, tel_r = [], [], [], [], []
-        for tel_name, data in self.telescope_model.items():
+        for tel_name, data in self.telescope_models.items():
             name.append(tel_name)
             xyz = data.position(coordinate_system=coordinate_system)
             pos_x.append(xyz[0])

simtools/model/calibration_model.py

@@ -4,8 +4,6 @@ import logging
 
 from simtools.model.model_parameter import ModelParameter
 
-__all__ = ["CalibrationModel"]
-
 
 class CalibrationModel(ModelParameter):
     """

simtools/model/camera.py

@@ -8,8 +8,6 @@ import numpy as np
 
 from simtools.utils.geometry import rotate
 
-__all__ = ["Camera"]
-
 
 class Camera:
     """

simtools/model/mirrors.py

@@ -8,8 +8,6 @@ import astropy.units as u
 import numpy as np
 from astropy.table import Table
 
-__all__ = ["InvalidMirrorListFileError", "Mirrors"]
-
 
 class InvalidMirrorListFileError(Exception):
     """Exception for invalid mirror list file."""

simtools/model/model_parameter.py

@@ -13,8 +13,6 @@ from simtools.io import ascii_handler, io_handler
 from simtools.simtel.simtel_config_writer import SimtelConfigWriter
 from simtools.utils import names
 
-__all__ = ["InvalidModelParameterError", "ModelParameter"]
-
 
 class InvalidModelParameterError(Exception):
     """Exception for invalid model parameter."""
@@ -198,6 +196,31 @@ class ModelParameter:
 
         return _parameter
 
+    def _create_quantity_for_value(self, value, unit):
+        """
+        Create an astropy quantity for a single value and unit.
+
+        Parameters
+        ----------
+        value: numeric or str
+            The value to create a quantity for.
+        unit: str or None
+            The unit string or None.
+
+        Returns
+        -------
+        astropy.Quantity or original value
+            Astropy quantity for numeric values with units,
+            original value for non-numeric values.
+        """
+        if not isinstance(value, int | float):
+            return value
+
+        if unit is None or unit == "null":
+            return value * u.dimensionless_unscaled
+
+        return value * u.Unit(unit)
+
     def get_parameter_value_with_unit(self, par_name):
         """
         Get the value of an existing parameter of the model as an Astropy Quantity with its unit.
@@ -226,15 +249,16 @@ class ModelParameter:
             if (isinstance(_value, (int | float))) or (len(_value) > len(_unit)):
                 return _value * u.Unit(_unit[0])
 
-            # entries with 'null' units should be returned as dimensionless
-            _astropy_units = [
-                u.Unit(item) if item != "null" else u.dimensionless_unscaled for item in _unit
+            # Create list of quantities for multiple values with different units
+            return [
+                self._create_quantity_for_value(_value[i], _unit[i] if i < len(_unit) else None)
+                for i in range(len(_value))
             ]
 
-            return [_value[i] * _astropy_units[i] for i in range(len(_value))]
-
         except (KeyError, TypeError, AttributeError) as exc:
-            self._logger.debug(f"{exc} encountered, returning only value without units.")
+            self._logger.debug(
+                f"{exc} encountered for parameter {par_name}, returning only value without units."
+            )
             return _value  # if unit is NoneType
 
     def get_parameter_type(self, par_name):
@@ -308,7 +332,7 @@ class ModelParameter:
             return
 
         self._config_file_directory = self.io_handler.get_model_configuration_directory(
-            label=self.label, model_version=self.model_version
+            model_version=self.model_version
         )
 
         # Setting file name and the location
@@ -495,23 +519,19 @@ class ModelParameter:
         )
         self._is_exported_model_files_up_to_date = True
 
-    def write_sim_telarray_config_file(self, additional_model=None):
+    def write_sim_telarray_config_file(self, additional_models=None):
         """
         Write the sim_telarray configuration file.
 
         Parameters
         ----------
-        additional_model: TelescopeModel or SiteModel
+        additional_models: TelescopeModel or SiteModel
             Model object for additional parameter to be written to the config file.
         """
         self.parameters.update(self._simulation_config_parameters.get("sim_telarray", {}))
         self.export_model_files(update_if_necessary=True)
 
-        if additional_model:
-            self.parameters.update(additional_model.parameters)
-            additional_model.export_model_files(
-                self.config_file_directory, update_if_necessary=True
-            )
+        self._add_additional_models(additional_models)
 
         self._load_simtel_config_writer()
         self.simtel_config_writer.write_telescope_config_file(
@@ -519,6 +539,19 @@ class ModelParameter:
             parameters=self.parameters,
         )
 
+    def _add_additional_models(self, additional_models):
+        """Add additional models to the current model parameters."""
+        if additional_models is None:
+            return
+
+        if isinstance(additional_models, dict):
+            for additional_model in additional_models.values():
+                self._add_additional_models(additional_model)
+            return
+
+        self.parameters.update(additional_models.parameters)
+        additional_models.export_model_files(self.config_file_directory, update_if_necessary=True)
+
     @property
     def config_file_directory(self):
         """Directory for configuration files. Configure if not yet set."""
@@ -539,6 +572,7 @@ class ModelParameter:
             self.simtel_config_writer = SimtelConfigWriter(
                 site=self.site,
                 telescope_model_name=self.name,
+                telescope_design_model=self.design_model,
                 model_version=self.model_version,
                 label=self.label,
             )