gammasimtools 0.18.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

simtools/{io_operations → io}/io_handler.py

@@ -97,11 +97,14 @@ class IOHandler(metaclass=IOHandlerSingleton):
                 path.joinpath(label_dir) if sub_dir is None else path.joinpath(label_dir, sub_dir)
             )
 
+        return self._mkdir(path)
+
+    def _mkdir(self, path):
+        """Create a directory and return path."""
         try:
             path.mkdir(parents=True, exist_ok=True)
-        except FileNotFoundError:
-            self._logger.error(f"Error creating directory {path!s}")
-            raise
+        except FileNotFoundError as exc:
+            raise FileNotFoundError(f"Error creating directory {path!s}") from exc
 
         return path.absolute()
 
@@ -156,3 +159,22 @@ class IOHandler(metaclass=IOHandlerSingleton):
         else:
             raise IncompleteIOHandlerInitError
         return file_prefix.joinpath(file_name).absolute()
+
+    def get_model_configuration_directory(self, label, model_version):
+        """
+        Get path of the simulation model configuration directory.
+
+        This is the directory where the sim_telarray configuration files will be stored.
+
+        Parameters
+        ----------
+        label: str
+            Instance label.
+        model_version: str
+            Model version.
+
+        Returns
+        -------
+        Path
+        """
+        return self._mkdir(self.get_output_directory(label=label).joinpath("model", model_version))

simtools/job_execution/htcondor_script_generator.py

@@ -17,13 +17,12 @@ def generate_submission_script(args_dict):
     args_dict: dict
         Arguments dictionary.
     """
-    _logger.info("Generating HT Condor submission scripts ")
-
     work_dir = Path(args_dict["output_path"])
     log_dir = work_dir / "logs"
     work_dir.mkdir(parents=True, exist_ok=True)
     log_dir.mkdir(parents=True, exist_ok=True)
     submit_file_name = "simulate_prod.submit"
+    _logger.info(f"Generating HT Condor submission scripts (path: {work_dir})")
 
     with open(work_dir / f"{submit_file_name}.condor", "w", encoding="utf-8") as submit_file_handle:
         submit_file_handle.write(
@@ -101,9 +100,20 @@ def _get_submit_script(args_dict):
     )
     core_scatter = args_dict["core_scatter"]
     core_scatter_string = f'"{core_scatter[0]} {core_scatter[1].to(u.m).value} m"'
+    view_cone = args_dict["view_cone"]
+    view_cone_string = f'"{view_cone[0].to(u.deg)} {view_cone[1].to(u.deg)}"'
 
     label = args_dict["label"] if args_dict["label"] else "simulate-prod"
 
+    array_layout_name = (
+        args_dict["array_layout_name"][0]
+        if isinstance(args_dict["array_layout_name"], list)
+        and len(args_dict["array_layout_name"]) == 1
+        else args_dict["array_layout_name"]
+    )
+
+    run_number_offset = args_dict["run_number_offset"] or 1
+
     return f"""#!/usr/bin/env bash
 
 # Process ID used to generate run number
@@ -116,15 +126,16 @@ simtools-simulate-prod \\
     --label {label} \\
     --model_version {args_dict["model_version"]} \\
     --site {args_dict["site"]} \\
-    --array_layout_name {args_dict["array_layout_name"]} \\
+    --array_layout_name {array_layout_name} \\
     --primary {args_dict["primary"]} \\
     --azimuth_angle {azimuth_angle_string} \\
     --zenith_angle {zenith_angle_string} \\
     --nshow {args_dict["nshow"]} \\
     --energy_range {energy_range_string} \\
     --core_scatter {core_scatter_string} \\
+    --view_cone {view_cone_string} \\
     --run_number $((process_id)) \\
-    --run_number_offset {args_dict["run_number_offset"]} \\
+    --run_number_offset {run_number_offset} \\
     --number_of_runs 1 \\
     --data_directory /tmp/simtools-data \\
     --output_path /tmp/simtools-output \\

simtools/layout/array_layout.py

@@ -10,7 +10,7 @@ from astropy.table import QTable
 
 import simtools.utils.general as gen
 from simtools.data_model import data_reader, schema
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.layout.geo_coordinates import GeoCoordinates
 from simtools.layout.telescope_position import TelescopePosition
 from simtools.model.site_model import SiteModel

simtools/layout/array_layout_utils.py

@@ -7,7 +7,7 @@ import simtools.utils.general as gen
 from simtools.data_model import data_reader
 from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.data_model.model_data_writer import ModelDataWriter
-from simtools.io_operations import io_handler
+from simtools.io import ascii_handler, io_handler
 from simtools.model.array_model import ArrayModel
 from simtools.model.site_model import SiteModel
 from simtools.utils import names
@@ -36,17 +36,19 @@ def retrieve_ctao_array_layouts(site, repository_url, branch_name="main"):
     _logger.info(f"Retrieving array layouts from {repository_url} on branch {branch_name}.")
 
     if gen.is_url(repository_url):
-        array_element_ids = gen.collect_data_from_http(
+        array_element_ids = ascii_handler.collect_data_from_http(
             url=f"{repository_url}/{branch_name}/array-element-ids.json"
         )
-        sub_arrays = gen.collect_data_from_http(
+        sub_arrays = ascii_handler.collect_data_from_http(
             url=f"{repository_url}/{branch_name}/subarray-ids.json"
         )
     else:
-        array_element_ids = gen.collect_data_from_file(
+        array_element_ids = ascii_handler.collect_data_from_file(
             Path(repository_url) / "array-element-ids.json"
         )
-        sub_arrays = gen.collect_data_from_file(Path(repository_url) / "subarray-ids.json")
+        sub_arrays = ascii_handler.collect_data_from_file(
+            Path(repository_url) / "subarray-ids.json"
+        )
 
     return _get_ctao_layouts_per_site(site, sub_arrays, array_element_ids)
 
@@ -232,7 +234,7 @@ def get_array_layouts_from_parameter_file(
     list
         List of dictionaries containing array layout names and their elements.
     """
-    array_layouts = gen.collect_data_from_file(file_path)
+    array_layouts = ascii_handler.collect_data_from_file(file_path)
     try:
         value = array_layouts["value"]
     except KeyError as exc:
@@ -243,7 +245,12 @@ def get_array_layouts_from_parameter_file(
     for layout in value:
         layouts.append(
             _get_array_layout_dict(
-                db_config, model_version, site, None, layout["name"], coordinate_system
+                db_config,
+                model_version,
+                site,
+                layout.get("elements"),
+                layout["name"],
+                coordinate_system,
             )
         )
     return layouts

simtools/model/array_model.py

@@ -8,11 +8,11 @@ from astropy.table import QTable
 
 from simtools.data_model import data_reader, schema
 from simtools.db import db_handler
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.model.site_model import SiteModel
 from simtools.model.telescope_model import TelescopeModel
 from simtools.simtel.simtel_config_writer import SimtelConfigWriter
-from simtools.utils import names
+from simtools.utils import general, names
 
 __all__ = ["ArrayModel"]
 
@@ -297,11 +297,30 @@ class ArrayModel:
             Path of the config directory path for sim_telarray.
         """
         if self._config_file_directory is None:
-            self._config_file_directory = self.io_handler.get_output_directory(
-                self.label, f"model/{self.model_version}"
+            self._config_file_directory = self.io_handler.get_model_configuration_directory(
+                self.label, self.model_version
             )
         return self._config_file_directory
 
+    def pack_model_files(self):
+        """
+        Pack all model files into a tar.gz archive.
+
+        Returns
+        -------
+        Path
+            Path of the packed model files archive.
+        """
+        model_files = list(Path(self.get_config_directory()).rglob("*"))
+        if not model_files:
+            self._logger.warning("No model files found to pack.")
+            return None
+
+        archive_name = self.get_config_directory() / "model_files.tar.gz"
+        general.pack_tar_file(archive_name, model_files)
+        self._logger.info(f"Packed model files into {archive_name}")
+        return archive_name
+
     def _load_array_element_positions_from_file(
         self, array_elements_file: str | Path, site: str
     ) -> dict:

simtools/model/flasher_model.py

@@ -0,0 +1,106 @@
+"""Definition and modeling of a flasher device."""
+
+import logging
+
+from simtools.model.model_parameter import ModelParameter
+
+__all__ = ["FlasherModel"]
+
+
+class FlasherModel(ModelParameter):
+    """
+    FlasherModel represents the MC model of an individual flasher device.
+
+    It provides functionality to read the required parameters from the DB.
+    Flasher devices are used for flat fielding of the camera pixels.
+
+    Parameters
+    ----------
+    site: str
+        Site name (e.g., South or North).
+    flasher_device_model_name: str
+        Flasher device model name (ex. FLSN-01, FLSS-01, ...).
+    mongo_db_config: dict
+        MongoDB configuration.
+    model_version: str
+        Model version.
+    label: str, optional
+        Instance label. Important for output file naming.
+    """
+
+    def __init__(
+        self,
+        site: str,
+        flasher_device_model_name: str,
+        mongo_db_config: dict,
+        model_version: str,
+        label: str | None = None,
+    ):
+        # Minimal dummy DB to avoid real DB access
+        class _DummyDB:  # pylint: disable=too-few-public-methods
+            def get_design_model(self, *_, **__):
+                return {}
+
+            def get_model_parameters(self, *_, **__):
+                return {}
+
+            def get_simulation_configuration_parameters(self, *_, **__):
+                return {}
+
+            def export_model_files(self, *_, **__):
+                return None
+
+        super().__init__(
+            site=site,
+            array_element_name=None,  # bypass validation (no flasher in array_elements)
+            collection="flasher_devices",
+            mongo_db_config=mongo_db_config,
+            model_version=model_version,
+            db=_DummyDB(),  # do not query DB
+            label=label,
+        )
+
+        self._logger = logging.getLogger(__name__)
+        self._logger.debug(f"Init FlasherModel {site} {flasher_device_model_name}")
+
+        # Keep provided flasher name for reference/logging only
+        self._flasher_device_model_name = flasher_device_model_name
+
+        # Inject defaults for MST (single-mirror) when DB entries are missing.
+        self._inject_mst_defaults_if_missing()
+
+    def _inject_mst_defaults_if_missing(self):
+        """Provide dummy defaults (here NectarCam) when flasher collection is absent."""
+        defaults = {
+            "photons_per_flasher": {"value": 2.5e6, "type": "float"},
+            # Position near optical axis (cm)
+            "flasher_position": {
+                "value": [0.0, 0.0],
+                "unit": "cm,cm",
+                "type": "float_list",
+            },
+            # Distance flasher window to Winston cones (16.75 m)
+            "flasher_depth": {"value": 1675.0, "unit": "cm", "type": "float"},
+            # Received wavelength at PMT (nm)
+            "spectrum": {"value": 392, "unit": "nm", "type": "float"},
+            # Simple Gaussian pulse width (ns)
+            "lightpulse": {"value": "Gauss:3.0", "type": "string"},
+            # Store rise/decay for future use
+            "rise_time_10_90": {"value": 2.5, "unit": "ns", "type": "float"},
+            "decay_time_90_10": {"value": 5.0, "unit": "ns", "type": "float"},
+            # Angular distribution width ~11 deg around axis
+            "angular_distribution": {"value": "gauss:11", "type": "string"},
+            "centroid_offset_deg": {"value": 0.5, "unit": "deg", "type": "float"},
+            # Bunch size for LE
+            "bunch_size": {"value": 1.0, "type": "float"},
+            # Placeholder for future spectral file usage
+            "spectrum_file": {"value": None, "type": "string"},
+        }
+
+        missing = []
+        for key, entry in defaults.items():
+            if key not in self.parameters:
+                self.parameters[key] = entry
+                missing.append(key)
+        if missing:
+            self._logger.info(f"Using built-in MST flasher defaults for: {', '.join(missing)}")

simtools/model/model_parameter.py

@@ -9,7 +9,7 @@ import astropy.units as u
 
 import simtools.utils.general as gen
 from simtools.db import db_handler
-from simtools.io_operations import io_handler
+from simtools.io import ascii_handler, io_handler
 from simtools.simtel.simtel_config_writer import SimtelConfigWriter
 from simtools.utils import names
 
@@ -307,8 +307,8 @@ class ModelParameter:
         if self.name is None and self.site is None:
             return
 
-        self._config_file_directory = self.io_handler.get_output_directory(
-            label=self.label, sub_dir=f"model/{self.model_version}"
+        self._config_file_directory = self.io_handler.get_model_configuration_directory(
+            label=self.label, model_version=self.model_version
         )
 
         # Setting file name and the location
@@ -431,7 +431,7 @@ class ModelParameter:
             "Insufficient validation of parameters."
         )
         self._logger.debug(f"Changing parameters from file {file_name}")
-        self.change_multiple_parameters(**gen.collect_data_from_file(file_name=file_name))
+        self.change_multiple_parameters(**ascii_handler.collect_data_from_file(file_name=file_name))
 
     def change_multiple_parameters(self, **kwargs):
         """

simtools/model/model_repository.py

@@ -5,10 +5,12 @@ a gitlab repository ('SimulationModels'). This module provides service
 functions to interact with and verify the repository.
 """
 
+import json
 import logging
+import shutil
 from pathlib import Path
 
-from simtools.utils import general as gen
+from simtools.io import ascii_handler
 from simtools.utils import names
 
 _logger = logging.getLogger(__name__)
@@ -76,7 +78,7 @@ def _verify_model_parameters_for_production(simulation_models_path, production_f
     tuple
         (missing_files_list, total_checked_count)
     """
-    production_table = gen.collect_data_from_file(production_file)
+    production_table = ascii_handler.collect_data_from_file(production_file)
     missing_files = []
     total_checked = 0
 
@@ -132,3 +134,196 @@ def _get_model_parameter_file_path(
         / parameter_name
         / f"{parameter_name}-{parameter_version}.json"
     )
+
+
+def copy_and_update_production_table(args_dict):
+    """
+    Copy and update simulation model production tables.
+
+    Parameters
+    ----------
+    args_dict: dict
+        Dictionary containing the arguments for copying and updating production tables.
+    """
+    modifications = ascii_handler.collect_data_from_file(args_dict["modifications"])
+    changes = modifications.get("changes", {})
+    model_version = modifications["model_version"]
+
+    simulation_models_path = Path(args_dict["simulation_models_path"])
+    source_prod_table_path = (
+        simulation_models_path / "productions" / args_dict["source_prod_table_dir"]
+    )
+    target_prod_table_path = simulation_models_path / "productions" / model_version
+    model_parameters_dir = simulation_models_path / "model_parameters"
+
+    _logger.info(
+        f"Copying production tables from {source_prod_table_path} to {target_prod_table_path}"
+    )
+
+    if Path(target_prod_table_path).exists():
+        raise FileExistsError(
+            f"The target production table directory '{target_prod_table_path}' already exists."
+        )
+    shutil.copytree(source_prod_table_path, target_prod_table_path)
+
+    _apply_changes_to_production_tables(target_prod_table_path, changes, model_version)
+
+    for telescope, parameters in changes.items():
+        for param, param_data in parameters.items():
+            if param_data.get("value"):
+                _create_new_parameter_entry(telescope, param, param_data, model_parameters_dir)
+
+
+def _apply_changes_to_production_tables(target_prod_table_path, changes, model_version):
+    """Apply changes to the production tables in the target directory."""
+    for file_path in Path(target_prod_table_path).rglob("*.json"):
+        if file_path.name.startswith("configuration"):
+            continue
+        data = ascii_handler.collect_data_from_file(file_path)
+        _apply_changes_to_production_table(data, changes, model_version)
+        with file_path.open("w", encoding="utf-8") as f:
+            json.dump(data, f, indent=4, sort_keys=True)
+            f.write("\n")
+
+
+def _apply_changes_to_production_table(data, changes, model_version):
+    """
+    Recursively apply changes to the new production tables.
+
+    Parameters
+    ----------
+    data: dict or list
+        The JSON data to be updated.
+    changes: dict
+        The changes to be applied.
+    model_version: str
+        The model version to be set in the JSON data.
+    """
+    if isinstance(data, dict):
+        if "model_version" in data:
+            data["model_version"] = model_version
+        _update_parameters(data.get("parameters", {}), changes)
+
+    elif isinstance(data, list):
+        for item in data:
+            _apply_changes_to_production_table(item, changes, model_version)
+
+
+def _update_parameters(params, changes):
+    """Update parameters in the given dictionary based on changes."""
+    for telescope, updates in changes.items():
+        if telescope not in params:
+            continue
+        for param, param_data in updates.items():
+            if param in params[telescope]:
+                old = params[telescope][param]
+                new = param_data["version"]
+                _logger.info(f"Updating '{telescope} - {param}' from {old} to {new}")
+                params[telescope][param] = new
+            else:
+                _logger.info(
+                    f"Adding new parameter '{telescope} - {param}' "
+                    f"with version {param_data['version']}"
+                )
+                params[telescope][param] = param_data["version"]
+
+
+def _create_new_parameter_entry(telescope, param, param_data, model_parameters_dir):
+    """
+    Create new model parameter JSON file by copying the latest version and updating fields.
+
+    Parameters
+    ----------
+    telescope: str
+        Name of the telescope.
+    param: str
+        Name of the parameter.
+    param_data: dict
+        Dictionary containing the parameter data including version and value.
+    model_parameters_dir: str
+        Path to the model parameters directory.
+    """
+    telescope_dir = Path(model_parameters_dir) / telescope
+    if not telescope_dir.exists():
+        raise FileNotFoundError(
+            f"Directory for telescope '{telescope}' does not exist in '{model_parameters_dir}'."
+        )
+
+    param_dir = telescope_dir / param
+    if not param_dir.exists():
+        raise FileNotFoundError(
+            f"Directory for parameter '{param}' does not exist in '{telescope}'."
+        )
+
+    latest_file = _get_latest_model_parameter_file(param_dir, param)
+    if not latest_file:
+        raise FileNotFoundError(
+            f"No files found for parameter '{param}' in directory '{param_dir}'."
+        )
+
+    json_data = ascii_handler.collect_data_from_file(latest_file)
+
+    json_data["parameter_version"] = _update_model_parameter_version(
+        json_data, param_data, param, telescope
+    )
+    json_data["value"] = param_data["value"]
+
+    new_file_name = f"{param}-{param_data['version']}.json"
+    new_file_path = param_dir / new_file_name
+
+    with new_file_path.open("w", encoding="utf-8") as f:
+        json.dump(json_data, f, indent=4)
+        f.write("\n")
+    _logger.info(f"Created new model parameter JSON file: {new_file_path}")
+
+
+def _get_latest_model_parameter_file(directory, parameter):
+    """
+    Get the latest model parameter JSON file for a parameter in the given directory.
+
+    Assume files are named in the format 'parameter-version.json'.
+
+    Parameters
+    ----------
+    directory: str
+        Path to the directory containing parameter JSON files.
+    parameter: str
+        Name of the parameter to find.
+
+    Returns
+    -------
+    str
+        Path to the latest JSON file for the parameter.
+
+    Raises
+    ------
+    FileNotFoundError
+        If no files for the parameter are found in the directory.
+    """
+    directory_path = Path(directory)
+    files = list(directory_path.glob(f"{parameter}-*.json"))
+    if not files:
+        raise FileNotFoundError(
+            f"No JSON files found for parameter '{parameter}' in directory '{directory}'."
+        )
+
+    # Sort files by version number (assumes version is part of the filename)
+    def safe_parse_version(filename):
+        version_str = filename.stem.split("-")[-1]
+        parts = version_str.split(".")
+        return tuple(part.zfill(8) for part in parts)
+
+    files.sort(key=safe_parse_version)
+    return str(files[-1])
+
+
+def _update_model_parameter_version(json_data, param_data, param, telescope):
+    """Check for major version jump and print a warning if necessary."""
+    latest_version = int(json_data.get("parameter_version", "0").split(".")[0])
+    new_version = int(param_data["version"].split(".")[0])
+    if new_version > latest_version + 1:
+        _logger.warning(
+            f"Major version jump from {latest_version} to {new_version} "
+            f"for parameter '{param}' in telescope '{telescope}'."
+        )
+    return param_data["version"]

simtools/model/telescope_model.py

@@ -174,7 +174,9 @@ class TelescopeModel(ModelParameter):
             eff_focal_length = eff_focal_length.to(unit).value
         except AttributeError:
             eff_focal_length = 0.0
-        if return_focal_length_if_zero and (eff_focal_length is None or eff_focal_length == 0.0):
+        if return_focal_length_if_zero and (
+            eff_focal_length is None or np.isclose(eff_focal_length, 0.0)
+        ):
             self._logger.warning("Using focal_length because effective_focal_length is 0")
             return self.get_parameter_value_with_unit("focal_length").to(unit).value
         return eff_focal_length