gammasimtools 0.26.0__py3-none-any.whl → 0.27.1__py3-none-any.whl

simtools/db/db_handler.py

@@ -260,9 +260,7 @@ class DatabaseHandler:
             collection,
         )
         if cache_dict:
-            self._logger.debug(f"Found {array_element} in cache (key: {cache_key})")
             return cache_dict
-        self._logger.debug(f"Did not find {array_element} in cache (key: {cache_key})")
 
         try:
             parameter_version_table = production_table["parameters"][array_element]
@@ -873,6 +871,10 @@ class DatabaseHandler:
                 array_element_name,
             ]
         except KeyError as exc:
+            # simplified model definitions when e.g. adding new telescopes without design model
+            if settings.config.args.get("ignore_missing_design_model", False):
+                element_type = names.get_array_element_type_from_name(array_element_name)
+                return [array_element_name, f"{element_type}-01", f"{element_type}-design"]
             raise KeyError(
                 f"Failed generated array element list for db query for {array_element_name}"
             ) from exc

simtools/db/mongo_db.py

@@ -4,7 +4,7 @@ import io
 import logging
 import re
 from pathlib import Path
-from threading import Lock
+from threading import Lock as _Lock
 
 import gridfs
 import jsonschema
@@ -126,7 +126,7 @@ class MongoDBHandler:  # pylint: disable=unsubscriptable-object
     """
 
     db_client: MongoClient = None
-    _lock = Lock()
+    _lock = _Lock()
     _logger = logging.getLogger(__name__)
 
     def __init__(self, db_config=None):

simtools/io/ascii_handler.py

@@ -10,7 +10,7 @@ import astropy.units as u
 import numpy as np
 import yaml
 
-from simtools.utils.general import is_url
+from simtools.utils.general import ensure_iterable, is_url
 
 _logger = logging.getLogger(__name__)
 
@@ -189,9 +189,9 @@ def is_utf8_file(file_name):
         return False
 
 
-def write_data_to_file(data, output_file, sort_keys=False, numpy_types=False):
+def write_data_to_file(data, output_file, sort_keys=False, numpy_types=False, unique_lines=False):
     """
-    Write structured data to JSON or YAML file.
+    Write structured data to JSON, YAML, or text file.
 
     The file type is determined by the file extension.
 
@@ -213,12 +213,43 @@ def write_data_to_file(data, output_file, sort_keys=False, numpy_types=False):
     if output_file.suffix.lower() in [".yml", ".yaml"]:
         _write_to_yaml(data, output_file, sort_keys)
         return
+    if output_file.suffix.lower() in [".txt", ".list"]:
+        _write_to_text_file(data, output_file, unique_lines)
+        return
 
     raise ValueError(
         f"Unsupported file type {output_file.suffix}. Only .json, .yml, and .yaml are supported."
     )
 
 
+def _write_to_text_file(data, output_file, unique_lines):
+    """
+    Write data to a text file.
+
+    Parameters
+    ----------
+    data: list
+        Data to be written to the file.
+    output_file: Path
+        Name of the file to be written.
+    unique_lines: bool
+        If True, write only unique lines.
+
+    """
+
+    def iter_lines(data):
+        for entry in ensure_iterable(data):
+            yield from entry.splitlines()
+
+    lines_to_write = (
+        list(dict.fromkeys(iter_lines(data))) if unique_lines else list(iter_lines(data))
+    )
+
+    with open(output_file, "w", encoding="utf-8") as file:
+        for line in lines_to_write:
+            file.write(f"{line}\n")
+
+
 def _write_to_json(data, output_file, sort_keys, numpy_types):
     """
     Write data to a JSON file.
@@ -259,8 +290,25 @@ def _write_to_yaml(data, output_file, sort_keys):
         If True, sort the keys.
 
     """
+    data = _to_builtin(data)
     with open(output_file, "w", encoding="utf-8") as file:
-        yaml.dump(data, file, indent=4, sort_keys=sort_keys, explicit_start=True)
+        yaml.dump(data, file, indent=2, sort_keys=sort_keys, explicit_start=True)
+
+
+def _to_builtin(data):
+    """Convert numpy types to native Python types for yaml output."""
+    if isinstance(data, u.Quantity):
+        return {
+            "value": float(data.value),
+            "unit": str(data.unit),
+        }
+    if isinstance(data, np.generic):
+        return data.item()
+    if isinstance(data, dict):
+        return {k: _to_builtin(v) for k, v in data.items()}
+    if isinstance(data, (list, tuple)):
+        return [_to_builtin(v) for v in data]
+    return data
 
 
 class JsonNumpyEncoder(json.JSONEncoder):

simtools/io/io_handler.py

@@ -22,12 +22,10 @@ class IOHandler(metaclass=IOHandlerSingleton):
     def __init__(self):
         """Initialize IOHandler."""
         self.logger = logging.getLogger(__name__)
-        self.logger.debug("Init IOHandler")
-
-        self.output_path = None
+        self.output_path = {}
         self.model_path = None
 
-    def set_paths(self, output_path=None, model_path=None):
+    def set_paths(self, output_path=None, model_path=None, output_path_label="default"):
         """
         Set paths for input and output.
 
@@ -37,11 +35,13 @@ class IOHandler(metaclass=IOHandlerSingleton):
             Path pointing to the output directory.
         model_path: str or Path
             Path pointing to the model file directory.
+        output_path_label: str
+            Label for the output path.
         """
-        self.output_path = output_path
+        self.output_path[output_path_label] = output_path
         self.model_path = model_path
 
-    def get_output_directory(self, sub_dir=None):
+    def get_output_directory(self, sub_dir=None, output_path_label="default"):
         """
         Create and get path of an output directory.
 
@@ -49,6 +49,8 @@ class IOHandler(metaclass=IOHandlerSingleton):
         ----------
         sub_dir: str or list of str, optional
             Name of the subdirectory (ray_tracing, model etc)
+        output_path_label: str
+            Label for the output path.
 
         Returns
         -------
@@ -65,16 +67,19 @@ class IOHandler(metaclass=IOHandlerSingleton):
             parts = sub_dir
         else:
             parts = [sub_dir]
-        path = Path(self.output_path, *parts)
+        try:
+            output_path = Path(self.output_path[output_path_label], *parts)
+        except KeyError as exc:
+            raise KeyError(f"Output path label '{output_path_label}' not found") from exc
 
         try:
-            path.mkdir(parents=True, exist_ok=True)
+            output_path.mkdir(parents=True, exist_ok=True)
         except FileNotFoundError as exc:
-            raise FileNotFoundError(f"Error creating directory {path!s}") from exc
+            raise FileNotFoundError(f"Error creating directory {output_path!s}") from exc
 
-        return path.resolve()
+        return output_path.resolve()
 
-    def get_output_file(self, file_name, sub_dir=None):
+    def get_output_file(self, file_name, sub_dir=None, output_path_label="default"):
         """
         Get path of an output file.
 
@@ -84,12 +89,18 @@ class IOHandler(metaclass=IOHandlerSingleton):
             File name.
         sub_dir: sub_dir: str or list of str, optional
             Name of the subdirectory (ray_tracing, model etc)
+        output_path_label: str
+            Label for the output path.
 
         Returns
         -------
         Path
         """
-        return self.get_output_directory(sub_dir).joinpath(file_name).absolute()
+        return (
+            self.get_output_directory(sub_dir, output_path_label=output_path_label)
+            .joinpath(file_name)
+            .absolute()
+        )
 
     def get_test_data_file(self, file_name=None):
         """

simtools/job_execution/job_manager.py

@@ -1,6 +1,8 @@
 """Interface to workload managers to run jobs on a compute node."""
 
 import logging
+import os
+import stat
 import subprocess
 import time
 from pathlib import Path
@@ -55,91 +57,164 @@ def retry_command(command, max_attempts=3, delay=10):
     return False
 
 
-class JobManager:
+def submit(
+    command,
+    stdin=None,
+    out_file=None,
+    err_file=None,
+    configuration=None,
+    application_log=None,
+    runtime_environment=None,
+    env=None,
+    test=False,
+):
     """
-    Job manager for submitting jobs to a compute node.
+    Submit a job described by a command or a shell script.
 
-    Expects that jobs can be described by shell scripts.
+    Allow to specify a runtime environment (e.g., Docker).
 
     Parameters
     ----------
-    test : bool
+    command: str
+        Command or shell script to execute.
+    stdin: str or Path
+        Input stream.
+    out_file: str or Path
+        Output stream (stdout if out_file and err_file are None).
+    err_file: str or Path
+        Error stream (stderr if out_file and err_file are None).
+    configuration: dict
+        Configuration for the 'command' execution.
+    runtime_environment: list
+        Command to run the application in the specified runtime environment.
+    env: dict
+        Environment variables to set for the job execution.
+    application_log: str or Path
+        The log file of the actual application.
+        Provided in order to print the log excerpt in case of run time error.
+    test: bool
         Testing mode without sub submission.
     """
+    command = _build_command(command, configuration, runtime_environment)
+
+    logger.info(f"Submitting command {command}")
+    logger.info(f"Job output/error streams {out_file} / {err_file}")
+
+    if test:
+        logger.info("Testing mode enabled")
+        return None
+
+    sub_process_env = os.environ.copy()
+    if env:
+        for key, value in env.items():
+            sub_process_env[key] = value
+    logger.debug(f"Setting environment variables for job execution: {sub_process_env}")
+
+    # disable pylint warning about not closing files here (explicitly closed in finally block)
+    stdout = open(out_file, "w", encoding="utf-8") if out_file else subprocess.PIPE  # pylint: disable=consider-using-with
+    stderr = open(err_file, "w", encoding="utf-8") if err_file else subprocess.PIPE  # pylint: disable=consider-using-with
+
+    try:
+        result = subprocess.run(
+            command,
+            shell=isinstance(command, str),
+            check=True,
+            text=True,
+            stdin=stdin,
+            stdout=stdout,
+            stderr=stderr,
+            env=sub_process_env,
+        )
+
+    except subprocess.CalledProcessError as exc:
+        _raise_job_execution_error(exc, out_file, err_file, application_log)
+    finally:
+        if stdout != subprocess.PIPE:
+            stdout.close()
+        if stderr != subprocess.PIPE:
+            stderr.close()
+
+    return result
+
+
+def _build_command(command, configuration=None, runtime_environment=None):
+    """Build command to run in the specified runtime environment."""
+    if isinstance(command, (str, Path)) and Path(command).is_file():
+        command = Path(command)
+        command.chmod(command.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP)
+        command = str(command)
+
+    if runtime_environment:
+        if isinstance(runtime_environment, list):
+            command = [*runtime_environment, command]
+        else:
+            command = [runtime_environment, command]
+
+    if configuration:
+        if isinstance(command, list):
+            command = command + _convert_dict_to_args(configuration)
+        else:
+            command = [command, *_convert_dict_to_args(configuration)]
+
+    return command
+
+
+def _convert_dict_to_args(parameters):
+    """
+    Convert a dictionary of parameters to a list of command line arguments.
 
-    def __init__(self, test=False):
-        """Initialize JobManager."""
-        self._logger = logging.getLogger(__name__)
-        self.test = test
-        self.run_script = None
-        self.run_out_file = None
-
-    def submit(self, run_script=None, run_out_file=None, log_file=None):
-        """
-        Submit a job described by a shell script.
-
-        Parameters
-        ----------
-        run_script: str
-            Shell script describing the job to be submitted.
-        run_out_file: str or Path
-            Redirect output/error/job stream to this file (out,err,job suffix).
-        log_file: str or Path
-            The log file of the actual simulator (CORSIKA or sim_telarray).
-            Provided in order to print the log excerpt in case of run time error.
-        """
-        self.run_script = str(run_script)
-        run_out_file = Path(run_out_file)
-        self.run_out_file = str(run_out_file.parent.joinpath(run_out_file.stem))
-
-        self._logger.info(f"Submitting script {self.run_script}")
-        self._logger.info(f"Job output stream {self.run_out_file + '.out'}")
-        self._logger.info(f"Job error stream {self.run_out_file + '.err'}")
-        self._logger.info(f"Job log stream {self.run_out_file + '.job'}")
-
-        submit_result = self.submit_local(log_file)
-        if submit_result != 0:
-            raise JobExecutionError(f"Job submission failed with return code {submit_result}")
-
-    def submit_local(self, log_file):
-        """
-        Run a job script on the command line.
-
-        Parameters
-        ----------
-        log_file: str or Path
-            The log file of the actual simulator (CORSIKA or sim_telarray).
-            Provided in order to print the log excerpt in case of run time error.
-
-        Returns
-        -------
-        int
-            Return code of the executed script
-        """
-        self._logger.info("Running script locally")
-
-        if self.test:
-            self._logger.info("Testing (local)")
-            return 0
-
-        result = None
-        try:
-            with (
-                open(f"{self.run_out_file}.out", "w", encoding="utf-8") as stdout,
-                open(f"{self.run_out_file}.err", "w", encoding="utf-8") as stderr,
-            ):
-                result = subprocess.run(
-                    f"{self.run_script}",
-                    shell=True,
-                    check=True,
-                    text=True,
-                    stdout=stdout,
-                    stderr=stderr,
-                )
-        except subprocess.CalledProcessError as exc:
-            self._logger.error(gen.get_log_excerpt(f"{self.run_out_file}.err"))
-            if log_file.exists() and gen.get_file_age(log_file) < 5:
-                self._logger.error(gen.get_log_excerpt(log_file))
-            raise JobExecutionError("See excerpt from log file above\n") from exc
+    Parameters
+    ----------
+    parameters : dict
+        Dictionary containing parameters to convert.
+
+    Returns
+    -------
+    list
+        List of command line arguments.
+    """
+    args = []
+    for key, value in parameters.items():
+        if isinstance(value, bool):
+            if value:
+                args.append(f"--{key}")
+        elif isinstance(value, list):
+            args.extend([f"--{key}", *(str(item) for item in value)])
+        else:
+            args.extend([f"--{key}", str(value)])
+    return args
+
+
+def _raise_job_execution_error(exc, out_file, err_file, application_log):
+    """
+    Raise job execution error with log excerpt.
+
+    Parameters
+    ----------
+    exc: subprocess.CalledProcessError
+        The caught exception.
+    out_file: str or Path
+        Output stream file path.
+    err_file: str or Path
+        Error stream file path.
+    application_log: str or Path
+        The log file of the actual application.
+    """
+    logger.error(f"Job execution failed with return code {exc.returncode}")
+    logger.error(f"stderr: {exc.stderr}")
+
+    if out_file:
+        logger.error(f"Output log excerpt from {out_file}:\n{gen.get_log_excerpt(out_file)}")
+
+    if err_file:
+        logger.error(f"Error log excerpt from {err_file}:\n{gen.get_log_excerpt(err_file)}")
+
+    if application_log:
+        log = Path(application_log)
+        if log.exists() and gen.get_file_age(log) < 5:
+            logger.error(
+                f"Application log excerpt from {application_log}:\n"
+                f"{gen.get_log_excerpt(application_log)}"
+            )
 
-        return result.returncode if result else 0
+    raise JobExecutionError("See excerpt from log file above") from exc

simtools/job_execution/process_pool.py

@@ -0,0 +1,137 @@
+"""Run work in parallel and keep results ordered.
+
+Notes
+-----
+This module provides small wrappers around ``concurrent.futures.ProcessPoolExecutor``
+that make it easy to:
+
+- preserve input order in the returned results;
+- choose how many worker processes to use;
+- optionally run per-process initialization code; and
+- select a multiprocessing start method (e.g. ``"fork"`` or ``"spawn"``).
+
+Examples
+--------
+Parallelize a function with multiple arguments using tuples:
+
+.. code-block:: python
+
+    from simtools.job_execution.process_pool import process_pool_map_ordered
+
+    def power(args):
+        base, exp = args
+        return base ** exp
+
+    inputs = [(2, 3), (3, 2), (4, 0)]
+    results = process_pool_map_ordered(power, inputs, max_workers=3)
+    # results == [8, 9, 1]
+
+Parallelize methods that require an object instance:
+
+.. code-block:: python
+
+    from simtools.ray_tracing import MirrorPanelPSF
+    from simtools.job_execution.process_pool import process_pool_map_ordered
+
+    def worker_function(args):
+        mirror_idx, instance = args
+        measured_psf_mm = float(instance.measured_data[mirror_idx])
+        return instance.optimize_single_mirror(mirror_idx, measured_psf_mm)
+
+    instance = MirrorPanelPSF(label="test", args_dict=args)
+    worker_inputs = [(i, instance) for i in range(n_mirrors)]
+    results = process_pool_map_ordered(worker_function, worker_inputs)
+    # results contains per-mirror optimization outputs
+"""
+
+import logging
+import os
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from multiprocessing import get_context
+from typing import Any, TypeVar
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")  # type of input items
+R = TypeVar("R")  # type of return values
+
+
+def process_pool_map_ordered(
+    func,
+    items,
+    max_workers=None,
+    mp_start_method="fork",
+    initializer=None,
+    initargs=(),
+):
+    """Apply ``func`` to each item in ``items`` using a process pool.
+
+    Parameters
+    ----------
+    func : callable
+        Function to apply to each item.
+    items : iterable
+        Items to process.
+    max_workers : int or None
+        Number of worker processes. If ``None`` or ``<= 0``, uses ``os.cpu_count()``.
+    mp_start_method : str or None
+        Multiprocessing start method (e.g. ``"fork"``, ``"spawn"``). If ``None``,
+        uses the default context.
+    initializer : callable or None
+        Optional per-process initializer.
+    initargs : tuple
+        Arguments passed to ``initializer``.
+
+    Returns
+    -------
+    list
+        Results ordered to match the input item order.
+
+    Raises
+    ------
+    Exception
+        Any exception raised by a worker process is re-raised when collecting
+        the corresponding future result.
+
+    Notes
+    -----
+    This helper submits all items at once and collects results as futures
+    complete.
+    """
+    item_list = list(items)
+    n_items = len(item_list)
+
+    if max_workers is None or int(max_workers) <= 0:
+        max_workers = os.cpu_count() or 1
+
+    # create a temporary list of Nones to hold results in input order
+    results: list[R] = [None] * n_items  # type: ignore[list-item]
+
+    ctx = None
+    if mp_start_method:
+        ctx = get_context(str(mp_start_method))
+
+    logger.info(
+        "Starting ProcessPoolExecutor: n_items=%d, max_workers=%s, start_method=%s",
+        n_items,
+        str(max_workers),
+        str(mp_start_method),
+    )
+
+    executor_kwargs: dict[str, Any] = {
+        "max_workers": int(max_workers),
+        "initializer": initializer,
+        "initargs": tuple(initargs),
+    }
+    if ctx is not None:
+        executor_kwargs["mp_context"] = ctx
+
+    with ProcessPoolExecutor(**executor_kwargs) as executor:
+        future_to_index = {
+            executor.submit(func, item): index for index, item in enumerate(item_list)
+        }
+        for fut in as_completed(future_to_index):
+            index = future_to_index[fut]
+            results[index] = fut.result()
+
+    return results

simtools/layout/array_layout.py

@@ -331,7 +331,6 @@ class ArrayLayout:
         astropy.table.QTable
             Table with the telescope layout information.
         """
-        self._logger.debug("Initializing array (site and telescope parameters)")
         self._initialize_site_parameters_from_db()
         self._initialize_coordinate_systems()