gammasimtools 0.18.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

simtools/ray_tracing/ray_tracing.py

@@ -13,7 +13,7 @@ import matplotlib.pyplot as plt
 import numpy as np
 from astropy.table import QTable
 
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.model.model_utils import compute_telescope_transmission
 from simtools.ray_tracing.psf_analysis import PSFImage
 from simtools.simtel.simulator_ray_tracing import SimulatorRayTracing
@@ -49,6 +49,8 @@ class RayTracing:
         Single mirror mode flag.
     use_random_focal_length: bool
         Use random focal length flag.
+    random_focal_length_seed: int
+        Seed for the random number generator used for focal length variation.
     mirror_numbers: list, str
         List of mirror numbers (or 'all').
     """
@@ -71,6 +73,7 @@ class RayTracing:
         source_distance=10.0 * u.km,
         single_mirror_mode=False,
         use_random_focal_length=False,
+        random_focal_length_seed=None,
         mirror_numbers="all",
     ):
         """Initialize RayTracing class."""
@@ -87,6 +90,7 @@ class RayTracing:
         self.off_axis_angle = np.around(off_axis_angle.to("deg").value, 5)
         self.single_mirror_mode = single_mirror_mode
         self.use_random_focal_length = use_random_focal_length
+        self.random_focal_length_seed = random_focal_length_seed
         self.mirrors = self._initialize_mirror_configuration(source_distance, mirror_numbers)
         self.output_directory = self._io_handler.get_output_directory(
             label=self.label, sub_dir="ray_tracing"
@@ -185,7 +189,7 @@ class RayTracing:
         """
         _focal_length = self.telescope_model.get_parameter_value("mirror_focal_length")
         if self.use_random_focal_length:
-            rng = np.random.default_rng()
+            rng = np.random.default_rng(self.random_focal_length_seed)
             _random_focal_length = self.telescope_model.get_parameter_value("random_focal_length")
             if _random_focal_length[0] > 0.0:
                 _focal_length += rng.normal(loc=0, scale=_random_focal_length[0])

simtools/reporting/docs_read_parameters.py

@@ -9,14 +9,14 @@ from collections import defaultdict
 from itertools import groupby
 from pathlib import Path
 
+import matplotlib.pyplot as plt
 import numpy as np
 
 from simtools.db import db_handler
-from simtools.io_operations import io_handler
+from simtools.io import ascii_handler, io_handler
 from simtools.model.telescope_model import TelescopeModel
-from simtools.utils import general as gen
 from simtools.utils import names
-from simtools.visualization import plot_pixels
+from simtools.visualization import plot_pixels, plot_tables
 
 logger = logging.getLogger()
 
@@ -64,72 +64,141 @@ class ReadParameters:
             )
         self._model_version = model_version
 
-    def _convert_to_md(self, parameter, parameter_version, input_file):
+    def _generate_plots(self, parameter, parameter_version, input_file, outpath, design_type):
+        """Generate plots based on the parameter type."""
+        plot_names = []
+
+        if parameter == "camera_config_file":
+            plot_names = self._plot_camera_config(parameter, parameter_version, input_file, outpath)
+        elif parameter_version:
+            plot_names = self._plot_parameter_tables(
+                parameter, parameter_version, outpath, design_type
+            )
+
+        return plot_names
+
+    def _plot_camera_config(self, parameter, parameter_version, input_file, outpath):
+        """Generate plots for camera configuration files."""
+        if not parameter_version:
+            return []
+
+        plot_names = []
+        plot_name = input_file.stem.replace(".", "-")
+        plot_path = Path(f"{outpath}/{plot_name}").with_suffix(".png")
+
+        if not plot_path.exists():
+            plot_config = {
+                "file_name": input_file.name,
+                "telescope": self.array_element,
+                "parameter_version": parameter_version,
+                "site": self.site,
+                "model_version": self.model_version,
+                "parameter": parameter,
+            }
+
+            plot_pixels.plot(
+                config=plot_config,
+                output_file=Path(f"{outpath}/{plot_name}"),
+                db_config=self.db_config,
+            )
+            plot_names.append(plot_name)
+        else:
+            logger.info(
+                "Camera configuration file plot already exists: %s",
+                plot_path,
+            )
+            plot_names.append(plot_name)
+
+        return plot_names
+
+    def _plot_parameter_tables(self, parameter, parameter_version, outpath, design_type):
+        """Generate plots for parameter tables."""
+        telescope_design = self.db.get_design_model(
+            self.model_version, self.array_element, collection="telescopes"
+        )
+
+        if not self.array_element:
+            tel = None
+        elif not design_type:
+            tel = telescope_design
+        else:
+            tel = self.array_element
+
+        config_data = plot_tables.generate_plot_configurations(
+            parameter=parameter,
+            parameter_version=parameter_version,
+            site=self.site,
+            telescope=tel,
+            output_path=outpath,
+            plot_type="all",
+            db_config=self.db_config,
+        )
+
+        if not config_data:
+            return []
+
+        plot_configs, output_files = config_data
+        plot_names = [i.stem for i in output_files]
+
+        for plot_config, output_file in zip(plot_configs, output_files):
+            image_output_file = outpath / output_file.name
+            if not image_output_file.with_suffix(".png").exists():
+                plot_tables.plot(
+                    config=plot_config,
+                    output_file=image_output_file,
+                    db_config=self.db_config,
+                )
+                plt.close("all")
+
+        return plot_names
+
+    def _convert_to_md(self, parameter, parameter_version, input_file, design_type=False):
         """Convert a file to a Markdown file, preserving formatting."""
         input_file = Path(input_file)
 
+        if not input_file.exists():
+            raise FileNotFoundError(f"Data file not found: {input_file}")
+
+        # Store the markdown output file path early and don't modify it
         output_data_path = Path(self.output_path / "_data_files")
         output_data_path.mkdir(parents=True, exist_ok=True)
         output_file_name = Path(input_file.stem + ".md")
-        output_file = output_data_path / output_file_name
-        image_name = f"{self.array_element}_{parameter}_{self.model_version.replace('.', '-')}"
-        outpath = Path(io_handler.IOHandler().get_output_directory().parent / "_images")
-        outpath.mkdir(parents=True, exist_ok=True)
-        image_path = Path(f"{outpath}/{image_name}")
-
-        if parameter == "camera_config_file" and parameter_version:
-            image_path = Path(f"{outpath}/{input_file.stem.replace('.', '-')}")
-            if not (image_path.with_suffix(".png")).exists():
-                logger.info("Plotting camera configuration file: %s", input_file.name)
-                plot_config = {
-                    "file_name": input_file.name,
-                    "telescope": self.array_element,
-                    "parameter_version": parameter_version,
-                    "site": self.site,
-                    "model_version": self.model_version,
-                    "parameter": parameter,
-                }
+        relative_path = f"_data_files/{output_file_name}"
+        markdown_output_file = output_data_path / output_file_name
 
-                plot_pixels.plot(
-                    config=plot_config,
-                    output_file=image_path,
-                    db_config=self.db_config,
-                )
-            else:
-                logger.info(
-                    "Camera configuration file plot already exists: %s",
-                    image_path.with_suffix(".png"),
-                )
+        if not markdown_output_file.exists():
+            outpath = Path(io_handler.IOHandler().get_output_directory().parent / "_images")
+            outpath.mkdir(parents=True, exist_ok=True)
 
-        try:
-            # with input_file.open("r", encoding="utf-8") as infile:
-            file_contents = gen.read_file_encoded_in_utf_or_latin(input_file)
-
-            if self.model_version is not None:
-                with output_file.open("w", encoding="utf-8") as outfile:
-                    outfile.write(f"# {input_file.stem}\n")
-                    outfile.write(f"![Parameter plot.]({image_path}.png)\n\n")
-                    outfile.write(
-                        "\n\nThe full file can be found in the Simulation Model repository [here]"
-                        "(https://gitlab.cta-observatory.org/cta-science/simulations/"
-                        "simulation-model/simulation-models/-/blob/main/simulation-models/"
-                        f"model_parameters/Files/{input_file.name}).\n\n"
-                    )
-                    outfile.write("\n\n")
-                    outfile.write("The first 30 lines of the file are:\n")
-                    outfile.write("```\n")
-                    first_30_lines = "".join(file_contents[:30])
-                    outfile.write(first_30_lines)
-                    outfile.write("\n```")
+            plot_names = self._generate_plots(
+                parameter, parameter_version, input_file, outpath, design_type
+            )
+            # Write markdown file using the stored path
+            file_contents = ascii_handler.read_file_encoded_in_utf_or_latin(input_file)
+
+            with markdown_output_file.open("w", encoding="utf-8") as outfile:
+                outfile.write(f"# {input_file.stem}\n")
 
-        except FileNotFoundError as exc:
-            logger.exception(f"Data file not found: {input_file}.")
-            raise FileNotFoundError(f"Data file not found: {input_file}.") from exc
+                for plot_name in plot_names:
+                    outfile.write(f"![Parameter plot.]({outpath}/{plot_name}.png)\n\n")
 
-        return f"_data_files/{output_file_name}"
+                outfile.write(
+                    "\n\nThe full file can be found in the Simulation Model repository [here]"
+                    "(https://gitlab.cta-observatory.org/cta-science/simulations/"
+                    "simulation-model/simulation-models/-/blob/main/simulation-models/"
+                    f"model_parameters/Files/{input_file.name}).\n\n"
+                )
+                outfile.write("\n\n")
+                outfile.write("The first 30 lines of the file are:\n")
+                outfile.write("```\n")
+                first_30_lines = "".join(file_contents[:30])
+                outfile.write(first_30_lines)
+                outfile.write("\n```")
+
+        return relative_path
 
     def _format_parameter_value(
-        self, parameter, value_data, unit, file_flag, parameter_version=None
+        self, parameter, value_data, unit, file_flag, parameter_version=None, design_type=False
     ):
         """Format parameter value based on type."""
         if file_flag:
@@ -140,7 +209,9 @@ class ReadParameters:
                     "cta-science/simulations/simulation-model/simulation-models/-/blob/main/"
                     f"simulation-models/model_parameters/Files/{value_data})"
                 ).strip()
-            output_file_name = self._convert_to_md(parameter, parameter_version, input_file_name)
+            output_file_name = self._convert_to_md(
+                parameter, parameter_version, input_file_name, design_type
+            )
             return f"[{Path(value_data).name}]({output_file_name})".strip()
         if isinstance(value_data, (str | int | float)):
             return f"{value_data} {unit}".strip()
@@ -326,9 +397,10 @@ class ReadParameters:
             if value_data is None:
                 continue
 
+            design_type = names.is_design_type(telescope_model.name)
             file_flag = parameter_data.get("file", False)
             value = self._format_parameter_value(
-                parameter, value_data, unit, file_flag, parameter_version
+                parameter, value_data, unit, file_flag, parameter_version, design_type
             )
 
             description = parameter_descriptions.get(parameter).get("description")
@@ -338,10 +410,10 @@ class ReadParameters:
             inst_class = parameter_descriptions.get(parameter).get("inst_class")
 
             matching_instrument = parameter_data["instrument"] == telescope_model.name
-            if not names.is_design_type(telescope_model.name) and matching_instrument:
+            if not design_type and matching_instrument:
                 parameter = f"***{parameter}***"
                 parameter_version = f"***{parameter_version}***"
-                if not re.match(r"^\[.*\]\(.*\)$", value.strip()):
+                if not self.is_markdown_link(value):
                     value = f"***{value}***"
                 description = f"***{description}***"
                 short_description = f"***{short_description}***"
@@ -723,7 +795,7 @@ class ReadParameters:
             if not names.is_design_type(array_element) and matching_instrument:
                 parameter = f"***{parameter}***"
                 parameter_version = f"***{parameter_version}***"
-                if not re.match(r"^\[.*\]\(.*\)$", value.strip()):
+                if not self.is_markdown_link(value):
                     value = f"***{value}***"
                 description = f"***{description}***"
                 short_description = f"***{short_description}***"
@@ -750,6 +822,22 @@ class ReadParameters:
 
         return data
 
+    def is_markdown_link(self, value):
+        """
+        Return True if the string is a Markdown-style link: [text](target).
+
+        Parameters
+        ----------
+        value : str
+            The string to check.
+
+        Returns
+        -------
+        bool
+            True if the string is a Markdown link, False otherwise.
+        """
+        return bool(re.fullmatch(r"\[[^\]]*\]\([^)]+\)", value.strip()))
+
     def produce_calibration_reports(self):
         """Write calibration reports."""
         calibration_array_elements = self.db.get_array_elements(

simtools/runners/corsika_runner.py

@@ -4,7 +4,7 @@ import logging
 import stat
 from pathlib import Path
 
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.runners.runner_services import RunnerServices
 
 __all__ = ["CorsikaRunner", "MissingRequiredEntryInCorsikaConfigError"]

simtools/runners/corsika_simtel_runner.py

@@ -43,6 +43,7 @@ class CorsikaSimtelRunner:
         use_multipipe=False,
         sim_telarray_seeds=None,
         sequential=False,
+        calibration_runner_args=None,
     ):
         self._logger = logging.getLogger(__name__)
         self.corsika_config = (
@@ -55,6 +56,7 @@ class CorsikaSimtelRunner:
         self.sim_telarray_seeds = sim_telarray_seeds
         self.label = label
         self.sequential = "--sequential" if sequential else ""
+        self.calibration_runner_args = calibration_runner_args
 
         self.base_corsika_config.set_output_file_and_directory(use_multipipe)
         self.corsika_runner = CorsikaRunner(
@@ -124,11 +126,18 @@ class CorsikaSimtelRunner:
 
         with open(multipipe_file, "w", encoding="utf-8") as file:
             for simulator_array in self.simulator_array:
-                run_command = simulator_array.make_run_command(
-                    run_number=run_number,
-                    input_file="-",  # instruct sim_telarray to take input from standard output
-                    weak_pointing=self._determine_pointing_option(self.label),
-                )
+                if self.calibration_runner_args:
+                    run_command = simulator_array.make_run_command_for_calibration_simulations(
+                        run_number=run_number,
+                        input_file="-",  # instruct sim_telarray to take input from standard output
+                        calibration_runner_args=self.calibration_runner_args,
+                    )
+                else:
+                    run_command = simulator_array.make_run_command(
+                        run_number=run_number,
+                        input_file="-",  # instruct sim_telarray to take input from standard output
+                        weak_pointing=self._determine_pointing_option(self.label),
+                    )
                 file.write(f"{run_command}")
                 file.write("\n")
         self._logger.info(f"Multipipe script: {multipipe_file}")

simtools/runners/runner_services.py

@@ -3,7 +3,7 @@
 import logging
 from pathlib import Path
 
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 
 _logger = logging.getLogger(__name__)
 
@@ -139,11 +139,16 @@ class RunnerServices:
         zenith = self.corsika_config.get_config_parameter("THETAP")[0]
         azimuth = self.corsika_config.azimuth_angle
         run_number_string = self._get_run_number_string(info_for_file_name["run_number"])
-        return (
+        prefix = (
             f"{info_for_file_name['primary']}_{run_number_string}_"
-            f"za{round(zenith):02}deg_azm{azimuth:03}deg_"
-            f"{info_for_file_name['site']}_{info_for_file_name['array_name']}_"
-            f"{info_for_file_name['model_version']}{file_label}"
+            if info_for_file_name["primary"]
+            else f"{run_number_string}_"
+        )
+        return (
+            prefix
+            + f"za{round(zenith):02}deg_azm{azimuth:03}deg_"
+            + f"{info_for_file_name['site']}_{info_for_file_name['array_name']}_"
+            + f"{info_for_file_name['model_version']}{file_label}"
         )
 
     def _get_log_file_path(self, file_type, file_name):

simtools/runners/simtools_runner.py

@@ -0,0 +1,267 @@
+"""Tools for running applications in the simtools framework."""
+
+import shutil
+import subprocess
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools import dependencies
+from simtools.io import ascii_handler
+
+
+def run_applications(args_dict, db_config, logger):
+    """
+    Run simtools applications step-by-step as defined in a configuration file.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Dictionary containing command line arguments.
+    db_config : dict
+        Database configuration
+    logger : logging.Logger
+        Logger for logging application output.
+    """
+    configurations, runtime_environment, log_file = _read_application_configuration(
+        args_dict["configuration_file"], args_dict.get("steps"), logger
+    )
+    run_time = (
+        read_runtime_environment(runtime_environment)
+        if not args_dict["ignore_runtime_environment"]
+        else []
+    )
+
+    with log_file.open("w", encoding="utf-8") as file:
+        file.write("Running simtools applications\n")
+        file.write(dependencies.get_version_string(db_config, run_time))
+
+        for config in configurations:
+            app = config.get("application")
+            if not config.get("run_application"):
+                logger.info(f"Skipping application: {app}")
+                continue
+            logger.info(f"Running application: {app}")
+            stdout, stderr = run_application(run_time, app, config.get("configuration"), logger)
+            file.write("=" * 80 + "\n")
+            file.write(f"Application: {app}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}\n")
+
+
+def run_application(runtime_environment, application, configuration, logger):
+    """
+    Run a simtools application and return stdout and stderr.
+
+    Allow to specify a runtime environment (e.g., Docker) and a working directory.
+
+    Parameters
+    ----------
+    runtime_environment : list
+        Command to run the application in the specified runtime environment.
+    application : str
+        Name of the application to run.
+    configuration : dict
+        Configuration for the application.
+    logger : logging.Logger
+        Logger for logging application output.
+
+    Returns
+    -------
+    tuple
+        stdout and stderr from the application run.
+
+    """
+    command = [application, *_convert_dict_to_args(configuration)]
+    if runtime_environment:
+        command = runtime_environment + command
+    try:
+        result = subprocess.run(
+            command,
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+    except subprocess.CalledProcessError as exc:
+        logger.error(f"Error running application {application}: {exc.stderr}")
+        raise exc
+
+    return result.stdout, result.stderr
+
+
+def _convert_dict_to_args(parameters):
+    """
+    Convert a dictionary of parameters to a list of command line arguments.
+
+    Parameters
+    ----------
+    parameters : dict
+        Dictionary containing parameters to convert.
+
+    Returns
+    -------
+    list
+        List of command line arguments.
+    """
+    args = []
+    for key, value in parameters.items():
+        if isinstance(value, bool):
+            if value:
+                args.append(f"--{key}")
+        elif isinstance(value, list):
+            args.extend([f"--{key}", *(str(item) for item in value)])
+        else:
+            args.extend([f"--{key}", str(value)])
+    return args
+
+
+def _read_application_configuration(configuration_file, steps, logger):
+    """
+    Read application configuration from file and modify for setting workflows.
+
+    Strong assumptions on the structure of input and output files:
+
+    - configuration file is expected to be in './input/<workflow directory>/<yaml file>'
+    - output files will be written out to './output/<workflow directory>/'
+
+    Replaces the placeholders in the configuration file with the actual values.
+    Sets 'USE_PLAIN_OUTPUT_PATH' to True for all applications.
+
+    Parameters
+    ----------
+    configuration_file : str
+        Configuration file name.
+    steps : list
+        List of steps to be executed (None: all steps).
+    logger : Logger
+        Logger object.
+
+    Returns
+    -------
+    dict
+        Application configuration.
+    dict:
+        Runtime environment configuration.
+    Path
+        Path to the log file.
+
+    """
+    job_configuration = ascii_handler.collect_data_from_file(configuration_file)
+    configurations = job_configuration.get("applications")
+    output_path, setting_workflow = _set_input_output_directories(configuration_file)
+    logger.info(f"Setting workflow output path to {output_path}")
+    for step_count, config in enumerate(configurations, start=1):
+        config["run_application"] = step_count in steps if steps else True
+        config = gen.change_dict_keys_case(config, True)
+        config["configuration"] = _replace_placeholders_in_configuration(
+            config.get("configuration", {}),
+            output_path,
+            setting_workflow,
+        )
+        configurations[step_count - 1] = config
+
+    return (
+        configurations,
+        job_configuration.get("runtime_environment"),
+        output_path / "simtools.log",
+    )
+
+
+def _replace_placeholders_in_configuration(
+    configuration, output_path, setting_workflow, place_holder="__SETTING_WORKFLOW__"
+):
+    """
+    Replace placeholders in the configuration dictionary.
+
+    Parameters
+    ----------
+    configuration : dict
+        Configuration dictionary.
+    output_path : Path
+        Path to the output directory.
+    setting_workflow : str
+        The setting workflow to replace the placeholder with.
+    place_holder : str
+        Placeholder to be replaced.
+
+    Returns
+    -------
+    dict
+        Configuration dictionary with placeholders replaced.
+    """
+    for key, value in configuration.items():
+        if isinstance(value, str):
+            configuration[key] = value.replace(place_holder, setting_workflow)
+        if isinstance(value, list):
+            configuration[key] = [
+                item.replace(place_holder, setting_workflow) if isinstance(item, str) else item
+                for item in value
+            ]
+    if output_path:
+        configuration["use_plain_output_path"] = True
+        configuration["output_path"] = str(output_path)
+
+    return configuration
+
+
+def _set_input_output_directories(path):
+    """
+    Set input and output directories based on the configuration file path.
+
+    Tuned to simulation models setting workflows.
+
+    Parameters
+    ----------
+    path : str or Path
+        Path to the configuration file.
+
+    Returns
+    -------
+    tuple
+        The first part is the 'input' directory, the second part is the subdirectory name
+    """
+    path = Path(path).resolve()
+    try:
+        input_index = path.parts.index("input")
+        # Get all parts after 'input', excluding the filename
+        subdirs = path.parts[input_index + 1 : -1]
+        setting_workflow = "/".join(subdirs)
+        workflow_dir = path.parts[input_index]
+    except (ValueError, IndexError) as exc:
+        raise ValueError(f"Could not find subdirectory under 'input': {exc}") from exc
+
+    output_path = Path(str(workflow_dir).replace("input", "output")) / Path(setting_workflow)
+    output_path.mkdir(parents=True, exist_ok=True)
+    return output_path, "/".join(subdirs)
+
+
+def read_runtime_environment(runtime_environment, workdir="/workdir/external/"):
+    """
+    Read the runtime environment (e.g. docker runtime) and generate the required command.
+
+    Parameters
+    ----------
+    runtime_environment : str or None
+        Path to the runtime environment configuration file.
+
+    Returns
+    -------
+    list
+        Runtime command.
+    """
+    if runtime_environment is None:
+        return []
+
+    engine = runtime_environment.get("container_engine", "docker")
+    if shutil.which(engine) is None:
+        raise RuntimeError(f"Container engine '{engine}' not found.")
+    cmd = [engine, "run", "--rm", "-v", f"{Path.cwd()}:{workdir}", "-w", workdir]
+
+    if options := runtime_environment.get("options"):
+        for opt in options:
+            cmd.extend(opt.split())
+
+    if env := runtime_environment.get("env_file"):
+        cmd += ["--env-file", env]
+    if net := runtime_environment.get("network"):
+        cmd += ["--network", net]
+
+    cmd.append(runtime_environment["image"])
+    return cmd